diff --git a/scripts/vllm_dissag/ARCHITECTURE.md b/scripts/vllm_dissag/ARCHITECTURE.md index 5e64b83..cc0396e 100644 --- a/scripts/vllm_dissag/ARCHITECTURE.md +++ b/scripts/vllm_dissag/ARCHITECTURE.md @@ -12,31 +12,31 @@ How the pieces fit, from sbatch down to the per-node `vllm serve` workers. ```mermaid flowchart TB - user([user / CI]) -->|"sbatch + env\n(CONNECTOR, WIDE_EP, EP_BACKEND,\nMODEL_NAME, xP, yD, RUN_MORI/RUN_DEEPEP)"| slurm + user([user / CI]) -->|"sbatch + env
(CONNECTOR, WIDE_EP, EP_BACKEND,
MODEL_NAME, xP, yD, RUN_MORI/RUN_DEEPEP)"| slurm subgraph host["Submit host"] - slurm["run_xPyD_models.slurm\n• resolve script dir\n• validate MODEL_NAME (VALID_MODELS)\n• axis resolution + back-compat shim\n• pick nodes, gather IPs\n• docker run per node (-e env plumb)"] + slurm["run_xPyD_models.slurm
• resolve script dir
• validate MODEL_NAME (VALID_MODELS)
• axis resolution + back-compat shim
• pick nodes, gather IPs
• docker run per node (-e env plumb)"] end - slurm -->|"srun + docker run\n(one container per node)"| n0 + slurm -->|"srun + docker run
(one container per node)"| n0 slurm --> n1 slurm --> n2 slurm --> n3 subgraph cluster["Allocated nodes (xP prefill + yD decode)"] - n0["NODE_RANK 0\nPrefill MASTER + Proxy"] - n1["NODE_RANK 1..xP-1\nPrefill CHILD"] - n2["NODE_RANK xP\nDecode MASTER"] - n3["NODE_RANK xP+1..end\nDecode CHILD"] + n0["NODE_RANK 0
Prefill MASTER + Proxy"] + n1["NODE_RANK 1..xP-1
Prefill CHILD"] + n2["NODE_RANK xP
Decode MASTER"] + n3["NODE_RANK xP+1..end
Decode CHILD"] end subgraph driverbox["Inside each container: vllm_disagg.sh (the one launcher)"] - driver["DRIVER\n• axis select + validate\n• topology math\n• models.yaml parse\n• role branch\n• barrier / benchmark / cleanup"] - para["parallelism.sh\nTP vs wideEP arg helpers"] - yaml[("models.yaml\nper-model flags + env")] + driver["DRIVER
• axis select + validate
• topology math
• models.yaml parse
• role branch
• barrier / benchmark / cleanup"] + para["parallelism.sh
TP vs wideEP arg helpers"] + yaml[("models.yaml
per-model flags + env")] conn{{"connectors/.sh"}} - rixl["rixl.sh\nNixlConnector\nTP + DeepEP"] - moriio["moriio.sh\nMoRIIOConnector\nMoRIIO+TP + MoRI-EP"] + rixl["rixl.sh
NixlConnector
TP + DeepEP"] + moriio["moriio.sh
MoRIIOConnector
MoRIIO+TP + MoRI-EP"] driver --> para driver --> yaml driver --> conn @@ -45,9 +45,9 @@ flowchart TB end n0 --> driverbox - driver -->|connector_launch_worker| vllm[["vllm serve\n(prefill / decode worker)"]] - driver -->|rank 0 only| proxy[["proxy / router\n(co-located)"]] - proxy --> bench["benchmark_xPyD.sh\n→ *_CONCURRENCY.log"] + driver -->|connector_launch_worker| vllm[["vllm serve
(prefill / decode worker)"]] + driver -->|rank 0 only| proxy[["proxy / router
(co-located)"]] + proxy --> bench["benchmark_xPyD.sh
→ *_CONCURRENCY.log"] ``` --- @@ -62,24 +62,24 @@ flowchart TD start([env in]) --> q0{CONNECTOR set?} q0 -->|no| shim{legacy flag?} - shim -->|RUN_MORI=1| m["CONNECTOR=moriio\nWIDE_EP=1\nEP_BACKEND=mori"] - shim -->|RUN_DEEPEP=1| d["CONNECTOR=rixl\nWIDE_EP=1\nEP_BACKEND=deepep"] - shim -->|neither| def["CONNECTOR=rixl\nWIDE_EP=0 (TP)"] - q0 -->|yes| explicit["use explicit\nCONNECTOR / WIDE_EP / EP_BACKEND"] + shim -->|RUN_MORI=1| m["CONNECTOR=moriio
WIDE_EP=1
EP_BACKEND=mori"] + shim -->|RUN_DEEPEP=1| d["CONNECTOR=rixl
WIDE_EP=1
EP_BACKEND=deepep"] + shim -->|neither| def["CONNECTOR=rixl
WIDE_EP=0 (TP)"] + q0 -->|yes| explicit["use explicit
CONNECTOR / WIDE_EP / EP_BACKEND"] m --> vconn d --> vconn def --> vconn explicit --> vconn - vconn{validate CONNECTOR\nin rixl|moriio} + vconn{"validate CONNECTOR
in rixl|moriio"} vconn -->|invalid| err1[["abort: invalid CONNECTOR"]] - vconn -->|valid| vwide{validate WIDE_EP\nin 0|1} + vconn -->|valid| vwide{"validate WIDE_EP
in 0|1"} vwide -->|invalid| err2[["abort: invalid WIDE_EP"]] vwide -->|valid| qwide{WIDE_EP == 1?} - qwide -->|no (TP)| okTP["EP_BACKEND = n/a"] - qwide -->|yes wideEP| qpair{connector ↔ EP_BACKEND} + qwide -->|"no (TP)"| okTP["EP_BACKEND = n/a"] + qwide -->|yes wideEP| qpair{"connector ↔ EP_BACKEND"} qpair -->|moriio + mori| okM["OK: all2all = mori_*"] qpair -->|rixl + deepep| okD["OK: all2all = deepep_*"] qpair -->|moriio + deepep| err3[["abort: cross-pair"]] @@ -95,12 +95,12 @@ flowchart TD ```mermaid flowchart LR subgraph TP["WIDE_EP=0 (TP, PARALLEL_MODE=tp)"] - rt["rixl + TP\nNixlConnector\n--tensor-parallel-size"] - mt["moriio + TP (NEW)\nMoRIIOConnector\n--tensor-parallel-size"] + rt["rixl + TP
NixlConnector
--tensor-parallel-size"] + mt["moriio + TP (NEW)
MoRIIOConnector
--tensor-parallel-size"] end subgraph EP["WIDE_EP=1 (wideEP, PARALLEL_MODE=dp)"] - rd["rixl + deepep\nNixlConnector\n-tp 1 --data-parallel-size\n--enable-expert-parallel\n--all2all-backend deepep_*"] - md["moriio + mori\nMoRIIOConnector\n-tp 1 --data-parallel-size\n--enable-expert-parallel\n--all2all-backend mori_*"] + rd["rixl + deepep
NixlConnector
-tp 1 --data-parallel-size
--enable-expert-parallel
--all2all-backend deepep_*"] + md["moriio + mori
MoRIIOConnector
-tp 1 --data-parallel-size
--enable-expert-parallel
--all2all-backend mori_*"] end ``` @@ -116,16 +116,16 @@ stateDiagram-v2 [*] --> ResolveAxes ResolveAxes --> ParseModel: CONNECTOR/WIDE_EP/EP_BACKEND valid ResolveAxes --> Abort: invalid / cross-pair - ParseModel: Parse models.yaml\n(export env: block,\nresolve prefill/decode flags) + ParseModel: Parse models.yaml
(export env: block,
resolve prefill/decode flags) ParseModel --> LoadProfiles - LoadProfiles: source parallelism.sh +\nconnectors/.sh →\nconnector_init + LoadProfiles: source parallelism.sh +
connectors/.sh →
connector_init LoadProfiles --> DryRunEmit: DRY_RUN=1 - DryRunEmit: emit assembled\nvllm serve argv + DryRunEmit: emit assembled
vllm serve argv DryRunEmit --> [*] LoadProfiles --> Barrier: normal run - Barrier: container barrier\n(socket_barrier.py)\n+ connector_runtime_patch + Barrier: container barrier
(socket_barrier.py)
+ connector_runtime_patch Barrier --> RoleBranch state RoleBranch <> @@ -134,16 +134,16 @@ stateDiagram-v2 RoleBranch --> DecodeMaster: RANK==xP RoleBranch --> DecodeChild: RANK>xP - PrefillMaster: launch worker (kv_producer)\n+ wait_workers_ready\n+ start_proxy + benchmark + PrefillMaster: launch worker (kv_producer)
+ wait_workers_ready
+ start_proxy + benchmark PrefillChild: launch worker (kv_producer, headless) DecodeMaster: launch worker (kv_consumer) DecodeChild: launch worker (kv_consumer, headless) - PrefillMaster --> Cleanup: bench done →\nkill proxy + worker + PrefillMaster --> Cleanup: bench done →
kill proxy + worker PrefillChild --> WaitProxy DecodeMaster --> WaitProxy DecodeChild --> WaitProxy - WaitProxy: wait_for_proxy_and_cleanup\n(barrier → wait close → kill) + WaitProxy: wait_for_proxy_and_cleanup
(barrier → wait close → kill) WaitProxy --> Cleanup Cleanup --> [*] Abort --> [*] @@ -189,12 +189,12 @@ sequenceDiagram | Hook | rixl.sh | moriio.sh | |------|---------|-----------| -| `connector_init` | ports 2584/router; PROXY_TYPE; barrier 5000/15000 | ports 20005/10001; per-role MoRI backend; barrier 2222 | +| `connector_init` | serve 2584; `ROUTER_PORT` 18001; barrier 5000 (TP) / 15000 (deepep); `PROXY_TYPE` vllm_router\|toy_proxy | serve 20005; `ROUTER_PORT` 30000, toy 10001; barrier 2222; `PROXY_TYPE` vllm_router\|moriio_toy | | `connector_setup_env` | UCX/NIXL (TP) or RocSHMEM/UCX/NIXL+#39276 sed (deepep) | MoRI/RDMA/RocSHMEM + caches | | `connector_runtime_patch` | no-op (deepep patches inline in setup_env) | no-op (disagg fixes are in-source in the image's vLLM) | | `connector_launch_worker` | TP server **or** deepep DP+EP server | MoRIIO+TP server **or** MoRI-EP DP+EP server | | `connector_wait_workers_ready` | TP: socket_barrier; deepep: log-signal | log-signal on prefill+decode masters | -| `connector_start_proxy` | vllm_router / toy_proxy over all P/D IPs | moriio_toy_proxy (or router) + curl probe | +| `connector_start_proxy` | vllm_router / toy_proxy over all P/D IPs | vllm_router / moriio_toy + curl probe | --- @@ -206,19 +206,19 @@ Where each piece of configuration lives, and the precedence when they overlap. flowchart TD subgraph yamlcat["models.yaml (per model)"] bf["base_flags"] - mf["tp_flags / dp_flags\n(by WIDE_EP)"] + mf["tp_flags / dp_flags
(by WIDE_EP)"] rf["prefill.{tp,dp} / decode.{tp,dp}"] ef["experimental_flags"] envb["env: { VAR: val }"] end - bf & mf & rf & ef --> compose["MODEL_CONFIG_PREFILL / _DECODE\n(passed to connector_launch_worker)"] + bf & mf & rf & ef --> compose["MODEL_CONFIG_PREFILL / _DECODE
(passed to connector_launch_worker)"] subgraph envlayer["ENV precedence (low → high)"] - c1["connector default\nexport VAR=\${VAR:-default}"] --> c2["models.yaml env:\n(exported before setup_env)"] --> c3["slurm -e VAR=…\n(submit/site override, wins)"] + c1["connector default
export VAR=\${VAR:-default}"] --> c2["models.yaml env:
(exported before setup_env)"] --> c3["slurm -e VAR=…
(submit/site override, wins)"] end envb -.-> c2 - note["Launcher owns: connector, transfer,\nparallelism DEGREE (--tensor/data-parallel-size,\n--all2all-backend, kv-transfer-config).\nyaml owns: model-tuning flags + env only."] + note["Launcher owns: connector, transfer,
parallelism DEGREE (--tensor/data-parallel-size,
--all2all-backend, kv-transfer-config).
yaml owns: model-tuning flags + env only."] ``` --- @@ -234,10 +234,5 @@ flowchart TD | `connectors/moriio.sh` | MoRIIOConnector profile (MoRIIO+TP + MoRI-EP) | | `connectors/{rixl,moriio}.env` | per-connector platform env (expandable_segments:False etc.), forwarded via `docker -e` | | `models.yaml` | per-model flags + env catalog | -| `tests/gate_check.sh` | combo-gate unit tests (model × connector × WIDE_EP allow/reject) | -| `tests/argv_assert.sh` | per-cell `vllm serve` flag/env assertions from the driver's `DRY_RUN=1` output | -| `tests/run_all.sh` | runs all offline gates (gate_check + argv_assert) | -| `tests/{drive_cell,harvest,run_interactive}.sh` | interactive-allocation live-test drivers | -| `tests/TEST_PLAN.md` | before/after verification plan | | `benchmark_xPyD.sh`, `benchmark_long_context.sh`, `benchmark_niah.{sh,py}`, `benchmark_parser.py`, `parse_to_csv.py` | benchmark + parsing (NIAH = long-context retrieval, vllm#47042) | | `socket_barrier.py`, `socket_wait.py`, `salloc_launch.sh` | node coordination + salloc helper | diff --git a/scripts/vllm_dissag/README.MD b/scripts/vllm_dissag/README.MD index 17660ab..19fe320 100644 --- a/scripts/vllm_dissag/README.MD +++ b/scripts/vllm_dissag/README.MD @@ -58,15 +58,16 @@ slurm gate rejects any other pairing. Dense models are TP-only; the DeepSeek fam - A Slurm cluster with `xP + yD` nodes (minimum 2: `xP=1`, `yD=1`), 8 GPUs/node. - A Docker image with vLLM + the KV connector(s) + NIC drivers. See **Building the Docker image**. -- Model weights present on all allocated nodes under `/mnt/m2m_nobackup/models_blog/` or - `/shared_inference/models_blog/` (or set `MODEL_DIR`). +- Model weights present on all allocated nodes at `$MODEL_DIR/$MODEL_NAME` (`MODEL_DIR` default + `/shared_inference/models_blog/`). The slurm resolves `MODEL_PATH` from these; override + `MODEL_DIR` to point elsewhere. - A shared filesystem for logs. ## Building the Docker image -No public prebuilt image ships — build your own from the one fullsource Dockerfile (MoRI + -AITER + vLLM + vllm-router + optional UCX/RIXL/rocSHMEM/DeepEP, from Git pins on an open -ROCm ci_base): +No public prebuilt image ships — build your own from the single Dockerfile +`docker/vllm_disagg_inference.ubuntu.amd.Dockerfile` (MoRI + AITER + vLLM + vllm-router + +optional UCX/RIXL/rocSHMEM/DeepEP, from Git pins on an open ROCm ci_base): ```bash cd MAD @@ -172,10 +173,12 @@ separate port from the vLLM server. **Port defaults by connector:** -| Connector | vLLM server port | Proxy port | -|-----------|------------------|------------| -| `rixl` (TP or DeepEP) | 2584 | `ROUTER_PORT` (30000 default) | -| `moriio` (MoRI-EP or TP) | 20005 | `ROUTER_PORT` (30000) with the default vllm_router; 10001 with `PROXY_TYPE=moriio_toy` | +| Connector | vLLM server port | Proxy port (`vllm_router`) | Toy-proxy port | +|-----------|------------------|----------------------------|----------------| +| `rixl` (TP or DeepEP) | 2584 | `ROUTER_PORT` (default 18001) | 18001 (`toy_proxy`) | +| `moriio` (MoRI-EP or TP) | 20005 | `ROUTER_PORT` (default 30000) | 10001 (`moriio_toy`) | + +Override any port with the matching env var (`ROUTER_PORT`, `MORI_SERVE_PORT`, etc.). ## Proxy options @@ -184,24 +187,22 @@ separate port from the vLLM server. `ROUTER_BINARY` (both connectors): path to the `vllm-router` binary. Images that bake the router ship it on `PATH`, so this can be left unset. Runtime-only images that do NOT ship a -router (e.g. the mori121 MoRI-EP image) require `ROUTER_BINARY=` — point it at a built +router require `ROUTER_BINARY=` — point it at a built `vllm-router` on a shared mount. If unset and none is on `PATH`, the launcher aborts with a clear error. -## Validation status (image capability) - -The consolidated launcher assembles correct argv for all 4 combos (offline gates in `tests/`), -but each combo also needs the connector's transport built into the image: +## Image capability by combo -| Combo | Live-validated | Image requirement | -|-------|----------------|-------------------| -| moriio + TP | ✅ | MoRI (mori121 image) | -| moriio + wideEP (mori) | ✅ 1P/1D + 2P/2D | MoRI (mori121 image) | -| rixl + TP | needs a NIXL-validated image | NixlConnector + working NCCL TP path | -| rixl + wideEP (deepep) | needs a DeepEP-validated image | DeepEP all2all kernels | +The launcher assembles correct argv for all 4 combos, but each combo needs its transport +built into the image. Build with `WITH_NIXL=1` (default) to get all four; `WITH_NIXL=0` gives +a MoRI-only image (moriio TP/wideEP). -The `mori_ep_fullsource` Dockerfile builds the MoRI stack; rixl/DeepEP combos need an image -where those transports are validated (e.g. develop's `vllm_disagg_inference` for NIXL). +| Combo | Transport needed in image | +|-------|---------------------------| +| moriio + TP | MoRI | +| moriio + wideEP (mori) | MoRI | +| rixl + TP | NixlConnector (+ working NCCL TP path) | +| rixl + wideEP (deepep) | DeepEP all2all kernels | ## DeepEP options (optional) @@ -226,6 +227,18 @@ Parse a results log: python3 benchmark_parser.py /benchmark_XXX_CONCURRENCY.log ``` +The default sweep runs `benchmark_xPyD.sh`. Two alternatives plug in via `BENCHMARK_SCRIPT_FILE`: + +- **`benchmark_long_context.sh`** — steady-state long-context throughput/latency with per-shape + warmup (each ISL/OSL cell is warmed before measurement, so JIT/cudagraph/handshake costs are + excluded). Use for large-ISL runs, e.g. `BENCHMARK_COMBINATIONS="28000/1000"`. +- **`benchmark_niah.sh`** — long-context *retrieval* accuracy (see below). + +```bash +export BENCHMARK_SCRIPT_FILE=benchmark_long_context.sh +export BENCHMARK_COMBINATIONS="28000/1000" BENCHMARK_CON="1 4 8" +``` + ### Long-context retrieval (NIAH) `benchmark_niah.{sh,py}` run a needle-in-a-haystack retrieval sweep (adapted from @@ -241,20 +254,10 @@ export NIAH_WORDS="2000,8000,20000,35000" # context sizes (words); optional Or run `benchmark_niah.py` standalone against any live OpenAI-compatible endpoint (`NIAH_URL`, `NIAH_MODEL`). -## Testing the launcher (no GPUs) - -Offline suites validate the launcher without a cluster — run after any change to the -launcher/connectors/yaml: - -```bash -bash tests/run_all.sh # gate_check + argv_assert; expect ALL OFFLINE SUITES PASSED -``` - -- `tests/gate_check.sh` — connector × WIDE_EP combo validation (valid/invalid pairings, back-compat shims). -- `tests/argv_assert.sh` — asserts expected `vllm serve` flags/env are present/absent per cell, from - the launcher's `DRY_RUN=1` output (no GPUs, no container). +## Inspecting the launcher (no GPUs) -You can also dry-run any single cell to inspect the assembled command: +Dry-run any cell to print the assembled `vllm serve` argv without launching — useful to +sanity-check flags after editing the launcher/connectors/`models.yaml`: ```bash CONNECTOR=moriio WIDE_EP=1 EP_BACKEND=mori DRY_RUN=1 NODE_RANK=0 \ diff --git a/scripts/vllm_dissag/connectors/moriio.env b/scripts/vllm_dissag/connectors/moriio.env index 29ed23b..e4045be 100644 --- a/scripts/vllm_dissag/connectors/moriio.env +++ b/scripts/vllm_dissag/connectors/moriio.env @@ -1,5 +1,5 @@ -# Platform-mandatory runtime env for the moriio (MoRIIO) connector on the -# ROCm-7.2.3 / mori121 image. Applies to ALL moriio runs (TP and wideEP, every model). +# Platform-mandatory runtime env for the moriio (MoRIIO) connector on ROCm 7.2.3. +# Applies to ALL moriio runs (TP and wideEP, every model). # # These must reach the container at PID 1 — PyTorch reads the alloc-conf at import, # so a late shell export inside the launcher is too late. The slurm sources this file diff --git a/scripts/vllm_dissag/connectors/moriio.sh b/scripts/vllm_dissag/connectors/moriio.sh index 44c64d6..db3788a 100644 --- a/scripts/vllm_dissag/connectors/moriio.sh +++ b/scripts/vllm_dissag/connectors/moriio.sh @@ -35,11 +35,11 @@ connector_init() { ROUTER_PORT="${ROUTER_PORT:-${VLLM_ROUTER_HTTP_PORT:-30000}}" [ "$PROXY_TYPE" == "vllm_router" ] && PROXY_PORT="${ROUTER_PORT}" - # Per-role MoRI all2all backend (wideEP only). Newer vLLM images (the v1.2.0 - # MoRI-EP image) split the kernel: prefill=high_throughput (InterNodeV1), - # decode=low_latency (InterNodeV1LL) and REJECT the bare "mori" alias. Default - # to the per-role names; override via PREFILL_MORI_BACKEND/DECODE_MORI_BACKEND - # (or VLLM_ALL2ALL_BACKEND for the prefill side). + # Per-role MoRI all2all backend (wideEP only). Newer MoRI-EP images split the + # kernel: prefill=high_throughput (InterNodeV1), decode=low_latency + # (InterNodeV1LL) and REJECT the bare "mori" alias. Default to the per-role + # names; override via PREFILL_MORI_BACKEND/DECODE_MORI_BACKEND (or + # VLLM_ALL2ALL_BACKEND for the prefill side). PREFILL_MORI_BACKEND="${PREFILL_MORI_BACKEND:-${VLLM_ALL2ALL_BACKEND:-mori_high_throughput}}" DECODE_MORI_BACKEND="${DECODE_MORI_BACKEND:-mori_low_latency}" } @@ -58,7 +58,7 @@ connector_setup_env() { export VLLM_LOGGING_LEVEL=INFO export VLLM_USE_V1=1 - export VLLM_ALL2ALL_BACKEND=mori + export VLLM_ALL2ALL_BACKEND="${VLLM_ALL2ALL_BACKEND:-mori}" export GLOO_SOCKET_IFNAME="${GLOO_SOCKET_IFNAME:-${MORI_SOCKET_IFNAME:-eth0}}" export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-${MORI_SOCKET_IFNAME:-eth0}}" @@ -114,7 +114,7 @@ connector_setup_env() { export ROCSHMEM_HEAP_SIZE="${ROCSHMEM_HEAP_SIZE:-8589934592}" export ROCSHMEM_MAX_NUM_CONTEXTS="${ROCSHMEM_MAX_NUM_CONTEXTS:-256}" - # MoRI shmem heap: 4 GiB default too small for EP>=32; 16 GiB (matches #324). + # MoRI shmem heap: 4 GiB default too small for EP>=32; use 16 GiB. export MORI_SHMEM_HEAP_SIZE="${MORI_SHMEM_HEAP_SIZE:-17179869184}" } @@ -139,16 +139,38 @@ connector_launch_worker() { connector_setup_env "${EP_BACKEND:-mori}" - # Patch PyTorch default_pg_timeout (DP Gloo groups) — wideEP only. + # Patch PyTorch default_pg_timeout (DP Gloo groups) — wideEP only. REQUIRED: the + # DP ranks form a Gloo PG whose default timeout is 30 min; the cold AITER JIT + # compile on first boot takes ~15-20 min, and ranks that finish early block on the + # slowest one — a 30-min PG timeout kills the launch mid-compile. There is no clean + # runtime hook: torch reads no env var for this, and the value is frozen as a + # default-arg at torch-import time, so a post-import monkeypatch is too late. We + # therefore rewrite the source constant BEFORE torch is imported. Safe because the + # container is --rm and site-packages is baked in the image (NOT host-mounted); do + # NOT host-mount dist-packages or this edit would corrupt the shared host copy. if parallelism_is_wide_ep; then local _timeout_s="${DISTRIBUTED_TIMEOUT_SECONDS:-7200}" - local _torch_const="/usr/local/lib/python3.12/dist-packages/torch/distributed/constants.py" - if [ -f "$_torch_const" ]; then + # Resolve torch's constants.py from the live interpreter (do not hardcode the + # python version dir — it drifts across images). + local _torch_const + _torch_const="$(python3 -c 'import os,torch.distributed as d; print(os.path.join(os.path.dirname(d.__file__),"constants.py"))' 2>/dev/null)" + if [[ -z "$_torch_const" || ! -f "$_torch_const" ]]; then + echo "WARN: could not locate torch/distributed/constants.py; Gloo PG timeout stays at the 30m default — long JIT compiles may time out." >&2 + elif grep -q "default_pg_timeout: timedelta = timedelta(seconds=" "$_torch_const"; then + echo "[moriio] Gloo PG timeout already patched in ${_torch_const}" + else sed -i "s/default_pg_timeout: timedelta = _DEFAULT_PG_TIMEOUT/default_pg_timeout: timedelta = timedelta(seconds=${_timeout_s})/" "$_torch_const" 2>/dev/null || true + # Verify the substitution actually landed — the target line changes across + # torch versions, and a silent no-op reintroduces the 30m timeout crash. + if grep -q "default_pg_timeout: timedelta = timedelta(seconds=${_timeout_s})" "$_torch_const"; then + echo "[moriio] Patched Gloo PG timeout -> ${_timeout_s}s in ${_torch_const}" + else + echo "WARN: failed to patch default_pg_timeout in ${_torch_const} (torch layout changed?); Gloo PG stays at the 30m default — long JIT compiles may time out. Update the sed target for this torch version." >&2 + fi fi fi - # Per-role execution mode. Ported from #324: NEVER use bare --enforce-eager. + # Per-role execution mode. NEVER use bare --enforce-eager. # On these AITER images an enforce-eager worker (no +quant_fp8 custom op) routes # fp8 quant through an AITER op whose signature mismatches the build # (dynamic_per_token_scaled_quant: out aiter_tensor_t) -> engine-init crash. @@ -178,7 +200,7 @@ connector_launch_worker() { if parallelism_is_wide_ep; then # ---- WIDE_EP=1 (MoriEP) ---- # Per-role all2all: prefill=high_throughput, decode=low_latency. The - # v1.2.0 image rejects the bare "mori" alias; these names are required. + # Newer MoRI-EP images reject the bare "mori" alias; these names are required. local _all2all="${PREFILL_MORI_BACKEND}" [[ "$log_prefix" == "decode" ]] && _all2all="${DECODE_MORI_BACKEND}" @@ -306,7 +328,6 @@ connector_wait_workers_ready() { } connector_start_proxy() { - # Ported faithfully from the validated MAD-private PR#324 mori launcher. # vllm_router: production router with --kv-connector moriio (needs the binary on # PATH or ROUTER_BINARY set); includes a registration gate so the benchmark # doesn't fire before prefill+decode register (else every request 503s). diff --git a/scripts/vllm_dissag/connectors/rixl.env b/scripts/vllm_dissag/connectors/rixl.env index 9d3cffb..1ef3f2b 100644 --- a/scripts/vllm_dissag/connectors/rixl.env +++ b/scripts/vllm_dissag/connectors/rixl.env @@ -1,5 +1,5 @@ -# Platform-mandatory runtime env for the rixl (NixlConnector) connector on the -# ROCm-7.2.3 / mori121 image. Applies to ALL rixl runs (TP and deepep wideEP, every model). +# Platform-mandatory runtime env for the rixl (NixlConnector) connector on ROCm 7.2.3. +# Applies to ALL rixl runs (TP and deepep wideEP, every model). # # These must reach the container at PID 1 — PyTorch reads the alloc-conf at import, # so a late shell export inside the launcher is too late. The slurm sources this file diff --git a/scripts/vllm_dissag/connectors/rixl.sh b/scripts/vllm_dissag/connectors/rixl.sh index e6dda9b..f7509c8 100644 --- a/scripts/vllm_dissag/connectors/rixl.sh +++ b/scripts/vllm_dissag/connectors/rixl.sh @@ -365,8 +365,8 @@ connector_start_proxy() { echo "Starting vLLM Router (Production Proxy) on port ${PROXY_PORT}..." [ -f /root/.cargo/env ] && source /root/.cargo/env # Resolve the router binary: ROUTER_BINARY (site override, e.g. a shared-FS - # build) wins, else the one on PATH. Images that don't ship a router (e.g. - # the mori121 runtime image) require ROUTER_BINARY to be set. + # build) wins, else the one on PATH. Runtime-only images that don't ship a + # router require ROUTER_BINARY to be set. local ROUTER_BIN="${ROUTER_BINARY:-$(command -v vllm-router 2>/dev/null || true)}" if [ -z "${ROUTER_BIN}" ] || [ ! -x "${ROUTER_BIN}" ]; then echo "Error: vllm-router not found. Set ROUTER_BINARY=, or PROXY_TYPE=toy_proxy." \ diff --git a/scripts/vllm_dissag/models.yaml b/scripts/vllm_dissag/models.yaml index 23d6605..eb9baa5 100644 --- a/scripts/vllm_dissag/models.yaml +++ b/scripts/vllm_dissag/models.yaml @@ -121,9 +121,9 @@ Qwen3-30B-A3B: # ============================ MoE (TP + wideEP) ============================ -# DeepSeek-V3 family. The env: block encodes the validated MoRI-EP recipe knobs -# (from MAD-private #324 DeepSeekV3_recipe.env). They are exported BEFORE the -# connector's setup_env, so they define the wideEP recipe by default while still +# DeepSeek-V3 family. The env: block encodes the validated MoRI-EP recipe knobs. +# They are exported BEFORE the connector's setup_env, so they define the wideEP +# recipe by default while still # yielding to a submit-time `-e` override. WIDE_EP=1 (wideEP) reads these; the # connector emits --block-size/--kv-cache-memory-bytes/--all2all-backend from them. # The tp: blocks are the TP-mode (WIDE_EP=0) flags (dense-style serving). diff --git a/scripts/vllm_dissag/tests/TEST_PLAN.md b/scripts/vllm_dissag/tests/TEST_PLAN.md deleted file mode 100644 index 705ff67..0000000 --- a/scripts/vllm_dissag/tests/TEST_PLAN.md +++ /dev/null @@ -1,114 +0,0 @@ -# vllm_dissag consolidation — before/after test plan - -Goal: prove the unified `vllm_disagg.sh` produces **the same behavior** as the three legacy launchers -it replaced (`vllm_disagg_server.sh`, `vllm_disagg_mori_ep.sh`, `vllm_disagg_server_deepep.sh`), and -that the one genuinely-new path (moriio + TP) works. - -There are two layers of testing: -1. **Offline checks** (no GPUs) — validate the assembled `vllm serve` argv + combo gate. -2. **Live cluster** (GPUs) — proves the launched servers actually serve + benchmark at par. - ---- - -## 0. What "at par" means - -| Cell | CONNECTOR / WIDE_EP / EP_BACKEND | Legacy equivalent | At-par check | -|------|----------------------------------|-------------------|--------------| -| 1 | `rixl` / 0 / — | `vllm_disagg_server.sh` | live boot+bench | -| 2 | `moriio` / 0 / — | none (NEW) | live boot+bench | -| 3 | `moriio` / 1 / `mori` | `vllm_disagg_mori_ep.sh` | live ITL vs baseline | -| 4 | `rixl` / 1 / `deepep` | `vllm_disagg_server_deepep.sh` | live boot+bench | - ---- - -## 1. Offline checks (run now, no GPUs) - -```bash -cd MAD/scripts/vllm_dissag -bash tests/run_all.sh # gate_check + argv_assert; expect ALL OFFLINE SUITES PASSED -``` - -- `gate_check.sh` — combo validation: valid/invalid connector × WIDE_EP pairings, back-compat shims, - invalid cross-pairings (`moriio`+`deepep`, `rixl`+`mori`) abort. -- `argv_assert.sh` — per-cell `vllm serve` flag/env assertions from the driver's `DRY_RUN=1` output - (e.g. moriio+TP has `--tensor-parallel-size` + `MoRIIOConnector` and no `--enable-expert-parallel`; - wideEP cells have `--enable-expert-parallel` + `--data-parallel-size`). - -Static checks: -```bash -for f in vllm_disagg.sh parallelism.sh connectors/*.sh run_xPyD_models.slurm; do - bash -n "$f" && echo "OK $f"; done -python3 -c "import yaml; yaml.safe_load(open('models.yaml')); print('yaml OK')" -``` - -### Back-compat equivalence (legacy flags) -Confirm the old flags resolve to the same argv as the new axes: -```bash -# RUN_MORI=1 must equal CONNECTOR=moriio WIDE_EP=1 EP_BACKEND=mori -diff <(RUN_MORI=1 DRY_RUN=1 NODE_RANK=0 MODEL_NAME=DeepSeek-V3 MODEL_PATH=/m NIXL_COOKBOOK_PATH=$PWD \ - xP=2 yD=2 IPADDRS=10.0.0.1,10.0.0.2,10.0.0.3,10.0.0.4 bash vllm_disagg.sh 2>/dev/null) \ - <(CONNECTOR=moriio WIDE_EP=1 EP_BACKEND=mori DRY_RUN=1 NODE_RANK=0 MODEL_NAME=DeepSeek-V3 MODEL_PATH=/m \ - NIXL_COOKBOOK_PATH=$PWD xP=2 yD=2 IPADDRS=10.0.0.1,10.0.0.2,10.0.0.3,10.0.0.4 bash vllm_disagg.sh 2>/dev/null) \ - && echo "RUN_MORI back-compat OK" -``` - ---- - -## 2. Live cluster validation (GPUs; run per matrix row) - -Pre: set `DOCKER_IMAGE_NAME`, pick nodes. `num_nodes = xP + yD`. The proxy is co-located on rank 0. - -For each row, a run is "at par / pass" when: -- all workers reach `Application startup complete.` within timeout, -- the rank-0 proxy answers the bring-up curl (moriio path) / barrier (rixl path), -- `benchmark_xPyD.sh` writes `/run_logs//..._CONCURRENCY.log` with non-empty results, 0 failed, -- ITL median is within ~5% of the documented baseline for the EP cells. - -### Row 1 — rixl + TP (1P/1D) -```bash -export DOCKER_IMAGE_NAME=; export CONNECTOR=rixl WIDE_EP=0 -export xP=1 yD=1 MODEL_NAME=DeepSeek-V3 -sbatch -N 2 -n 2 --nodelist= run_xPyD_models.slurm -``` - -### Row 2 — moriio + TP (1P/1D) ← NEW, highest-risk -```bash -export DOCKER_IMAGE_NAME=; export CONNECTOR=moriio WIDE_EP=0 -export xP=1 yD=1 MODEL_NAME=DeepSeek-V3 -sbatch -N 2 -n 2 --nodelist= run_xPyD_models.slurm -``` -Watch for: does `MoRIIOConnector` initialize without `--enable-expert-parallel`? Is `kv_parallel_size` -correct in TP? Does the moriio toy proxy / notify path assume DP ranks? If it wedges at connector init -or KV handshake, capture `prefill_NODE0.log` / `decode_NODE1.log` — this is the cell with no precedent. - -### Row 3 — MoRI-EP wideEP (2P/2D) ← compare ITL to baseline -```bash -export DOCKER_IMAGE_NAME=; export CONNECTOR=moriio WIDE_EP=1 # or legacy: RUN_MORI=1 -export xP=2 yD=2 MODEL_NAME=DeepSeek-V3 -sbatch -N 4 -n 4 --nodelist= run_xPyD_models.slurm -``` -At-par bar: ITL median ~48–51 ms across 1024/1024, 4096/4096, 8192/1024 (per the recipe baselines). - -### Row 4 — DeepEP wideEP (2P/2D) -```bash -export DOCKER_IMAGE_NAME=; export CONNECTOR=rixl WIDE_EP=1 EP_BACKEND=deepep # or legacy: RUN_DEEPEP=1 -export xP=2 yD=2 MODEL_NAME=DeepSeek-V3 -sbatch -N 4 -n 4 --nodelist= run_xPyD_models.slurm -``` - -### A/B confirmation against the OLD launchers (strongest proof) -The legacy launchers were deleted on this branch. To A/B live, check out the parent commit (or -`develop`) in a second worktree and run the same MODEL/xP/yD there; compare the resulting -`_CONCURRENCY.log` medians. Identical-within-noise ITL/throughput = behavior preserved end-to-end. - ---- - -## 3. Sign-off checklist - -- [ ] `tests/run_all.sh` green (offline: gate_check + argv_assert) -- [ ] static checks (`bash -n`, yaml) green -- [ ] back-compat: `RUN_MORI=1` / `RUN_DEEPEP=1` argv-equal to new axes -- [ ] Row 1 rixl-TP live: boots + benchmarks -- [ ] Row 3 moriio-wideEP live: ITL within ~5% of baseline (vs OLD launcher A/B) -- [ ] Row 4 deepep live: boots + benchmarks -- [ ] Row 2 moriio-TP live: boots + benchmarks (NEW capability) diff --git a/scripts/vllm_dissag/tests/argv_assert.sh b/scripts/vllm_dissag/tests/argv_assert.sh deleted file mode 100755 index 9c85da8..0000000 --- a/scripts/vllm_dissag/tests/argv_assert.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# Offline argv + env assertions for the unified launcher: checks that each connector × -# WIDE_EP × role cell emits the expected `vllm serve` flags/env (and omits the wrong ones). -# No cluster / no GPUs. Exits 0 if all assertions hold. -# -# Covers: -# - moriio+TP (Llama): exactly ONE --compilation-config, has --disable-custom-all-reduce, -# has --tensor-parallel-size, NO -tp 1 / --enable-expert-parallel / --all2all-backend -# - moriio+wideEP (DSV3): -tp 1 + --data-parallel-size + --enable-expert-parallel + -# --all2all-backend mori_high_throughput + --block-size 16, exactly ONE --compilation-config -# - slurm docker -e forwards the RDMA-fix env (expandable_segments:False x2, IPC_MODE_LEGACY=0) -set -u -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SLURM="$DIR/run_xPyD_models.slurm" -pass=0; fail=0 - -# emit argv for a cell -_argv() { # connector wide_ep ep_backend model model_path - env -i PATH="$PATH" HOME="$HOME" NIXL_COOKBOOK_PATH="$DIR" \ - DRY_RUN=1 NODE_RANK=0 xP=1 yD=1 CONNECTOR="$1" WIDE_EP="$2" EP_BACKEND="$3" \ - MODEL_NAME="$4" MODEL_PATH="$5" MASTER_ADDR=10.0.0.1 IPADDRS=10.0.0.1,10.0.0.2 \ - GPUS_PER_NODE=8 SLURM_JOB_ID=ASSERT PROXY_TYPE=vllm_router ROUTER_PORT=30000 \ - bash "$DIR/vllm_disagg.sh" 2>/dev/null | awk '/^===DRYRUN/{f=1;next} /^===END===/{f=0} f' -} - -_has() { grep -qF -- "$2" <<<"$1" && { printf " PASS %s\n" "$3"; pass=$((pass+1)); } || { printf " FAIL %s (missing: %s)\n" "$3" "$2"; fail=$((fail+1)); }; } -_hasnot(){ grep -qF -- "$2" <<<"$1" && { printf " FAIL %s (unexpected: %s)\n" "$3" "$2"; fail=$((fail+1)); } || { printf " PASS %s\n" "$3"; pass=$((pass+1)); }; } -_count() { local n; n="$(grep -cF -- "$2" <<<"$1")"; [[ "$n" == "$3" ]] && { printf " PASS %s (=%s)\n" "$4" "$n"; pass=$((pass+1)); } || { printf " FAIL %s (got %s want %s)\n" "$4" "$n" "$3"; fail=$((fail+1)); }; } - -echo "=== moriio + TP (Llama-70B) ===" -A="$(_argv moriio 0 '' amd-Llama-3.3-70B-Instruct-FP8-KV /m/Llama)" -_has "$A" "--tensor-parallel-size" "has --tensor-parallel-size" -_has "$A" "--disable-custom-all-reduce" "has --disable-custom-all-reduce" -_count "$A" "--compilation-config" 1 "exactly one --compilation-config" -_hasnot "$A" "--enable-expert-parallel" "no --enable-expert-parallel" -_hasnot "$A" "--all2all-backend" "no --all2all-backend" -_hasnot "$A" "--data-parallel-size" "no --data-parallel-size" - -echo "" -echo "=== moriio + wideEP (DeepSeek-V3, EP) ===" -B="$(_argv moriio 1 mori DeepSeek-V3 /m/DSV3)" -_has "$B" "--enable-expert-parallel" "has --enable-expert-parallel" -_has "$B" "--data-parallel-size" "has --data-parallel-size" -_has "$B" "mori_high_throughput" "prefill all2all = mori_high_throughput" -_has "$B" "--block-size" "has --block-size" -_has "$B" "16" "block-size value 16 present" -_count "$B" "--compilation-config" 1 "exactly one --compilation-config" -_hasnot "$B" "--tensor-parallel-size" "no --tensor-parallel-size (uses -tp 1)" - -echo "" -echo "=== connector platform env files carry the RDMA-fix env ===" -# The ROCm-7.2.3 GPU-RDMA env now lives in per-connector .env files; the slurm -# sources connectors/.env and forwards each var via docker -e. -S="$(cat "$SLURM")" -_has "$S" 'CONNECTOR_ENV_FILE="${SCRIPT_DIR}/connectors/${CONNECTOR}.env"' "slurm sources connector .env" -_has "$S" '${CONNECTOR_ENV_ARGS}' "slurm forwards CONNECTOR_ENV_ARGS in docker run" -for cf in moriio rixl; do - F="$DIR/connectors/${cf}.env" - if [[ -f "$F" ]]; then - E="$(cat "$F")" - _has "$E" "PYTORCH_ALLOC_CONF=expandable_segments:False" "${cf}.env: PYTORCH_ALLOC_CONF" - _has "$E" "PYTORCH_HIP_ALLOC_CONF=expandable_segments:False" "${cf}.env: PYTORCH_HIP_ALLOC_CONF" - _has "$E" "HSA_ENABLE_IPC_MODE_LEGACY=0" "${cf}.env: IPC_MODE_LEGACY=0" - _has "$E" "MORI_GPU_ARCHS=gfx942" "${cf}.env: MORI_GPU_ARCHS" - else - printf " FAIL connectors/%s.env missing\n" "$cf"; fail=$((fail+1)) - fi -done -# parse check: the slurm's KEY=${KEY:-VAL} loop yields correct -e args (+ override wins) -_parse() { # $1=connector ; reads its .env with same logic as the slurm - local A="" l k v - while IFS= read -r l; do - [[ "$l" =~ ^[[:space:]]*# || -z "${l// }" ]] && continue - k="${l%%=*}"; v="${l#*=}"; A+=" -e ${k}=${!k:-$v}" - done < "$DIR/connectors/$1.env" - printf '%s' "$A" -} -_has "$(_parse moriio)" "-e PYTORCH_HIP_ALLOC_CONF=expandable_segments:False" "parse yields HIP_ALLOC -e arg" -_has "$(PYTORCH_HIP_ALLOC_CONF=expandable_segments:True _parse moriio)" "-e PYTORCH_HIP_ALLOC_CONF=expandable_segments:True" "submit-time override wins" - -echo "" -echo "======================================================" -echo " argv_assert: ${pass} passed, ${fail} failed" -echo "======================================================" -[[ "$fail" == "0" ]] diff --git a/scripts/vllm_dissag/tests/drive_cell.sh b/scripts/vllm_dissag/tests/drive_cell.sh deleted file mode 100755 index 13a286b..0000000 --- a/scripts/vllm_dissag/tests/drive_cell.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# Generic interactive-allocation driver for one test cell. -# Drives an existing Slurm allocation via `srun --overlap` (no sbatch), one -# container per node, NODE_RANK assigned per node. -# -# Required env: -# JOBID slurm job to --overlap onto -# NODES comma list of node names (or NODE_PREFIX-relative suffixes) — len = xP+yD -# IPS comma list of node IPs, same order as NODES -# MODEL_NAME, MODEL_PATH -# CONNECTOR (rixl|moriio), WIDE_EP (0|1); EP_BACKEND for wideEP -# xP, yD -# TAG short run tag (log dir suffix) -# DOCKER_IMAGE_NAME the image to run (build from docker/*_mori_ep_fullsource*.Dockerfile) -# Optional: NODE_PREFIX (prepended to each NODES entry; default empty = full names), -# BENCHMARK_CON, BENCHMARK_COMBINATIONS, PROXY_TYPE, ROUTER_BINARY, -# ROUTER_PORT, RUN_MORI, RUN_DEEPEP -set -u -: "${JOBID:?} ${NODES:?} ${IPS:?} ${MODEL_NAME:?} ${MODEL_PATH:?} ${CONNECTOR:?} ${WIDE_EP:?} ${xP:?} ${yD:?} ${TAG:?} ${DOCKER_IMAGE_NAME:?}" - -NODE_PREFIX="${NODE_PREFIX:-}" # e.g. "mycluster-node-"; empty => NODES are full names -BENCHMARK_CON="${BENCHMARK_CON:-8 16}" -BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS:-512/512}" -BENCHMARK_ITR="${BENCHMARK_ITR:-1}" -PROXY_TYPE="${PROXY_TYPE:-vllm_router}" -ROUTER_PORT="${ROUTER_PORT:-30000}" -# Path to a built vllm-router binary (from vllm-project/router PR#181). Required only if -# the image doesn't already ship vllm-router on PATH. -ROUTER_BINARY="${ROUTER_BINARY:-}" -EP_BACKEND="${EP_BACKEND:-}" - -IFS=',' read -ra NODE_ARR <<< "$NODES" -IFS=',' read -ra IP_ARR <<< "$IPS" -NNODES=$(( xP + yD )) -IPADDRS="$IPS" -MASTER_ADDR="${IP_ARR[0]}" -MASTER_PORT=39566 -RUN_TAG="int${JOBID}_${TAG}" -HOSTRUN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/run_interactive.sh" - -# env forwarded into each srun step (SLURM_JOB_ID set to RUN_TAG only INSIDE bash -c). -FWD="DOCKER_IMAGE_NAME MODEL_NAME MODEL_PATH CONNECTOR WIDE_EP EP_BACKEND xP yD NNODES IPADDRS MASTER_ADDR MASTER_PORT BENCHMARK_CON BENCHMARK_COMBINATIONS BENCHMARK_ITR PROXY_TYPE ROUTER_PORT ROUTER_BINARY RUN_MORI RUN_DEEPEP" -EXPORTS="" -for v in $FWD; do EXPORTS="$EXPORTS $v=\"${!v:-}\""; done -EXPORTS="$EXPORTS SLURM_JOB_ID=\"$RUN_TAG\"" - -echo "=== cell $TAG: $CONNECTOR WIDE_EP=$WIDE_EP ${EP_BACKEND:+EP_BACKEND=$EP_BACKEND} model=$MODEL_NAME xP=$xP yD=$yD ===" -echo " nodes=$NODES ips=$IPS con='$BENCHMARK_CON' combos='$BENCHMARK_COMBINATIONS'" - -pids=() -# launch decode + child nodes first (ranks 1..N-1), then prefill master (rank 0) last -for (( r=NNODES-1; r>=1; r-- )); do - node="${NODE_PREFIX}${NODE_ARR[$r]}" - srun --jobid="$JOBID" --overlap -w "$node" bash -c "export $EXPORTS; $HOSTRUN $r" \ - > "/tmp/${RUN_TAG}_rank${r}.log" 2>&1 & - pids+=($!) -done -sleep 5 -srun --jobid="$JOBID" --overlap -w "${NODE_PREFIX}${NODE_ARR[0]}" bash -c "export $EXPORTS; $HOSTRUN 0" \ - > "/tmp/${RUN_TAG}_rank0.log" 2>&1 & -master_pid=$! - -echo "logs: /tmp/${RUN_TAG}_rank*.log container logs: /shared_inference/${USER}/model_blog_logs/${RUN_TAG}/" -wait "$master_pid" -echo "=== $TAG: rank0 exited; stopping others ===" -for p in "${pids[@]}"; do kill "$p" 2>/dev/null || true; done -echo "DONE $TAG" diff --git a/scripts/vllm_dissag/tests/gate_check.sh b/scripts/vllm_dissag/tests/gate_check.sh deleted file mode 100755 index 66f6167..0000000 --- a/scripts/vllm_dissag/tests/gate_check.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# Offline combo-gate unit tests. No cluster / no GPUs. -# Verifies the (MODEL x CONNECTOR x WIDE_EP x EP_BACKEND) enablement gate in -# run_xPyD_models.slurm accepts exactly the supported combos and rejects the rest. -# -# Strategy: source the slurm's model lists + axis/validation logic in a harness that -# stops right after the gate (never reaches docker/srun), then assert exit status. -# This runs the REAL gate code path, so a future edit to the lists/gate is caught. -# -# Usage: bash tests/gate_check.sh (exit 0 = all pass) -set -u -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -SLURM="${SCRIPT_DIR}/run_xPyD_models.slurm" -[[ -f "$SLURM" ]] || { echo "FATAL: $SLURM not found"; exit 1; } - -# --------------------------------------------------------------------------- -# Extract the gate: everything from the VALID_MODELS decl through the allowlist -# gate `fi`, i.e. the model+axis validation, WITHOUT the docker run below it. -# We re-run that slice per case with MODEL_NAME/CONNECTOR/WIDE_EP/EP_BACKEND set. -# --------------------------------------------------------------------------- -_run_gate() { - # $1=MODEL $2=CONNECTOR(optional) $3=WIDE_EP(optional) $4=EP_BACKEND(optional) - local M="$1" C="${2:-}" W="${3:-}" B="${4:-}" - env -i PATH="$PATH" HOME="$HOME" \ - MODEL_NAME="$M" CONNECTOR="$C" WIDE_EP="$W" EP_BACKEND="$B" \ - RUN_MORI="${RUN_MORI:-}" RUN_DEEPEP="${RUN_DEEPEP:-}" \ - bash -c ' - set -u - # --- mirror of run_xPyD_models.slurm gate (keep in sync) --- - VALID_MODELS=( "Llama-3.1-405B-Instruct-FP8-KV" "amd-Llama-3.3-70B-Instruct-FP8-KV" \ - "DeepSeek-V3" "DeepSeek-V3-5layer" "gpt-oss-120b" "DeepSeek-R1" "Qwen3-32B" "Qwen3-30B-A3B" ) - MORI_EP_VALID_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) - DEEPEP_VALID_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) - WIDE_EP_ONLY_MODELS=( "DeepSeek-V3" "DeepSeek-V3-5layer" "DeepSeek-R1" ) - MODEL_NAME="${MODEL_NAME:-None}" - _in(){ local n="$1"; shift; for x in "$@"; do [[ "$n" == "$x" ]] && return 0; done; return 1; } - _in "$MODEL_NAME" "${VALID_MODELS[@]}" || { echo REJECT; exit 0; } - # axis resolution + shim - if [[ -z "${CONNECTOR:-}" ]]; then - if [[ "${RUN_MORI:-0}" == "1" ]]; then CONNECTOR=moriio; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-mori}"; - elif [[ "${RUN_DEEPEP:-0}" == "1" ]]; then CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-1}"; EP_BACKEND="${EP_BACKEND:-deepep}"; - else CONNECTOR=rixl; WIDE_EP="${WIDE_EP:-0}"; fi - fi - WIDE_EP="${WIDE_EP:-0}" - case "$CONNECTOR" in rixl|moriio) ;; *) echo REJECT; exit 0 ;; esac - case "$WIDE_EP" in 0|1) ;; *) echo REJECT; exit 0 ;; esac - if [[ "$WIDE_EP" == "1" ]]; then - if [[ "$CONNECTOR" == "moriio" ]]; then EP_BACKEND="${EP_BACKEND:-mori}"; [[ "$EP_BACKEND" == "mori" ]] || { echo REJECT; exit 0; } - else EP_BACKEND="${EP_BACKEND:-deepep}"; [[ "$EP_BACKEND" == "deepep" ]] || { echo REJECT; exit 0; }; fi - fi - # model x combo allowlist gate - if [[ "$WIDE_EP" == "0" ]]; then - _in "$MODEL_NAME" "${WIDE_EP_ONLY_MODELS[@]}" && { echo REJECT; exit 0; } - elif [[ "$WIDE_EP" == "1" && "$CONNECTOR" == "moriio" ]]; then - _in "$MODEL_NAME" "${MORI_EP_VALID_MODELS[@]}" || { echo REJECT; exit 0; } - elif [[ "$WIDE_EP" == "1" && "$CONNECTOR" == "rixl" ]]; then - _in "$MODEL_NAME" "${DEEPEP_VALID_MODELS[@]}" || { echo REJECT; exit 0; } - fi - echo ALLOW - ' -} - -pass=0; fail=0 -_case() { # $1=expect $2..=args to _run_gate ; last-ish: description via $DESC - local expect="$1"; shift - local desc="$1"; shift - local got; got="$(_run_gate "$@")" - if [[ "$got" == "$expect" ]]; then - printf " PASS %-52s -> %s\n" "$desc" "$got"; pass=$((pass+1)) - else - printf " FAIL %-52s -> got %s, want %s\n" "$desc" "$got" "$expect"; fail=$((fail+1)) - fi -} - -echo "=== combo-gate unit tests ===" -# dense TP — both connectors -_case ALLOW "Llama-70B rixl TP" amd-Llama-3.3-70B-Instruct-FP8-KV rixl 0 -_case ALLOW "Llama-70B moriio TP" amd-Llama-3.3-70B-Instruct-FP8-KV moriio 0 -_case ALLOW "Qwen3-32B moriio TP" Qwen3-32B moriio 0 -_case ALLOW "gpt-oss-120b rixl TP" gpt-oss-120b rixl 0 -# dense must NOT run wideEP -_case REJECT "Llama-70B moriio wideEP" amd-Llama-3.3-70B-Instruct-FP8-KV moriio 1 -_case REJECT "Qwen3-32B rixl wideEP" Qwen3-32B rixl 1 -# DeepSeek family — wideEP only -_case ALLOW "DSV3 moriio wideEP(mori)" DeepSeek-V3 moriio 1 -_case ALLOW "DSV3 rixl wideEP(deepep)" DeepSeek-V3 rixl 1 -_case ALLOW "R1 moriio wideEP" DeepSeek-R1 moriio 1 -_case REJECT "DSV3 moriio TP" DeepSeek-V3 moriio 0 -_case REJECT "DSV3 rixl TP" DeepSeek-V3 rixl 0 -# cross-pairs -_case REJECT "DSV3 moriio+deepep xpair" DeepSeek-V3 moriio 1 deepep -_case REJECT "DSV3 rixl+mori xpair" DeepSeek-V3 rixl 1 mori -# unknown model -_case REJECT "bogus model" not-a-real-model moriio 0 -# invalid axis values -_case REJECT "bad connector" DeepSeek-V3 banana 1 -_case REJECT "bad wide_ep" Qwen3-32B rixl 2 - -echo "" -echo "=== legacy shim tests (RUN_MORI / RUN_DEEPEP) ===" -RUN_MORI=1 _case ALLOW "RUN_MORI=1 DSV3 (->moriio wideEP)" DeepSeek-V3 "" "" -RUN_DEEPEP=1 _case ALLOW "RUN_DEEPEP=1 DSV3 (->rixl wideEP)" DeepSeek-V3 "" "" -RUN_MORI=1 _case REJECT "RUN_MORI=1 Llama (dense no wideEP)" amd-Llama-3.3-70B-Instruct-FP8-KV "" "" - -echo "" -echo "=== REQUIRED_FILES completeness ===" -_missing=0 -for f in $(grep -oE 'REQUIRED_FILES=\([^)]*\)' "$SLURM" | tr -d '()"' | sed 's/REQUIRED_FILES=//'); do - [[ -f "${SCRIPT_DIR}/${f}" ]] || { echo " FAIL REQUIRED_FILES entry missing on disk: $f"; _missing=$((_missing+1)); fail=$((fail+1)); } -done -[[ "$_missing" == "0" ]] && { echo " PASS all REQUIRED_FILES present"; pass=$((pass+1)); } - -echo "" -echo "======================================================" -echo " gate_check: ${pass} passed, ${fail} failed" -echo "======================================================" -[[ "$fail" == "0" ]] diff --git a/scripts/vllm_dissag/tests/harvest.sh b/scripts/vllm_dissag/tests/harvest.sh deleted file mode 100755 index fea84f7..0000000 --- a/scripts/vllm_dissag/tests/harvest.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# harvest.sh — print pass/fail + per-cell benchmark summary -RUN_TAG="$1" -D=/shared_inference/${USER}/model_blog_logs/${RUN_TAG} -echo "### $RUN_TAG ###" -[ -d "$D" ] || { echo " no log dir yet"; exit 0; } -# readiness + errors -pc=$(grep -l 'Application startup complete' $D/prefill_NODE0.log 2>/dev/null | wc -l) -rdma=$(grep -h 'RegisterRdmaMemoryRegion failed' $D/*.log 2>/dev/null | wc -l) -died=$(grep -h 'died unexpectedly' $D/*.log 2>/dev/null | wc -l) -echo " startup-complete(prefill)=$pc rdma-fail=$rdma worker-died=$died" -# benchmark cells -grep -aE 'Maximum request concurrency:|Successful requests:|Failed requests:|Output token throughput|Median TTFT|Median ITL' $D/*CONCURRENCY.log 2>/dev/null \ - | grep -vE "namespace|Namespace" \ - | awk '/Maximum request concurrency:/{c=$4} /Successful requests:/{s=$3} /Failed requests:/{f=$3} /Output token throughput/{o=$5} /Median TTFT/{t=$4} /Median ITL/{print " con="c" ok="s" fail="f" tok/s="o" TTFT="t" ITL="$4}' diff --git a/scripts/vllm_dissag/tests/run_all.sh b/scripts/vllm_dissag/tests/run_all.sh deleted file mode 100755 index f75601d..0000000 --- a/scripts/vllm_dissag/tests/run_all.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Run all offline test suites (no cluster). Exit 0 iff all pass. -set -u -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -rc=0 -for t in gate_check.sh argv_assert.sh; do - echo "############### $t ###############" - env -i PATH="$PATH" HOME="$HOME" bash "$DIR/$t" || rc=1 - echo "" -done -[[ "$rc" == "0" ]] && echo "ALL OFFLINE SUITES PASSED ✅" || echo "SOME SUITES FAILED ❌" -exit $rc diff --git a/scripts/vllm_dissag/tests/run_interactive.sh b/scripts/vllm_dissag/tests/run_interactive.sh deleted file mode 100755 index d6e1e67..0000000 --- a/scripts/vllm_dissag/tests/run_interactive.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Interactive multi-node launcher (mirrors run_xPyD_models.slurm's docker run, but -# NODE_RANK is passed per node instead of being derived from SLURM_PROCID). Used by -# tests/drive_cell.sh to drive an existing allocation via `srun --overlap`. -# Usage: run_interactive.sh -# Env expected: DOCKER_IMAGE_NAME, MODEL_NAME, MODEL_PATH, IPADDRS, MASTER_ADDR, -# MASTER_PORT, NNODES, xP, yD, CONNECTOR, WIDE_EP, BENCHMARK_CON, -# BENCHMARK_COMBINATIONS, SLURM_JOB_ID, plus any recipe/-e overrides. -set -u -NODE_RANK="$1" - -NIXL_COOKBOOK_PATH="/opt/nixl-vllm-cookbook" -# repo dir = the vllm_dissag dir containing this tests/ folder (override NIXL_REPO_DIR to relocate) -NIXL_REPO_DIR="${NIXL_REPO_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" -LOG_PATH="${LOG_PATH:-/shared_inference/${USER}/model_blog_logs}" -mkdir -p "$LOG_PATH" 2>/dev/null || true -DOCKER_CONT_NAME="container_${MODEL_NAME}_${SLURM_JOB_ID}" -RUN_FILE_FULL="$NIXL_COOKBOOK_PATH/vllm_disagg.sh" - -# cleanup any stale container/ports on this node -docker rm -f "$DOCKER_CONT_NAME" 2>/dev/null || true -fuser -k 5000/tcp 2>/dev/null || true -fuser -k 2222/tcp 2>/dev/null || true -fuser -k 15000/tcp 2>/dev/null || true -fuser -k 30000/tcp 2>/dev/null || true -fuser -k 36367/tcp 2>/dev/null || true # router discovery / moriio proxy_ping -fuser -k 20005/tcp 2>/dev/null || true # serve port -sleep 2 - -mkdir -p /tmp/vllm_cache/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true -# Persistent JIT cache: the image points AITER_JIT_DIR/TRITON_CACHE_DIR/VLLM_CACHE_ROOT/ -# COMGR_CACHE_DIR at /opt/vllm_cache. Mount a host dir there so AITER CK kernels compile -# ONCE and are reused across runs (cold compile is ~15 min; warm boot is ~1 min). Host dir -# on local NVMe for speed. Keyed by the image ID so a new image (different kernels/ABI) -# starts a fresh cache instead of reusing stale .so's; set JIT_CACHE_HOST to override, or -# JIT_CACHE_PERSIST=0 to disable and fall back to an ephemeral in-container cache. -if [[ "${JIT_CACHE_PERSIST:-1}" == "1" ]]; then - _IMG_KEY="$(docker image inspect --format '{{.Id}}' "$DOCKER_IMAGE_NAME" 2>/dev/null | sed 's/^sha256://; s/[^a-f0-9]//g' | cut -c1-12)" - _IMG_KEY="${_IMG_KEY:-noimg}" - _JIT_CACHE_HOST="${JIT_CACHE_HOST:-/mnt/m2m_nobackup/${USER}/vllm_jit_cache/${_IMG_KEY}}" - mkdir -p "$_JIT_CACHE_HOST"/{aiter_jit,triton,vllm,comgr} 2>/dev/null || true - _JIT_CACHE_MOUNT="-v ${_JIT_CACHE_HOST}:/opt/vllm_cache" - echo "JIT cache (persistent, image ${_IMG_KEY}): ${_JIT_CACHE_HOST} -> /opt/vllm_cache" -else - _JIT_CACHE_MOUNT="" -fi - -# host RDMA library mounts -_RDMA_MOUNTS="" -_LIBDIR=/usr/lib/x86_64-linux-gnu -for _lib in libibverbs.so libibverbs.so.1 librdmacm.so librdmacm.so.1; do - [ -e "$_LIBDIR/$_lib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/$_lib:$_LIBDIR/$_lib:ro" -done -for _vlib in $_LIBDIR/libibverbs.so.1.* $_LIBDIR/librdmacm.so.1.*; do - [ -e "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" -done -for _pattern in libmlx5.so* libionic*.so* libbnxt_re*.so* libefa.so* libhns.so*; do - for _vlib in $_LIBDIR/${_pattern}; do - [ -e "$_vlib" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_vlib:$_vlib:ro" - done -done -[ -d "$_LIBDIR/libibverbs" ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v $_LIBDIR/libibverbs:$_LIBDIR/libibverbs:ro" -[ -d /etc/libibverbs.d ] && _RDMA_MOUNTS="$_RDMA_MOUNTS -v /etc/libibverbs.d:/etc/libibverbs.d:ro" - -docker run --rm \ - --device /dev/dri --device /dev/kfd --device /dev/infiniband \ - --network host --ipc host --group-add video \ - --cap-add SYS_PTRACE --security-opt seccomp=unconfined --privileged \ - -v $HOME:$HOME \ - -v /shared_inference:/shared_inference \ - -v /mnt/m2m_nobackup:/mnt/m2m_nobackup \ - -v $HOME/.ssh:/root/.ssh \ - --shm-size "${DOCKER_SHM_SIZE:-256G}" --ulimit nofile=524288:524288 --ulimit memlock=-1:-1 \ - -v ${LOG_PATH}:/run_logs \ - -v $NIXL_REPO_DIR:$NIXL_COOKBOOK_PATH \ - -v /tmp/vllm_cache:/tmp/vllm_cache \ - ${_JIT_CACHE_MOUNT} \ - $_RDMA_MOUNTS \ - --entrypoint /bin/bash \ - -e SLURM_JOB_ID=$SLURM_JOB_ID \ - -e NNODES=$NNODES \ - -e NODE_RANK=$NODE_RANK \ - -e MASTER_ADDR=$MASTER_ADDR \ - -e MASTER_PORT=$MASTER_PORT \ - -e MODEL_PATH=$MODEL_PATH \ - -e NIXL_COOKBOOK_PATH=$NIXL_COOKBOOK_PATH \ - -e xP=$xP -e yD=$yD \ - -e USER_NAME=$USER \ - -e MODEL_NAME=$MODEL_NAME \ - -e BENCHMARK_ITR=${BENCHMARK_ITR:-1} \ - -e BENCHMARK_CON="${BENCHMARK_CON}" \ - -e BENCHMARK_COMBINATIONS="${BENCHMARK_COMBINATIONS}" \ - -e IPADDRS=$IPADDRS \ - -e CONNECTOR=$CONNECTOR \ - -e WIDE_EP=$WIDE_EP \ - ${EP_BACKEND:+-e EP_BACKEND=$EP_BACKEND} \ - -e PROXY_TYPE=${PROXY_TYPE:-vllm_router} \ - -e ROUTER_PORT=${ROUTER_PORT:-30000} \ - ${ROUTER_BINARY:+-e ROUTER_BINARY=$ROUTER_BINARY} \ - -e GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8} \ - -e GPUS_PER_NODE=${GPUS_PER_NODE:-8} \ - -e MORI_SOCKET_IFNAME=${MORI_SOCKET_IFNAME:-eth0} \ - -e DISTRIBUTED_TIMEOUT_SECONDS=${DISTRIBUTED_TIMEOUT_SECONDS:-7200} \ - -e VLLM_RPC_TIMEOUT=${VLLM_RPC_TIMEOUT:-300000} \ - -e VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=${VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS:-3600} \ - -e PYTORCH_ALLOC_CONF=${PYTORCH_ALLOC_CONF:-expandable_segments:False} \ - -e PYTORCH_HIP_ALLOC_CONF=${PYTORCH_HIP_ALLOC_CONF:-expandable_segments:False} \ - -e HSA_ENABLE_IPC_MODE_LEGACY=${HSA_ENABLE_IPC_MODE_LEGACY:-0} \ - -e MORI_GPU_ARCHS=${MORI_GPU_ARCHS:-gfx942} \ - -e HSA_NO_SCRATCH_RECLAIM=${HSA_NO_SCRATCH_RECLAIM:-1} \ - --name $DOCKER_CONT_NAME \ - $DOCKER_IMAGE_NAME -c " - mkdir -p /run_logs/${SLURM_JOB_ID} - $RUN_FILE_FULL 2>&1 | tee /run_logs/${SLURM_JOB_ID}/pd_vllm_bench_NODE${NODE_RANK}.log - "