From 456836b7c7294467a4f08b9577b8fb909ad8634d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 9 Oct 2026 11:26:23 -0700 Subject: [PATCH] Fix the CI jobs that fail on the OSDC images --- .../examples/scripts/test_examples.py | 8 +++++++ .github/unittest/helpers/reap_strays.sh | 21 +++++++++++++++++++ .github/unittest/linux/scripts/run_all.sh | 20 +----------------- .../scripts_craftground/run_test.sh | 1 + .../linux_libs/scripts_habitat/run_all.sh | 2 +- .../linux_libs/scripts_habitat/setup_env.sh | 7 ++++--- .../linux_libs/scripts_libero/run_test.sh | 1 + .../linux_libs/scripts_minari/setup_env.sh | 6 ++++-- .../linux_libs/scripts_openx/setup_env.sh | 3 ++- .../linux_libs/scripts_procgen/setup_env.sh | 3 ++- .../scripts_safety_gymnasium/setup_env.sh | 3 ++- .../scripts_gym_0_13/run_test.sh | 5 +++++ .github/unittest/llm/scripts_llm/setup_env.sh | 7 ++----- .../unittest/llm/scripts_sglang/install.sh | 2 +- .../unittest/llm/scripts_sglang/setup_env.sh | 3 ++- 15 files changed, 57 insertions(+), 35 deletions(-) create mode 100755 .github/unittest/helpers/reap_strays.sh diff --git a/.github/unittest/examples/scripts/test_examples.py b/.github/unittest/examples/scripts/test_examples.py index 81d3f3e9733..ffb957a6c45 100644 --- a/.github/unittest/examples/scripts/test_examples.py +++ b/.github/unittest/examples/scripts/test_examples.py @@ -650,6 +650,14 @@ def test_run_example_terminates_timeouts(tmp_path: Path) -> None: os.kill(child_pid, 0) except ProcessLookupError: break + # A killed child re-parented to a PID 1 that never reaps orphans (OSDC's + # `tail -f /dev/null`) stays a zombie, which os.kill still reports. + try: + stat = Path(f"/proc/{child_pid}/stat").read_text() + if stat.rsplit(")", 1)[1].split()[0] == "Z": + break + except (FileNotFoundError, IndexError): + pass time.sleep(0.01) else: pytest.fail(f"child process {child_pid} survived timeout cleanup") diff --git a/.github/unittest/helpers/reap_strays.sh b/.github/unittest/helpers/reap_strays.sh new file mode 100755 index 00000000000..09be0cfb85f --- /dev/null +++ b/.github/unittest/helpers/reap_strays.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# On OSDC the step runs under run_with_env_secrets.py, which drains our stdout +# until EOF. EOF only arrives once every process holding the write end has +# closed it, so a single xdist worker, Ray actor or Xvfb that outlives the tests +# keeps the job alive until its 120-minute timeout, long after the script exits. +# Under linux_job_v2 the surrounding `docker run` reaped these for us. + +echo "::group::Processes still alive before exit" +ps -eo pid,ppid,etimes,rss,args --sort=-rss | head -40 || true +echo "::endgroup::" + +# Anything still running that is not this shell or ps itself. +strays="$(pgrep -f 'pytest|ray::|Xvfb' 2>/dev/null | grep -v "^$$\$" || true)" +if [ -n "${strays}" ]; then + echo "Reaping strays holding the step open: ${strays}" + # shellcheck disable=SC2086 + kill -TERM ${strays} 2>/dev/null || true + sleep 5 + # shellcheck disable=SC2086 + kill -KILL ${strays} 2>/dev/null || true +fi diff --git a/.github/unittest/linux/scripts/run_all.sh b/.github/unittest/linux/scripts/run_all.sh index 89a15e0f6ec..e3bee12c50c 100755 --- a/.github/unittest/linux/scripts/run_all.sh +++ b/.github/unittest/linux/scripts/run_all.sh @@ -618,25 +618,7 @@ bash ${this_dir}/post_process.sh # ==================================================================================== # # ================================ Reap strays ======================================= # -# On OSDC the step runs under run_with_env_secrets.py, which drains our stdout -# until EOF. EOF only arrives once every process holding the write end has -# closed it, so a single xdist worker or Ray actor that outlives pytest keeps -# the job alive until its 120-minute timeout, long after this script exits 0. -# Under linux_job_v2 the surrounding `docker run` reaped these for us. -echo "::group::Processes still alive before exit" -ps -eo pid,ppid,etimes,rss,args --sort=-rss | head -40 || true -echo "::endgroup::" - -# Anything still running that is not this shell or ps itself. -strays="$(pgrep -f 'pytest|ray::|Xvfb' 2>/dev/null | grep -v "^$$\$" || true)" -if [ -n "${strays}" ]; then - echo "Reaping strays holding the step open: ${strays}" - # shellcheck disable=SC2086 - kill -TERM ${strays} 2>/dev/null || true - sleep 5 - # shellcheck disable=SC2086 - kill -KILL ${strays} 2>/dev/null || true -fi +bash "${root_dir}/.github/unittest/helpers/reap_strays.sh" # Exit with failure if any tests failed exit $EXIT_STATUS diff --git a/.github/unittest/linux_libs/scripts_craftground/run_test.sh b/.github/unittest/linux_libs/scripts_craftground/run_test.sh index 797a0fefaaf..9e300b1d2b3 100755 --- a/.github/unittest/linux_libs/scripts_craftground/run_test.sh +++ b/.github/unittest/linux_libs/scripts_craftground/run_test.sh @@ -26,6 +26,7 @@ java -version # The Minecraft client needs an X display; use a virtual framebuffer with # Mesa software rendering. Xvfb :99 -screen 0 1024x768x24 & +trap 'bash .github/unittest/helpers/reap_strays.sh' EXIT export DISPLAY=:99 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/libs/test_craftground.py --instafail -v --durations 200 --capture no -k TestCraftGround --error-for-skips --runslow diff --git a/.github/unittest/linux_libs/scripts_habitat/run_all.sh b/.github/unittest/linux_libs/scripts_habitat/run_all.sh index 6a627cb8d6b..9a390812c96 100755 --- a/.github/unittest/linux_libs/scripts_habitat/run_all.sh +++ b/.github/unittest/linux_libs/scripts_habitat/run_all.sh @@ -12,7 +12,7 @@ apt-get install -y vim git wget cmake ninja-build # and both resolve on jammy too. apt-get install -y libglfw3 libglfw3-dev libgl1 libglx-mesa0 libosmesa6 libosmesa6-dev libglew-dev apt-get install -y libxinerama-dev libxcursor-dev libxi-dev libxrandr-dev libxxf86vm-dev -apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 +apt-get install -y libglvnd0 libgl1 libglx0 libegl1 libgles2 libglib2.0-0 apt-get install -y libegl1-mesa-dev libgles2-mesa-dev # Build tools and libraries for habitat-sim diff --git a/.github/unittest/linux_libs/scripts_habitat/setup_env.sh b/.github/unittest/linux_libs/scripts_habitat/setup_env.sh index 9c00c5c60af..3556f194921 100755 --- a/.github/unittest/linux_libs/scripts_habitat/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_habitat/setup_env.sh @@ -89,11 +89,12 @@ cd habitat-sim # Build with headless (EGL) and bullet physics support # Ensure system cmake is used (pip cmake 4.x is incompatible with habitat-sim's CMake files) -# Put /usr/bin at the front of PATH to prefer system cmake over any pip-installed cmake -export PATH="/usr/bin:$PATH" +# Put /usr/bin at the front of PATH for this build only to prefer system cmake +# over any pip-installed cmake; exporting it would make later pip3 calls resolve +# to the externally-managed system pip instead of the conda env's # Also set CMAKE_EXECUTABLE to explicitly use system cmake export CMAKE_EXECUTABLE=/usr/bin/cmake -pip3 install . --no-build-isolation +PATH="/usr/bin:$PATH" "${env_dir}/bin/pip3" install . --no-build-isolation cd "${root_dir}" diff --git a/.github/unittest/linux_libs/scripts_libero/run_test.sh b/.github/unittest/linux_libs/scripts_libero/run_test.sh index 5d19e2e9138..d1c6455d5ae 100755 --- a/.github/unittest/linux_libs/scripts_libero/run_test.sh +++ b/.github/unittest/linux_libs/scripts_libero/run_test.sh @@ -22,6 +22,7 @@ timeout 300s python -m torch.utils.collect_env git config --global --add safe.directory '*' Xvfb :99 -screen 0 1024x768x24 & +trap 'bash .github/unittest/helpers/reap_strays.sh' EXIT timeout 120s python -c 'import torch; t = torch.ones([2, 2], device="cuda:0" if torch.cuda.is_available() else "cpu"); print(t); print("tensor device:" + str(t.device))' timeout 120s python -c 'from torchrl.envs.libs.libero import _ensure_libero_config; _ensure_libero_config()' diff --git a/.github/unittest/linux_libs/scripts_minari/setup_env.sh b/.github/unittest/linux_libs/scripts_minari/setup_env.sh index 081f32199c9..575ee2474b5 100755 --- a/.github/unittest/linux_libs/scripts_minari/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_minari/setup_env.sh @@ -8,7 +8,8 @@ set -e set -v -apt-get update && apt-get upgrade -y && apt-get install -y git cmake +# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI +apt-get update && apt-get upgrade -y && apt-get install -y git cmake python3-venv # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' apt-get install -y wget \ @@ -26,7 +27,8 @@ apt-get install -y wget \ libgl1 \ libglx0 \ libegl1 \ - libgles2 + libgles2 \ + python3-dev # Upgrade specific package apt-get upgrade -y libstdc++6 diff --git a/.github/unittest/linux_libs/scripts_openx/setup_env.sh b/.github/unittest/linux_libs/scripts_openx/setup_env.sh index a7afd646646..2e1be5677da 100755 --- a/.github/unittest/linux_libs/scripts_openx/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_openx/setup_env.sh @@ -8,7 +8,8 @@ set -e set -v -apt-get update && apt-get upgrade -y && apt-get install -y git cmake +# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI +apt-get update && apt-get upgrade -y && apt-get install -y git cmake python3-venv # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' apt-get install -y wget \ diff --git a/.github/unittest/linux_libs/scripts_procgen/setup_env.sh b/.github/unittest/linux_libs/scripts_procgen/setup_env.sh index 0f1210f3a54..c612ae0f099 100644 --- a/.github/unittest/linux_libs/scripts_procgen/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_procgen/setup_env.sh @@ -27,7 +27,8 @@ apt-get install -y wget \ libglx0 \ libegl1 \ libgles2 \ - libglib2.0-0 + libglib2.0-0 \ + python3-dev # Upgrade specific package apt-get upgrade -y libstdc++6 diff --git a/.github/unittest/linux_libs/scripts_safety_gymnasium/setup_env.sh b/.github/unittest/linux_libs/scripts_safety_gymnasium/setup_env.sh index 0f1210f3a54..c612ae0f099 100755 --- a/.github/unittest/linux_libs/scripts_safety_gymnasium/setup_env.sh +++ b/.github/unittest/linux_libs/scripts_safety_gymnasium/setup_env.sh @@ -27,7 +27,8 @@ apt-get install -y wget \ libglx0 \ libegl1 \ libgles2 \ - libglib2.0-0 + libglib2.0-0 \ + python3-dev # Upgrade specific package apt-get upgrade -y libstdc++6 diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh b/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh index 1ac223003e1..95c91408714 100755 --- a/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh +++ b/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh @@ -12,6 +12,11 @@ python -m torch.utils.collect_env # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' +# Resolve the OSDC container's hostname for torchrl/_comm/distributed.py; see +# linux/scripts/run_all.sh. +echo "127.0.0.1 $(hostname)" >> /etc/hosts +getent hosts "$(hostname)" + root_dir="$(git rev-parse --show-toplevel)" env_dir="${root_dir}/env" lib_dir="${env_dir}/lib" diff --git a/.github/unittest/llm/scripts_llm/setup_env.sh b/.github/unittest/llm/scripts_llm/setup_env.sh index 0245a7fe790..491cce19671 100644 --- a/.github/unittest/llm/scripts_llm/setup_env.sh +++ b/.github/unittest/llm/scripts_llm/setup_env.sh @@ -9,13 +9,10 @@ set -e export DEBIAN_FRONTEND=noninteractive export TZ=UTC apt-get update -apt-get install -yq --no-install-recommends git wget unzip curl patchelf +# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI +apt-get install -yq --no-install-recommends git wget unzip curl patchelf cmake python3-venv # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*' -# The base PyTorch devel image provides compilers, CMake >= 3.22, and most build deps. -# Install only minimal utilities not guaranteed to be present. - -# CMake available in the PyTorch devel image (Ubuntu 22.04) is sufficient. # Cleanup APT cache apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/.github/unittest/llm/scripts_sglang/install.sh b/.github/unittest/llm/scripts_sglang/install.sh index d9d9a16d961..69270a8487e 100755 --- a/.github/unittest/llm/scripts_sglang/install.sh +++ b/.github/unittest/llm/scripts_sglang/install.sh @@ -69,7 +69,7 @@ fi printf "* Installing tensordict\n" # Install tensordict dependencies first (pyvers is required but --no-deps skips it) uv pip install cloudpickle packaging importlib_metadata numpy orjson "pyvers>=0.2.3,<0.3.0" -uv pip install "pybind11[global]" ninja +uv pip install "pybind11[global]" ninja cmake if [[ "$RELEASE" == 0 ]]; then uv pip install --no-build-isolation --no-deps git+https://github.com/pytorch/tensordict.git@$(cat .github/ci_commit_pins/tensordict.txt) else diff --git a/.github/unittest/llm/scripts_sglang/setup_env.sh b/.github/unittest/llm/scripts_sglang/setup_env.sh index 0f6a18b17c7..2d6a2eb6b3e 100755 --- a/.github/unittest/llm/scripts_sglang/setup_env.sh +++ b/.github/unittest/llm/scripts_sglang/setup_env.sh @@ -8,7 +8,8 @@ export DEBIAN_FRONTEND=noninteractive export TZ=UTC apt-get update -apt-get install -yq --no-install-recommends git wget unzip curl patchelf +# python3-venv lets linux_job_v3's HF cache sync install the AWS CLI +apt-get install -yq --no-install-recommends git wget unzip curl patchelf python3-dev python3-venv # Avoid error: "fatal: unsafe repository" git config --global --add safe.directory '*'