diff --git a/.azure-pipelines/templates/ut.yml b/.azure-pipelines/templates/ut.yml index 743c66e6e..81a74cf85 100644 --- a/.azure-pipelines/templates/ut.yml +++ b/.azure-pipelines/templates/ut.yml @@ -40,8 +40,8 @@ steps: name: PyTests displayName: Run pytests remoteScript: | - mpirun --allow-run-as-root -tag-output -x MSCCLPP_HOME=/root/mscclpp -x GPU_MAX_HW_QUEUES=8 -np 8 python3 -m pytest ./python/test/test_mscclpp.py -x - mpirun --allow-run-as-root -tag-output -x MSCCLPP_HOME=/root/mscclpp -x GPU_MAX_HW_QUEUES=8 -np 8 python3 -m pytest ./python/test/test_fp8_accum.py -x + mpirun --allow-run-as-root -tag-output -x MSCCLPP_HOME=/root/mscclpp -x HSA_NO_SCRATCH_RECLAIM=1 -x GPU_MAX_HW_QUEUES=8 -np 8 python3 -m pytest ./python/test/test_mscclpp.py -x + mpirun --allow-run-as-root -tag-output -x MSCCLPP_HOME=/root/mscclpp -x HSA_NO_SCRATCH_RECLAIM=1 -x GPU_MAX_HW_QUEUES=8 -np 8 python3 -m pytest ./python/test/test_fp8_accum.py -x - template: stop.yml parameters: diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b8316fb6..c3fb8fba5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,11 +112,20 @@ if(MSCCLPP_ENABLE_COVERAGE) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") message(STATUS "Code coverage enabled") - # Add coverage flags to C++ targets only (not CUDA) - add_compile_options($<$:--coverage>) + # Add coverage flags to C++ targets only (not CUDA). For ROCm, keep + # coverage on the host compilation so HIP device linking does not look + # for gcov runtime symbols. + if(MSCCLPP_USE_ROCM) + add_compile_options($<$:-Xarch_host>) + add_compile_options($<$:--coverage>) + add_link_options($<$:-Xarch_host>) + add_link_options($<$:--coverage>) + else() + add_compile_options($<$:--coverage>) + add_link_options($<$:--coverage>) + endif() add_compile_options($<$:-O0>) add_compile_options($<$:-g>) - add_link_options($<$:--coverage>) # Find lcov find_program(LCOV_PATH lcov) diff --git a/src/core/gpu_ipc_mem.cc b/src/core/gpu_ipc_mem.cc index 49dfcf822..b885a1ac4 100644 --- a/src/core/gpu_ipc_mem.cc +++ b/src/core/gpu_ipc_mem.cc @@ -186,6 +186,8 @@ UniqueGpuIpcMemHandle GpuIpcMemHandle::create(const CUdeviceptr ptr) { if (err == cudaSuccess) { handle->typeFlags |= GpuIpcMemHandle::Type::RuntimeIpc; } else { + // The VMM fallback below handles this failure, so do not leak it as CUDA's last error. + (void)cudaGetLastError(); WARN(GPU, "Failed to create runtime CUDA IPC handle for pointer ", (void*)basePtr, ": error=", static_cast(err), " (", std::string(cudaGetErrorString(err)), ")"); }