feat(s3): add optional S3-over-RDMA (cuObject) data plane

.gitignore

@@ -57,3 +57,6 @@ multi-storage-client/src/multistorageclient/explorer/static/
 .windsurf/
 CLAUDE.md
 .mcp.json
+
+# Local cuObject SDK copy for the optional MSC `rdma` feature (never committed)
+multi-storage-client/rust/vendor/

.release_notes/.unreleased.md

@@ -4,4 +4,6 @@
 
 ## New Features
 
+- Add an optional S3-over-RDMA data plane to the `s3` provider (`rdma` option), backed by NVIDIA cuObject for direct buffer transfers to RDMA-capable endpoints.
+
 ## Bug Fixes

multi-storage-client/examples/rdma_roundtrip.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+End-to-end validation of the S3-over-RDMA (NVIDIA cuObject) data plane for the
+``s3`` storage provider, mirroring the PyTorch cuObject checkpoint roundtrip.
+
+This requires an RDMA-capable S3 endpoint and the cuObject runtime, so it is a
+standalone script rather than a CI test. Run it on a cluster client (e.g. coe09)
+against the RDMA MinIO endpoint (e.g. coe01:9000):
+
+    # Build the wheel once on a host with the cuObject runtime, enabling the
+    # crate's `rdma` feature so the cuObject shim is compiled into the extension:
+    #   maturin develop --features rdma        # into the active venv, or
+    #   maturin build   --features rdma        # a distributable wheel
+
+    export CUFILE_ENV_PATH_JSON=/path/to/cuobj.json   # rdma_dev_addr_list, use_pci_p2pdma, ...
+    export LD_LIBRARY_PATH=/path/to/sdklib:$LD_LIBRARY_PATH   # version-matched libcufile/libcuobjclient
+    export S3_ENDPOINT=http://coe01:9000
+    export S3_BUCKET=rdma-test
+    export AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin
+
+    python examples/rdma_roundtrip.py
+"""
+
+import os
+
+import multistorageclient as msc
+
+_ENDPOINT = os.environ["S3_ENDPOINT"]
+_BUCKET = os.environ["S3_BUCKET"]
+_PROFILE = "rdma-test"
+
+
+def _config() -> dict:
+    return {
+        "profiles": {
+            _PROFILE: {
+                "storage_provider": {
+                    # The `s3_cuobject` provider moves payloads over RDMA via
+                    # cuObject; it subclasses `s3` and forces the empty-body,
+                    # unsigned-payload wire contract.
+                    "type": "s3_cuobject",
+                    "options": {
+                        "base_path": _BUCKET,
+                        "endpoint_url": _ENDPOINT,
+                        "region_name": "us-east-1",
+                        # MinIO AIStor RDMA endpoints are path-addressed; the
+                        # provider leaves addressing style to the user.
+                        "s3": {"addressing_style": "path"},
+                        # Optional: tune the RDMA multipart part size.
+                        # "rdma": {"multipart_chunksize": 536870912},
+                    },
+                }
+            }
+        }
+    }
+
+
+def main() -> None:
+    config = msc.StorageClientConfig.from_dict(_config(), profile=_PROFILE)
+    client = msc.StorageClient(config)
+
+    # A few sizes spanning the old multipart threshold (64 MiB); all must take
+    # the single-shot RDMA path.
+    for mib in (1, 64, 256):
+        size = mib * 1024 * 1024
+        payload = os.urandom(size)
+        key = f"rdma_roundtrip/blob_{mib}mib.bin"
+
+        client.write(key, payload)
+        reloaded = bytes(client.read(key))
+
+        assert len(reloaded) == size, f"{key}: size {len(reloaded)} != {size}"
+        assert reloaded == payload, f"{key}: byte mismatch after RDMA roundtrip"
+        print(f"OK  {key}  ({mib} MiB) byte-identical over RDMA")
+
+    print("\nAll cuObject RDMA roundtrips passed.")
+
+
+if __name__ == "__main__":
+    main()

multi-storage-client/rust/Cargo.toml

@@ -13,6 +13,17 @@ publish = false
 name = "multistorageclient_rust"
 crate-type = ["cdylib"]
 
+[features]
+default = []
+# NVIDIA cuObject (S3-over-RDMA) data plane. Off by default; the default build
+# needs no C++ compiler or cuObject SDK. When enabled, build.rs compiles the
+# extern "C" shim (csrc/cuobj_shim.cc) against the vendored cuObject headers/libs
+# (vendor/cuobj/) and links libcuobjclient/libcufile. Linux only.
+rdma = ["dep:libc"]
+
+[build-dependencies]
+cc = "1.2"
+
 [dependencies]
 pyo3 = "0.29"
 pyo3-async-runtimes = { version = "0.29", features = ["tokio-runtime"] }
@@ -31,3 +42,4 @@ async-trait = "0.1.89"
 aws-config = { version = "1.8.18", default-features = false, features = ["rt-tokio", "credentials-process", "sso"] }
 aws-credential-types = "1.2.14"
 aws-smithy-http-client = { version = "1.1.13", default-features = false, features = ["rustls-ring"] }
+libc = { version = "0.2", optional = true }

multi-storage-client/rust/build.rs

@@ -0,0 +1,125 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::path::PathBuf;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=csrc/cuobj_shim.cc");
+    println!("cargo:rerun-if-changed=csrc/cuobj_shim.h");
+    println!("cargo:rerun-if-env-changed=MSC_CUOBJ_HOME");
+    println!("cargo:rerun-if-env-changed=CUDA_HOME");
+    println!("cargo:rerun-if-env-changed=CUDA_PATH");
+    println!("cargo:rerun-if-env-changed=CUDA_ROOT");
+
+    // The cuObject (RDMA) data plane is opt-in. Without the `rdma` feature the
+    // crate builds as a pure-Rust cdylib with no C++ compiler or cuObject SDK,
+    // so build.rs is a no-op.
+    if env::var("CARGO_FEATURE_RDMA").is_err() {
+        return;
+    }
+
+    if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("linux") {
+        panic!("`rdma` feature is only supported on Linux");
+    }
+
+    let arch = match env::var("CARGO_CFG_TARGET_ARCH").as_deref() {
+        Ok("x86_64") => "x86_64",
+        Ok("aarch64") => "aarch64",
+        Ok(other) => panic!("`rdma` feature: unsupported target arch `{other}`"),
+        Err(_) => panic!("CARGO_CFG_TARGET_ARCH not set"),
+    };
+
+    // Resolve the cuObject SDK (headers + libs) at build time. Nothing is
+    // vendored into the repository: the RDMA feature expects the NVIDIA
+    // cuObject SDK (CUDA Toolkit >= 13.1) to be installed. Resolution order:
+    //   1. MSC_CUOBJ_HOME    -> {include, lib}
+    //   2. CUDA_HOME/CUDA_PATH -> targets/<triple>/{include, lib}
+    //   3. ./vendor/cuobj    -> {include, lib/<arch>}  (local dev only)
+    let (include_dir, lib_dir) = resolve_cuobj_dirs(arch)
+        .expect(
+            "`rdma` feature: could not locate the cuObject SDK. Set MSC_CUOBJ_HOME (a directory \
+             with include/ and lib/), or CUDA_HOME/CUDA_PATH to a CUDA Toolkit (>= 13.1) that \
+             ships cuobjclient.h + libcuobjclient.",
+        );
+
+    cc::Build::new()
+        .cpp(true)
+        .std("c++17")
+        .file("csrc/cuobj_shim.cc")
+        .include(&include_dir)
+        .flag_if_supported("-Wno-unused-parameter")
+        .flag_if_supported("-fvisibility=hidden")
+        .compile("miniors_cuobj_shim");
+
+    println!("cargo:rustc-link-search=native={}", lib_dir.display());
+    println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display());
+
+    for lib in &[
+        "cuobjclient",
+        "cufile",
+        "ibverbs",
+        "rdmacm",
+        "numa",
+        "pthread",
+        "dl",
+        "rt",
+    ] {
+        println!("cargo:rustc-link-lib=dylib={lib}");
+    }
+    println!("cargo:rustc-link-lib=dylib=stdc++");
+}
+
+fn resolve_cuobj_dirs(arch: &str) -> Option<(PathBuf, PathBuf)> {
+    let has_header = |inc: &PathBuf| inc.join("cuobjclient.h").exists();
+
+    if let Ok(home) = env::var("MSC_CUOBJ_HOME") {
+        let base = PathBuf::from(home);
+        let inc = base.join("include");
+        let lib = base.join("lib");
+        if has_header(&inc) {
+            return Some((inc, lib));
+        }
+    }
+
+    let triple = match arch {
+        "x86_64" => "x86_64-linux",
+        "aarch64" => "sbsa-linux",
+        _ => "x86_64-linux",
+    };
+    for var in ["CUDA_HOME", "CUDA_PATH", "CUDA_ROOT"] {
+        if let Ok(cuda) = env::var(var) {
+            let base = PathBuf::from(cuda).join("targets").join(triple);
+            let inc = base.join("include");
+            let lib = base.join("lib");
+            if has_header(&inc) {
+                return Some((inc, lib));
+            }
+        }
+    }
+
+    // Local dev fallback: a vendored SDK copy (git-ignored, never committed).
+    let vendor = PathBuf::from(env::var("CARGO_MANIFEST_DIR").ok()?)
+        .join("vendor")
+        .join("cuobj");
+    let inc = vendor.join("include");
+    let lib = vendor.join("lib").join(arch);
+    if has_header(&inc) {
+        return Some((inc, lib));
+    }
+
+    None
+}