feat(vllm): share one engine across multiple HTTP frontends

examples/configs/distillation_math.yaml

@@ -245,6 +245,7 @@ policy: &POLICY_BASE
         # LoRA refit representation for vLLM. Use merged only as an explicit opt-in.
         lora_refit_mode: native
         vllm_cfg:
+            http_api_server_count: 1 # Positive process count; values >1 require async_engine: true and expose_http_server: true.
             async_engine: false
             precision: ${...precision}
             kv_cache_dtype: "auto"

examples/configs/evals/eval.yaml

@@ -20,6 +20,7 @@ generation:
   stop_token_ids: null
   stop_strings: null
   vllm_cfg:
+    http_api_server_count: 1 # Positive process count; values >1 require async_engine: true and expose_http_server: true.
     async_engine: false
     precision: "bfloat16"
     fp32_lm_head: false # Compute Nemotron-H logits with an fp32 LM head.

examples/configs/evals/mmau.yaml

@@ -19,6 +19,7 @@ generation:
   stop_token_ids: null
   stop_strings: null
   vllm_cfg:
+    http_api_server_count: 1 # Positive process count; values >1 require async_engine: true and expose_http_server: true.
     async_engine: false
     precision: "bfloat16"
     tensor_parallel_size: 1

examples/configs/grpo_math_1B.yaml

@@ -570,6 +570,7 @@ policy:
       expose_http_server: false  # Start an OpenAI-compatible HTTP server alongside the persistent engine. Required for NeMo-Gym.
       http_server_num_replicas: 8  # HTTP frontend event loops per model-parallel coordinator.
     vllm_cfg:
+      http_api_server_count: 1 # Positive process count; values >1 require async_engine: true and expose_http_server: true.
       http_server_api_key_required: false  # Require a per-job bearer token on the exposed server's /v1/* and /tokenize routes; the token is generated at startup and handed to the NeMo-Gym model server.
       async_engine: false
       precision: ${policy.precision}

examples/configs/ppo_math_1B.yaml

@@ -323,6 +323,7 @@ policy:
       logprobs_mode: processed_logprobs
       http_server_num_replicas: 8
     vllm_cfg:
+      http_api_server_count: 1 # Positive process count; values >1 require async_engine: true and expose_http_server: true.
       async_engine: false
       precision: ${policy.precision}
       kv_cache_dtype: "auto"

nemo_rl/algorithms/single_controller_utils/config.py

@@ -1603,6 +1603,16 @@ def validate_single_controller_config(master_config: MasterConfig) -> None:
         )
 
     generation_cfg = master_config.policy["generation"]
+    if (
+        generation_cfg["backend"] == "vllm"
+        and generation_cfg["vllm_cfg"].get("http_api_server_count", 1) > 1
+        and async_config.generation_fleet_health.enabled
+    ):
+        raise ValueError(
+            "Multiple HTTP API servers do not support fault tolerance with "
+            "async_rl.generation_fleet_health.enabled=true. "
+            "See https://github.com/NVIDIA-NeMo/RL/pull/5075."
+        )
     top_logprobs = generation_cfg.get("top_logprobs", 0)
     if (
         isinstance(top_logprobs, bool)

nemo_rl/models/generation/vllm/config.py

@@ -120,6 +120,9 @@ class VllmSpecificArgs(TypedDict):
     # Exposing vLLM as a server is useful in instances where the multi-turn rollout is performed with utilities outside of NeMo RL, but the user still wants to take advantage of the refit logic in NeMo RL that keeps the policy and generation up to date.
     # Currently it will expose the /tokenize and /v1/chat/completions endpoints. Later on we may expose /v1/completions or /v1/responses.
     expose_http_server: NotRequired[bool]
+    # Positive frontend process count sharing one engine and socket; 1
+    # keeps a single frontend. Multiple processes require async HTTP serving.
+    http_api_server_count: NotRequired[int]
     # Require `Authorization: Bearer <key>` on the exposed server's OpenAI routes
     # (/v1/* and /tokenize). VllmGeneration generates one random key per job, hands
     # it to the workers, and exposes it as `http_server_api_key` for the NeMo-Gym
@@ -299,6 +302,14 @@ def validate_vllm_async_actor_settings(config: VllmConfig) -> None:
     vllm_cfg = config.get("vllm_cfg")
     if vllm_cfg is None:
         return
+    api_server_count = vllm_cfg.get("http_api_server_count", 1)
+    if api_server_count > 1:
+        if not (vllm_cfg.get("async_engine") and vllm_cfg.get("expose_http_server")):
+            raise ValueError(
+                "Multiple API servers require async_engine=true and expose_http_server=true"
+            )
+        if config.get("refit_transport") in VLLM_SPARSE_REFIT_TRANSPORTS:
+            raise ValueError("Multiple HTTP API servers do not support sparse refit")
     if vllm_cfg.get("flat_logprobs") and not (
         vllm_cfg.get("async_engine") and vllm_cfg.get("expose_http_server")
     ):