From 5e4ec51f7d850a5c8e6425f503f854e1efcd535a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Wed, 24 Jun 2026 23:52:37 -0700 Subject: [PATCH 01/10] docs: strengthen rust quick start examples Signed-off-by: mnajafian-nv --- docs/getting-started/quick-start/rust.mdx | 78 ++++++++++++++++++----- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/docs/getting-started/quick-start/rust.mdx b/docs/getting-started/quick-start/rust.mdx index 347375b17..bd4c1dcf4 100644 --- a/docs/getting-started/quick-start/rust.mdx +++ b/docs/getting-started/quick-start/rust.mdx @@ -6,7 +6,7 @@ position: 3 {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} -This quick start shows the smallest Rust workflow that emits scope and mark events. +This quick start shows the smallest Rust workflow that emits scope, tool, and LLM events. ## Choose an Install Path @@ -19,8 +19,8 @@ Use the published crates when you are consuming a release: ```bash cargo add nemo-relay@0.5.0 -cargo add nemo-relay-adaptive@0.5.0 cargo add serde_json +cargo add tokio --features macros,rt-multi-thread ``` Install the published NeMo Relay CLI separately when you need coding-agent hook @@ -37,28 +37,36 @@ Use a path dependency when your application is consuming a local checkout: ```toml [dependencies] nemo-relay = { path = "../NeMo-Relay/crates/core" } -nemo-relay-adaptive = { path = "../NeMo-Relay/crates/adaptive" } serde_json = "1" +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } ``` - `nemo-relay` is the core Rust runtime surface. -- `nemo-relay-adaptive` is the companion crate for adaptive runtime primitives and Redis-backed learning components. -- `nemo-relay-cli` is a binary crate. Use `cargo install nemo-relay-cli@0.5.0` when - you need the NeMo Relay CLI. +- `tokio` provides the async runtime used by the managed tool and LLM helpers in + this quick start. +- `nemo-relay-adaptive` is a companion crate for adaptive runtime primitives and + Redis-backed learning components when you need adaptive behavior later, but it + is not required for this minimal direct-runtime example. +- `nemo-relay-cli` is a binary crate. Use `cargo install nemo-relay-cli@0.5.0` + when you need the NeMo Relay CLI. -## Push a Scope and Emit a Mark +## Run One Scope, One Tool Call, and One LLM Call -The example below creates a scope and records a mark event from Rust. +The example below creates a scope, records an initialization mark, runs one +managed tool call, and runs one managed LLM call through the public Rust APIs. ```rust +use nemo_relay::api::llm::{llm_call_execute, LlmCallExecuteParams, LlmRequest}; use nemo_relay::api::scope::{ self, EmitMarkEventParams, PopScopeParams, PushScopeParams, ScopeAttributes, ScopeType, }; use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; +use nemo_relay::api::tool::{tool_call_execute, ToolCallExecuteParams}; use serde_json::json; use std::sync::Arc; -fn main() -> Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { register_subscriber( "quickstart-printer", Arc::new(|event| { @@ -82,6 +90,41 @@ fn main() -> Result<(), Box> { .data(json!({"ok": true})) .build(), )?; + + let tool_result = tool_call_execute( + ToolCallExecuteParams::builder() + .name("search") + .args(json!({"query": "hello"})) + .func(Arc::new(|args| Box::pin(async move { + Ok(json!({"echo": args["query"]})) + }))) + .parent(handle.clone()) + .build(), + ) + .await?; + + let llm_result = llm_call_execute( + LlmCallExecuteParams::builder() + .name("demo-provider") + .request(LlmRequest { + headers: Default::default(), + content: json!({"messages": [{"role": "user", "content": "hi"}]}), + }) + .func(Arc::new(|request| Box::pin(async move { + Ok(json!({ + "messages": request.content["messages"], + "ok": true + })) + }))) + .parent(handle.clone()) + .model_name("demo-model") + .build(), + ) + .await?; + + println!("{tool_result}"); + println!("{llm_result}"); + scope::pop_scope(PopScopeParams::builder().handle_uuid(&handle.uuid).build())?; flush_subscribers()?; let _ = deregister_subscriber("quickstart-printer")?; @@ -91,15 +134,20 @@ fn main() -> Result<(), Box> { ## What Success Looks Like -The script should exit cleanly and print lifecycle lines from the subscriber. -Native subscriber delivery is asynchronous, so examples flush before checking -subscriber output. -You should see one line for the scope start event, one for the `initialized` -mark, and one for the scope end event. +You should see: + +- event lines for the scope, tool, LLM, and mark lifecycle +- `{"echo":"hello"}` from the tool call +- a final object containing `ok: true` and the echoed message payload from the + LLM callback + +Native subscriber delivery is asynchronous, so flush before checking subscriber +output. If you only see the returned JSON values and no event lines, the +business callbacks ran but you did not verify instrumentation. That tells you two things: -- The scope API ran successfully. +- The direct Rust runtime APIs ran successfully. - Emitted events were observable through the subscriber system. ## What to Learn Next From 4b47b5ac0348b330909a231f7ba7830077361e3a Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 00:00:52 -0700 Subject: [PATCH 02/10] docs: fix rust quick start bullet style Signed-off-by: mnajafian-nv --- docs/getting-started/quick-start/rust.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/quick-start/rust.mdx b/docs/getting-started/quick-start/rust.mdx index bd4c1dcf4..ffd18364e 100644 --- a/docs/getting-started/quick-start/rust.mdx +++ b/docs/getting-started/quick-start/rust.mdx @@ -136,9 +136,9 @@ async fn main() -> Result<(), Box> { You should see: -- event lines for the scope, tool, LLM, and mark lifecycle +- Event lines for the scope, tool, LLM, and mark lifecycle - `{"echo":"hello"}` from the tool call -- a final object containing `ok: true` and the echoed message payload from the +- A final object containing `ok: true` and the echoed message payload from the LLM callback Native subscriber delivery is asynchronous, so flush before checking subscriber From 477b695e9863181043a63f6c31d9ad39a3b6f4de Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 00:16:34 -0700 Subject: [PATCH 03/10] docs: clarify plugin-managed example paths Signed-off-by: mnajafian-nv --- .../plugin-configuration-files.mdx | 37 +++++++++++++++++++ docs/observability-plugin/about.mdx | 23 ++++++++++++ 2 files changed, 60 insertions(+) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index 803cb7e0d..b8704df0e 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -24,6 +24,43 @@ generic plugin document and component-local `config` objects use canonical +## Shortest Path Example + +Use this minimal `plugins.toml` when you want the CLI gateway to start with one +plugin-managed observability exporter and no extra layering: + +```toml +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 1 + +[components.config.atof] +enabled = true +output_directory = "logs" +filename = "events.jsonl" +mode = "append" +``` + +Place the file at `./.nemo-relay/plugins.toml` or another discovered location, +then start the gateway normally. If you want an explicit config root, pass +`--config path/to/config.toml` and place `plugins.toml` next to that file. + +## What Success Looks Like + +The shortest-path setup is working when: + +- `nemo-relay` starts without plugin validation or activation errors. +- The observability component activates exactly one exporter path from the file. +- An instrumented gateway run writes ATOF JSONL output to `logs/events.jsonl`. + +After that path works, expand the config with additional components, policies, +or higher-precedence files as needed. + ## File Shape `plugins.toml` uses the canonical plugin document shape: diff --git a/docs/observability-plugin/about.mdx b/docs/observability-plugin/about.mdx index b9fb46a0d..7248f6740 100644 --- a/docs/observability-plugin/about.mdx +++ b/docs/observability-plugin/about.mdx @@ -11,6 +11,16 @@ Use the Observability plugin when you need to inspect NeMo Relay lifecycle event in process or export agent activity to tracing, trajectory, or analysis systems from one plugin configuration document. +## Shortest Path + +Start with one plugin-managed exporter before layering in additional formats or +manual subscribers. A common first path is: + +1. Instrument one scope, tool call, or LLM call in your application. +2. Enable the Observability plugin with only one exporter, such as ATOF JSONL. +3. Run one request through the instrumented path. +4. Confirm the emitted output before adding more exporters or sanitization rules. + Observability in NeMo Relay starts with events. Scopes, marks, managed tool calls, managed LLM calls, middleware, and manual lifecycle APIs emit the canonical Agent Trajectory Observability Format (ATOF) event stream. @@ -85,6 +95,19 @@ Choose the exporter based on the downstream system: Start with in-process event inspection before exporting externally. Add sanitize guardrails before exporters receive sensitive payloads. +## What Success Looks Like + +Your first exporter path is wired correctly when: + +- The instrumented request still completes successfully. +- The chosen exporter receives scope data for that request, plus tool or LLM + lifecycle data when the request exercises those paths. +- The exported output shows the same NeMo Relay scope hierarchy you expect from the runtime contract. + +If that basic check fails, use the +[Trace Incident Runbook](/resources/troubleshooting/trace-incident-runbook) +before adding more exporters or extra config layers. + For trace incidents involving missing traces, wrong scope attachment, export failures, duplicate events, or sensitive telemetry, use the [Trace Incident Runbook](/resources/troubleshooting/trace-incident-runbook). From 6695a3c62a3b86821e3a22f712e19cd3f73e973c Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 00:18:53 -0700 Subject: [PATCH 04/10] docs: add integration verification notes Signed-off-by: mnajafian-nv --- docs/supported-integrations/deepagents.mdx | 8 ++++++++ docs/supported-integrations/langchain.mdx | 8 ++++++++ docs/supported-integrations/langgraph.mdx | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/docs/supported-integrations/deepagents.mdx b/docs/supported-integrations/deepagents.mdx index fa247fe23..265bad979 100644 --- a/docs/supported-integrations/deepagents.mdx +++ b/docs/supported-integrations/deepagents.mdx @@ -87,6 +87,14 @@ final_message = result["messages"][-1] print(f"Final response: {final_message.content}") ``` +## Verify the Integration + +The integration is wired correctly when: + +- The Deep Agents run completes and prints a final response. +- The `deepagents-request` scope contains the top-level agent execution. +- Skill, subagent, and human-in-the-loop marks appear when those features are exercised. + ## Observability The integration composes the existing NeMo Relay LangChain and LangGraph hooks, diff --git a/docs/supported-integrations/langchain.mdx b/docs/supported-integrations/langchain.mdx index f9d830111..76c70b065 100644 --- a/docs/supported-integrations/langchain.mdx +++ b/docs/supported-integrations/langchain.mdx @@ -88,6 +88,14 @@ final_message = result["messages"][-1] print(f"Final response: {final_message.content}") ``` +## Verify the Integration + +The integration is wired correctly when: + +- The agent run completes and prints a final response. +- The `langchain-request` scope contains the managed model call. +- Tool activity appears under the same request when the agent decides to use `get_weather`. + ## Observability Refer to [Observability](/observability-plugin/about) for details on exporting NeMo Relay observability data to third-party systems. diff --git a/docs/supported-integrations/langgraph.mdx b/docs/supported-integrations/langgraph.mdx index b3723263c..f617cc233 100644 --- a/docs/supported-integrations/langgraph.mdx +++ b/docs/supported-integrations/langgraph.mdx @@ -101,6 +101,14 @@ pip install "nemo-relay[langgraph,langchain-nvidia]" +## Verify the Integration + +The integration is wired correctly when: + +- `graph.invoke(...)` returns the incremented state from the example. +- The `langgraph-request` scope contains the LangGraph run. +- Nested LangChain agents inherit the same callback config when you pass the LangGraph `config` through. + ## Observability Refer to [Observability](/observability-plugin/about) for details on exporting NeMo Relay observability data to third-party systems. From 4bb9f81afd327d181f77327c639f50331c30ff39 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 08:39:54 -0700 Subject: [PATCH 05/10] docs: clarify example execution paths Signed-off-by: mnajafian-nv --- .../plugin-configuration-files.mdx | 10 +++++++--- docs/getting-started/quick-start/index.mdx | 20 +++++++++++++++++-- docs/getting-started/quick-start/nodejs.mdx | 2 ++ docs/getting-started/quick-start/python.mdx | 3 +++ docs/getting-started/quick-start/rust.mdx | 2 ++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index b8704df0e..2c118d5ae 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -46,9 +46,13 @@ filename = "events.jsonl" mode = "append" ``` -Place the file at `./.nemo-relay/plugins.toml` or another discovered location, -then start the gateway normally. If you want an explicit config root, pass -`--config path/to/config.toml` and place `plugins.toml` next to that file. +Place the file at `./.nemo-relay/plugins.toml` or another discovered location. +If you want an explicit config root, pass `--config path/to/config.toml` and +place `plugins.toml` next to that file. For a quick verification path, run one +of the documented wrapper commands from the same project root, such as +`nemo-relay codex` or `nemo-relay run -- codex`, so +`.nemo-relay/plugins.toml` is discoverable by the gateway startup path +described in [CLI Basic Usage](/nemo-relay-cli/basic-usage). ## What Success Looks Like diff --git a/docs/getting-started/quick-start/index.mdx b/docs/getting-started/quick-start/index.mdx index edbdac5d9..9267e68fc 100644 --- a/docs/getting-started/quick-start/index.mdx +++ b/docs/getting-started/quick-start/index.mdx @@ -18,6 +18,22 @@ If you still need the shared runtime vocabulary for scopes, middleware, events, or plugins, refer to the [Agent Runtime Primer](/getting-started/agent-runtime-primer) or [Concepts](/about-nemo-relay/concepts) first. +## Examples By Integration Layer + +Use this map when you know what Relay boundary you want to verify, but you do +not yet know which guide owns the working path. + +| Layer | Use When | Start Here | Success Check | +|---|---|---|---| +| CLI and gateway | A coding-agent harness owns invocation and provider routing | [CLI Basic Usage](/nemo-relay-cli/basic-usage) | The wrapped agent runs with Relay active and emits hook plus LLM lifecycle output. | +| Direct Python or Node.js application APIs | Your application owns the tool or LLM callback | [Python Quick Start](/getting-started/quick-start/python) or [Node.js Quick Start](/getting-started/quick-start/nodejs) | The sample prints event lines plus tool and LLM results. | +| Direct Rust application APIs | Your Rust application owns the tool or LLM callback | [Rust Quick Start](/getting-started/quick-start/rust) | The sample prints scope, tool, LLM, and mark lifecycle output. | +| Plugin-managed runtime setup | You need process-level exporter or plugin behavior from `plugins.toml` | [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The selected plugin path activates and writes the expected output or behavior. | +| Managed middleware | You want policy, redaction, routing, or execution wrapping around managed calls | [Add Middleware](/instrument-applications/advanced-guide) | One allowed request succeeds, one rejected request stops before execution, and observed payloads match the policy. | +| Framework integrations | LangChain, LangGraph, or Deep Agents owns callbacks or scheduling | [Supported Integrations](/supported-integrations/about) | The integration guide's verify step confirms the expected framework-owned output. | +| OpenClaw plugin path | OpenClaw owns plugin setup and Relay observes the OpenClaw-managed boundary | [OpenClaw](/supported-integrations/openclaw-plugin) | The OpenClaw guide's verify step confirms plugin setup and runtime output. | +| Manual or CI workflows | You need explicit config files, deterministic commands, or non-interactive automation | [CLI Basic Usage](/nemo-relay-cli/basic-usage) and [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The explicit command uses the intended config files without interactive setup. | + ## Local Coding-Agent Runs Use the NVIDIA NeMo Relay CLI when you want to observe a local Codex, Claude @@ -114,8 +130,8 @@ Smallest Node.js workflow that emits scope, tool, and LLM events. href="/getting-started/quick-start/rust" > -Smallest Rust workflow for direct runtime ownership, starting from scope events -and mark-based checkpoints. +Smallest Rust workflow for direct runtime ownership that emits scope, tool, and +LLM events. diff --git a/docs/getting-started/quick-start/nodejs.mdx b/docs/getting-started/quick-start/nodejs.mdx index 7807d1644..83023a8a3 100644 --- a/docs/getting-started/quick-start/nodejs.mdx +++ b/docs/getting-started/quick-start/nodejs.mdx @@ -37,6 +37,8 @@ This path is for local source development when you need to build the binding fro ## Run One Scope, One Tool Call, and One LLM Call The example below runs one minimal instrumented workflow through the binding. +Save it as `quickstart.js` in the same project where you installed +`nemo-relay-node`, then run `node quickstart.js`. ```js const { diff --git a/docs/getting-started/quick-start/python.mdx b/docs/getting-started/quick-start/python.mdx index ec6262847..54a63c7fc 100644 --- a/docs/getting-started/quick-start/python.mdx +++ b/docs/getting-started/quick-start/python.mdx @@ -52,6 +52,9 @@ This records the local source in the application's `pyproject.toml` through ## Run One Scope, One Tool Call, and One LLM Call The example below runs one minimal instrumented workflow through the binding. +Save it as `quickstart.py` in the same Python project where you installed +`nemo-relay`, then run `uv run python quickstart.py`. If you are not using +`uv`, run it with the same Python environment where you installed the package. ```python import asyncio diff --git a/docs/getting-started/quick-start/rust.mdx b/docs/getting-started/quick-start/rust.mdx index ffd18364e..9b42bcc88 100644 --- a/docs/getting-started/quick-start/rust.mdx +++ b/docs/getting-started/quick-start/rust.mdx @@ -54,6 +54,8 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread"] } The example below creates a scope, records an initialization mark, runs one managed tool call, and runs one managed LLM call through the public Rust APIs. +Place it in `src/main.rs` in the same Cargo project where you added the +dependencies above, then run `cargo run`. ```rust use nemo_relay::api::llm::{llm_call_execute, LlmCallExecuteParams, LlmRequest}; From 47cd310f474e8a0fe21df37539421df349cdc9b7 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 09:14:34 -0700 Subject: [PATCH 06/10] docs: tighten example verification wording Signed-off-by: mnajafian-nv --- docs/build-plugins/plugin-configuration-files.mdx | 3 ++- docs/getting-started/quick-start/index.mdx | 6 +++--- docs/getting-started/quick-start/nodejs.mdx | 1 + docs/getting-started/quick-start/rust.mdx | 3 ++- docs/observability-plugin/about.mdx | 4 ---- docs/supported-integrations/deepagents.mdx | 4 +++- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index 2c118d5ae..25f8230ee 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -59,7 +59,8 @@ described in [CLI Basic Usage](/nemo-relay-cli/basic-usage). The shortest-path setup is working when: - `nemo-relay` starts without plugin validation or activation errors. -- The observability component activates exactly one exporter path from the file. +- The observability component activates the ATOF exporter configured in this + file. - An instrumented gateway run writes ATOF JSONL output to `logs/events.jsonl`. After that path works, expand the config with additional components, policies, diff --git a/docs/getting-started/quick-start/index.mdx b/docs/getting-started/quick-start/index.mdx index 9267e68fc..bd941ce90 100644 --- a/docs/getting-started/quick-start/index.mdx +++ b/docs/getting-started/quick-start/index.mdx @@ -18,19 +18,19 @@ If you still need the shared runtime vocabulary for scopes, middleware, events, or plugins, refer to the [Agent Runtime Primer](/getting-started/agent-runtime-primer) or [Concepts](/about-nemo-relay/concepts) first. -## Examples By Integration Layer +## Examples by Integration Layer Use this map when you know what Relay boundary you want to verify, but you do not yet know which guide owns the working path. | Layer | Use When | Start Here | Success Check | |---|---|---|---| -| CLI and gateway | A coding-agent harness owns invocation and provider routing | [CLI Basic Usage](/nemo-relay-cli/basic-usage) | The wrapped agent runs with Relay active and emits hook plus LLM lifecycle output. | +| CLI and gateway | A coding-agent harness owns invocation and provider routing | [CLI Basic Usage](/nemo-relay-cli/basic-usage) | The wrapped agent runs with Relay active and emits hook output, plus gateway-routed LLM lifecycle output when provider routing is active. | | Direct Python or Node.js application APIs | Your application owns the tool or LLM callback | [Python Quick Start](/getting-started/quick-start/python) or [Node.js Quick Start](/getting-started/quick-start/nodejs) | The sample prints event lines plus tool and LLM results. | | Direct Rust application APIs | Your Rust application owns the tool or LLM callback | [Rust Quick Start](/getting-started/quick-start/rust) | The sample prints scope, tool, LLM, and mark lifecycle output. | | Plugin-managed runtime setup | You need process-level exporter or plugin behavior from `plugins.toml` | [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The selected plugin path activates and writes the expected output or behavior. | | Managed middleware | You want policy, redaction, routing, or execution wrapping around managed calls | [Add Middleware](/instrument-applications/advanced-guide) | One allowed request succeeds, one rejected request stops before execution, and observed payloads match the policy. | -| Framework integrations | LangChain, LangGraph, or Deep Agents owns callbacks or scheduling | [Supported Integrations](/supported-integrations/about) | The integration guide's verify step confirms the expected framework-owned output. | +| Framework integrations | A framework such as LangChain, LangGraph, or Deep Agents owns callbacks or scheduling | [Supported Integrations](/supported-integrations/about) | The integration guide's verify step confirms the expected framework-owned output. | | OpenClaw plugin path | OpenClaw owns plugin setup and Relay observes the OpenClaw-managed boundary | [OpenClaw](/supported-integrations/openclaw-plugin) | The OpenClaw guide's verify step confirms plugin setup and runtime output. | | Manual or CI workflows | You need explicit config files, deterministic commands, or non-interactive automation | [CLI Basic Usage](/nemo-relay-cli/basic-usage) and [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The explicit command uses the intended config files without interactive setup. | diff --git a/docs/getting-started/quick-start/nodejs.mdx b/docs/getting-started/quick-start/nodejs.mdx index 83023a8a3..349d4ad64 100644 --- a/docs/getting-started/quick-start/nodejs.mdx +++ b/docs/getting-started/quick-start/nodejs.mdx @@ -87,6 +87,7 @@ async function main() { }); flushSubscribers(); + await new Promise((resolve) => setImmediate(resolve)); deregisterSubscriber("quickstart-printer"); } diff --git a/docs/getting-started/quick-start/rust.mdx b/docs/getting-started/quick-start/rust.mdx index 9b42bcc88..8da571f4e 100644 --- a/docs/getting-started/quick-start/rust.mdx +++ b/docs/getting-started/quick-start/rust.mdx @@ -138,7 +138,8 @@ async fn main() -> Result<(), Box> { You should see: -- Event lines for the scope, tool, LLM, and mark lifecycle +- Event lines for the root scope, managed tool span, managed LLM span, and + mark lifecycle - `{"echo":"hello"}` from the tool call - A final object containing `ok: true` and the echoed message payload from the LLM callback diff --git a/docs/observability-plugin/about.mdx b/docs/observability-plugin/about.mdx index 7248f6740..0afcfe8d6 100644 --- a/docs/observability-plugin/about.mdx +++ b/docs/observability-plugin/about.mdx @@ -108,10 +108,6 @@ If that basic check fails, use the [Trace Incident Runbook](/resources/troubleshooting/trace-incident-runbook) before adding more exporters or extra config layers. -For trace incidents involving missing traces, wrong scope attachment, export -failures, duplicate events, or sensitive telemetry, use the -[Trace Incident Runbook](/resources/troubleshooting/trace-incident-runbook). - ## Correlating Trajectories And Traces When ATIF and trace exporters observe the same NeMo Relay events, they share diff --git a/docs/supported-integrations/deepagents.mdx b/docs/supported-integrations/deepagents.mdx index 265bad979..126d17fef 100644 --- a/docs/supported-integrations/deepagents.mdx +++ b/docs/supported-integrations/deepagents.mdx @@ -63,7 +63,6 @@ agent = create_deep_agent( **add_nemo_relay_integration( model="nvidia:nvidia/nemotron-3-nano-30b-a3b", tools=[], - skills=["/skills/research/"], name="main-agent", ) ) @@ -87,6 +86,9 @@ final_message = result["messages"][-1] print(f"Final response: {final_message.content}") ``` +Add `skills=[...]` or subagent configuration after this minimal path is working +if you want to capture Deep Agents skill or subagent marks too. + ## Verify the Integration The integration is wired correctly when: From c3441648cdbf56d254a440d68fecabbb3547b672 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 09:29:10 -0700 Subject: [PATCH 07/10] docs: make plugin config verification deterministic Signed-off-by: mnajafian-nv --- .../plugin-configuration-files.mdx | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index 25f8230ee..9598c85bc 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -46,13 +46,19 @@ filename = "events.jsonl" mode = "append" ``` -Place the file at `./.nemo-relay/plugins.toml` or another discovered location. -If you want an explicit config root, pass `--config path/to/config.toml` and -place `plugins.toml` next to that file. For a quick verification path, run one -of the documented wrapper commands from the same project root, such as -`nemo-relay codex` or `nemo-relay run -- codex`, so -`.nemo-relay/plugins.toml` is discoverable by the gateway startup path -described in [CLI Basic Usage](/nemo-relay-cli/basic-usage). +For the most deterministic verification path, create `path/to/config.toml`, +place `plugins.toml` next to it, and launch the wrapper with that explicit +config root. For example: + +```bash +nemo-relay --config path/to/config.toml run -- codex +``` + +This keeps the gateway on the colocated `plugins.toml` path instead of relying +on system or user plugin discovery. If you prefer implicit discovery, place the +file at `./.nemo-relay/plugins.toml` or another discovered location and ensure +no higher-precedence plugin file overrides the exporter you want to verify. See +[CLI Basic Usage](/nemo-relay-cli/basic-usage) for the wrapper command shapes. ## What Success Looks Like From 6e28eb9275c63450de87712cc4b865c1d551a8f6 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Thu, 25 Jun 2026 09:54:33 -0700 Subject: [PATCH 08/10] docs: clarify example ownership paths Signed-off-by: mnajafian-nv --- .../plugin-configuration-files.mdx | 19 +++++++++++-------- docs/getting-started/quick-start/index.mdx | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index 9598c85bc..cf865ef71 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -46,19 +46,22 @@ filename = "events.jsonl" mode = "append" ``` -For the most deterministic verification path, create `path/to/config.toml`, -place `plugins.toml` next to it, and launch the wrapper with that explicit -config root. For example: +For the most deterministic verification path, place `plugins.toml` next to the +gateway `config.toml` you want this run to use, then launch the wrapper with +that explicit config root. For example: ```bash nemo-relay --config path/to/config.toml run -- codex ``` -This keeps the gateway on the colocated `plugins.toml` path instead of relying -on system or user plugin discovery. If you prefer implicit discovery, place the -file at `./.nemo-relay/plugins.toml` or another discovered location and ensure -no higher-precedence plugin file overrides the exporter you want to verify. See -[CLI Basic Usage](/nemo-relay-cli/basic-usage) for the wrapper command shapes. +This keeps plugin discovery scoped to the colocated `plugins.toml` instead of +relying on implicit project, user, or system plugin files. The plugin file is +the configuration being demonstrated here; `--config` only tells the gateway +which config root to use for this run. If you prefer implicit discovery, place +the file at `./.nemo-relay/plugins.toml` or another discovered location and +ensure no higher-precedence plugin file overrides the exporter you want to +verify. See [CLI Basic Usage](/nemo-relay-cli/basic-usage) for the wrapper +command shapes. ## What Success Looks Like diff --git a/docs/getting-started/quick-start/index.mdx b/docs/getting-started/quick-start/index.mdx index bd941ce90..d241de468 100644 --- a/docs/getting-started/quick-start/index.mdx +++ b/docs/getting-started/quick-start/index.mdx @@ -26,6 +26,7 @@ not yet know which guide owns the working path. | Layer | Use When | Start Here | Success Check | |---|---|---|---| | CLI and gateway | A coding-agent harness owns invocation and provider routing | [CLI Basic Usage](/nemo-relay-cli/basic-usage) | The wrapped agent runs with Relay active and emits hook output, plus gateway-routed LLM lifecycle output when provider routing is active. | +| Persistent host-plugin installs | You want the maintained install path for Codex or Claude Code instead of the transparent wrapper | [Plugin Installation](/nemo-relay-cli/plugin-installation) | `nemo-relay doctor --plugin ` confirms the installed host plugin is registered and ready. | | Direct Python or Node.js application APIs | Your application owns the tool or LLM callback | [Python Quick Start](/getting-started/quick-start/python) or [Node.js Quick Start](/getting-started/quick-start/nodejs) | The sample prints event lines plus tool and LLM results. | | Direct Rust application APIs | Your Rust application owns the tool or LLM callback | [Rust Quick Start](/getting-started/quick-start/rust) | The sample prints scope, tool, LLM, and mark lifecycle output. | | Plugin-managed runtime setup | You need process-level exporter or plugin behavior from `plugins.toml` | [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The selected plugin path activates and writes the expected output or behavior. | From dc5acada758ad218b8bef35bf90b4e8270566587 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 29 Jun 2026 11:47:08 -0700 Subject: [PATCH 09/10] docs: tighten plugin configuration guide wording Signed-off-by: mnajafian-nv --- .../plugin-configuration-files.mdx | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index cf865ef71..fb408136e 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -257,10 +257,8 @@ Tables inside component config merge recursively. ## Configuration Layering -Plugin settings can come from files and the code. The files are the -configuration base layer and the code sits on top of it. If there is a -conflict, the code will always take precedence over the files. The layering -process is as follows: +Plugin settings come from files and code. Files form the base layer, and code +sits on top. When the two conflict, code wins. Layering works as follows: 1. Discover and merge the `plugins.toml` files from lowest to highest precedence (system → project → user), using the [Precedence And Merge Behavior](#precedence-and-merge-behavior) rules above. @@ -275,12 +273,12 @@ Files and code differ only in how they treat a setting you **omit**: | `version`, `policy`, or the `enabled` flag of a component you declare | Inherited from a lower-precedence file | **Always taken from code** — its default if you did not set it | | A whole component kind, or a key inside a component's `config` | Inherited from a lower-precedence file | Inherited from the file layer | -In other words, a file's gaps are filled by the layers beneath it, but code's -typed fields are never gaps — they always carry a value, so they always take precedence. -Only the open-ended parts of your code config (which components you include, and -the keys within each component's `config`) merge with the files. +In other words, the layers beneath a file fill its gaps. Code's typed fields +are never gaps because they always carry a value, so they always take +precedence. Only the open-ended parts of your code config (which components you +include and the keys within each component's `config`) merge with the files. -Without a filesystem — for example, a WebAssembly (wasm) build — no files are +Without a filesystem, such as a WebAssembly (wasm) build, no files are read, so the base is empty and only your `initialize` config applies. ## Explicit Defaults And Overrides @@ -300,7 +298,7 @@ enabled = false mode = "append" ``` -The merged config may still contain inherited ATOF sibling fields such as +The merged config may still contain inherited ATOF sibling fields, such as `output_directory`, but the runtime ignores the section because `enabled = false`. @@ -335,7 +333,7 @@ endpoints, and checks reachable OTLP endpoints when those settings are present. For Pricing, doctor validates enabled file and inline sources and fails when a source is unreadable or the catalog schema is invalid. -## Relationship To `config.toml` +## Relationship to `config.toml` `config.toml` owns gateway and agent setup, such as upstream provider base URLs and agent command configuration. `plugins.toml` owns reusable runtime behavior From 62430368eb0508e072f117b5c35da82ac7f2a2b2 Mon Sep 17 00:00:00 2001 From: mnajafian-nv Date: Mon, 29 Jun 2026 12:06:35 -0700 Subject: [PATCH 10/10] docs: clarify example verification wording Signed-off-by: mnajafian-nv --- docs/build-plugins/plugin-configuration-files.mdx | 3 ++- docs/getting-started/quick-start/index.mdx | 2 +- docs/getting-started/quick-start/nodejs.mdx | 2 +- docs/getting-started/quick-start/python.mdx | 2 +- docs/getting-started/quick-start/rust.mdx | 8 ++++---- docs/observability-plugin/about.mdx | 2 +- docs/supported-integrations/deepagents.mdx | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/build-plugins/plugin-configuration-files.mdx b/docs/build-plugins/plugin-configuration-files.mdx index fb408136e..f6536baf7 100644 --- a/docs/build-plugins/plugin-configuration-files.mdx +++ b/docs/build-plugins/plugin-configuration-files.mdx @@ -258,7 +258,8 @@ Tables inside component config merge recursively. ## Configuration Layering Plugin settings come from files and code. Files form the base layer, and code -sits on top. When the two conflict, code wins. Layering works as follows: +sits on top. When the two conflict, code takes precedence. Layering works as +follows: 1. Discover and merge the `plugins.toml` files from lowest to highest precedence (system → project → user), using the [Precedence And Merge Behavior](#precedence-and-merge-behavior) rules above. diff --git a/docs/getting-started/quick-start/index.mdx b/docs/getting-started/quick-start/index.mdx index d241de468..017d691f3 100644 --- a/docs/getting-started/quick-start/index.mdx +++ b/docs/getting-started/quick-start/index.mdx @@ -28,7 +28,7 @@ not yet know which guide owns the working path. | CLI and gateway | A coding-agent harness owns invocation and provider routing | [CLI Basic Usage](/nemo-relay-cli/basic-usage) | The wrapped agent runs with Relay active and emits hook output, plus gateway-routed LLM lifecycle output when provider routing is active. | | Persistent host-plugin installs | You want the maintained install path for Codex or Claude Code instead of the transparent wrapper | [Plugin Installation](/nemo-relay-cli/plugin-installation) | `nemo-relay doctor --plugin ` confirms the installed host plugin is registered and ready. | | Direct Python or Node.js application APIs | Your application owns the tool or LLM callback | [Python Quick Start](/getting-started/quick-start/python) or [Node.js Quick Start](/getting-started/quick-start/nodejs) | The sample prints event lines plus tool and LLM results. | -| Direct Rust application APIs | Your Rust application owns the tool or LLM callback | [Rust Quick Start](/getting-started/quick-start/rust) | The sample prints scope, tool, LLM, and mark lifecycle output. | +| Direct Rust application APIs | Your Rust application owns the tool or LLM callback | [Rust Quick Start](/getting-started/quick-start/rust) | The sample prints scope, tool, and LLM lifecycle output plus the `initialized` mark event. | | Plugin-managed runtime setup | You need process-level exporter or plugin behavior from `plugins.toml` | [Plugin Configuration Files](/build-plugins/plugin-configuration-files) | The selected plugin path activates and writes the expected output or behavior. | | Managed middleware | You want policy, redaction, routing, or execution wrapping around managed calls | [Add Middleware](/instrument-applications/advanced-guide) | One allowed request succeeds, one rejected request stops before execution, and observed payloads match the policy. | | Framework integrations | A framework such as LangChain, LangGraph, or Deep Agents owns callbacks or scheduling | [Supported Integrations](/supported-integrations/about) | The integration guide's verify step confirms the expected framework-owned output. | diff --git a/docs/getting-started/quick-start/nodejs.mdx b/docs/getting-started/quick-start/nodejs.mdx index 349d4ad64..dacd01a7d 100644 --- a/docs/getting-started/quick-start/nodejs.mdx +++ b/docs/getting-started/quick-start/nodejs.mdx @@ -101,7 +101,7 @@ main().catch((error) => { You should see: -- Event lines for the scope, tool, LLM, and mark lifecycle +- Event lines for the scope, tool, and LLM lifecycles, plus the `initialized` mark event - `{ echo: 'hello' }` from the tool call - An object containing `ok: true` and the echoed message payload from the LLM callback diff --git a/docs/getting-started/quick-start/python.mdx b/docs/getting-started/quick-start/python.mdx index 54a63c7fc..3b048cfa1 100644 --- a/docs/getting-started/quick-start/python.mdx +++ b/docs/getting-started/quick-start/python.mdx @@ -100,7 +100,7 @@ asyncio.run(main()) You should see: -- Event lines for the scope, tool, LLM, and mark lifecycle +- Event lines for the scope, tool, and LLM lifecycles, plus the `initialized` mark event - `{'echo': 'hello'}` from the tool call - A final object containing `ok: True` and the echoed message payload from the LLM callback diff --git a/docs/getting-started/quick-start/rust.mdx b/docs/getting-started/quick-start/rust.mdx index 8da571f4e..3d4498fcd 100644 --- a/docs/getting-started/quick-start/rust.mdx +++ b/docs/getting-started/quick-start/rust.mdx @@ -138,15 +138,15 @@ async fn main() -> Result<(), Box> { You should see: -- Event lines for the root scope, managed tool span, managed LLM span, and - mark lifecycle +- Event lines for the root scope, managed tool and LLM lifecycles, plus the + `initialized` mark event - `{"echo":"hello"}` from the tool call - A final object containing `ok: true` and the echoed message payload from the LLM callback Native subscriber delivery is asynchronous, so flush before checking subscriber -output. If you only see the returned JSON values and no event lines, the -business callbacks ran but you did not verify instrumentation. +output. If you only see the returned JSON values and no event lines, the tool +and LLM callbacks ran but you did not verify instrumentation. That tells you two things: diff --git a/docs/observability-plugin/about.mdx b/docs/observability-plugin/about.mdx index 0afcfe8d6..a6a526e22 100644 --- a/docs/observability-plugin/about.mdx +++ b/docs/observability-plugin/about.mdx @@ -102,7 +102,7 @@ Your first exporter path is wired correctly when: - The instrumented request still completes successfully. - The chosen exporter receives scope data for that request, plus tool or LLM lifecycle data when the request exercises those paths. -- The exported output shows the same NeMo Relay scope hierarchy you expect from the runtime contract. +- The exported output shows the same NeMo Relay scope hierarchy you expect from that request. If that basic check fails, use the [Trace Incident Runbook](/resources/troubleshooting/trace-incident-runbook) diff --git a/docs/supported-integrations/deepagents.mdx b/docs/supported-integrations/deepagents.mdx index 126d17fef..24bf37db9 100644 --- a/docs/supported-integrations/deepagents.mdx +++ b/docs/supported-integrations/deepagents.mdx @@ -87,7 +87,7 @@ print(f"Final response: {final_message.content}") ``` Add `skills=[...]` or subagent configuration after this minimal path is working -if you want to capture Deep Agents skill or subagent marks too. +when you need to capture Deep Agents skill or subagent marks. ## Verify the Integration