Project tool observability into reviewer transcripts

.github/scripts/run-agent-task/execute-native-agent-task.mjs

@@ -139,6 +139,56 @@ function record(value) {
   return value && typeof value === "object" && !Array.isArray(value) ? value : {}
 }
 
+function canonicalToolObservability(metadata) {
+  const source = record(record(record(metadata).agents_api).tool_observability)
+  if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > 64) return undefined
+  const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
+  return calls.length ? { version: 1, calls } : undefined
+}
+
+function projectCanonicalToolCall(value) {
+  const call = record(value)
+  const argumentsSummary = record(call.arguments)
+  const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
+  if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1
+    || !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name)
+    || !["succeeded", "failed", "rejected", "pending"].includes(call.status)
+    || argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0
+    || argumentsSummary.count !== keys.length || keys.length > 32 || !keys.every(safeToolIdentifier)) return undefined
+  const resultSummary = projectCanonicalToolResult(call.result)
+  if (call.result !== undefined && !resultSummary) return undefined
+  return Object.fromEntries(Object.entries({
+    sequence: call.sequence, turn: call.turn, tool_call_id: call.tool_call_id, tool_name: call.tool_name, status: call.status,
+    arguments: { keys, count: argumentsSummary.count, redacted: true }, result: resultSummary,
+    error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
+      : call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." } : undefined,
+  }).filter(([, item]) => item !== undefined))
+}
+
+function projectCanonicalToolResult(value) {
+  const result = record(value)
+  if (Object.keys(result).length === 0) return undefined
+  if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined
+  if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined
+  return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined
+}
+
+function safeToolIdentifier(value) {
+  return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
+}
+
+function removeRawToolObservability(metadata) {
+  const visit = (value) => {
+    if (Array.isArray(value)) return value.forEach(visit)
+    const entry = record(value)
+    if (!Object.keys(entry).length) return
+    const agentsApi = record(record(entry.metadata).agents_api)
+    delete agentsApi.tool_observability
+    Object.values(entry).forEach(visit)
+  }
+  visit(metadata)
+}
+
 function string(value) {
   return typeof value === "string" ? value.trim() : ""
 }
@@ -519,7 +569,17 @@ const nativeRuntimeResult = request.run_agent && !request.dry_run
   : {}
 await rm(nativeResultPath, { force: true })
 reviewerEvidence = await canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath)
+const toolObservability = canonicalToolObservability(record(nativeRuntimeResult).metadata)
+  ?? canonicalToolObservability(record(record(nativeRuntimeResult).agent_task_run_result).metadata)
 let runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization)
+removeRawToolObservability(runtimeResult)
+const normalizedAgentTaskResult = record(record(runtimeResult).agent_task_run_result)
+if (toolObservability) {
+  normalizedAgentTaskResult.metadata = {
+    ...record(normalizedAgentTaskResult.metadata),
+    tool_observability: toolObservability,
+  }
+}
 assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization)
 let workspaceApply = { status: "no-op", changedFiles: [] }
 let runnerWorkspaceCore = null

.github/scripts/run-agent-task/prepare-agent-task-upload.mjs

@@ -8,6 +8,7 @@ import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson } from "./runtime
 const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024
 const MAX_TRANSCRIPT_EXECUTIONS = 64
 const MAX_REVIEW_TEXT_BYTES = 32 * 1024
+const MAX_TOOL_ARGUMENT_KEYS = 32
 const workspace = resolve(process.env.AGENT_TASK_WORKSPACE || process.cwd())
 const uploadPath = resolve(process.env.AGENT_TASK_UPLOAD_PATH || join(workspace, ".codebox", "agent-task-upload"))
 const requestPath = resolve(process.env.AGENT_TASK_REQUEST_PATH || join(workspace, ".codebox", "agent-task-request.json"))
@@ -208,15 +209,64 @@ function projectParsed(value) {
   const results = Array.isArray(parsed.tool_results) ? parsed.tool_results : Array.isArray(parsed.toolResults) ? parsed.toolResults : []
   const errors = Array.isArray(parsed.errors) ? parsed.errors : parsed.error ? [parsed.error] : []
   const agent = record(parsed.agent ?? parsed.agent_metadata)
+  const toolObservability = canonicalToolObservability(parsed.metadata)
+    ?? canonicalToolObservability(record(parsed.agent_runtime).result?.metadata)
   return Object.fromEntries(Object.entries({
-    agent: Object.fromEntries(["id", "name", "model", "provider", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])),
+    agent: Object.fromEntries(["id", "name", "status"].flatMap((key) => boundedText(agent[key]) ? [[key, boundedText(agent[key])]] : [])),
     model_messages: messages.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((message) => boundedText(record(message).content ?? record(message).text ?? message) ? [{ role: boundedText(record(message).role), content: boundedText(record(message).content ?? record(message).text ?? message) }] : []),
     tool_calls: tools.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall),
     tool_results: results.slice(0, MAX_TRANSCRIPT_EXECUTIONS).map(projectToolCall),
     errors: errors.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((error) => boundedText(record(error).message ?? error) ? [boundedText(record(error).message ?? error)] : []),
+    tool_observability: toolObservability,
   }).filter(([, item]) => Array.isArray(item) ? item.length > 0 : Object.keys(item).length > 0))
 }
 
+// This is deliberately limited to the public Agents API summary. Tool payloads
+// and provider-specific tool records are not inputs to reviewer artifacts.
+function canonicalToolObservability(metadata) {
+  const source = record(record(record(metadata).agents_api).tool_observability)
+  if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TRANSCRIPT_EXECUTIONS) return undefined
+  const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
+  return calls.length ? { version: 1, calls } : undefined
+}
+
+function projectCanonicalToolCall(value) {
+  const call = record(value)
+  const argumentsSummary = record(call.arguments)
+  const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
+  if (!Number.isSafeInteger(call.sequence) || call.sequence < 1 || !Number.isSafeInteger(call.turn) || call.turn < 1
+    || !safeToolIdentifier(call.tool_call_id) || !safeToolIdentifier(call.tool_name)
+    || !["succeeded", "failed", "rejected", "pending"].includes(call.status)
+    || argumentsSummary.redacted !== true || !Number.isSafeInteger(argumentsSummary.count) || argumentsSummary.count < 0
+    || argumentsSummary.count !== keys.length || keys.length > MAX_TOOL_ARGUMENT_KEYS || !keys.every(safeToolIdentifier)) return undefined
+  const resultSummary = projectCanonicalToolResult(call.result)
+  if (call.result !== undefined && !resultSummary) return undefined
+  return Object.fromEntries(Object.entries({
+    sequence: call.sequence,
+    turn: call.turn,
+    tool_call_id: call.tool_call_id,
+    tool_name: call.tool_name,
+    status: call.status,
+    arguments: { keys, count: argumentsSummary.count, redacted: true },
+    result: resultSummary,
+    error: call.status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
+      : call.status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." }
+        : undefined,
+  }).filter(([, item]) => item !== undefined))
+}
+
+function projectCanonicalToolResult(value) {
+  const result = record(value)
+  if (Object.keys(result).length === 0) return undefined
+  if (["array", "object"].includes(result.type)) return Number.isSafeInteger(result.count) && result.count >= 0 ? { type: result.type, count: result.count } : undefined
+  if (result.type === "string") return Number.isSafeInteger(result.size) && result.size >= 0 ? { type: result.type, size: result.size } : undefined
+  return ["integer", "double", "boolean", "null"].includes(result.type) ? { type: result.type } : undefined
+}
+
+function safeToolIdentifier(value) {
+  return typeof value === "string" && value.length <= 256 && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
+}
+
 async function trustedTranscriptFile(path) {
   const root = await realpath(artifactsPath).catch(() => "")
   if (!root) return { unavailable: "artifact-root-missing" }

packages/runtime-core/src/agent-task-run-result.ts

@@ -2,6 +2,7 @@ import { isPlainObject, numberValue, objectValue, stringValue, stripUndefined }
 import { normalizeAgentTerminalResult, type AgentTerminalResult } from "./agent-terminal-result.js"
 import { RUNTIME_ACCESS_SCHEMA, normalizeRuntimeAccess, type RuntimeAccess } from "./runtime-boundary-contracts.js"
 import { normalizeAgentTaskStatus } from "./status-taxonomy.js"
+import { normalizeToolObservability } from "./tool-observability.js"
 
 export const AGENT_TASK_RUN_RESULT_SCHEMA = "wp-codebox/agent-task-run-result/v1" as const
 
@@ -137,6 +138,7 @@ export function normalizeAgentTaskRunResult(raw: unknown, options: AgentTaskRunR
       provider_error: objectValue(result.provider_error),
       timeout: result.timeout === true ? true : undefined,
       failure_evidence: objectValue(result.failure_evidence),
+      tool_observability: normalizeToolObservability(result.metadata) ?? normalizeToolObservability(agentResult.metadata),
     }),
     terminal_result: terminalResult,
     runtime_access: agentTaskRuntimeAccess(result),

packages/runtime-core/src/tool-observability.ts

@@ -0,0 +1,92 @@
+import { isPlainObject, objectValue, stripUndefined } from "./object-utils.js"
+
+export const TOOL_OBSERVABILITY_VERSION = 1 as const
+const MAX_TOOL_CALLS = 64
+const MAX_IDENTIFIER_LENGTH = 256
+const MAX_ARGUMENT_KEYS = 32
+const RESULT_TYPES = ["array", "object", "string", "integer", "double", "boolean", "null"] as const
+type ResultType = typeof RESULT_TYPES[number]
+
+export interface ToolObservabilityCall {
+  sequence: number
+  turn: number
+  tool_call_id: string
+  tool_name: string
+  status: "succeeded" | "failed" | "rejected" | "pending"
+  arguments: { keys: string[], count: number, redacted: true }
+  result?: { type: ResultType, count?: number, size?: number }
+  error?: { code: string, message: string }
+}
+
+export interface ToolObservability {
+  version: typeof TOOL_OBSERVABILITY_VERSION
+  calls: ToolObservabilityCall[]
+}
+
+/** Projects the public Agents API lifecycle without retaining tool payloads. */
+export function normalizeToolObservability(metadata: unknown): ToolObservability | undefined {
+  const observability = objectValue(objectValue(metadata).agents_api).tool_observability
+  const source = objectValue(observability)
+  if (source.version !== TOOL_OBSERVABILITY_VERSION || !Array.isArray(source.calls) || source.calls.length > MAX_TOOL_CALLS) return undefined
+
+  const calls = source.calls.map(projectCall).filter((call): call is ToolObservabilityCall => call !== undefined)
+  return calls.length > 0 ? { version: TOOL_OBSERVABILITY_VERSION, calls } : undefined
+}
+
+function projectCall(value: unknown): ToolObservabilityCall | undefined {
+  if (!isPlainObject(value)) return undefined
+  const call = objectValue(value)
+  const status = call.status
+  const argumentsSummary = objectValue(call.arguments)
+  const keys = Array.isArray(argumentsSummary.keys) ? argumentsSummary.keys : []
+  const count = argumentsSummary.count
+  if (!isPositiveInteger(call.sequence) || !isPositiveInteger(call.turn)
+    || !safeIdentifier(call.tool_call_id) || !safeIdentifier(call.tool_name)
+    || !["succeeded", "failed", "rejected", "pending"].includes(String(status))
+    || argumentsSummary.redacted !== true || !isNonNegativeInteger(count) || count !== keys.length || keys.length > MAX_ARGUMENT_KEYS
+    || !keys.every(safeIdentifier)) return undefined
+
+  const result = projectResult(call.result)
+  if (call.result !== undefined && !result) return undefined
+  return stripUndefined({
+    sequence: call.sequence,
+    turn: call.turn,
+    tool_call_id: call.tool_call_id,
+    tool_name: call.tool_name,
+    status: status as ToolObservabilityCall["status"],
+    arguments: { keys, count, redacted: true },
+    result,
+    error: status === "failed" ? { code: "tool_call_failed", message: "Tool call failed." }
+      : status === "rejected" ? { code: "tool_call_rejected", message: "Tool call was rejected." }
+        : undefined,
+  }) as ToolObservabilityCall
+}
+
+function projectResult(value: unknown): ToolObservabilityCall["result"] | undefined {
+  const result = objectValue(value)
+  if (Object.keys(result).length === 0) return undefined
+  if (!isResultType(result.type)) return undefined
+  if (result.type === "array" || result.type === "object") {
+    return isNonNegativeInteger(result.count) ? { type: result.type, count: result.count } : undefined
+  }
+  if (result.type === "string") {
+    return isNonNegativeInteger(result.size) ? { type: result.type, size: result.size } : undefined
+  }
+  return { type: result.type }
+}
+
+function safeIdentifier(value: unknown): value is string {
+  return typeof value === "string" && value.length <= MAX_IDENTIFIER_LENGTH && /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value)
+}
+
+function isResultType(value: unknown): value is ResultType {
+  return typeof value === "string" && RESULT_TYPES.includes(value as ResultType)
+}
+
+function isPositiveInteger(value: unknown): value is number {
+  return typeof value === "number" && Number.isSafeInteger(value) && value > 0
+}
+
+function isNonNegativeInteger(value: unknown): value is number {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
+}

tests/agent-task-contracts.test.ts

@@ -18,6 +18,41 @@ assert.equal(succeeded.schema, AGENT_TASK_RUN_RESULT_SCHEMA)
 assert.equal(succeeded.status, "succeeded")
 assert.equal(agentTaskRunExitCode({ success: true, agent_task_run_result: succeeded }), 0)
 
+const toolObservability = normalizeAgentTaskRunResult({
+  success: true,
+  metadata: {
+    agents_api: {
+      tool_observability: {
+        version: 1,
+        calls: [
+          { sequence: 1, turn: 1, tool_call_id: "call-success", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 1, redacted: true }, result: { type: "object", count: 2 }, result_body: "secret-sentinel" },
+          { sequence: 2, turn: 1, tool_call_id: "call-failure", tool_name: "workspace.write", status: "failed", arguments: { keys: [], count: 0, redacted: true }, error: { code: "raw-secret", message: "secret-sentinel" } },
+          { sequence: 3, turn: 2, tool_call_id: "call-rejected", tool_name: "workspace.delete", status: "rejected", arguments: { keys: ["path"], count: 1, redacted: true } },
+          { sequence: 4, turn: 2, tool_call_id: "call-pending", tool_name: "workspace.grep", status: "pending", arguments: { keys: ["query"], count: 1, redacted: true } },
+          { sequence: 5, turn: 2, tool_call_id: "malformed", tool_name: "workspace.read", status: "succeeded", arguments: { keys: ["path"], count: 2, redacted: true } },
+          { sequence: 6, turn: 2, tool_call_id: "untrusted secret sentinel", tool_name: "workspace.read", status: "pending", arguments: { keys: [], count: 0, redacted: true } },
+          { sequence: 7, turn: 2, tool_call_id: "call-secret-name", tool_name: "untrusted secret sentinel", status: "pending", arguments: { keys: [], count: 0, redacted: true } },
+          { sequence: 8, turn: 2, tool_call_id: "call-secret-key", tool_name: "workspace.read", status: "pending", arguments: { keys: ["untrusted secret sentinel"], count: 1, redacted: true } },
+          { sequence: 9, turn: 2, tool_call_id: "call-secret-type", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: { type: "untrusted secret sentinel", count: 1 } },
+        ],
+      },
+    },
+  },
+})
+assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls.map((call: any) => call.status), ["succeeded", "failed", "rejected", "pending"])
+assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[0].result, { type: "object", count: 2 })
+assert.deepEqual((toolObservability.metadata.tool_observability as any)?.calls[1].error, { code: "tool_call_failed", message: "Tool call failed." })
+assert.equal(JSON.stringify(toolObservability).includes("secret-sentinel"), false, "normalized tool observability never retains payloads or raw errors")
+assert.equal(JSON.stringify(toolObservability).includes("untrusted secret sentinel"), false, "normalized tool observability rejects non-identifier summary strings")
+for (const resultShape of [{ type: "string", size: 12 }, { type: "integer" }]) {
+  const normalized = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: 1, calls: [{ sequence: 1, turn: 1, tool_call_id: "call-shape", tool_name: "workspace.read", status: "succeeded", arguments: { keys: [], count: 0, redacted: true }, result: resultShape }] } } } })
+  assert.deepEqual((normalized.metadata.tool_observability as any)?.calls[0].result, resultShape)
+}
+for (const unsupported of [0, 2, "1"]) {
+  const result = normalizeAgentTaskRunResult({ metadata: { agents_api: { tool_observability: { version: unsupported, calls: [{ sequence: 1, turn: 1, tool_call_id: "call", tool_name: "tool", status: "pending", arguments: { keys: [], count: 0, redacted: true } }] } } } })
+  assert.equal("tool_observability" in result.metadata, false, "only canonical version 1 tool observability is consumed")
+}
+
 const resultFileDirectory = mkdtempSync(join(tmpdir(), "wp-codebox-agent-task-result-file-"))
 const resultFilePath = join(resultFileDirectory, "result.json")
 const atomicResult = { schema: "wp-codebox/agent-task-run/v1", success: true, status: "succeeded", padding: "x".repeat(40 * 1024) }