diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d8a9d4..187fc4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- **Async fixture responses** — Fixture responses can now be sync or async functions that receive the request and return the response dynamically. Enables awaiting side effects (database writes, API calls) before constructing the response — eliminating race conditions in complex multi-turn E2E tests. Works with all providers, streaming, and convenience methods (`on()`, `onMessage()`, `onTurn()`). (Feature request by @5ebastianMeier, issue #154) - **Snapshot-style recording** — When `X-Test-Id` is present, recorded fixtures are saved to `//.json` instead of timestamp-based filenames. Multiple fixtures for the same test+provider merge into one file. Stable paths enable meaningful PR diffs and easy test-to-fixture mapping. (Feature request by @jantimon, issue #155) ## [1.18.0] - 2026-05-04 diff --git a/docs/examples/index.html b/docs/examples/index.html index 4b32b7f..4ffe1e6 100644 --- a/docs/examples/index.html +++ b/docs/examples/index.html @@ -607,6 +607,52 @@

Tool-call cycle with hasToolResult

] } + + + +

Dynamic / Async Responses

+ +

+ Fixture responses can be functions — sync or async — that receive the request + and return the response dynamically. Use this when you need to await side effects, compute + responses based on request content, or inject runtime data into fixtures. +

+ +

Async response with side-effect

+

+ Wait for an external operation to complete before constructing the fixture response. + Eliminates race conditions in multi-turn E2E tests where entity creation happens + out-of-band. +

+
+
async-side-effect.ts ts
+
mock.on(
+  { toolCallId: "call_create_entity" },
+  async (req) => {
+    const entity = await createEntityPromise;
+    return {
+      content: `Entity "${entity.name}" created!`,
+      toolCalls: [{
+        name: "next_step",
+        arguments: JSON.stringify({ entityId: entity.id }),
+      }],
+    };
+  },
+);
+
+ +

Request-aware response

+

+ Compute the response from the incoming request content. Useful for echo-style fixtures, + transformations, or conditional logic that goes beyond what match fields can express. +

+
+
request-aware.ts ts
+
mock.onMessage("translate", (req) => {
+  const text = req.messages.at(-1)?.content ?? "";
+  return { content: `Translated: ${text.toUpperCase()}` };
+});
+
diff --git a/docs/fixtures/index.html b/docs/fixtures/index.html index 2bea28f..d9189c2 100644 --- a/docs/fixtures/index.html +++ b/docs/fixtures/index.html @@ -355,6 +355,14 @@

Response Types

+
+

+ Dynamic responses: Responses can also be sync or async functions that + receive the request and return the response dynamically. See + Dynamic Responses on the Examples page. +

+
+

Response Override Fields

Fixture responses can include optional fields to override auto-generated envelope values. diff --git a/docs/multi-turn/index.html b/docs/multi-turn/index.html index 1fa7890..922f340 100644 --- a/docs/multi-turn/index.html +++ b/docs/multi-turn/index.html @@ -182,6 +182,18 @@

hasToolResult — match by tool execution state

} +
+

+ Async fixture responses for race-free multi-turn tests. When a + multi-turn test depends on side effects between turns (database writes, entity creation, + external API calls), async fixture responses let you await those operations + before constructing the response — eliminating race conditions without + setTimeout hacks. See + Dynamic / Async Responses on the + Examples page. +

+
+

Programmatic API

The onTurn() convenience method combines turnIndex with a diff --git a/src/__tests__/async-fixture-response.test.ts b/src/__tests__/async-fixture-response.test.ts new file mode 100644 index 0000000..f4aa0e6 --- /dev/null +++ b/src/__tests__/async-fixture-response.test.ts @@ -0,0 +1,251 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { LLMock } from "../llmock.js"; +import type { ChatCompletionRequest, SSEChunk } from "../types.js"; + +function parseSSEChunks(body: string): SSEChunk[] { + return body + .split("\n\n") + .filter((line) => line.startsWith("data: ") && !line.includes("[DONE]")) + .map((line) => JSON.parse(line.slice(6)) as SSEChunk); +} + +describe("async fixture response (function responses)", () => { + let mock: LLMock | null = null; + + afterEach(async () => { + if (mock) { + await mock.stop(); + mock = null; + } + }); + + it("resolves a sync function response", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "sync-fn" }, () => ({ content: "sync-factory-result" })); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "sync-fn" }], + stream: false, + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.choices[0].message.content).toBe("sync-factory-result"); + }); + + it("resolves an async function response", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "async-fn" }, async () => { + return { content: "async-factory-result" }; + }); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "async-fn" }], + stream: false, + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.choices[0].message.content).toBe("async-factory-result"); + }); + + it("receives the request object in the factory function", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "echo-model" }, (req: ChatCompletionRequest) => ({ + content: `model=${req.model}`, + })); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "echo-model" }], + stream: false, + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.choices[0].message.content).toBe("model=gpt-4o-mini"); + }); + + it("works with streaming responses from a factory", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "stream-fn" }, () => ({ content: "streamed-from-factory" })); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "stream-fn" }], + stream: true, + }), + }); + + expect(res.status).toBe(200); + const chunks = parseSSEChunks(await res.text()); + const content = chunks.map((c) => c.choices?.[0]?.delta?.content ?? "").join(""); + expect(content).toBe("streamed-from-factory"); + }); + + it("works with onMessage convenience method", async () => { + mock = new LLMock({ port: 0 }); + mock.onMessage("convenience-fn", (req: ChatCompletionRequest) => ({ + content: `msg-count=${req.messages.length}`, + })); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [ + { role: "system", content: "you are helpful" }, + { role: "user", content: "convenience-fn" }, + ], + stream: false, + }), + }); + + expect(res.status).toBe(200); + const json = await res.json(); + expect(json.choices[0].message.content).toBe("msg-count=2"); + }); + + it("static response still works alongside function responses", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "static" }, { content: "plain-static" }); + mock.on({ userMessage: "dynamic" }, () => ({ content: "from-function" })); + await mock.start(); + + const [staticRes, dynamicRes] = await Promise.all([ + fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "static" }], + stream: false, + }), + }), + fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4o", + messages: [{ role: "user", content: "dynamic" }], + stream: false, + }), + }), + ]); + + expect(staticRes.status).toBe(200); + expect(dynamicRes.status).toBe(200); + + const staticJson = await staticRes.json(); + const dynamicJson = await dynamicRes.json(); + + expect(staticJson.choices[0].message.content).toBe("plain-static"); + expect(dynamicJson.choices[0].message.content).toBe("from-function"); + }); + + it("returns 500 when factory throws", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "boom" }, () => { + throw new Error("factory exploded"); + }); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4", + messages: [{ role: "user", content: "boom" }], + stream: false, + }), + }); + + expect(res.status).toBe(500); + }); + + it("returns 500 when async factory rejects", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "reject" }, async () => { + throw new Error("async rejection"); + }); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4", + messages: [{ role: "user", content: "reject" }], + stream: false, + }), + }); + + expect(res.status).toBe(500); + }); + + it("returns 500 when factory returns invalid response shape", async () => { + mock = new LLMock({ port: 0 }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + mock.on({ userMessage: "bad" }, () => ({ notAValidField: true }) as any); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4", + messages: [{ role: "user", content: "bad" }], + stream: false, + }), + }); + + expect(res.status).toBe(500); + }); + + it("works with async factory and streaming", async () => { + mock = new LLMock({ port: 0 }); + mock.on({ userMessage: "async-stream" }, async () => { + await new Promise((r) => setTimeout(r, 10)); + return { content: "async-streamed-result" }; + }); + await mock.start(); + + const res = await fetch(`${mock.url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json", Authorization: "Bearer test" }, + body: JSON.stringify({ + model: "gpt-4", + messages: [{ role: "user", content: "async-stream" }], + stream: true, + }), + }); + + expect(res.status).toBe(200); + const chunks = parseSSEChunks(await res.text()); + const content = chunks.map((c) => c.choices?.[0]?.delta?.content ?? "").join(""); + expect(content).toBe("async-streamed-result"); + }); +}); diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index 92bcab6..175622f 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -26,6 +26,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -659,7 +660,7 @@ export async function handleConverse( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); // Error response if (isErrorResponse(response)) { @@ -923,7 +924,7 @@ export async function handleConverseStream( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/bedrock.ts b/src/bedrock.ts index 5bfb354..ea93f84 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -37,6 +37,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -472,7 +473,7 @@ export async function handleBedrock( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); // Error response if (isErrorResponse(response)) { @@ -1069,7 +1070,7 @@ export async function handleBedrockStream( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/cohere.ts b/src/cohere.ts index deebd08..8704eb2 100644 --- a/src/cohere.ts +++ b/src/cohere.ts @@ -30,6 +30,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -863,7 +864,7 @@ export async function handleCohere( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/elevenlabs-audio.ts b/src/elevenlabs-audio.ts index e5e274c..4d8e43c 100644 --- a/src/elevenlabs-audio.ts +++ b/src/elevenlabs-audio.ts @@ -6,6 +6,7 @@ import { isErrorResponse, FORMAT_TO_CONTENT_TYPE, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -152,7 +153,7 @@ export async function handleElevenLabsAudio( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); // Error fixture if (isErrorResponse(response)) { diff --git a/src/embeddings.ts b/src/embeddings.ts index 1b25103..93558aa 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -20,6 +20,7 @@ import { buildEmbeddingResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -152,7 +153,7 @@ export async function handleEmbeddings( return; if (fixture) { - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); // Error response if (isErrorResponse(response)) { diff --git a/src/fal-audio.ts b/src/fal-audio.ts index 5c0a23b..3414f3a 100644 --- a/src/fal-audio.ts +++ b/src/fal-audio.ts @@ -1,7 +1,13 @@ import type http from "node:http"; import crypto from "node:crypto"; import type { AudioResponse, ChatCompletionRequest, Fixture, HandlerDefaults } from "./types.js"; -import { isAudioResponse, isErrorResponse, FORMAT_TO_CONTENT_TYPE, getTestId } from "./helpers.js"; +import { + isAudioResponse, + isErrorResponse, + FORMAT_TO_CONTENT_TYPE, + getTestId, + resolveResponse, +} from "./helpers.js"; import { matchFixture } from "./router.js"; import { proxyAndRecord } from "./recorder.js"; import type { Journal } from "./journal.js"; @@ -295,7 +301,7 @@ async function handleQueueSubmit( } journal.incrementFixtureMatchCount(fixture, fixtures, testId); - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; @@ -577,7 +583,7 @@ async function handleSyncRun( } journal.incrementFixtureMatchCount(fixture, fixtures, getTestId(req)); - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/fal.ts b/src/fal.ts index 85389bd..52c29c3 100644 --- a/src/fal.ts +++ b/src/fal.ts @@ -7,6 +7,7 @@ import { isJSONResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { proxyAndRecord } from "./recorder.js"; @@ -361,7 +362,7 @@ export async function handleFal( } journal.incrementFixtureMatchCount(fixture, fixtures, testId); - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index 26d1ecb..9b5410e 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -237,242 +237,248 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { const f = fixtures[i]; const response = f.response; - // --- Error checks --- - - // Response type recognition - // Note: isContentWithToolCallsResponse must be checked before isTextResponse - // and isToolCallResponse since it is a structural superset of both. - if ( - !isContentWithToolCallsResponse(response) && - !isTextResponse(response) && - !isToolCallResponse(response) && - !isErrorResponse(response) && - !isEmbeddingResponse(response) && - !isImageResponse(response) && - !isAudioResponse(response) && - !isTranscriptionResponse(response) && - !isVideoResponse(response) && - !isJSONResponse(response) - ) { - results.push({ - severity: "error", - fixtureIndex: i, - message: - "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, or json)", - }); - } + // Skip response-shape validation for function responses — they are + // evaluated at runtime so we cannot statically inspect them. + if (typeof response === "function") { + // Still validate match fields and numeric options below. + } else { + // --- Error checks --- - // Text response checks - if (isTextResponse(response)) { - if (response.content === "") { + // Response type recognition + // Note: isContentWithToolCallsResponse must be checked before isTextResponse + // and isToolCallResponse since it is a structural superset of both. + if ( + !isContentWithToolCallsResponse(response) && + !isTextResponse(response) && + !isToolCallResponse(response) && + !isErrorResponse(response) && + !isEmbeddingResponse(response) && + !isImageResponse(response) && + !isAudioResponse(response) && + !isTranscriptionResponse(response) && + !isVideoResponse(response) && + !isJSONResponse(response) + ) { results.push({ severity: "error", fixtureIndex: i, - message: "content is empty string", + message: + "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, or json)", }); } - validateReasoning(response, i, results); - validateWebSearches(response, i, results); - } - // ContentWithToolCalls response checks - if (isContentWithToolCallsResponse(response)) { - if (response.content === "") { - results.push({ - severity: "error", - fixtureIndex: i, - message: "content is empty string", - }); - } - if (response.toolCalls.length === 0) { - results.push({ - severity: "warning", - fixtureIndex: i, - message: "toolCalls array is empty — fixture will never produce tool calls", - }); - } - for (let j = 0; j < response.toolCalls.length; j++) { - const tc = response.toolCalls[j]; - if (!tc.name) { + // Text response checks + if (isTextResponse(response)) { + if (response.content === "") { results.push({ severity: "error", fixtureIndex: i, - message: `toolCalls[${j}].name is empty`, + message: "content is empty string", }); } - try { - JSON.parse(tc.arguments); - } catch { + validateReasoning(response, i, results); + validateWebSearches(response, i, results); + } + + // ContentWithToolCalls response checks + if (isContentWithToolCallsResponse(response)) { + if (response.content === "") { results.push({ severity: "error", fixtureIndex: i, - message: `toolCalls[${j}].arguments is not valid JSON: ${tc.arguments}`, + message: "content is empty string", }); } + if (response.toolCalls.length === 0) { + results.push({ + severity: "warning", + fixtureIndex: i, + message: "toolCalls array is empty — fixture will never produce tool calls", + }); + } + for (let j = 0; j < response.toolCalls.length; j++) { + const tc = response.toolCalls[j]; + if (!tc.name) { + results.push({ + severity: "error", + fixtureIndex: i, + message: `toolCalls[${j}].name is empty`, + }); + } + try { + JSON.parse(tc.arguments); + } catch { + results.push({ + severity: "error", + fixtureIndex: i, + message: `toolCalls[${j}].arguments is not valid JSON: ${tc.arguments}`, + }); + } + } + validateReasoning(response, i, results); + validateWebSearches(response, i, results); } - validateReasoning(response, i, results); - validateWebSearches(response, i, results); - } - // Tool call response checks - if (isToolCallResponse(response)) { - if (response.toolCalls.length === 0) { - results.push({ - severity: "warning", - fixtureIndex: i, - message: "toolCalls array is empty — fixture will never produce tool calls", - }); + // Tool call response checks + if (isToolCallResponse(response)) { + if (response.toolCalls.length === 0) { + results.push({ + severity: "warning", + fixtureIndex: i, + message: "toolCalls array is empty — fixture will never produce tool calls", + }); + } + for (let j = 0; j < response.toolCalls.length; j++) { + const tc = response.toolCalls[j]; + if (!tc.name) { + results.push({ + severity: "error", + fixtureIndex: i, + message: `toolCalls[${j}].name is empty`, + }); + } + try { + JSON.parse(tc.arguments); + } catch { + results.push({ + severity: "error", + fixtureIndex: i, + message: `toolCalls[${j}].arguments is not valid JSON: ${tc.arguments}`, + }); + } + } } - for (let j = 0; j < response.toolCalls.length; j++) { - const tc = response.toolCalls[j]; - if (!tc.name) { + + // Error response checks + if (isErrorResponse(response)) { + if (!response.error.message) { results.push({ severity: "error", fixtureIndex: i, - message: `toolCalls[${j}].name is empty`, + message: "error.message is empty", }); } - try { - JSON.parse(tc.arguments); - } catch { + if (response.status !== undefined && (response.status < 100 || response.status > 599)) { results.push({ severity: "error", fixtureIndex: i, - message: `toolCalls[${j}].arguments is not valid JSON: ${tc.arguments}`, + message: `error status ${response.status} is not a valid HTTP status code`, }); } } - } - - // Error response checks - if (isErrorResponse(response)) { - if (!response.error.message) { - results.push({ - severity: "error", - fixtureIndex: i, - message: "error.message is empty", - }); - } - if (response.status !== undefined && (response.status < 100 || response.status > 599)) { - results.push({ - severity: "error", - fixtureIndex: i, - message: `error status ${response.status} is not a valid HTTP status code`, - }); - } - } - // Embedding response checks - if (isEmbeddingResponse(response)) { - if (response.embedding.length === 0) { - results.push({ - severity: "error", - fixtureIndex: i, - message: "embedding array is empty", - }); - } - for (let j = 0; j < response.embedding.length; j++) { - if (typeof response.embedding[j] !== "number") { + // Embedding response checks + if (isEmbeddingResponse(response)) { + if (response.embedding.length === 0) { results.push({ severity: "error", fixtureIndex: i, - message: `embedding[${j}] is not a number`, + message: "embedding array is empty", }); - break; // one error is enough + } + for (let j = 0; j < response.embedding.length; j++) { + if (typeof response.embedding[j] !== "number") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `embedding[${j}] is not a number`, + }); + break; // one error is enough + } } } - } - // Audio response checks — validate object-form audio - if (isAudioResponse(response) && typeof response.audio === "object") { - const audioObj = response.audio; - if (typeof audioObj.b64Json !== "string" || audioObj.b64Json === "") { - results.push({ - severity: "error", - fixtureIndex: i, - message: "audio.b64Json must be a non-empty string", - }); - } - if (audioObj.contentType !== undefined && typeof audioObj.contentType !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `audio.contentType must be a string, got ${typeof audioObj.contentType}`, - }); + // Audio response checks — validate object-form audio + if (isAudioResponse(response) && typeof response.audio === "object") { + const audioObj = response.audio; + if (typeof audioObj.b64Json !== "string" || audioObj.b64Json === "") { + results.push({ + severity: "error", + fixtureIndex: i, + message: "audio.b64Json must be a non-empty string", + }); + } + if (audioObj.contentType !== undefined && typeof audioObj.contentType !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `audio.contentType must be a string, got ${typeof audioObj.contentType}`, + }); + } } - } - // Validate ResponseOverrides fields - if ( - isTextResponse(response) || - isToolCallResponse(response) || - isContentWithToolCallsResponse(response) - ) { - const r = response as ResponseOverrides; - if (r.id !== undefined && typeof r.id !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "id" must be a string, got ${typeof r.id}`, - }); - } - if (r.created !== undefined && (typeof r.created !== "number" || r.created < 0)) { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "created" must be a non-negative number`, - }); - } - if (r.model !== undefined && typeof r.model !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "model" must be a string, got ${typeof r.model}`, - }); - } - if (r.finishReason !== undefined && typeof r.finishReason !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "finishReason" must be a string, got ${typeof r.finishReason}`, - }); - } - if (r.role !== undefined && typeof r.role !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "role" must be a string, got ${typeof r.role}`, - }); - } - if (r.systemFingerprint !== undefined && typeof r.systemFingerprint !== "string") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "systemFingerprint" must be a string, got ${typeof r.systemFingerprint}`, - }); - } - if (r.usage !== undefined) { - if (typeof r.usage !== "object" || r.usage === null || Array.isArray(r.usage)) { + // Validate ResponseOverrides fields + if ( + isTextResponse(response) || + isToolCallResponse(response) || + isContentWithToolCallsResponse(response) + ) { + const r = response as ResponseOverrides; + if (r.id !== undefined && typeof r.id !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "id" must be a string, got ${typeof r.id}`, + }); + } + if (r.created !== undefined && (typeof r.created !== "number" || r.created < 0)) { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "created" must be a non-negative number`, + }); + } + if (r.model !== undefined && typeof r.model !== "string") { results.push({ severity: "error", fixtureIndex: i, - message: `override "usage" must be an object`, + message: `override "model" must be a string, got ${typeof r.model}`, }); - } else { - // Check all known usage fields are numbers if present - for (const key of Object.keys(r.usage)) { - const val = (r.usage as Record)[key]; - if (val !== undefined && typeof val !== "number") { - results.push({ - severity: "error", - fixtureIndex: i, - message: `override "usage.${key}" must be a number, got ${typeof val}`, - }); + } + if (r.finishReason !== undefined && typeof r.finishReason !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "finishReason" must be a string, got ${typeof r.finishReason}`, + }); + } + if (r.role !== undefined && typeof r.role !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "role" must be a string, got ${typeof r.role}`, + }); + } + if (r.systemFingerprint !== undefined && typeof r.systemFingerprint !== "string") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "systemFingerprint" must be a string, got ${typeof r.systemFingerprint}`, + }); + } + if (r.usage !== undefined) { + if (typeof r.usage !== "object" || r.usage === null || Array.isArray(r.usage)) { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "usage" must be an object`, + }); + } else { + // Check all known usage fields are numbers if present + for (const key of Object.keys(r.usage)) { + const val = (r.usage as Record)[key]; + if (val !== undefined && typeof val !== "number") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `override "usage.${key}" must be a number, got ${typeof val}`, + }); + } } } } } - } + } // end: skip response-shape validation for function responses // Numeric sanity checks if (f.latency !== undefined && f.latency < 0) { diff --git a/src/gemini-interactions.ts b/src/gemini-interactions.ts index ffa577a..d47944f 100644 --- a/src/gemini-interactions.ts +++ b/src/gemini-interactions.ts @@ -27,6 +27,7 @@ import { generateToolCallId, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -749,7 +750,7 @@ export async function handleGeminiInteractions( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/gemini.ts b/src/gemini.ts index e15727d..d9eb4e5 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -30,6 +30,7 @@ import { generateToolCallId, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -676,7 +677,7 @@ export async function handleGemini( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/helpers.ts b/src/helpers.ts index bec3e32..e8db542 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,7 +1,10 @@ import { createHash, randomBytes } from "node:crypto"; import type * as http from "node:http"; import type { + ChatCompletionRequest, + Fixture, FixtureResponse, + ResponseFactory, TextResponse, ToolCallResponse, ContentWithToolCallsResponse, @@ -33,6 +36,37 @@ export function flattenHeaders(headers: http.IncomingHttpHeaders): Record { + if (typeof fixture.response === "function") { + const raw = await fixture.response(request); + return normalizeFactoryResponse(raw); + } + return fixture.response; +} + +function normalizeFactoryResponse(raw: FixtureResponse): FixtureResponse { + const r = { ...raw } as Record; + if (typeof r.content === "object" && r.content !== null) { + r.content = JSON.stringify(r.content); + } + if (Array.isArray(r.toolCalls)) { + r.toolCalls = (r.toolCalls as Array>).map((tc) => { + if (typeof tc.arguments === "object" && tc.arguments !== null) { + return { ...tc, arguments: JSON.stringify(tc.arguments) }; + } + return tc; + }); + } + return r as unknown as FixtureResponse; +} + export function generateId(prefix = "chatcmpl"): string { return `${prefix}-${randomBytes(12).toString("base64url")}`; } diff --git a/src/images.ts b/src/images.ts index e90c8c1..cb5696c 100644 --- a/src/images.ts +++ b/src/images.ts @@ -1,6 +1,12 @@ import type * as http from "node:http"; import type { ChatCompletionRequest, Fixture, HandlerDefaults } from "./types.js"; -import { isImageResponse, isErrorResponse, flattenHeaders, getTestId } from "./helpers.js"; +import { + isImageResponse, + isErrorResponse, + flattenHeaders, + getTestId, + resolveResponse, +} from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; import type { Journal } from "./journal.js"; @@ -171,7 +177,7 @@ export async function handleImages( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/llmock.ts b/src/llmock.ts index d9228d0..8f540f1 100644 --- a/src/llmock.ts +++ b/src/llmock.ts @@ -12,6 +12,7 @@ import type { MockServerOptions, Mountable, RecordConfig, + ResponseFactory, TranscriptionResponse, VideoResponse, } from "./types.js"; @@ -99,21 +100,29 @@ export class LLMock { // ---- Convenience ---- - on(match: FixtureMatch, response: FixtureFileResponse, opts?: FixtureOpts): this { + on( + match: FixtureMatch, + response: FixtureFileResponse | ResponseFactory, + opts?: FixtureOpts, + ): this { return this.addFixture({ match, - response: normalizeResponse(response), + response: typeof response === "function" ? response : normalizeResponse(response), ...opts, }); } - onMessage(pattern: string | RegExp, response: FixtureFileResponse, opts?: FixtureOpts): this { + onMessage( + pattern: string | RegExp, + response: FixtureFileResponse | ResponseFactory, + opts?: FixtureOpts, + ): this { return this.on({ userMessage: pattern }, response, opts); } onEmbedding( pattern: string | RegExp, - response: FixtureFileResponse, + response: FixtureFileResponse | ResponseFactory, opts?: EmbeddingFixtureOpts, ): this { return this.on({ inputText: pattern }, response, opts); @@ -124,18 +133,26 @@ export class LLMock { return this.on({ userMessage: pattern, responseFormat: "json_object" }, { content }, opts); } - onToolCall(name: string, response: FixtureFileResponse, opts?: FixtureOpts): this { + onToolCall( + name: string, + response: FixtureFileResponse | ResponseFactory, + opts?: FixtureOpts, + ): this { return this.on({ toolName: name }, response, opts); } - onToolResult(id: string, response: FixtureFileResponse, opts?: FixtureOpts): this { + onToolResult( + id: string, + response: FixtureFileResponse | ResponseFactory, + opts?: FixtureOpts, + ): this { return this.on({ toolCallId: id }, response, opts); } onTurn( turn: number, pattern: string | RegExp, - response: FixtureFileResponse, + response: FixtureFileResponse | ResponseFactory, opts?: FixtureOpts, ): this { return this.on({ userMessage: pattern, turnIndex: turn }, response, opts); diff --git a/src/messages.ts b/src/messages.ts index a1465df..99b0740 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -27,6 +27,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -819,7 +820,7 @@ export async function handleMessages( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/ollama.ts b/src/ollama.ts index a88f3f2..11af03a 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -28,6 +28,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -573,7 +574,7 @@ export async function handleOllama( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); @@ -879,7 +880,7 @@ export async function handleOllamaGenerate( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/responses.ts b/src/responses.ts index 82ded10..98174b9 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -27,6 +27,7 @@ import { isErrorResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse, delay, calculateDelay } from "./sse-writer.js"; @@ -1029,7 +1030,7 @@ export async function handleResponses( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/router.ts b/src/router.ts index f5b55e4..ccbe861 100644 --- a/src/router.ts +++ b/src/router.ts @@ -59,17 +59,20 @@ export function matchFixture( if (match.endpoint !== reqEndpoint) continue; } else if (reqEndpoint && reqEndpoint !== "chat" && reqEndpoint !== "embedding") { // Fixture has no endpoint restriction but request is multimedia — - // only match if the response type is compatible + // only match if the response type is compatible. + // Function responses cannot be checked statically, so treat them as compatible. const r = fixture.response; - const compatible = - (reqEndpoint === "image" && isImageResponse(r)) || - (reqEndpoint === "speech" && isAudioResponse(r)) || - (reqEndpoint === "audio-gen" && isAudioResponse(r)) || - (reqEndpoint === "fal-audio" && isAudioResponse(r)) || - (reqEndpoint === "fal" && (isJSONResponse(r) || isErrorResponse(r))) || - (reqEndpoint === "transcription" && isTranscriptionResponse(r)) || - (reqEndpoint === "video" && isVideoResponse(r)); - if (!compatible) continue; + if (typeof r !== "function") { + const compatible = + (reqEndpoint === "image" && isImageResponse(r)) || + (reqEndpoint === "speech" && isAudioResponse(r)) || + (reqEndpoint === "audio-gen" && isAudioResponse(r)) || + (reqEndpoint === "fal-audio" && isAudioResponse(r)) || + (reqEndpoint === "fal" && (isJSONResponse(r) || isErrorResponse(r))) || + (reqEndpoint === "transcription" && isTranscriptionResponse(r)) || + (reqEndpoint === "video" && isVideoResponse(r)); + if (!compatible) continue; + } } // userMessage — case-sensitive match against the last user message content. diff --git a/src/server.ts b/src/server.ts index 0eb6a07..5f7c92d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -28,6 +28,7 @@ import { isAudioResponse, flattenHeaders, getTestId, + resolveResponse, } from "./helpers.js"; import { handleResponses } from "./responses.js"; import { handleMessages } from "./messages.js"; @@ -631,7 +632,7 @@ async function handleCompletions( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, body); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/speech.ts b/src/speech.ts index e37d95e..5a97404 100644 --- a/src/speech.ts +++ b/src/speech.ts @@ -6,6 +6,7 @@ import { flattenHeaders, getTestId, FORMAT_TO_CONTENT_TYPE, + resolveResponse, } from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; @@ -155,7 +156,7 @@ export async function handleSpeech( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/transcription.ts b/src/transcription.ts index 6a99396..430d8c5 100644 --- a/src/transcription.ts +++ b/src/transcription.ts @@ -1,6 +1,12 @@ import type * as http from "node:http"; import type { ChatCompletionRequest, Fixture, HandlerDefaults } from "./types.js"; -import { isTranscriptionResponse, isErrorResponse, flattenHeaders, getTestId } from "./helpers.js"; +import { + isTranscriptionResponse, + isErrorResponse, + flattenHeaders, + getTestId, + resolveResponse, +} from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; import type { Journal } from "./journal.js"; @@ -125,7 +131,7 @@ export async function handleTranscription( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/types.ts b/src/types.ts index 8a2f670..7318de9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -235,11 +235,17 @@ export interface ChaosConfig { export type ChaosAction = "drop" | "malformed" | "disconnect"; +// Response factory — allows dynamic fixture responses based on the incoming request + +export type ResponseFactory = ( + req: ChatCompletionRequest, +) => FixtureResponse | Promise; + // Fixture export interface Fixture { match: FixtureMatch; - response: FixtureResponse; + response: FixtureResponse | ResponseFactory; latency?: number; chunkSize?: number; truncateAfterChunks?: number; diff --git a/src/video.ts b/src/video.ts index e784428..598b81b 100644 --- a/src/video.ts +++ b/src/video.ts @@ -1,6 +1,12 @@ import type * as http from "node:http"; import type { ChatCompletionRequest, Fixture, HandlerDefaults, VideoResponse } from "./types.js"; -import { isVideoResponse, isErrorResponse, flattenHeaders, getTestId } from "./helpers.js"; +import { + isVideoResponse, + isErrorResponse, + flattenHeaders, + getTestId, + resolveResponse, +} from "./helpers.js"; import { matchFixture } from "./router.js"; import { writeErrorResponse } from "./sse-writer.js"; import type { Journal } from "./journal.js"; @@ -202,7 +208,7 @@ export async function handleVideoCreate( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, syntheticReq); if (isErrorResponse(response)) { const status = response.status ?? 500; diff --git a/src/ws-gemini-live.ts b/src/ws-gemini-live.ts index 0e9daad..5b08be0 100644 --- a/src/ws-gemini-live.ts +++ b/src/ws-gemini-live.ts @@ -22,6 +22,7 @@ import { isAudioResponse, formatToMime, generateToolCallId, + resolveResponse, } from "./helpers.js"; import { createInterruptionSignal } from "./interruption.js"; import { delay } from "./sse-writer.js"; @@ -373,7 +374,7 @@ async function processMessage( // Commit messages to conversation history only after successful fixture match session.conversationHistory.push(...newMessages); - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/ws-realtime.ts b/src/ws-realtime.ts index 8d09fb9..9576e65 100644 --- a/src/ws-realtime.ts +++ b/src/ws-realtime.ts @@ -14,6 +14,7 @@ import { isTextResponse, isToolCallResponse, isErrorResponse, + resolveResponse, } from "./helpers.js"; import { createInterruptionSignal } from "./interruption.js"; import { delay } from "./sse-writer.js"; @@ -334,7 +335,7 @@ async function handleResponseCreate( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); diff --git a/src/ws-responses.ts b/src/ws-responses.ts index 40ddcb6..b5c36df 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -14,7 +14,7 @@ import { buildToolCallStreamEvents, type ResponsesSSEEvent, } from "./responses.js"; -import { isTextResponse, isToolCallResponse, isErrorResponse } from "./helpers.js"; +import { isTextResponse, isToolCallResponse, isErrorResponse, resolveResponse } from "./helpers.js"; import { createInterruptionSignal } from "./interruption.js"; import { delay } from "./sse-writer.js"; import { DEFAULT_TEST_ID, type Journal } from "./journal.js"; @@ -185,7 +185,7 @@ async function processMessage( return; } - const response = fixture.response; + const response = await resolveResponse(fixture, completionReq); const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize);