From 5f5e8f5ba108272af36e9654e4b1329323b27fbf Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:19:31 -0400 Subject: [PATCH 01/16] Keep Slack terminal delivery in durable custody --- .changeset/calm-slack-continuations.md | 5 + .../a2a-continuation-processor.spec.ts | 489 ++++++++++++++++-- .../a2a-continuation-processor.ts | 408 +++++++++++---- .../a2a-continuations-store.spec.ts | 371 +++++++++++++ .../integrations/a2a-continuations-store.ts | 321 +++++++++++- packages/core/src/integrations/plugin.spec.ts | 57 +- packages/core/src/integrations/plugin.ts | 46 +- .../src/integrations/task-queue-stats.spec.ts | 46 ++ .../core/src/integrations/task-queue-stats.ts | 146 ++++++ .../webhook-handler-engine.spec.ts | 26 +- .../core/src/integrations/webhook-handler.ts | 23 +- ...ies-recover-after-connected-app-updates.md | 6 + 12 files changed, 1770 insertions(+), 174 deletions(-) create mode 100644 .changeset/calm-slack-continuations.md create mode 100644 templates/dispatch/changelog/2026-07-26-slack-replies-recover-after-connected-app-updates.md diff --git a/.changeset/calm-slack-continuations.md b/.changeset/calm-slack-continuations.md new file mode 100644 index 0000000000..562d6984ce --- /dev/null +++ b/.changeset/calm-slack-continuations.md @@ -0,0 +1,5 @@ +--- +"@agent-native/core": patch +--- + +Keep verified downstream mutation receipts and delivered integration replies in durable custody until provider delivery and conversation history are confirmed. diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index a7332d1709..07a60662d7 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -21,13 +21,18 @@ const completeIntegrationCampaignTaskAfterA2AMock = vi.hoisted(() => ); const claimA2AContinuationDeliveryMock = vi.hoisted(() => vi.fn()); const completeA2AContinuationMock = vi.hoisted(() => vi.fn()); -const hasActiveA2AContinuationsForIntegrationTaskMock = vi.hoisted(() => +const recordA2ATerminalDeliveryReceiptMock = vi.hoisted(() => vi.fn()); +const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => + vi.fn(async () => "terminal-delivered"), +); +const hasPendingConfirmedA2ADeliveryForIntegrationTaskMock = vi.hoisted(() => vi.fn(async () => false), ); const failA2AContinuationMock = vi.hoisted(() => vi.fn()); const failA2AContinuationsForIntegrationTaskMock = vi.hoisted(() => vi.fn()); const getA2AContinuationMock = vi.hoisted(() => vi.fn()); const rescheduleA2AContinuationMock = vi.hoisted(() => vi.fn()); +const saveA2AVerifiedArtifactCheckpointMock = vi.hoisted(() => vi.fn()); const getTaskMock = vi.hoisted(() => vi.fn()); const signA2ATokenMock = vi.hoisted(() => vi.fn(async () => "signed-a2a-token"), @@ -45,16 +50,19 @@ vi.mock("./a2a-continuations-store.js", () => ({ claimA2AContinuation: claimA2AContinuationMock, claimA2AContinuationDelivery: claimA2AContinuationDeliveryMock, claimDueA2AContinuations: vi.fn(async () => []), - completeA2AContinuation: completeA2AContinuationMock, + finalizeA2ATerminalHistory: completeA2AContinuationMock, failA2AContinuation: failA2AContinuationMock, failA2AContinuationsForIntegrationTask: failA2AContinuationsForIntegrationTaskMock, getA2AContinuation: getA2AContinuationMock, - hasActiveA2AContinuationsForIntegrationTask: - hasActiveA2AContinuationsForIntegrationTaskMock, + getA2AContinuationTaskOutcome: getA2AContinuationTaskOutcomeMock, + hasPendingConfirmedA2ADeliveryForIntegrationTask: + hasPendingConfirmedA2ADeliveryForIntegrationTaskMock, listRecoverableA2AIntegrationTasks: listRecoverableA2ATasksMock, recoverDueA2AContinuationIds: recoverDueA2AContinuationIdsMock, + recordA2ATerminalDeliveryReceipt: recordA2ATerminalDeliveryReceiptMock, rescheduleA2AContinuation: rescheduleA2AContinuationMock, + saveA2AVerifiedArtifactCheckpoint: saveA2AVerifiedArtifactCheckpointMock, })); vi.mock("./pending-tasks-store.js", () => ({ @@ -123,6 +131,10 @@ function continuation( agentUrl: "https://slides.agent-native.test", a2aTaskId: "a2a-task-1", a2aAuthToken: null, + verifiedArtifactCheckpoint: null, + terminalDeliveryKind: null, + terminalDeliveryConfirmedAt: null, + terminalHistoryPayload: null, status: "processing", attempts: 1, nextCheckAt: 1, @@ -169,8 +181,31 @@ describe("A2A continuation processor", () => { continuation({ id, status: "pending" }), ); completeA2AContinuationMock.mockResolvedValue(undefined); + getA2AContinuationTaskOutcomeMock.mockResolvedValue("terminal-delivered"); + hasPendingConfirmedA2ADeliveryForIntegrationTaskMock.mockResolvedValue( + false, + ); failA2AContinuationMock.mockResolvedValue(undefined); + recordA2ATerminalDeliveryReceiptMock.mockImplementation( + async ( + id: string, + kind: "success" | "failure", + terminalHistoryPayload: A2AContinuation["terminalHistoryPayload"], + errorMessage?: string, + ) => + continuation({ + id, + status: "delivering", + terminalDeliveryKind: kind, + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload, + errorMessage: errorMessage ?? null, + }), + ); rescheduleA2AContinuationMock.mockResolvedValue(undefined); + saveA2AVerifiedArtifactCheckpointMock.mockImplementation( + async (_id: string, checkpoint: string) => checkpoint, + ); getThreadMappingMock.mockResolvedValue(null); getThreadMock.mockResolvedValue(null); updateThreadDataMock.mockResolvedValue(undefined); @@ -185,6 +220,7 @@ describe("A2A continuation processor", () => { externalThreadId: "slack:team:C123:1", dispatchScope: "C123", status: "processing", + hasPendingConfirmedDelivery: false, }, { id: "task-2", @@ -192,6 +228,7 @@ describe("A2A continuation processor", () => { externalThreadId: "slack:team:C123:2", dispatchScope: "C123", status: "processing", + hasPendingConfirmedDelivery: false, }, ]); getPendingTaskMock.mockResolvedValue({ @@ -295,6 +332,57 @@ describe("A2A continuation processor", () => { ); }); + it("releases an escaped processor failure for durable retry below the attempt bound", async () => { + getA2AContinuationMock.mockResolvedValueOnce( + continuation({ status: "processing", attempts: 2 }), + ); + const { recoverA2AContinuationAfterProcessorFailure } = + await import("./a2a-continuation-processor.js"); + + await recoverA2AContinuationAfterProcessorFailure("cont-1", { + adapters: new Map([["slack", adapter()]]), + reason: "temporary database outage", + }); + + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( + "cont-1", + 20_000, + ); + expect(fetch).toHaveBeenCalled(); + expect(failA2AContinuationMock).not.toHaveBeenCalled(); + }); + + it("delivers the durable checkpoint when an escaped processor failure exhausts attempts", async () => { + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + getA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + attempts: 30, + verifiedArtifactCheckpoint: "Verified Content: /page/content-1", + }), + ); + claimA2AContinuationDeliveryMock.mockResolvedValueOnce( + continuation({ status: "delivering", attempts: 30 }), + ); + const { recoverA2AContinuationAfterProcessorFailure } = + await import("./a2a-continuation-processor.js"); + + await recoverA2AContinuationAfterProcessorFailure("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + reason: "processor failed after mutation", + }); + + expect(sendResponse).toHaveBeenCalledWith( + expect.objectContaining({ + text: expect.stringContaining("/page/content-1"), + }), + expect.any(Object), + { placeholderRef: undefined }, + ); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); + expect(failA2AContinuationMock).not.toHaveBeenCalled(); + }); + it("allows one durable wake-up dispatch failure without stranding the rest", async () => { recoverDueA2AContinuationIdsMock.mockResolvedValue([ "cont-fail", @@ -335,6 +423,41 @@ describe("A2A continuation processor", () => { expect(vi.mocked(fetch)).not.toHaveBeenCalled(); }); + it("still wakes receipt-confirmed history when the rollout scope is disabled", async () => { + durableDispatchEnabledMock.mockReturnValue(false); + listRecoverableA2ATasksMock.mockResolvedValueOnce([ + { + id: "task-confirmed", + platform: "slack", + externalThreadId: "slack:team:C123:confirmed", + dispatchScope: "C123", + status: "processing", + hasPendingConfirmedDelivery: true, + }, + ]); + recoverDueA2AContinuationIdsMock + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(["cont-confirmed"]); + const { recoverDueA2AContinuations } = + await import("./a2a-continuation-processor.js"); + + await expect(recoverDueA2AContinuations()).resolves.toEqual({ + dispatched: 1, + failed: 0, + }); + + expect(recoverDueA2AContinuationIdsMock).toHaveBeenNthCalledWith(1, 5, []); + expect(recoverDueA2AContinuationIdsMock).toHaveBeenNthCalledWith( + 2, + 5, + ["task-confirmed"], + true, + ); + expect(failA2AContinuationsForIntegrationTaskMock).not.toHaveBeenCalled(); + expect(failDisabledIntegrationCampaignTaskMock).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledOnce(); + }); + it("surfaces a durable-store failure for the next scheduler run", async () => { recoverDueA2AContinuationIdsMock.mockRejectedValueOnce( new Error("example database timeout"), @@ -393,6 +516,92 @@ describe("A2A continuation processor", () => { }); }); + it("cancels only an unconfirmed sibling while confirmed history still owns custody", async () => { + const claimed = continuation({ id: "cont-unconfirmed" }); + claimA2AContinuationMock.mockResolvedValueOnce(claimed); + getIntegrationCampaignForTaskMock.mockResolvedValueOnce({ + id: "campaign-1", + status: "waiting", + }); + durableDispatchEnabledMock.mockReturnValue(false); + hasPendingConfirmedA2ADeliveryForIntegrationTaskMock.mockResolvedValueOnce( + true, + ); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById(claimed.id, { + adapters: new Map([["slack", adapter()]]), + }); + + expect(failA2AContinuationMock).toHaveBeenCalledWith( + "cont-unconfirmed", + expect.stringContaining("disabled before this continuation delivered"), + ); + expect(failA2AContinuationsForIntegrationTaskMock).not.toHaveBeenCalled(); + expect(failDisabledIntegrationCampaignTaskMock).not.toHaveBeenCalled(); + expect(getTaskMock).not.toHaveBeenCalled(); + }); + + it("finishes confirmed sibling history before closing a disabled mixed parent", async () => { + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + id: "cont-confirmed", + status: "processing", + terminalDeliveryKind: "success", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [], + }, + }), + ); + getIntegrationCampaignForTaskMock.mockResolvedValue({ + id: "campaign-1", + integrationTaskId: "task-1", + status: "waiting", + }); + getA2AContinuationTaskOutcomeMock.mockResolvedValue( + "terminal-without-delivery", + ); + durableDispatchEnabledMock.mockReturnValue(false); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-confirmed", { adapters: new Map() }); + + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-confirmed"); + expect(failDisabledIntegrationCampaignTaskMock).toHaveBeenCalledWith( + "task-1", + expect.stringContaining("disabled"), + ); + expect(getTaskMock).not.toHaveBeenCalled(); + }); + + it("repairs a disabled terminal-without-delivery parent on re-entry", async () => { + getA2AContinuationTaskOutcomeMock.mockResolvedValue( + "terminal-without-delivery", + ); + durableDispatchEnabledMock.mockReturnValue(false); + const { reconcileTerminalA2AParentIfDisabled } = + await import("./a2a-continuation-processor.js"); + + await expect(reconcileTerminalA2AParentIfDisabled("task-1")).resolves.toBe( + true, + ); + + expect(failA2AContinuationsForIntegrationTaskMock).toHaveBeenCalledWith( + "task-1", + expect.stringContaining("disabled"), + ); + expect(failDisabledIntegrationCampaignTaskMock).toHaveBeenCalledWith( + "task-1", + expect.stringContaining("disabled"), + ); + }); + it.each(["failed", "completed"] as const)( "does not poll an A2A row after its owning campaign is %s", async (status) => { @@ -508,7 +717,7 @@ describe("A2A continuation processor", () => { integrationTaskId: claimed.integrationTaskId, status: "waiting", }); - hasActiveA2AContinuationsForIntegrationTaskMock.mockResolvedValueOnce(true); + getA2AContinuationTaskOutcomeMock.mockResolvedValueOnce("active"); const { processA2AContinuationById } = await import("./a2a-continuation-processor.js"); @@ -621,7 +830,7 @@ describe("A2A continuation processor", () => { }); }); - it("does not redeliver when post-delivery history persistence fails", async () => { + it("keeps provider-confirmed history retryable without redelivering", async () => { getThreadMappingMock.mockResolvedValue({ internalThreadId: "thread-123", }); @@ -636,7 +845,19 @@ describe("A2A continuation processor", () => { .spyOn(console, "error") .mockImplementation(() => undefined); const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); - claimA2AContinuationMock.mockResolvedValueOnce(continuation()); + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + terminalDeliveryKind: "success", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [], + }, + }), + ); const { processA2AContinuationById } = await import("./a2a-continuation-processor.js"); @@ -644,14 +865,126 @@ describe("A2A continuation processor", () => { adapters: new Map([["slack", adapter(sendResponse)]]), }); - expect(sendResponse).toHaveBeenCalledTimes(1); + expect(sendResponse).not.toHaveBeenCalled(); + expect(getTaskMock).not.toHaveBeenCalled(); expect(updateThreadDataMock).toHaveBeenCalledTimes(3); - expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); - expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( + "cont-1", + 20_000, + ); + expect(completeA2AContinuationMock).not.toHaveBeenCalled(); expect(consoleError).toHaveBeenCalledWith( - expect.stringContaining("could not persist its thread history"), - expect.any(Error), + expect.stringContaining("history remains retryable"), + "Error", + ); + }); + + it("recovers provider-confirmed history in a fresh invocation without polling or sending again", async () => { + const history = { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [ + { + id: "content-1", + resourceType: "document", + sourceAction: "call-agent", + }, + ], + }; + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + terminalDeliveryKind: "success", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: history, + }), + ); + getIntegrationCampaignForTaskMock.mockResolvedValue({ + id: "campaign-1", + integrationTaskId: "task-1", + status: "waiting", + }); + durableDispatchEnabledMock.mockReturnValue(false); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { adapters: new Map() }); + + expect(getTaskMock).not.toHaveBeenCalled(); + expect(durableDispatchEnabledMock).not.toHaveBeenCalled(); + expect(failDisabledIntegrationCampaignTaskMock).not.toHaveBeenCalled(); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); + expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledWith( + "task-1", + ); + }); + + it("uses a stable assistant message id when a committed history write is retried", async () => { + const history = { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [ + { + id: "content-1", + resourceType: "document", + sourceAction: "call-agent", + }, + ], + }; + const existingMessage = { + id: "msg-cont-1-assistant-continuation", + role: "assistant", + content: [{ type: "text", text: history.text }], + }; + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + terminalDeliveryKind: "success", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: history, + }), + ); + getThreadMappingMock.mockResolvedValue({ internalThreadId: "thread-123" }); + getThreadMock.mockResolvedValue({ + id: "thread-123", + title: "Slack thread", + preview: "Create the request", + threadData: JSON.stringify({ messages: [existingMessage] }), + }); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { adapters: new Map() }); + + const persisted = JSON.parse(updateThreadDataMock.mock.calls[0][1]); + expect(persisted.messages).toHaveLength(1); + expect(persisted.messages[0].id).toBe("msg-cont-1-assistant-continuation"); + expect(getTaskMock).not.toHaveBeenCalled(); + }); + + it("recovers a provider-confirmed failure notice through the same history-only path", async () => { + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + terminalDeliveryKind: "failure", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: { + text: "The Content agent could not finish this request.", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-failure"], + artifacts: [], + }, + }), ); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { adapters: new Map() }); + + expect(getTaskMock).not.toHaveBeenCalled(); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); it("finishes the resumed native progress stream when the remote task completes", async () => { @@ -1018,7 +1351,7 @@ describe("A2A continuation processor", () => { expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); - it("leaves completion failures in delivery for stale retry recovery", async () => { + it("reschedules history finalization failures without redelivering", async () => { const consoleError = vi .spyOn(console, "error") .mockImplementation(() => undefined); @@ -1037,12 +1370,15 @@ describe("A2A continuation processor", () => { expect(sendResponse).toHaveBeenCalledTimes(1); expect(completeA2AContinuationMock).toHaveBeenCalledTimes(3); - expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( + "cont-1", + 20_000, + ); expect(failA2AContinuationMock).not.toHaveBeenCalled(); - expect(fetch).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledOnce(); expect(consoleError).toHaveBeenCalledWith( - expect.stringContaining("marking it completed failed"), - expect.any(Error), + expect.stringContaining("history remains retryable"), + "Error", ); }); @@ -1230,11 +1566,13 @@ describe("A2A continuation processor", () => { expect.any(Object), { placeholderRef: undefined }, ); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), "The deck export failed", ); - expect(completeA2AContinuationMock).not.toHaveBeenCalled(); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); it("retries a terminal failure notification until delivery is confirmed", async () => { @@ -1269,7 +1607,7 @@ describe("A2A continuation processor", () => { expect(completeIntegrationCampaignTaskAfterA2AMock).not.toHaveBeenCalled(); }); - it("releases parent custody when failure notification delivery exhausts its bound", async () => { + it("retains parent custody when failure notification delivery exhausts its bound", async () => { const exhausted = continuation({ attempts: 30 }); const sendResponse = vi.fn(async () => { throw new Error("Slack delivery unavailable"); @@ -1302,14 +1640,12 @@ describe("A2A continuation processor", () => { adapters: new Map([["slack", adapter(sendResponse)]]), }); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( exhausted.id, - "The deck export failed", - ); - expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledWith( - exhausted.integrationTaskId, + 20_000, ); - expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(failA2AContinuationMock).not.toHaveBeenCalled(); + expect(completeIntegrationCampaignTaskAfterA2AMock).not.toHaveBeenCalled(); }); it("rechecks durable scope before claiming a terminal failure notification", async () => { @@ -1453,8 +1789,10 @@ describe("A2A continuation processor", () => { expect(sentText).not.toContain("ANTHROPIC_API_KEY"); expect(sentText).toContain("Error code: `missing_credentials`"); expect(sentText).toContain("Request ID: `task-1`"); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), "ANTHROPIC_API_KEY is not set", ); }); @@ -1519,8 +1857,10 @@ describe("A2A continuation processor", () => { expect.any(Object), { placeholderRef: undefined }, ); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), expect.stringContaining( "Timed out polling the Slides A2A task a2a-task-1 after 30 attempts", ), @@ -1662,6 +2002,10 @@ describe("A2A continuation processor", () => { await processing; expect(getTaskMock).toHaveBeenCalledTimes(2); + expect(saveA2AVerifiedArtifactCheckpointMock).toHaveBeenCalledWith( + "cont-1", + expect.stringContaining("request_checkpoint_a"), + ); expect(sendResponse).toHaveBeenCalledWith( expect.objectContaining({ text: expect.stringContaining("request_final_b"), @@ -1676,6 +2020,83 @@ describe("A2A continuation processor", () => { expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); }); + it("delivers a verified artifact checkpoint recovered by a fresh processor invocation", async () => { + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + const persistedCheckpoint = + "The agent is still working on the full response, but these verified artifacts already exist:\n- Content: /page/content-1"; + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + attempts: 30, + verifiedArtifactCheckpoint: persistedCheckpoint, + }), + ); + claimA2AContinuationDeliveryMock.mockResolvedValueOnce( + continuation({ status: "delivering" }), + ); + getTaskMock.mockResolvedValueOnce({ + id: "a2a-task-1", + status: { + state: "failed", + message: { + role: "agent", + parts: [{ type: "text", text: "remote worker exited" }], + }, + timestamp: new Date().toISOString(), + }, + }); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledWith( + expect.objectContaining({ + text: expect.stringContaining("/page/content-1"), + }), + expect.any(Object), + { placeholderRef: undefined }, + ); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); + expect(failA2AContinuationMock).not.toHaveBeenCalled(); + }); + + it("delivers a durable checkpoint instead of a failure notice after an exhausted non-transient poll error", async () => { + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + attempts: 30, + verifiedArtifactCheckpoint: "Verified Content: /page/content-1", + }), + ); + claimA2AContinuationDeliveryMock.mockResolvedValueOnce( + continuation({ status: "delivering", attempts: 30 }), + ); + getTaskMock.mockRejectedValueOnce(new Error("unexpected response shape")); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledWith( + expect.objectContaining({ + text: expect.stringContaining("/page/content-1"), + }), + expect.any(Object), + { placeholderRef: undefined }, + ); + expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( + "cont-1", + "success", + expect.any(Object), + undefined, + ); + }); + it("delivers the latest signed checkpoint only when remote polling is exhausted", async () => { vi.useFakeTimers(); process.env.A2A_SECRET = "test-a2a-secret-for-signed-checkpoints"; @@ -1838,8 +2259,10 @@ describe("A2A continuation processor", () => { expect.any(Object), { placeholderRef: undefined }, ); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), expect.stringContaining( "Timed out polling the Slides A2A task a2a-task-1 after 30 attempts", ), @@ -1917,8 +2340,10 @@ describe("A2A continuation processor", () => { "The Slides agent could not finish this request: Timed out polling the Slides A2A task a2a-task-1 after 20 minutes", ); expect(sentText).not.toContain("This operation was aborted"); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), expect.stringContaining( "Timed out polling the Slides A2A task a2a-task-1 after 20 minutes", ), @@ -2015,8 +2440,10 @@ describe("A2A continuation processor", () => { expect.any(Object), { placeholderRef: undefined }, ); - expect(failA2AContinuationMock).toHaveBeenCalledWith( + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", + "failure", + expect.any(Object), expect.stringContaining( "Timed out polling the Slides A2A task a2a-task-1 after 20 minutes", ), diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index ec6f819d6d..0fd590fac0 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -2,6 +2,7 @@ import { appendA2AArtifactLinks, extractA2AArtifactIdentities, stripA2APersistedArtifactMarkers, + type A2AArtifactIdentity, } from "../a2a/artifact-response.js"; import { A2AClient, signA2AToken } from "../a2a/client.js"; import type { Task } from "../a2a/types.js"; @@ -18,15 +19,20 @@ import { claimA2AContinuation, claimA2AContinuationDelivery, claimDueA2AContinuations, - completeA2AContinuation, failA2AContinuationsForIntegrationTask, failA2AContinuation, + finalizeA2ATerminalHistory, getA2AContinuation, - hasActiveA2AContinuationsForIntegrationTask, + getA2AContinuationTaskOutcome, + hasPendingConfirmedA2ADeliveryForIntegrationTask, listRecoverableA2AIntegrationTasks, recoverDueA2AContinuationIds, + recordA2ATerminalDeliveryReceipt, rescheduleA2AContinuation, + saveA2AVerifiedArtifactCheckpoint, type A2AContinuation, + type A2ATerminalDeliveryKind, + type A2ATerminalHistoryPayload, type RecoverableA2AIntegrationTask, } from "./a2a-continuations-store.js"; import { @@ -67,6 +73,23 @@ const PLATFORM_SEND_TIMEOUT_MS = 12_000; const DISPATCH_SETTLE_WAIT_MS = 2_000; const COMPLETE_AFTER_DELIVERY_ATTEMPTS = 3; +function logA2AContinuationTransition( + event: string, + continuation: Pick< + A2AContinuation, + "id" | "integrationTaskId" | "a2aTaskId" | "attempts" | "status" + >, +): void { + console.info("[integrations] a2a-continuation-transition", { + event, + continuationId: continuation.id, + integrationTaskId: continuation.integrationTaskId, + downstreamTaskId: continuation.a2aTaskId, + attempt: continuation.attempts, + status: continuation.status, + }); +} + export async function dispatchA2AContinuation( continuationId: string, webhookBaseUrl?: string, @@ -153,17 +176,82 @@ export async function processA2AContinuationById( await processClaimedContinuation(continuation, options); } +export async function recoverA2AContinuationAfterProcessorFailure( + continuationId: string, + options: { + adapters: Map; + reason: string; + }, +): Promise { + const continuation = await getA2AContinuation(continuationId); + if ( + !continuation || + continuation.status === "completed" || + continuation.status === "failed" + ) { + return; + } + if ( + continuation.terminalDeliveryConfirmedAt != null && + continuation.terminalHistoryPayload + ) { + await persistAndFinalizeConfirmedA2ADelivery(continuation); + return; + } + const adapter = options.adapters.get(continuation.platform); + if (continuation.attempts < MAX_ATTEMPTS || !adapter) { + logA2AContinuationTransition("processor_released", continuation); + await rescheduleAndRedispatchA2AContinuation(continuation.id); + return; + } + + const progress = await resumeA2AContinuationProgress(continuation, adapter); + if (continuation.verifiedArtifactCheckpoint) { + logA2AContinuationTransition( + "processor_exhausted_checkpoint_delivery", + continuation, + ); + await deliverAndCompleteA2AContinuation( + continuation, + adapter, + formatRecoverableArtifactFallbackText( + continuation.verifiedArtifactCheckpoint, + ), + progress, + ); + return; + } + logA2AContinuationTransition( + "processor_exhausted_failure_notice", + continuation, + ); + await notifyAndFailA2AContinuation( + continuation, + adapter, + options.reason, + progress, + ); +} + export async function processDueA2AContinuations(options: { adapters: Map; limit?: number; }): Promise { const continuations = await claimDueA2AContinuations(options.limit ?? 5); for (const continuation of continuations) { - await processClaimedContinuation(continuation, options).catch((err) => - console.error( - `[integrations] A2A continuation ${continuation.id} failed:`, - err, - ), + await processClaimedContinuation(continuation, options).catch( + async (err) => { + console.error( + `[integrations] A2A continuation ${continuation.id} failed; durable recovery requested`, + ); + await recoverA2AContinuationAfterProcessorFailure(continuation.id, { + adapters: options.adapters, + reason: + err instanceof Error + ? err.message.slice(0, 500) + : "continuation processing failed", + }); + }, ); } } @@ -182,6 +270,7 @@ export async function recoverDueA2AContinuations(options?: { const limit = options?.limit ?? 5; const candidateTasks = await listRecoverableA2AIntegrationTasks(200); const eligibleTaskIds: string[] = []; + const confirmedHistoryTaskIds: string[] = []; for (const task of candidateTasks) { const enabled = isIntegrationDurableDispatchEnabledForTask({ platform: task.platform, @@ -192,12 +281,24 @@ export async function recoverDueA2AContinuations(options?: { }); if (enabled) { eligibleTaskIds.push(task.id); - if (eligibleTaskIds.length >= limit) break; + } else if (task.hasPendingConfirmedDelivery) { + confirmedHistoryTaskIds.push(task.id); } else { await failDisabledDurableA2ATask(task); } + if (eligibleTaskIds.length + confirmedHistoryTaskIds.length >= limit) break; } const ids = await recoverDueA2AContinuationIds(limit, eligibleTaskIds); + const remaining = Math.max(0, limit - ids.length); + const confirmedHistoryIds = + remaining > 0 && confirmedHistoryTaskIds.length > 0 + ? await recoverDueA2AContinuationIds( + remaining, + confirmedHistoryTaskIds, + true, + ) + : []; + ids.push(...confirmedHistoryIds); let dispatched = 0; let failed = 0; @@ -223,6 +324,13 @@ async function processClaimedContinuation( continuation: A2AContinuation, options: { adapters: Map }, ): Promise { + if ( + continuation.terminalDeliveryConfirmedAt != null && + continuation.terminalHistoryPayload + ) { + await persistAndFinalizeConfirmedA2ADelivery(continuation); + return; + } if (!(await durableContinuationScopeStillEnabled(continuation))) return; const adapter = options.adapters.get(continuation.platform); if (!adapter) { @@ -244,7 +352,7 @@ async function processClaimedContinuation( const recoverableArtifactSecrets = await resolveContinuationArtifactSecrets(continuation); let task: Task | null = null; - let latestRecoverableArtifactText: string | null = null; + let latestRecoverableArtifactText = continuation.verifiedArtifactCheckpoint; try { while (Date.now() < deadline) { @@ -255,7 +363,13 @@ async function processClaimedContinuation( recoverableArtifactSecrets, ); if (recoverableArtifactText) { - latestRecoverableArtifactText = recoverableArtifactText; + latestRecoverableArtifactText = await saveA2AVerifiedArtifactCheckpoint( + continuation.id, + recoverableArtifactText, + ); + if (latestRecoverableArtifactText) { + logA2AContinuationTransition("checkpoint_persisted", continuation); + } } if (TERMINAL_STATES.has(task.status.state)) break; await reportA2AContinuationProgress(continuation, progress, task); @@ -287,6 +401,15 @@ async function processClaimedContinuation( return; } if (continuation.attempts >= MAX_ATTEMPTS) { + if (latestRecoverableArtifactText) { + await deliverAndCompleteA2AContinuation( + continuation, + adapter, + formatRecoverableArtifactFallbackText(latestRecoverableArtifactText), + progress, + ); + return; + } await notifyAndFailA2AContinuation( continuation, adapter, @@ -396,6 +519,19 @@ async function durableContinuationScopeStillEnabled( }); if (enabled) return true; + if ( + await hasPendingConfirmedA2ADeliveryForIntegrationTask( + continuation.integrationTaskId, + ) + ) { + await failA2AContinuation( + continuation.id, + "Durable integration campaign was disabled before this continuation delivered", + ); + await reconcileTerminalA2AParentIfDisabled(continuation.integrationTaskId); + return false; + } + await failDisabledDurableA2ATask({ id: continuation.integrationTaskId, platform: task?.platform ?? continuation.platform, @@ -430,6 +566,38 @@ async function failDisabledDurableA2ATask( } } +export async function reconcileTerminalA2AParentIfDisabled( + integrationTaskId: string, +): Promise { + if ( + (await getA2AContinuationTaskOutcome(integrationTaskId)) !== + "terminal-without-delivery" + ) { + return false; + } + const task = await getPendingTask(integrationTaskId); + if ( + !task || + isIntegrationDurableDispatchEnabledForTask({ + platform: task.platform, + externalThreadId: task.externalThreadId, + platformContext: task.dispatchScope + ? { channelId: task.dispatchScope } + : undefined, + }) + ) { + return false; + } + await failDisabledDurableA2ATask({ + id: task.id, + platform: task.platform, + externalThreadId: task.externalThreadId, + dispatchScope: task.dispatchScope, + status: task.status, + }); + return true; +} + async function resumeA2AContinuationProgress( continuation: A2AContinuation, adapter: PlatformAdapter, @@ -507,14 +675,20 @@ async function notifyAndFailA2AContinuation( continuation.id, ); if (!deliveryContinuation) return; + logA2AContinuationTransition( + "failure_delivery_claimed", + deliveryContinuation, + ); const message = formatContinuationFailureMessage( deliveryContinuation, reason, ); + let outgoing: OutgoingMessage; + let deliveryReceipt: PlatformDeliveryReceipt; try { - const outgoing = adapter.formatAgentResponse(message); - await withTimeout( + outgoing = adapter.formatAgentResponse(message); + deliveryReceipt = await withTimeout( deliverA2AContinuationResponse( adapter, deliveryContinuation, @@ -530,17 +704,20 @@ async function notifyAndFailA2AContinuation( `[integrations] Failed to notify ${deliveryContinuation.platform} about failed A2A continuation ${deliveryContinuation.id}:`, err, ); - if (deliveryContinuation.attempts >= MAX_ATTEMPTS) { - await failA2AContinuation(deliveryContinuation.id, reason); - await completeParentCampaignAfterTerminalA2A(deliveryContinuation); - return; - } await rescheduleAndRedispatchA2AContinuation(deliveryContinuation.id); return; } - await failA2AContinuation(deliveryContinuation.id, reason); - await completeParentCampaignAfterTerminalA2A(deliveryContinuation); + const confirmed = await recordTerminalA2ADelivery( + deliveryContinuation, + "failure", + outgoing, + deliveryReceipt, + [], + reason, + ); + if (!confirmed) return; + await persistAndFinalizeConfirmedA2ADelivery(confirmed); } async function deliverAndCompleteA2AContinuation( @@ -554,12 +731,24 @@ async function deliverAndCompleteA2AContinuation( continuation.id, ); if (!deliveryContinuation) return; + logA2AContinuationTransition( + "response_delivery_claimed", + deliveryContinuation, + ); + let outgoing: OutgoingMessage; + let deliveryReceipt: PlatformDeliveryReceipt; + const artifactSecrets = + await resolveContinuationArtifactSecrets(deliveryContinuation); + const artifacts = extractA2AArtifactIdentities( + [{ tool: "call-agent", result: text }], + { persistedArtifactSecrets: artifactSecrets }, + ); try { - const outgoing = adapter.formatAgentResponse( + outgoing = adapter.formatAgentResponse( stripA2APersistedArtifactMarkers(text), ); - const deliveryReceipt = await withTimeout( + deliveryReceipt = await withTimeout( deliverA2AContinuationResponse( adapter, deliveryContinuation, @@ -570,43 +759,19 @@ async function deliverAndCompleteA2AContinuation( PLATFORM_SEND_TIMEOUT_MS, `${deliveryContinuation.platform} response delivery timed out`, ); - let persistenceError: unknown; - const artifactSecrets = - await resolveContinuationArtifactSecrets(deliveryContinuation); - for (let attempt = 0; attempt < 3; attempt += 1) { - try { - await persistA2AContinuationDelivery( - deliveryContinuation, - outgoing, - deliveryReceipt, - text, - artifactSecrets, - ); - persistenceError = undefined; - break; - } catch (err) { - persistenceError = err; - } - } - if (persistenceError) { - console.error( - `[integrations] Delivered A2A continuation ${deliveryContinuation.id} but could not persist its thread history:`, - persistenceError, - ); - } - } catch (err) { - if (deliveryContinuation.attempts >= MAX_ATTEMPTS) { - await failA2AContinuation( - deliveryContinuation.id, - err instanceof Error ? err.message : String(err), - ); - return; - } + const confirmed = await recordTerminalA2ADelivery( + deliveryContinuation, + "success", + outgoing, + deliveryReceipt, + artifacts, + ); + if (!confirmed) return; + await persistAndFinalizeConfirmedA2ADelivery(confirmed); + } catch { await rescheduleAndRedispatchA2AContinuation(deliveryContinuation.id); return; } - - await completeAfterSuccessfulDelivery(deliveryContinuation); } async function deliverA2AContinuationResponse( @@ -639,6 +804,7 @@ async function deliverA2AContinuationResponse( } catch { // The thread reply below is still the authoritative final answer. } + logA2AContinuationTransition("native_progress_fallback", continuation); } } const receipt = await adapter.sendResponse(message, continuation.incoming, { @@ -652,10 +818,7 @@ async function deliverA2AContinuationResponse( async function persistA2AContinuationDelivery( continuation: A2AContinuation, - outgoing: OutgoingMessage, - receipt: PlatformDeliveryReceipt, - artifactText: string, - artifactSecrets: readonly string[], + history: A2ATerminalHistoryPayload, ): Promise { const mapping = await getThreadMapping( continuation.platform, @@ -673,33 +836,32 @@ async function persistA2AContinuationDelivery( } if (!Array.isArray(repo.messages)) repo.messages = []; - const artifacts = extractA2AArtifactIdentities( - [{ tool: "call-agent", result: artifactText }], - { - persistedArtifactSecrets: artifactSecrets, - }, - ); const metadata: Record = { integrationDeliveryAttempted: true, integrationDelivery: { platform: continuation.platform, status: "delivered", - text: outgoing.text, - deliveredAt: new Date().toISOString(), - ...(receipt.messageRefs?.length - ? { messageRefs: receipt.messageRefs } + text: history.text, + deliveredAt: history.deliveredAt, + ...(history.messageRefs.length + ? { messageRefs: history.messageRefs } : {}), }, }; - if (artifacts.length > 0) metadata.integrationArtifacts = artifacts; - - repo.messages.push({ - id: `msg-${Date.now()}-assistant-continuation`, - role: "assistant", - content: [{ type: "text", text: outgoing.text }], - createdAt: new Date().toISOString(), - metadata, - }); + if (history.artifacts.length > 0) { + metadata.integrationArtifacts = history.artifacts; + } + + const messageId = `msg-${continuation.id}-assistant-continuation`; + if (!repo.messages.some((message: any) => message?.id === messageId)) { + repo.messages.push({ + id: messageId, + role: "assistant", + content: [{ type: "text", text: history.text }], + createdAt: history.deliveredAt, + metadata, + }); + } const meta = extractThreadMeta(repo); await updateThreadData( mapping.internalThreadId, @@ -722,25 +884,75 @@ async function rescheduleAndRedispatchA2AContinuation( }); } -async function completeAfterSuccessfulDelivery( +async function recordTerminalA2ADelivery( + continuation: A2AContinuation, + kind: A2ATerminalDeliveryKind, + outgoing: OutgoingMessage, + receipt: PlatformDeliveryReceipt, + artifacts: A2AArtifactIdentity[], + errorMessage?: string, +): Promise { + const historyPayload: A2ATerminalHistoryPayload = { + text: outgoing.text, + deliveredAt: new Date().toISOString(), + messageRefs: receipt.messageRefs ?? [], + artifacts, + }; + for (let attempt = 0; attempt < COMPLETE_AFTER_DELIVERY_ATTEMPTS; attempt++) { + try { + const confirmed = await recordA2ATerminalDeliveryReceipt( + continuation.id, + kind, + historyPayload, + errorMessage, + ); + logA2AContinuationTransition( + kind === "success" + ? "response_delivery_confirmed" + : "failure_delivery_confirmed", + continuation, + ); + return confirmed; + } catch {} + } + + console.error( + `[integrations] ${continuation.platform} accepted terminal A2A delivery for ${continuation.id}, ` + + `but recording its receipt failed after ${COMPLETE_AFTER_DELIVERY_ATTEMPTS} attempts. Leaving it in delivering for stale recovery.`, + ); + return null; +} + +async function persistAndFinalizeConfirmedA2ADelivery( continuation: A2AContinuation, ): Promise { + const history = continuation.terminalHistoryPayload; + if (!history || continuation.terminalDeliveryConfirmedAt == null) { + throw new Error("Confirmed A2A delivery is missing durable history"); + } let lastError: unknown; - for (let attempt = 0; attempt < COMPLETE_AFTER_DELIVERY_ATTEMPTS; attempt++) { + for (let attempt = 0; attempt < 3; attempt += 1) { try { - await completeA2AContinuation(continuation.id); - await completeParentCampaignAfterTerminalA2A(continuation); + await persistA2AContinuationDelivery(continuation, history); + await finalizeA2ATerminalHistory(continuation.id); + logA2AContinuationTransition("terminal_history_persisted", continuation); + await completeParentCampaignAfterTerminalA2A({ + ...continuation, + status: + continuation.terminalDeliveryKind === "success" + ? "completed" + : "failed", + }); return; } catch (err) { lastError = err; } } - console.error( - `[integrations] ${continuation.platform} accepted A2A continuation ${continuation.id}, ` + - "but marking it completed failed. Leaving it in delivering for stale-delivery recovery.", - lastError, + `[integrations] A2A continuation ${continuation.id} has a provider receipt but its history remains retryable:`, + lastError instanceof Error ? lastError.name : "persistence_error", ); + await rescheduleAndRedispatchA2AContinuation(continuation.id); } async function completeParentCampaignAfterTerminalA2A( @@ -750,17 +962,31 @@ async function completeParentCampaignAfterTerminalA2A( continuation.integrationTaskId, ); if (!campaign) return; - if ( - await hasActiveA2AContinuationsForIntegrationTask( - continuation.integrationTaskId, - ) - ) { + const outcome = await getA2AContinuationTaskOutcome( + continuation.integrationTaskId, + ); + if (outcome !== "terminal-delivered") { + if (outcome === "terminal-without-delivery") { + if ( + await reconcileTerminalA2AParentIfDisabled( + continuation.integrationTaskId, + ) + ) { + return; + } + } + if (outcome !== "active") { + console.warn( + `[integrations] Refusing to complete A2A parent ${continuation.integrationTaskId} with outcome ${outcome}`, + ); + } return; } const completed = await completeIntegrationCampaignTaskAfterA2A( continuation.integrationTaskId, ); if (!completed) return; + logA2AContinuationTransition("parent_completed", continuation); const nextTask = await getNextPendingTaskForThread( continuation.platform, diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 14cf0e2857..4ad31645e6 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -52,6 +52,10 @@ function continuationRow(overrides: Record = {}) { dedupe_key: "message-hash-1", a2a_task_id: "a2a-task-1", a2a_auth_token: null, + verified_artifact_checkpoint: null, + terminal_delivery_kind: null, + terminal_delivery_confirmed_at: null, + terminal_history_payload: null, status: "processing", attempts: 1, next_check_at: 1, @@ -88,6 +92,21 @@ describe("A2A continuations store", () => { expect(calls).toContainEqual( expect.stringContaining("ADD COLUMN progress_ref"), ); + expect(calls).toContainEqual( + expect.stringContaining("ADD COLUMN verified_artifact_checkpoint"), + ); + expect(calls).toContainEqual( + expect.stringContaining("ADD COLUMN terminal_delivery_kind"), + ); + expect(calls).toContainEqual( + expect.stringContaining("ADD COLUMN terminal_delivery_confirmed_at"), + ); + expect(calls).toContainEqual( + expect.stringContaining("ADD COLUMN terminal_history_payload"), + ); + expect(calls).toContainEqual( + expect.stringContaining("SET terminal_delivery_kind = 'success'"), + ); const progressOwnerAlterIndex = calls.findIndex((sql) => sql.includes("ADD COLUMN progress_ref_claimed"), ); @@ -106,6 +125,330 @@ describe("A2A continuations store", () => { expect(progressOwnerBackfillIndex).toBeLessThan(progressOwnerIndexIndex); }); + it("applies terminal receipt and history migrations on Postgres", async () => { + isPostgresMock.mockReturnValue(true); + executeMock.mockResolvedValue({ rows: [], rowsAffected: 0 }); + const { getA2AContinuationForIntegrationTask } = await loadStore(); + + await getA2AContinuationForIntegrationTask("task-existing"); + + const calls = executeMock.mock.calls.map(([query]) => querySql(query)); + expect(calls).toContainEqual( + expect.stringContaining( + "ADD COLUMN IF NOT EXISTS terminal_delivery_confirmed_at", + ), + ); + expect(calls).toContainEqual( + expect.stringContaining( + "ADD COLUMN IF NOT EXISTS terminal_history_payload", + ), + ); + expect(calls).toContainEqual( + expect.stringContaining("SET terminal_delivery_kind = 'success'"), + ); + }); + + it("persists and verifies a bounded artifact checkpoint on an active continuation", async () => { + const { saveA2AVerifiedArtifactCheckpoint } = await loadStore(); + let checkpoint: string | null = null; + executeMock.mockImplementation( + async (query: string | { sql: string; args?: unknown[] }) => { + const sql = querySql(query); + const args = queryArgs(query); + if (sql.includes("SET verified_artifact_checkpoint = ?")) { + checkpoint = String(args[0]); + return { rows: [], rowsAffected: 1 }; + } + if ( + sql.includes( + "SELECT * FROM integration_a2a_continuations WHERE id = ?", + ) + ) { + return { + rows: [ + continuationRow({ verified_artifact_checkpoint: checkpoint }), + ], + rowsAffected: 0, + }; + } + return { rows: [], rowsAffected: 0 }; + }, + ); + + await expect( + saveA2AVerifiedArtifactCheckpoint( + "cont-1", + " verified /page/content-1 ", + ), + ).resolves.toBe("verified /page/content-1"); + }); + + it("rejects an oversized verified artifact checkpoint", async () => { + const { saveA2AVerifiedArtifactCheckpoint } = await loadStore(); + executeMock.mockResolvedValue({ rows: [], rowsAffected: 0 }); + + await expect( + saveA2AVerifiedArtifactCheckpoint("cont-1", "x".repeat(16_001)), + ).rejects.toThrow("exceeds 16000 characters"); + }); + + it.each([ + [[], "missing"], + [ + [{ status: "processing", terminal_delivery_confirmed_at: null }], + "active", + ], + [ + [{ status: "completed", terminal_delivery_confirmed_at: 10 }], + "terminal-delivered", + ], + [ + [{ status: "failed", terminal_delivery_confirmed_at: null }], + "terminal-without-delivery", + ], + ])("classifies task continuation custody as %s", async (rows, expected) => { + const { getA2AContinuationTaskOutcome } = await loadStore(); + executeMock.mockImplementation(async (query: string | { sql: string }) => { + if ( + querySql(query).includes( + "SELECT status, terminal_delivery_confirmed_at", + ) + ) { + return { rows, rowsAffected: 0 }; + } + return { rows: [], rowsAffected: 0 }; + }); + + await expect(getA2AContinuationTaskOutcome("task-1")).resolves.toBe( + expected, + ); + }); + + it("records provider-confirmed terminal delivery without terminalizing or scrubbing history custody", async () => { + const persisted = new Map>(); + executeMock.mockImplementation( + async (query: string | { sql: string; args?: unknown[] }) => { + const sql = querySql(query); + const args = queryArgs(query); + if (sql.includes("terminal_delivery_confirmed_at = COALESCE")) { + persisted.set(String(args[6]), { + status: "delivering", + terminal_delivery_kind: args[0], + terminal_delivery_confirmed_at: args[1], + terminal_history_payload: args[2], + error_message: args[5], + }); + return { rows: [], rowsAffected: 1 }; + } + if ( + sql.includes( + "SELECT * FROM integration_a2a_continuations WHERE id = ?", + ) + ) { + return { + rows: [continuationRow(persisted.get(String(args[0])) ?? {})], + rowsAffected: 0, + }; + } + return { rows: [], rowsAffected: 0 }; + }, + ); + const { recordA2ATerminalDeliveryReceipt } = await loadStore(); + const history = { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [ + { + id: "content-1", + resourceType: "document", + sourceAction: "call-agent", + }, + ], + }; + + await recordA2ATerminalDeliveryReceipt("cont-success", "success", history); + await recordA2ATerminalDeliveryReceipt( + "cont-failure", + "failure", + history, + "remote failed", + ); + + const terminalUpdates = executeMock.mock.calls + .map(([query]) => query) + .filter( + (query): query is { sql: string; args: unknown[] } => + typeof query !== "string" && + query.sql.includes("terminal_delivery_confirmed_at = COALESCE"), + ); + expect(terminalUpdates).toHaveLength(2); + expect(terminalUpdates[0].args).toEqual([ + "success", + expect.any(Number), + JSON.stringify(history), + expect.any(Number), + expect.any(Number), + null, + "cont-success", + "success", + ]); + expect(terminalUpdates[1].args).toEqual([ + "failure", + expect.any(Number), + JSON.stringify(history), + expect.any(Number), + expect.any(Number), + "remote failed", + "cont-failure", + "failure", + ]); + }); + + it("fails loud when terminal delivery confirmation does not persist", async () => { + executeMock.mockImplementation(async (query: string | { sql: string }) => { + if ( + querySql(query).includes( + "SELECT * FROM integration_a2a_continuations WHERE id = ?", + ) + ) { + return { + rows: [continuationRow({ status: "delivering" })], + rowsAffected: 0, + }; + } + return { rows: [], rowsAffected: 0 }; + }); + const { recordA2ATerminalDeliveryReceipt } = await loadStore(); + + await expect( + recordA2ATerminalDeliveryReceipt("cont-unconfirmed", "success", { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: [], + artifacts: [], + }), + ).rejects.toThrow("did not persist"); + }); + + it("rejects an oversized terminal history payload before writing a receipt", async () => { + executeMock.mockResolvedValue({ rows: [], rowsAffected: 0 }); + const { recordA2ATerminalDeliveryReceipt } = await loadStore(); + + await expect( + recordA2ATerminalDeliveryReceipt("cont-1", "success", { + text: "x".repeat(64_001), + deliveredAt: new Date().toISOString(), + messageRefs: [], + artifacts: [], + }), + ).rejects.toThrow("exceeds 64000 characters"); + }); + + it("terminalizes and scrubs only after durable history persistence", async () => { + let finalized = false; + executeMock.mockImplementation( + async (query: string | { sql: string; args?: unknown[] }) => { + const sql = querySql(query); + if (sql.includes("terminal_history_payload = NULL")) { + finalized = true; + return { rows: [], rowsAffected: 1 }; + } + if ( + sql.includes( + "SELECT * FROM integration_a2a_continuations WHERE id = ?", + ) + ) { + return { + rows: [ + continuationRow({ + status: finalized ? "completed" : "delivering", + terminal_delivery_kind: "success", + terminal_delivery_confirmed_at: 10, + terminal_history_payload: finalized + ? null + : JSON.stringify({ + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: [], + artifacts: [], + }), + }), + ], + rowsAffected: 0, + }; + } + return { rows: [], rowsAffected: 0 }; + }, + ); + const { finalizeA2ATerminalHistory } = await loadStore(); + + await expect(finalizeA2ATerminalHistory("cont-1")).resolves.toBeUndefined(); + expect(finalized).toBe(true); + }); + + it("finalizes receipt-backed history after stale delivery recovery and a fresh processing claim", async () => { + const history = JSON.stringify({ + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [], + }); + const state = continuationRow({ + status: "delivering", + terminal_delivery_kind: "success", + terminal_delivery_confirmed_at: 10, + terminal_history_payload: history, + next_check_at: 0, + }); + executeMock.mockImplementation( + async (query: string | { sql: string; args?: unknown[] }) => { + const sql = querySql(query); + if ( + sql.includes("WHERE status = 'delivering'") && + sql.includes("terminal_delivery_confirmed_at") + ) { + state.status = "pending"; + return { rows: [], rowsAffected: 1 }; + } + if (sql.includes("SELECT id FROM integration_a2a_continuations")) { + return { rows: [{ id: state.id }], rowsAffected: 0 }; + } + if (sql.includes("attempts = attempts + 1")) { + state.status = "processing"; + return { rows: [], rowsAffected: 1 }; + } + if (sql.includes("terminal_history_payload = NULL")) { + state.status = "completed"; + state.terminal_history_payload = null; + return { rows: [], rowsAffected: 1 }; + } + if ( + sql.includes( + "SELECT * FROM integration_a2a_continuations WHERE id = ?", + ) + ) { + return { rows: [{ ...state }], rowsAffected: 0 }; + } + return { rows: [], rowsAffected: 0 }; + }, + ); + const { + claimA2AContinuation, + finalizeA2ATerminalHistory, + recoverDueA2AContinuationIds, + } = await loadStore(); + + await expect(recoverDueA2AContinuationIds(1)).resolves.toEqual(["cont-1"]); + await expect(claimA2AContinuation("cont-1")).resolves.toMatchObject({ + status: "processing", + terminalDeliveryConfirmedAt: 10, + }); + await expect(finalizeA2ATerminalHistory("cont-1")).resolves.toBeUndefined(); + expect(state.status).toBe("completed"); + expect(state.terminal_history_payload).toBeNull(); + }); + it("loads recoverable continuation owners and scope without task N+1 reads", async () => { const { listRecoverableA2AIntegrationTasks } = await loadStore(); executeMock.mockImplementation(async (query: string | { sql: string }) => { @@ -118,6 +461,7 @@ describe("A2A continuations store", () => { external_thread_id: "C123:123.456", dispatch_scope: "C123", status: "processing", + has_pending_confirmed_delivery: 1, }, ], }; @@ -132,6 +476,7 @@ describe("A2A continuations store", () => { externalThreadId: "C123:123.456", dispatchScope: "C123", status: "processing", + hasPendingConfirmedDelivery: true, }, ]); const joinedReads = executeMock.mock.calls.filter(([query]) => @@ -156,6 +501,9 @@ describe("A2A continuations store", () => { expect(querySql(update!)).toContain( "status IN ('pending', 'processing', 'delivering')", ); + expect(querySql(update!)).toContain( + "terminal_delivery_confirmed_at IS NULL", + ); expect(queryArgs(update!)).toEqual([ "durable scope disabled", expect.any(Number), @@ -686,6 +1034,7 @@ describe("A2A continuations store", () => { expect.any(Number), expect.any(Number), "{}", + expect.any(Number), "cont-completed", ]); expect(terminalUpdates[1].args).toEqual([ @@ -858,6 +1207,7 @@ describe("A2A continuations store", () => { expect.any(Number), expect.any(Number), expect.any(Number), + expect.any(Number), ], }), ); @@ -865,6 +1215,27 @@ describe("A2A continuations store", () => { expect(querySql(recoveryCall![0])).not.toContain("completed_at"); }); + it("limits disabled-scope recovery to receipt-confirmed continuation rows", async () => { + const { recoverDueA2AContinuationIds } = await loadStore(); + executeMock.mockResolvedValue({ rows: [], rowsAffected: 0 }); + + await expect( + recoverDueA2AContinuationIds(5, ["task-mixed"], true), + ).resolves.toEqual([]); + + const custodyQueries = executeMock.mock.calls + .map(([query]) => querySql(query)) + .filter( + (sql) => + sql.includes("integration_a2a_continuations") && + (sql.includes("SET status = ?") || sql.includes("SELECT id FROM")), + ); + expect(custodyQueries).toHaveLength(3); + for (const sql of custodyQueries) { + expect(sql).toContain("terminal_delivery_confirmed_at IS NOT NULL"); + } + }); + it("lists due/stale scheduler recovery ids without claiming terminal rows", async () => { const { recoverDueA2AContinuationIds } = await loadStore(); executeMock.mockImplementation( diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 181bf70728..b8dcebc78b 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -1,3 +1,4 @@ +import type { A2AArtifactIdentity } from "../a2a/artifact-response.js"; import { getDbExec, isPostgres, @@ -15,6 +16,8 @@ import type { IncomingMessage, PlatformRunProgressRef } from "./types.js"; let _initPromise: Promise | undefined; const PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000; const PROCESSING_NEXT_CHECK_STALE_AFTER_MS = 60 * 1000; +const MAX_VERIFIED_ARTIFACT_CHECKPOINT_CHARS = 16_000; +const MAX_TERMINAL_HISTORY_PAYLOAD_CHARS = 64_000; // Build the CREATE SQL lazily (not at module scope) so intType() runs at // RUNTIME, not import time — a module-scope call breaks any consumer whose @@ -37,6 +40,10 @@ function buildCreateSql(): string { dedupe_key TEXT, a2a_task_id TEXT NOT NULL, a2a_auth_token TEXT, + verified_artifact_checkpoint TEXT, + terminal_delivery_kind TEXT, + terminal_delivery_confirmed_at ${intType()}, + terminal_history_payload TEXT, status TEXT NOT NULL, attempts ${intType()} NOT NULL DEFAULT 0, next_check_at ${intType()} NOT NULL, @@ -88,6 +95,27 @@ async function ensureTable(): Promise { "progress_ref_claimed", `ALTER TABLE integration_a2a_continuations ADD COLUMN IF NOT EXISTS progress_ref_claimed ${intType()} NOT NULL DEFAULT 0`, ); + await ensureColumnExists( + "integration_a2a_continuations", + "verified_artifact_checkpoint", + `ALTER TABLE integration_a2a_continuations ADD COLUMN IF NOT EXISTS verified_artifact_checkpoint TEXT`, + ); + await ensureColumnExists( + "integration_a2a_continuations", + "terminal_delivery_kind", + `ALTER TABLE integration_a2a_continuations ADD COLUMN IF NOT EXISTS terminal_delivery_kind TEXT`, + ); + await ensureColumnExists( + "integration_a2a_continuations", + "terminal_delivery_confirmed_at", + `ALTER TABLE integration_a2a_continuations ADD COLUMN IF NOT EXISTS terminal_delivery_confirmed_at ${intType()}`, + ); + await ensureColumnExists( + "integration_a2a_continuations", + "terminal_history_payload", + `ALTER TABLE integration_a2a_continuations ADD COLUMN IF NOT EXISTS terminal_history_payload TEXT`, + ); + await backfillLegacyCompletedDeliveries(client); await backfillProgressRefOwners(client); await ensureIndexExists( "idx_a2a_continuations_dedupe_key", @@ -123,6 +151,11 @@ async function ensureTable(): Promise { "progress_ref_claimed", `${intType()} NOT NULL DEFAULT 0`, ); + await addColumnIfMissing("verified_artifact_checkpoint", "TEXT"); + await addColumnIfMissing("terminal_delivery_kind", "TEXT"); + await addColumnIfMissing("terminal_delivery_confirmed_at", intType()); + await addColumnIfMissing("terminal_history_payload", "TEXT"); + await backfillLegacyCompletedDeliveries(client); await backfillProgressRefOwners(client); await retryOnDdlRace(() => client.execute( @@ -143,6 +176,10 @@ async function ensureTable(): Promise { return _initPromise; } +export async function ensureA2AContinuationsTable(): Promise { + await ensureTable(); +} + async function addColumnIfMissing(name: string, definition: string) { try { await retryOnDdlRace(() => @@ -183,6 +220,18 @@ async function backfillProgressRefOwners( `); } +async function backfillLegacyCompletedDeliveries( + client: ReturnType, +): Promise { + await client.execute(` + UPDATE integration_a2a_continuations + SET terminal_delivery_kind = 'success', + terminal_delivery_confirmed_at = COALESCE(completed_at, updated_at) + WHERE status = 'completed' + AND terminal_delivery_confirmed_at IS NULL + `); +} + export type A2AContinuationStatus = | "pending" | "processing" @@ -190,6 +239,21 @@ export type A2AContinuationStatus = | "completed" | "failed"; +export type A2ATerminalDeliveryKind = "success" | "failure"; + +export interface A2ATerminalHistoryPayload { + text: string; + deliveredAt: string; + messageRefs: string[]; + artifacts: A2AArtifactIdentity[]; +} + +export type A2AContinuationTaskOutcome = + | "active" + | "terminal-delivered" + | "terminal-without-delivery" + | "missing"; + export interface A2AContinuation { id: string; integrationTaskId: string; @@ -206,6 +270,10 @@ export interface A2AContinuation { dedupeKey: string | null; a2aTaskId: string; a2aAuthToken: string | null; + verifiedArtifactCheckpoint: string | null; + terminalDeliveryKind: A2ATerminalDeliveryKind | null; + terminalDeliveryConfirmedAt: number | null; + terminalHistoryPayload: A2ATerminalHistoryPayload | null; status: A2AContinuationStatus; attempts: number; nextCheckAt: number; @@ -270,6 +338,20 @@ function rowToContinuation(row: Record): A2AContinuation { dedupeKey: (row.dedupe_key as string | null) ?? null, a2aTaskId: row.a2a_task_id as string, a2aAuthToken: (row.a2a_auth_token as string | null) ?? null, + verifiedArtifactCheckpoint: + (row.verified_artifact_checkpoint as string | null) ?? null, + terminalDeliveryKind: + row.terminal_delivery_kind === "success" || + row.terminal_delivery_kind === "failure" + ? row.terminal_delivery_kind + : null, + terminalDeliveryConfirmedAt: + row.terminal_delivery_confirmed_at == null + ? null + : Number(row.terminal_delivery_confirmed_at), + terminalHistoryPayload: parseTerminalHistoryPayload( + row.terminal_history_payload, + ), status: row.status as A2AContinuationStatus, attempts: Number(row.attempts ?? 0), nextCheckAt: Number(row.next_check_at ?? 0), @@ -281,6 +363,42 @@ function rowToContinuation(row: Record): A2AContinuation { }; } +function parseTerminalHistoryPayload( + value: unknown, +): A2ATerminalHistoryPayload | null { + if (typeof value !== "string" || !value) return null; + try { + const parsed = JSON.parse(value) as Partial; + if ( + typeof parsed.text !== "string" || + typeof parsed.deliveredAt !== "string" || + !Array.isArray(parsed.messageRefs) || + !parsed.messageRefs.every((ref) => typeof ref === "string") || + !Array.isArray(parsed.artifacts) + ) { + return null; + } + return { + text: parsed.text, + deliveredAt: parsed.deliveredAt, + messageRefs: parsed.messageRefs, + artifacts: parsed.artifacts.filter( + (artifact): artifact is A2AArtifactIdentity => + !!artifact && + typeof artifact === "object" && + !Array.isArray(artifact) && + typeof (artifact as { id?: unknown }).id === "string" && + typeof (artifact as { resourceType?: unknown }).resourceType === + "string" && + typeof (artifact as { sourceAction?: unknown }).sourceAction === + "string", + ), + }; + } catch { + return null; + } +} + export async function insertA2AContinuation(input: { integrationTaskId: string; platform: string; @@ -432,6 +550,44 @@ export async function hasActiveA2AContinuationsForIntegrationTask( return rows.length > 0; } +export async function getA2AContinuationTaskOutcome( + integrationTaskId: string, +): Promise { + await ensureTable(); + const { rows } = await getDbExec().execute({ + sql: `SELECT status, terminal_delivery_confirmed_at + FROM integration_a2a_continuations + WHERE integration_task_id = ?`, + args: [integrationTaskId], + }); + if (rows.length === 0) return "missing"; + if ( + rows.some((row) => + ["pending", "processing", "delivering"].includes(String(row.status)), + ) + ) { + return "active"; + } + return rows.every((row) => row.terminal_delivery_confirmed_at != null) + ? "terminal-delivered" + : "terminal-without-delivery"; +} + +export async function hasPendingConfirmedA2ADeliveryForIntegrationTask( + integrationTaskId: string, +): Promise { + await ensureTable(); + const { rows } = await getDbExec().execute({ + sql: `SELECT 1 AS confirmed FROM integration_a2a_continuations + WHERE integration_task_id = ? + AND status IN ('pending', 'processing', 'delivering') + AND terminal_delivery_confirmed_at IS NOT NULL + LIMIT 1`, + args: [integrationTaskId], + }); + return rows.length > 0; +} + export async function failA2AContinuationsForIntegrationTask( integrationTaskId: string, errorMessage: string, @@ -440,9 +596,11 @@ export async function failA2AContinuationsForIntegrationTask( const now = Date.now(); await getDbExec().execute({ sql: `UPDATE integration_a2a_continuations - SET status = 'failed', error_message = ?, updated_at = ?, completed_at = ? + SET status = 'failed', error_message = ?, updated_at = ?, completed_at = ?, + verified_artifact_checkpoint = NULL WHERE integration_task_id = ? - AND status IN ('pending', 'processing', 'delivering')`, + AND status IN ('pending', 'processing', 'delivering') + AND terminal_delivery_confirmed_at IS NULL`, args: [errorMessage.slice(0, 2000), now, now, integrationTaskId], }); } @@ -580,6 +738,7 @@ export async function claimDueA2AContinuations( export async function recoverDueA2AContinuationIds( limit = 5, integrationTaskIds?: string[], + confirmedDeliveryOnly = false, ): Promise { await ensureTable(); const client = getDbExec(); @@ -591,20 +750,25 @@ export async function recoverDueA2AContinuationIds( ? ` AND integration_task_id IN (${integrationTaskIds.map(() => "?").join(", ")})` : ""; const taskArgs = integrationTaskIds ?? []; - // If a processor dies while holding a delivery claim, retry the final send. - // The stale cutoff preserves the in-flight delivery guard while keeping - // final integration replies at-least-once. + const receiptFilter = confirmedDeliveryOnly + ? " AND terminal_delivery_confirmed_at IS NOT NULL" + : ""; + // If a processor dies after a provider receipt, retry history-only custody + // as soon as its short follow-up deadline passes. A pre-receipt delivery + // claim retains the longer stale cutoff before an at-least-once resend. await client.execute({ sql: `UPDATE integration_a2a_continuations SET status = ?, next_check_at = ?, updated_at = ? - WHERE status = 'delivering' AND updated_at <= ?${taskFilter}`, - args: ["pending", now, now, now - 5 * 60 * 1000, ...taskArgs], + WHERE status = 'delivering' + AND ((terminal_delivery_confirmed_at IS NOT NULL AND next_check_at <= ?) + OR updated_at <= ?)${taskFilter}${receiptFilter}`, + args: ["pending", now, now, now, now - 5 * 60 * 1000, ...taskArgs], }); await client.execute({ sql: `UPDATE integration_a2a_continuations SET status = ?, next_check_at = ?, updated_at = ? WHERE status = 'processing' - AND (updated_at <= ? OR next_check_at <= ?)${taskFilter}`, + AND (updated_at <= ? OR next_check_at <= ?)${taskFilter}${receiptFilter}`, args: [ "pending", now, @@ -616,7 +780,7 @@ export async function recoverDueA2AContinuationIds( }); const { rows } = await client.execute({ sql: `SELECT id FROM integration_a2a_continuations - WHERE status = 'pending' AND next_check_at <= ?${taskFilter} + WHERE status = 'pending' AND next_check_at <= ?${taskFilter}${receiptFilter} ORDER BY next_check_at ASC LIMIT ?`, args: [now, ...taskArgs, limit], @@ -637,6 +801,7 @@ export interface RecoverableA2AIntegrationTask { externalThreadId: string; dispatchScope: string | null; status: string; + hasPendingConfirmedDelivery: boolean; } /** @@ -650,7 +815,13 @@ export async function listRecoverableA2AIntegrationTasks( const now = Date.now(); const { rows } = await getDbExec().execute({ sql: `SELECT DISTINCT c.integration_task_id, t.platform, - t.external_thread_id, t.dispatch_scope, t.status + t.external_thread_id, t.dispatch_scope, t.status, + EXISTS ( + SELECT 1 FROM integration_a2a_continuations receipt + WHERE receipt.integration_task_id = c.integration_task_id + AND receipt.terminal_delivery_confirmed_at IS NOT NULL + AND receipt.status IN ('pending', 'processing', 'delivering') + ) AS has_pending_confirmed_delivery FROM integration_a2a_continuations c INNER JOIN integration_pending_tasks t ON t.id = c.integration_task_id @@ -658,13 +829,16 @@ export async function listRecoverableA2AIntegrationTasks( AND ((c.status = 'pending' AND c.next_check_at <= ?) OR (c.status = 'processing' AND (c.updated_at <= ? OR c.next_check_at <= ?)) - OR (c.status = 'delivering' AND c.updated_at <= ?)) + OR (c.status = 'delivering' AND + ((c.terminal_delivery_confirmed_at IS NOT NULL AND c.next_check_at <= ?) + OR c.updated_at <= ?))) ORDER BY c.integration_task_id ASC LIMIT ?`, args: [ now, now - PROCESSING_STUCK_AFTER_MS, now - PROCESSING_NEXT_CHECK_STALE_AFTER_MS, + now, now - 5 * 60 * 1000, Math.max(1, Math.min(Math.floor(limit), 200)), ], @@ -675,6 +849,9 @@ export async function listRecoverableA2AIntegrationTasks( externalThreadId: String(row.external_thread_id), dispatchScope: (row.dispatch_scope as string | null) ?? null, status: String(row.status), + hasPendingConfirmedDelivery: + row.has_pending_confirmed_delivery === true || + Number(row.has_pending_confirmed_delivery ?? 0) === 1, })); } @@ -723,6 +900,117 @@ export async function rescheduleA2AContinuation( }); } +export async function saveA2AVerifiedArtifactCheckpoint( + id: string, + checkpoint: string, +): Promise { + await ensureTable(); + const normalized = checkpoint.trim(); + if (!normalized) return null; + if (normalized.length > MAX_VERIFIED_ARTIFACT_CHECKPOINT_CHARS) { + throw new Error( + `Verified artifact checkpoint exceeds ${MAX_VERIFIED_ARTIFACT_CHECKPOINT_CHARS} characters`, + ); + } + const now = Date.now(); + await getDbExec().execute({ + sql: `UPDATE integration_a2a_continuations + SET verified_artifact_checkpoint = ?, updated_at = ? + WHERE id = ? AND status IN ('pending', 'processing', 'delivering')`, + args: [normalized, now, id], + }); + const persisted = await getA2AContinuation(id); + return persisted?.verifiedArtifactCheckpoint === normalized + ? normalized + : null; +} + +export async function recordA2ATerminalDeliveryReceipt( + id: string, + kind: A2ATerminalDeliveryKind, + historyPayload: A2ATerminalHistoryPayload, + errorMessage?: string, +): Promise { + await ensureTable(); + const serializedPayload = JSON.stringify(historyPayload); + if (serializedPayload.length > MAX_TERMINAL_HISTORY_PAYLOAD_CHARS) { + throw new Error( + `Terminal A2A history payload exceeds ${MAX_TERMINAL_HISTORY_PAYLOAD_CHARS} characters`, + ); + } + const now = Date.now(); + await getDbExec().execute({ + sql: `UPDATE integration_a2a_continuations + SET status = 'delivering', terminal_delivery_kind = ?, + terminal_delivery_confirmed_at = COALESCE(terminal_delivery_confirmed_at, ?), + terminal_history_payload = ?, next_check_at = ?, updated_at = ?, + error_message = ? + WHERE id = ? + AND status IN ('processing', 'delivering') + AND (terminal_delivery_confirmed_at IS NULL + OR terminal_delivery_kind = ?)`, + args: [ + kind, + now, + serializedPayload, + now, + now, + errorMessage?.slice(0, 2000) ?? null, + id, + kind, + ], + }); + const persisted = await getA2AContinuation(id); + if ( + persisted?.status !== "delivering" || + persisted.terminalDeliveryKind !== kind || + persisted.terminalDeliveryConfirmedAt == null || + JSON.stringify(persisted.terminalHistoryPayload) !== serializedPayload + ) { + throw new Error("Terminal A2A delivery confirmation did not persist"); + } + return persisted; +} + +export async function finalizeA2ATerminalHistory(id: string): Promise { + await ensureTable(); + const continuation = await getA2AContinuation(id); + if ( + continuation && + (continuation.status === "completed" || continuation.status === "failed") && + continuation.terminalDeliveryConfirmedAt != null && + continuation.terminalHistoryPayload == null + ) { + return; + } + if ( + !continuation?.terminalDeliveryKind || + continuation.terminalDeliveryConfirmedAt == null || + !continuation.terminalHistoryPayload + ) { + throw new Error("Terminal A2A history cannot finalize without a receipt"); + } + const now = Date.now(); + const status = + continuation.terminalDeliveryKind === "success" ? "completed" : "failed"; + await getDbExec().execute({ + sql: `UPDATE integration_a2a_continuations + SET status = ?, updated_at = ?, completed_at = ?, incoming_payload = ?, + a2a_auth_token = NULL, progress_ref = NULL, + verified_artifact_checkpoint = NULL, terminal_history_payload = NULL + WHERE id = ? AND status IN ('processing', 'delivering') + AND terminal_delivery_confirmed_at IS NOT NULL`, + args: [status, now, now, "{}", id], + }); + const persisted = await getA2AContinuation(id); + if ( + persisted?.status !== status || + persisted.terminalHistoryPayload != null + ) { + throw new Error("Terminal A2A history finalization did not persist"); + } +} + export async function completeA2AContinuation(id: string): Promise { await ensureTable(); const client = getDbExec(); @@ -730,9 +1018,13 @@ export async function completeA2AContinuation(id: string): Promise { await client.execute({ sql: `UPDATE integration_a2a_continuations SET status = ?, updated_at = ?, completed_at = ?, - incoming_payload = ?, a2a_auth_token = NULL, progress_ref = NULL + incoming_payload = ?, a2a_auth_token = NULL, progress_ref = NULL, + verified_artifact_checkpoint = NULL, + terminal_delivery_kind = COALESCE(terminal_delivery_kind, 'success'), + terminal_delivery_confirmed_at = COALESCE(terminal_delivery_confirmed_at, ?), + terminal_history_payload = NULL WHERE id = ? AND status IN ('processing', 'delivering', 'completed')`, - args: ["completed", now, now, "{}", id], + args: ["completed", now, now, "{}", now, id], }); } @@ -746,7 +1038,8 @@ export async function failA2AContinuation( await client.execute({ sql: `UPDATE integration_a2a_continuations SET status = ?, updated_at = ?, error_message = ?, - incoming_payload = ?, a2a_auth_token = NULL, progress_ref = NULL + incoming_payload = ?, a2a_auth_token = NULL, progress_ref = NULL, + verified_artifact_checkpoint = NULL WHERE id = ? AND status <> 'completed'`, args: ["failed", now, errorMessage.slice(0, 2000), "{}", id], }); diff --git a/packages/core/src/integrations/plugin.spec.ts b/packages/core/src/integrations/plugin.spec.ts index ef548225ae..385acd302a 100644 --- a/packages/core/src/integrations/plugin.spec.ts +++ b/packages/core/src/integrations/plugin.spec.ts @@ -53,6 +53,14 @@ const recoverDueA2AContinuationsMock = vi.hoisted(() => vi.fn(async () => ({ dispatched: 0, failed: 0 })), ); const processDueA2AContinuationsMock = vi.hoisted(() => vi.fn(async () => {})); +const processA2AContinuationByIdMock = vi.hoisted(() => vi.fn()); +const recoverA2AContinuationAfterProcessorFailureMock = vi.hoisted(() => + vi.fn(), +); +const reconcileTerminalA2AParentIfDisabledMock = vi.hoisted(() => + vi.fn(async () => false), +); +const failA2AContinuationMock = vi.hoisted(() => vi.fn()); const failDisabledIntegrationCampaignTaskMock = vi.hoisted(() => vi.fn()); const completeIntegrationCampaignTaskAfterA2AMock = vi.hoisted(() => vi.fn(async () => true), @@ -70,8 +78,8 @@ const transitionIntegrationCampaignTaskToDeliveryRetryMock = vi.hoisted(() => const waitForA2AIntegrationCampaignMock = vi.hoisted(() => vi.fn(async () => true), ); -const hasActiveA2AContinuationsForIntegrationTaskMock = vi.hoisted(() => - vi.fn(async () => true), +const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => + vi.fn(async () => "active"), ); const claimIntegrationCampaignDeliveryForTaskMock = vi.hoisted(() => vi.fn()); const completeIntegrationCampaignTaskMock = vi.hoisted(() => @@ -139,15 +147,19 @@ vi.mock("./integration-campaigns-store.js", () => ({ })); vi.mock("./a2a-continuations-store.js", () => ({ - failA2AContinuation: vi.fn(), - hasActiveA2AContinuationsForIntegrationTask: - hasActiveA2AContinuationsForIntegrationTaskMock, + ensureA2AContinuationsTable: vi.fn(async () => {}), + failA2AContinuation: failA2AContinuationMock, + getA2AContinuationTaskOutcome: getA2AContinuationTaskOutcomeMock, })); vi.mock("./a2a-continuation-processor.js", () => ({ - processA2AContinuationById: vi.fn(), + processA2AContinuationById: processA2AContinuationByIdMock, processDueA2AContinuations: processDueA2AContinuationsMock, + recoverA2AContinuationAfterProcessorFailure: + recoverA2AContinuationAfterProcessorFailureMock, recoverDueA2AContinuations: recoverDueA2AContinuationsMock, + reconcileTerminalA2AParentIfDisabled: + reconcileTerminalA2AParentIfDisabledMock, })); vi.mock("./integration-durable-dispatch.js", async () => { @@ -811,8 +823,8 @@ describe("integrations plugin routes", () => { process.env.NETLIFY = "true"; process.env.A2A_SECRET = "test-secret"; process.env.AGENT_INTEGRATION_DURABLE_DISPATCH = "true"; - hasActiveA2AContinuationsForIntegrationTaskMock.mockResolvedValueOnce( - false, + getA2AContinuationTaskOutcomeMock.mockResolvedValueOnce( + "terminal-delivered", ); const baseTask = claimedTask(1); const task = { @@ -1736,6 +1748,35 @@ describe("integrations plugin routes", () => { expect(markTaskFailedMock).not.toHaveBeenCalled(); }); + it("keeps an escaped A2A continuation processor failure in durable recovery custody", async () => { + process.env.NODE_ENV = "development"; + delete process.env.A2A_SECRET; + processA2AContinuationByIdMock.mockRejectedValueOnce( + new Error("temporary continuation store outage"), + ); + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + const result = await dispatch( + nitroApp, + "/_agent-native/integrations/process-a2a-continuation", + "POST", + { continuationId: "cont-boundary-failure" }, + ); + + expect(result.status).toBe(500); + expect( + recoverA2AContinuationAfterProcessorFailureMock, + ).toHaveBeenCalledWith( + "cont-boundary-failure", + expect.objectContaining({ + adapters: expect.any(Map), + reason: "temporary continuation store outage", + }), + ); + expect(failA2AContinuationMock).not.toHaveBeenCalled(); + }); + it("terminally fails a processor task only after its retry budget is exhausted", async () => { process.env.NODE_ENV = "development"; claimPendingTaskMock.mockResolvedValueOnce(claimedTask(3)); diff --git a/packages/core/src/integrations/plugin.ts b/packages/core/src/integrations/plugin.ts index 8b900b2cd3..af7f576004 100644 --- a/packages/core/src/integrations/plugin.ts +++ b/packages/core/src/integrations/plugin.ts @@ -36,12 +36,11 @@ import { runWithRequestContext } from "../server/request-context.js"; import { processA2AContinuationById, processDueA2AContinuations, + reconcileTerminalA2AParentIfDisabled, + recoverA2AContinuationAfterProcessorFailure, recoverDueA2AContinuations, } from "./a2a-continuation-processor.js"; -import { - failA2AContinuation, - hasActiveA2AContinuationsForIntegrationTask, -} from "./a2a-continuations-store.js"; +import { getA2AContinuationTaskOutcome } from "./a2a-continuations-store.js"; import { mergeIntegrationAdapters } from "./adapter-overrides.js"; import { discordAdapter } from "./adapters/discord.js"; import { emailAdapter } from "./adapters/email.js"; @@ -2113,17 +2112,27 @@ export function createIntegrationsPlugin( ); if (!waiting) return "campaign-active" as const; } - if ( - !(await hasActiveA2AContinuationsForIntegrationTask( - task.id, - )) - ) { + const a2aOutcome = await getA2AContinuationTaskOutcome( + task.id, + ); + if (a2aOutcome === "terminal-delivered") { const completed = await completeIntegrationCampaignTaskAfterA2A(task.id); return completed ? ("completed" as const) : ("campaign-active" as const); } + if ( + a2aOutcome === "terminal-without-delivery" && + (await reconcileTerminalA2AParentIfDisabled(task.id)) + ) { + return "campaign-failed" as const; + } + if (a2aOutcome !== "active") { + console.warn( + `[integrations] Waiting campaign ${task.id} has A2A outcome ${a2aOutcome} without terminal delivery proof`, + ); + } return "campaign-active" as const; } if (campaignContinuation) { @@ -2387,15 +2396,18 @@ export function createIntegrationsPlugin( adapters: adapterMap, }); } catch (err: any) { - // Mark the continuation failed so it isn't left dangling, and surface - // a 500 to the caller instead of leaking an unhandled rejection. - await failA2AContinuation( - continuationId, - err?.message?.slice(0, 500) || "continuation processing failed", - ).catch(() => {}); + const reason = + err?.message?.slice(0, 500) || "continuation processing failed"; + await recoverA2AContinuationAfterProcessorFailure(continuationId, { + adapters: adapterMap, + reason, + }).catch(() => { + console.error( + `[integrations] A2A continuation ${continuationId} recovery scheduling failed`, + ); + }); console.error( - "[integrations] process-a2a-continuation failure:", - err, + `[integrations] process-a2a-continuation failure for ${continuationId}; durable recovery requested`, ); setResponseStatus(event, 500); return { error: "Failed to process A2A continuation" }; diff --git a/packages/core/src/integrations/task-queue-stats.spec.ts b/packages/core/src/integrations/task-queue-stats.spec.ts index 5cf1964b33..5a0a8eeacf 100644 --- a/packages/core/src/integrations/task-queue-stats.spec.ts +++ b/packages/core/src/integrations/task-queue-stats.spec.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const executeMock = vi.hoisted(() => vi.fn()); const ensureTableMock = vi.hoisted(() => vi.fn()); +const ensureA2ATableMock = vi.hoisted(() => vi.fn()); vi.mock("../db/client.js", () => ({ getDbExec: () => ({ execute: executeMock }), @@ -11,10 +12,15 @@ vi.mock("./pending-tasks-store.js", () => ({ ensurePendingTasksTable: ensureTableMock, })); +vi.mock("./a2a-continuations-store.js", () => ({ + ensureA2AContinuationsTable: ensureA2ATableMock, +})); + describe("integration task queue stats", () => { beforeEach(() => { vi.clearAllMocks(); ensureTableMock.mockResolvedValue(undefined); + ensureA2ATableMock.mockResolvedValue(undefined); }); it("returns dispatch diagnostics without selecting payloads or user text", async () => { @@ -35,6 +41,28 @@ describe("integration task queue stats", () => { created_at: Date.now() - 5_000, }, ], + }) + .mockResolvedValueOnce({ rows: [{ c: 1 }] }) + .mockResolvedValueOnce({ + rows: [ + { + active_count: 2, + oldest_created_at: Date.now() - 10_000, + orphan_count: 1, + }, + ], + }) + .mockResolvedValueOnce({ + rows: [ + { + id: "cont-orphan-1", + integration_task_id: "task-1", + status: "failed", + attempts: 3, + created_at: Date.now() - 15_000, + error_message: "Slack delivery timed out", + }, + ], }); const { getTaskQueueStats } = await import("./task-queue-stats.js"); @@ -52,6 +80,24 @@ describe("integration task queue stats", () => { last_dispatch_outcome: "portable-unconfirmed", }), ); + expect(result).toEqual( + expect.objectContaining({ + waiting_campaigns: 1, + active_a2a_continuations: 2, + terminal_a2a_without_delivery: 1, + }), + ); + expect(result.recent_a2a_orphans).toEqual([ + expect.objectContaining({ + continuation_id: "cont-orphan-1", + integration_task_id: "task-1", + reason_class: "timeout", + }), + ]); + expect(ensureA2ATableMock).toHaveBeenCalledOnce(); + expect(ensureA2ATableMock.mock.invocationCallOrder[0]).toBeLessThan( + executeMock.mock.invocationCallOrder[0], + ); const sql = executeMock.mock.calls .map(([query]) => (query as { sql: string }).sql) .join("\n"); diff --git a/packages/core/src/integrations/task-queue-stats.ts b/packages/core/src/integrations/task-queue-stats.ts index 53274cb50c..9f8c3c773a 100644 --- a/packages/core/src/integrations/task-queue-stats.ts +++ b/packages/core/src/integrations/task-queue-stats.ts @@ -8,6 +8,7 @@ * columns before the SELECTs execute. */ import { getDbExec } from "../db/client.js"; +import { ensureA2AContinuationsTable } from "./a2a-continuations-store.js"; import { ensurePendingTasksTable } from "./pending-tasks-store.js"; export interface RecentFailure { @@ -23,6 +24,18 @@ export interface TaskQueueStats { completed_last_hour: number; failed_last_hour: number; oldest_pending_age_seconds: number; + waiting_campaigns: number; + active_a2a_continuations: number; + oldest_active_a2a_age_seconds: number; + terminal_a2a_without_delivery: number; + recent_a2a_orphans: Array<{ + continuation_id: string; + integration_task_id: string; + status: string; + attempts: number; + age_seconds: number; + reason_class: string; + }>; recent_failures: RecentFailure[]; recent_tasks: Array<{ id: string; @@ -46,6 +59,11 @@ const ZERO_STATS: TaskQueueStats = { completed_last_hour: 0, failed_last_hour: 0, oldest_pending_age_seconds: 0, + waiting_campaigns: 0, + active_a2a_continuations: 0, + oldest_active_a2a_age_seconds: 0, + terminal_a2a_without_delivery: 0, + recent_a2a_orphans: [], recent_failures: [], recent_tasks: [], }; @@ -66,11 +84,14 @@ export async function getTaskQueueStats( scope: TaskQueueStatsScope, ): Promise { await ensurePendingTasksTable(); + await ensureA2AContinuationsTable(); const client = getDbExec(); const now = Date.now(); const oneHourAgo = now - 60 * 60 * 1000; const scopeSql = `owner_email = ? AND ((org_id IS NULL AND CAST(? AS TEXT) IS NULL) OR org_id = ?)`; + const joinedScopeSql = `t.owner_email = ? + AND ((t.org_id IS NULL AND CAST(? AS TEXT) IS NULL) OR t.org_id = ?)`; const scopeArgs = [scope.ownerEmail, scope.orgId, scope.orgId]; try { @@ -178,6 +199,17 @@ export async function getTaskQueueStats( ), }), ); + const waitingCampaigns = await readWaitingCampaignCount( + client, + joinedScopeSql, + scopeArgs, + ); + const a2aStats = await readA2AContinuationStats( + client, + joinedScopeSql, + scopeArgs, + now, + ); return { pending, @@ -185,6 +217,8 @@ export async function getTaskQueueStats( completed_last_hour: completedLastHour, failed_last_hour: failedLastHour, oldest_pending_age_seconds: oldestPendingAgeSeconds, + waiting_campaigns: waitingCampaigns, + ...a2aStats, recent_failures: recentFailures, recent_tasks: recentTasks, }; @@ -195,3 +229,115 @@ export async function getTaskQueueStats( throw err; } } + +async function readWaitingCampaignCount( + client: ReturnType, + scopeSql: string, + scopeArgs: Array, +): Promise { + try { + const result = await client.execute({ + sql: `SELECT COUNT(*) AS c + FROM integration_campaigns c + INNER JOIN integration_pending_tasks t + ON t.id = c.integration_task_id + WHERE ${scopeSql} + AND c.status = 'waiting'`, + args: scopeArgs, + }); + return Number(result.rows[0]?.c ?? 0); + } catch (err) { + if (isMissingTableError(err)) return 0; + throw err; + } +} + +async function readA2AContinuationStats( + client: ReturnType, + scopeSql: string, + scopeArgs: Array, + now: number, +): Promise< + Pick< + TaskQueueStats, + | "active_a2a_continuations" + | "oldest_active_a2a_age_seconds" + | "terminal_a2a_without_delivery" + | "recent_a2a_orphans" + > +> { + try { + const live = await client.execute({ + sql: `SELECT + COUNT(CASE WHEN c.status IN ('pending', 'processing', 'delivering') THEN 1 END) AS active_count, + MIN(CASE WHEN c.status IN ('pending', 'processing', 'delivering') THEN c.created_at END) AS oldest_created_at, + COUNT(CASE WHEN c.status IN ('completed', 'failed') + AND c.terminal_delivery_confirmed_at IS NULL THEN 1 END) AS orphan_count + FROM integration_a2a_continuations c + INNER JOIN integration_pending_tasks t + ON t.id = c.integration_task_id + WHERE ${scopeSql}`, + args: scopeArgs, + }); + const orphaned = await client.execute({ + sql: `SELECT c.id, c.integration_task_id, c.status, c.attempts, + c.created_at, c.error_message + FROM integration_a2a_continuations c + INNER JOIN integration_pending_tasks t + ON t.id = c.integration_task_id + WHERE ${scopeSql} + AND c.status IN ('completed', 'failed') + AND c.terminal_delivery_confirmed_at IS NULL + ORDER BY c.updated_at DESC + LIMIT 5`, + args: scopeArgs, + }); + const activeCount = Number(live.rows[0]?.active_count ?? 0); + const orphanCount = Number(live.rows[0]?.orphan_count ?? 0); + const oldestCreatedAt = Number(live.rows[0]?.oldest_created_at ?? now); + const recentOrphans = (orphaned.rows as Array>).map( + (row) => ({ + continuation_id: String(row.id ?? ""), + integration_task_id: String(row.integration_task_id ?? ""), + status: String(row.status ?? ""), + attempts: Number(row.attempts ?? 0), + age_seconds: Math.max( + 0, + Math.floor((now - Number(row.created_at ?? now)) / 1000), + ), + reason_class: classifyA2AOrphanReason(row.error_message), + }), + ); + return { + active_a2a_continuations: activeCount, + oldest_active_a2a_age_seconds: + activeCount > 0 + ? Math.max(0, Math.floor((now - oldestCreatedAt) / 1000)) + : 0, + terminal_a2a_without_delivery: orphanCount, + recent_a2a_orphans: recentOrphans, + }; + } catch (err) { + if (isMissingTableError(err)) { + return { + active_a2a_continuations: 0, + oldest_active_a2a_age_seconds: 0, + terminal_a2a_without_delivery: 0, + recent_a2a_orphans: [], + }; + } + throw err; + } +} + +function classifyA2AOrphanReason(value: unknown): string { + const message = String(value ?? ""); + if (!message) return "missing_terminal_delivery"; + if (/timeout|timed out|abort/i.test(message)) return "timeout"; + if (/token|secret|auth|credential/i.test(message)) return "authentication"; + if (/database|sql|store|persist/i.test(message)) return "persistence"; + if (/slack|provider|deliver|stream|response/i.test(message)) { + return "provider_delivery"; + } + return "processor_failure"; +} diff --git a/packages/core/src/integrations/webhook-handler-engine.spec.ts b/packages/core/src/integrations/webhook-handler-engine.spec.ts index b3fb059e97..89dc74fc96 100644 --- a/packages/core/src/integrations/webhook-handler-engine.spec.ts +++ b/packages/core/src/integrations/webhook-handler-engine.spec.ts @@ -39,7 +39,10 @@ const completeIntegrationCampaignTaskMock = vi.hoisted(() => vi.fn()); const heartbeatIntegrationCampaignMock = vi.hoisted(() => vi.fn()); const waitForA2AIntegrationCampaignMock = vi.hoisted(() => vi.fn()); const failIntegrationCampaignMock = vi.hoisted(() => vi.fn()); -const hasActiveA2AContinuationsMock = vi.hoisted(() => vi.fn()); +const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => vi.fn()); +const reconcileTerminalA2AParentIfDisabledMock = vi.hoisted(() => + vi.fn(async () => false), +); const dispatchPendingIntegrationTaskMock = vi.hoisted(() => vi.fn()); const appendDurableContinuationContextMock = vi.hoisted(() => vi.fn()); const originalNodeEnv = process.env.NODE_ENV; @@ -87,7 +90,12 @@ vi.mock("./integration-campaigns-store.js", () => ({ })); vi.mock("./a2a-continuations-store.js", () => ({ - hasActiveA2AContinuationsForIntegrationTask: hasActiveA2AContinuationsMock, + getA2AContinuationTaskOutcome: getA2AContinuationTaskOutcomeMock, +})); + +vi.mock("./a2a-continuation-processor.js", () => ({ + reconcileTerminalA2AParentIfDisabled: + reconcileTerminalA2AParentIfDisabledMock, })); vi.mock("./integration-durable-dispatch.js", () => ({ @@ -307,7 +315,7 @@ describe("integration webhook handler engine resolution", () => { heartbeatIntegrationCampaignMock.mockResolvedValue(true); waitForA2AIntegrationCampaignMock.mockResolvedValue(true); failIntegrationCampaignMock.mockResolvedValue(true); - hasActiveA2AContinuationsMock.mockResolvedValue(false); + getA2AContinuationTaskOutcomeMock.mockResolvedValue("terminal-delivered"); dispatchPendingIntegrationTaskMock.mockResolvedValue( "background-acknowledged", ); @@ -2018,7 +2026,7 @@ describe("integration webhook handler engine resolution", () => { ); }); - it("finishes a waiting campaign only after downstream A2A is terminal", async () => { + it("does not finish a waiting campaign merely because no active A2A continuation remains", async () => { const { processIntegrationTask } = await import("./webhook-handler.js"); createIntegrationCampaignMock.mockResolvedValueOnce({ id: "campaign-qa", @@ -2037,7 +2045,9 @@ describe("integration webhook handler engine resolution", () => { checkpoint: JSON.stringify({ waitingForA2A: true }), }, }); - hasActiveA2AContinuationsMock.mockResolvedValueOnce(false); + getA2AContinuationTaskOutcomeMock.mockResolvedValueOnce( + "terminal-without-delivery", + ); const result = await processIntegrationTask( pendingTask({ id: "task-a2a" }), @@ -2052,9 +2062,9 @@ describe("integration webhook handler engine resolution", () => { { enabled: true, continuationInvocation: true }, ); - expect(result).toEqual({ status: "completed" }); - expect(hasActiveA2AContinuationsMock).toHaveBeenCalledWith("task-a2a"); - expect(completeIntegrationCampaignTaskMock).toHaveBeenCalled(); + expect(result).toEqual({ status: "campaign-pending" }); + expect(getA2AContinuationTaskOutcomeMock).toHaveBeenCalledWith("task-a2a"); + expect(completeIntegrationCampaignTaskMock).not.toHaveBeenCalled(); expect(startRunMock).not.toHaveBeenCalled(); }); diff --git a/packages/core/src/integrations/webhook-handler.ts b/packages/core/src/integrations/webhook-handler.ts index 4be2bd0d25..c2ea027e8f 100644 --- a/packages/core/src/integrations/webhook-handler.ts +++ b/packages/core/src/integrations/webhook-handler.ts @@ -60,7 +60,8 @@ import { import { runWithRequestContext } from "../server/request-context.js"; import { normalizeReasoningEffortForRequest } from "../shared/reasoning-effort.js"; import { A2A_CONTINUATION_QUEUED_MARKER } from "./a2a-continuation-marker.js"; -import { hasActiveA2AContinuationsForIntegrationTask } from "./a2a-continuations-store.js"; +import { reconcileTerminalA2AParentIfDisabled } from "./a2a-continuation-processor.js"; +import { getA2AContinuationTaskOutcome } from "./a2a-continuations-store.js"; import { clearIntegrationAwaitingInput, setIntegrationAwaitingInput, @@ -1071,10 +1072,22 @@ async function processIncomingMessage( JSON.parse(campaign.row.checkpoint).waitingForA2A === true; } catch {} if (waitingForA2A) { - const activeA2A = await hasActiveA2AContinuationsForIntegrationTask( - opts.taskId, - ); - if (activeA2A) { + const a2aOutcome = await getA2AContinuationTaskOutcome(opts.taskId); + if (a2aOutcome !== "terminal-delivered") { + if ( + a2aOutcome === "terminal-without-delivery" && + (await reconcileTerminalA2AParentIfDisabled(opts.taskId)) + ) { + await releaseApplicableIntegrationBudgets( + budgetReservations.reservations, + ); + return { status: "campaign-failed" }; + } + if (a2aOutcome !== "active") { + console.warn( + `[integrations] Waiting campaign ${campaign.row.id} has A2A outcome ${a2aOutcome} without terminal delivery proof`, + ); + } const waiting = await waitForA2AIntegrationCampaign(campaign.row.id, { runId: campaign.runId, leaseToken: campaign.leaseToken, diff --git a/templates/dispatch/changelog/2026-07-26-slack-replies-recover-after-connected-app-updates.md b/templates/dispatch/changelog/2026-07-26-slack-replies-recover-after-connected-app-updates.md new file mode 100644 index 0000000000..82d2cb637c --- /dev/null +++ b/templates/dispatch/changelog/2026-07-26-slack-replies-recover-after-connected-app-updates.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-26 +--- + +Slack replies now recover after a connected app finishes an update, without repeating the update or losing the delivered reply from future thread context. From 0f2f24953b2d05b6a908b6efc7790ab0914869a0 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:22:43 -0400 Subject: [PATCH 02/16] fix(core): narrow disabled A2A task input --- packages/core/src/integrations/a2a-continuation-processor.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 0fd590fac0..7fc8068249 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -543,7 +543,10 @@ async function durableContinuationScopeStillEnabled( } async function failDisabledDurableA2ATask( - task: RecoverableA2AIntegrationTask, + task: Pick< + RecoverableA2AIntegrationTask, + "id" | "platform" | "externalThreadId" | "dispatchScope" | "status" + >, ): Promise { const message = "Durable integration campaign was disabled for this scope"; await failA2AContinuationsForIntegrationTask(task.id, message); From ff1693371c5f626ee6b54640f3aa884d97d514f3 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:32:24 -0400 Subject: [PATCH 03/16] fix(core): keep parent completion retryable --- .../a2a-continuation-processor.spec.ts | 37 +++++++++++++++++++ .../a2a-continuation-processor.ts | 4 +- .../a2a-continuations-store.spec.ts | 11 ++++++ .../integrations/a2a-continuations-store.ts | 6 ++- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 07a60662d7..3393f5c80b 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -1382,6 +1382,43 @@ describe("A2A continuation processor", () => { ); }); + it("keeps receipt-backed parent completion recoverable before finalizing history", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + claimA2AContinuationMock.mockResolvedValueOnce(continuation()); + getIntegrationCampaignForTaskMock.mockResolvedValue({ + id: "campaign-1", + integrationTaskId: "task-1", + status: "waiting", + }); + completeIntegrationCampaignTaskAfterA2AMock + .mockRejectedValueOnce(new Error("db unavailable")) + .mockRejectedValueOnce(new Error("db unavailable")) + .mockRejectedValueOnce(new Error("db unavailable")); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledTimes(1); + expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledTimes( + 3, + ); + expect(completeA2AContinuationMock).not.toHaveBeenCalled(); + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( + "cont-1", + 20_000, + ); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining("history remains retryable"), + "Error", + ); + }); + it("does not post completed text when another processor already claimed delivery", async () => { const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); claimA2AContinuationMock.mockResolvedValueOnce(continuation()); diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 7fc8068249..ddf4a788a3 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -937,8 +937,6 @@ async function persistAndFinalizeConfirmedA2ADelivery( for (let attempt = 0; attempt < 3; attempt += 1) { try { await persistA2AContinuationDelivery(continuation, history); - await finalizeA2ATerminalHistory(continuation.id); - logA2AContinuationTransition("terminal_history_persisted", continuation); await completeParentCampaignAfterTerminalA2A({ ...continuation, status: @@ -946,6 +944,8 @@ async function persistAndFinalizeConfirmedA2ADelivery( ? "completed" : "failed", }); + await finalizeA2ATerminalHistory(continuation.id); + logA2AContinuationTransition("terminal_history_persisted", continuation); return; } catch (err) { lastError = err; diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 4ad31645e6..4162d3ef0c 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -202,6 +202,17 @@ describe("A2A continuations store", () => { [{ status: "completed", terminal_delivery_confirmed_at: 10 }], "terminal-delivered", ], + [ + [{ status: "delivering", terminal_delivery_confirmed_at: 10 }], + "terminal-delivered", + ], + [ + [ + { status: "delivering", terminal_delivery_confirmed_at: 10 }, + { status: "processing", terminal_delivery_confirmed_at: null }, + ], + "active", + ], [ [{ status: "failed", terminal_delivery_confirmed_at: null }], "terminal-without-delivery", diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index b8dcebc78b..910979d21b 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -562,8 +562,10 @@ export async function getA2AContinuationTaskOutcome( }); if (rows.length === 0) return "missing"; if ( - rows.some((row) => - ["pending", "processing", "delivering"].includes(String(row.status)), + rows.some( + (row) => + ["pending", "processing", "delivering"].includes(String(row.status)) && + row.terminal_delivery_confirmed_at == null, ) ) { return "active"; From b94d21720e98b46f419b6eab22fb0c5a74af7b55 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:41:28 -0400 Subject: [PATCH 04/16] fix(core): close A2A receipt custody gaps --- .../a2a-continuation-processor.spec.ts | 83 ++++++++++++++++++- .../a2a-continuation-processor.ts | 27 +++++- .../a2a-continuations-store.spec.ts | 5 ++ .../integrations/a2a-continuations-store.ts | 2 +- 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 3393f5c80b..c8ca406530 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -16,6 +16,9 @@ const dispatchPendingIntegrationTaskMock = vi.hoisted(() => vi.fn()); const getNextPendingTaskForThreadMock = vi.hoisted(() => vi.fn()); const getIntegrationCampaignForTaskMock = vi.hoisted(() => vi.fn()); const failDisabledIntegrationCampaignTaskMock = vi.hoisted(() => vi.fn()); +const failIntegrationCampaignTaskDeliveryContainmentMock = vi.hoisted(() => + vi.fn(async () => true), +); const completeIntegrationCampaignTaskAfterA2AMock = vi.hoisted(() => vi.fn(async () => true), ); @@ -80,6 +83,8 @@ vi.mock("./integration-campaigns-store.js", () => ({ completeIntegrationCampaignTaskAfterA2AMock, getIntegrationCampaignForTask: getIntegrationCampaignForTaskMock, failDisabledIntegrationCampaignTask: failDisabledIntegrationCampaignTaskMock, + failIntegrationCampaignTaskDeliveryContainment: + failIntegrationCampaignTaskDeliveryContainmentMock, })); vi.mock("../server/core-routes-plugin.js", () => ({ @@ -206,8 +211,15 @@ describe("A2A continuation processor", () => { saveA2AVerifiedArtifactCheckpointMock.mockImplementation( async (_id: string, checkpoint: string) => checkpoint, ); - getThreadMappingMock.mockResolvedValue(null); - getThreadMock.mockResolvedValue(null); + getThreadMappingMock.mockResolvedValue({ + internalThreadId: "thread-123", + }); + getThreadMock.mockResolvedValue({ + id: "thread-123", + title: "Slack thread", + preview: "Integration request", + threadData: JSON.stringify({ messages: [] }), + }); updateThreadDataMock.mockResolvedValue(undefined); claimA2AContinuationDeliveryMock.mockImplementation(async (id: string) => continuation({ id, status: "delivering" }), @@ -383,6 +395,29 @@ describe("A2A continuation processor", () => { expect(failA2AContinuationMock).not.toHaveBeenCalled(); }); + it("bounds exhausted recovery when its platform adapter is unavailable", async () => { + getA2AContinuationMock.mockResolvedValueOnce( + continuation({ status: "processing", attempts: 30 }), + ); + const { recoverA2AContinuationAfterProcessorFailure } = + await import("./a2a-continuation-processor.js"); + + await recoverA2AContinuationAfterProcessorFailure("cont-1", { + adapters: new Map(), + reason: "processor failed after its adapter was removed", + }); + + expect(failA2AContinuationsForIntegrationTaskMock).toHaveBeenCalledWith( + "task-1", + "Unknown platform: slack", + ); + expect( + failIntegrationCampaignTaskDeliveryContainmentMock, + ).toHaveBeenCalledWith("task-1", "Unknown platform: slack"); + expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(fetch).not.toHaveBeenCalled(); + }); + it("allows one durable wake-up dispatch failure without stranding the rest", async () => { recoverDueA2AContinuationIdsMock.mockResolvedValue([ "cont-fail", @@ -879,6 +914,50 @@ describe("A2A continuation processor", () => { ); }); + it.each(["mapping", "thread"] as const)( + "retains receipt custody while the integration %s is unavailable", + async (missing) => { + if (missing === "mapping") { + getThreadMappingMock.mockResolvedValue(null); + } else { + getThreadMock.mockResolvedValue(null); + } + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + claimA2AContinuationMock.mockResolvedValueOnce( + continuation({ + status: "processing", + terminalDeliveryKind: "success", + terminalDeliveryConfirmedAt: Date.now(), + terminalHistoryPayload: { + text: "Created /page/content-1", + deliveredAt: new Date().toISOString(), + messageRefs: ["slack-message-1"], + artifacts: [], + }, + }), + ); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { adapters: new Map() }); + + expect(completeA2AContinuationMock).not.toHaveBeenCalled(); + expect( + completeIntegrationCampaignTaskAfterA2AMock, + ).not.toHaveBeenCalled(); + expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( + "cont-1", + 20_000, + ); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining("history remains retryable"), + "Error", + ); + }, + ); + it("recovers provider-confirmed history in a fresh invocation without polling or sending again", async () => { const history = { text: "Created /page/content-1", diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index ddf4a788a3..6c1cec28bf 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -38,6 +38,7 @@ import { import { completeIntegrationCampaignTaskAfterA2A, failDisabledIntegrationCampaignTask, + failIntegrationCampaignTaskDeliveryContainment, getIntegrationCampaignForTask, } from "./integration-campaigns-store.js"; import { @@ -199,11 +200,27 @@ export async function recoverA2AContinuationAfterProcessorFailure( return; } const adapter = options.adapters.get(continuation.platform); - if (continuation.attempts < MAX_ATTEMPTS || !adapter) { + if (continuation.attempts < MAX_ATTEMPTS) { logA2AContinuationTransition("processor_released", continuation); await rescheduleAndRedispatchA2AContinuation(continuation.id); return; } + if (!adapter) { + const reason = `Unknown platform: ${continuation.platform}`; + logA2AContinuationTransition( + "processor_exhausted_without_adapter", + continuation, + ); + await failA2AContinuationsForIntegrationTask( + continuation.integrationTaskId, + reason, + ); + await failIntegrationCampaignTaskDeliveryContainment( + continuation.integrationTaskId, + reason, + ); + return; + } const progress = await resumeA2AContinuationProgress(continuation, adapter); if (continuation.verifiedArtifactCheckpoint) { @@ -827,9 +844,13 @@ async function persistA2AContinuationDelivery( continuation.platform, continuation.externalThreadId, ); - if (!mapping) return; + if (!mapping) { + throw new Error("Integration thread mapping is not available"); + } const thread = await getThread(mapping.internalThreadId); - if (!thread) return; + if (!thread) { + throw new Error("Integration chat thread is not available"); + } let repo: any; try { diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 4162d3ef0c..a6d5cec0df 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -358,10 +358,12 @@ describe("A2A continuations store", () => { it("terminalizes and scrubs only after durable history persistence", async () => { let finalized = false; + let finalizationSql = ""; executeMock.mockImplementation( async (query: string | { sql: string; args?: unknown[] }) => { const sql = querySql(query); if (sql.includes("terminal_history_payload = NULL")) { + finalizationSql = sql; finalized = true; return { rows: [], rowsAffected: 1 }; } @@ -396,6 +398,9 @@ describe("A2A continuations store", () => { await expect(finalizeA2ATerminalHistory("cont-1")).resolves.toBeUndefined(); expect(finalized).toBe(true); + expect(finalizationSql).toContain( + "status IN ('pending', 'processing', 'delivering')", + ); }); it("finalizes receipt-backed history after stale delivery recovery and a fresh processing claim", async () => { diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 910979d21b..34a8526783 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -1000,7 +1000,7 @@ export async function finalizeA2ATerminalHistory(id: string): Promise { SET status = ?, updated_at = ?, completed_at = ?, incoming_payload = ?, a2a_auth_token = NULL, progress_ref = NULL, verified_artifact_checkpoint = NULL, terminal_history_payload = NULL - WHERE id = ? AND status IN ('processing', 'delivering') + WHERE id = ? AND status IN ('pending', 'processing', 'delivering') AND terminal_delivery_confirmed_at IS NOT NULL`, args: [status, now, now, "{}", id], }); From b9022ef291303f973234ee7324d5a677a9aa3622 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:54:22 -0400 Subject: [PATCH 05/16] fix(core): recover parent after history finalization --- .../a2a-continuation-processor.spec.ts | 19 +++++++---- .../a2a-continuation-processor.ts | 33 ++++++++++++++++--- .../a2a-continuations-store.spec.ts | 5 +-- .../integrations/a2a-continuations-store.ts | 6 ++-- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index c8ca406530..24b14425bc 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -1461,7 +1461,7 @@ describe("A2A continuation processor", () => { ); }); - it("keeps receipt-backed parent completion recoverable before finalizing history", async () => { + it("wakes receipt-backed parent recovery after finalizing history", async () => { const consoleError = vi .spyOn(console, "error") .mockImplementation(() => undefined); @@ -1487,13 +1487,18 @@ describe("A2A continuation processor", () => { expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledTimes( 3, ); - expect(completeA2AContinuationMock).not.toHaveBeenCalled(); - expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( - "cont-1", - 20_000, - ); + expect(completeA2AContinuationMock).toHaveBeenCalledOnce(); + expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(dispatchPendingIntegrationTaskMock).toHaveBeenCalledWith({ + taskId: "task-1", + task: { + platform: "slack", + externalThreadId: "C123:123.456", + platformContext: { channelId: "C123", threadTs: "123.456" }, + }, + }); expect(consoleError).toHaveBeenCalledWith( - expect.stringContaining("history remains retryable"), + expect.stringContaining("parent completion remains retryable"), "Error", ); }); diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 6c1cec28bf..7ccfdf4a39 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -955,9 +955,18 @@ async function persistAndFinalizeConfirmedA2ADelivery( throw new Error("Confirmed A2A delivery is missing durable history"); } let lastError: unknown; + let historyFinalized = false; for (let attempt = 0; attempt < 3; attempt += 1) { try { - await persistA2AContinuationDelivery(continuation, history); + if (!historyFinalized) { + await persistA2AContinuationDelivery(continuation, history); + await finalizeA2ATerminalHistory(continuation.id); + historyFinalized = true; + logA2AContinuationTransition( + "terminal_history_persisted", + continuation, + ); + } await completeParentCampaignAfterTerminalA2A({ ...continuation, status: @@ -965,17 +974,33 @@ async function persistAndFinalizeConfirmedA2ADelivery( ? "completed" : "failed", }); - await finalizeA2ATerminalHistory(continuation.id); - logA2AContinuationTransition("terminal_history_persisted", continuation); return; } catch (err) { lastError = err; } } console.error( - `[integrations] A2A continuation ${continuation.id} has a provider receipt but its history remains retryable:`, + historyFinalized + ? `[integrations] A2A continuation ${continuation.id} finalized history but parent completion remains retryable:` + : `[integrations] A2A continuation ${continuation.id} has a provider receipt but its history remains retryable:`, lastError instanceof Error ? lastError.name : "persistence_error", ); + if (historyFinalized) { + await dispatchPendingIntegrationTask({ + taskId: continuation.integrationTaskId, + task: { + platform: continuation.platform, + externalThreadId: continuation.externalThreadId, + platformContext: continuation.incoming.platformContext, + }, + }).catch((err) => { + console.error( + `[integrations] Failed to wake A2A parent ${continuation.integrationTaskId} after terminal history finalization:`, + err, + ); + }); + return; + } await rescheduleAndRedispatchA2AContinuation(continuation.id); } diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index a6d5cec0df..62dbd38136 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -202,10 +202,7 @@ describe("A2A continuations store", () => { [{ status: "completed", terminal_delivery_confirmed_at: 10 }], "terminal-delivered", ], - [ - [{ status: "delivering", terminal_delivery_confirmed_at: 10 }], - "terminal-delivered", - ], + [[{ status: "delivering", terminal_delivery_confirmed_at: 10 }], "active"], [ [ { status: "delivering", terminal_delivery_confirmed_at: 10 }, diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 34a8526783..1f9e130aeb 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -562,10 +562,8 @@ export async function getA2AContinuationTaskOutcome( }); if (rows.length === 0) return "missing"; if ( - rows.some( - (row) => - ["pending", "processing", "delivering"].includes(String(row.status)) && - row.terminal_delivery_confirmed_at == null, + rows.some((row) => + ["pending", "processing", "delivering"].includes(String(row.status)), ) ) { return "active"; From e65d6da79c65cafa7290edb277a996462f152403 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:59:55 -0400 Subject: [PATCH 06/16] fix(core): lease terminal history finalization --- .../core/src/integrations/a2a-continuations-store.spec.ts | 7 +++++++ packages/core/src/integrations/a2a-continuations-store.ts | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 62dbd38136..0d1bb82534 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -275,6 +275,7 @@ describe("A2A continuations store", () => { ], }; + const beforeReceipt = Date.now(); await recordA2ATerminalDeliveryReceipt("cont-success", "success", history); await recordA2ATerminalDeliveryReceipt( "cont-failure", @@ -311,6 +312,12 @@ describe("A2A continuations store", () => { "cont-failure", "failure", ]); + expect(Number(terminalUpdates[0].args[3])).toBeGreaterThanOrEqual( + beforeReceipt + 60_000, + ); + expect(Number(terminalUpdates[1].args[3])).toBeGreaterThanOrEqual( + beforeReceipt + 60_000, + ); }); it("fails loud when terminal delivery confirmation does not persist", async () => { diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 1f9e130aeb..4c7b50b5fc 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -16,6 +16,7 @@ import type { IncomingMessage, PlatformRunProgressRef } from "./types.js"; let _initPromise: Promise | undefined; const PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000; const PROCESSING_NEXT_CHECK_STALE_AFTER_MS = 60 * 1000; +const TERMINAL_HISTORY_FINALIZATION_LEASE_MS = 60 * 1000; const MAX_VERIFIED_ARTIFACT_CHECKPOINT_CHARS = 16_000; const MAX_TERMINAL_HISTORY_PAYLOAD_CHARS = 64_000; @@ -953,7 +954,7 @@ export async function recordA2ATerminalDeliveryReceipt( kind, now, serializedPayload, - now, + now + TERMINAL_HISTORY_FINALIZATION_LEASE_MS, now, errorMessage?.slice(0, 2000) ?? null, id, From 43ec75cb6d5be0d3d1249714c4b3005ef9fd3008 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:07:46 -0400 Subject: [PATCH 07/16] fix(core): isolate A2A recovery failures --- .../a2a-continuation-processor.spec.ts | 66 ++++++++++++++++++- .../a2a-continuation-processor.ts | 35 ++++++++-- .../a2a-continuations-store.spec.ts | 45 +++++++++++++ .../integrations/a2a-continuations-store.ts | 23 +++++++ 4 files changed, 161 insertions(+), 8 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 24b14425bc..b3173b6680 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -8,6 +8,7 @@ import type { A2AContinuation } from "./a2a-continuations-store.js"; import type { PlatformAdapter } from "./types.js"; const claimA2AContinuationMock = vi.hoisted(() => vi.fn()); +const claimDueA2AContinuationsMock = vi.hoisted(() => vi.fn(async () => [])); const recoverDueA2AContinuationIdsMock = vi.hoisted(() => vi.fn()); const listRecoverableA2ATasksMock = vi.hoisted(() => vi.fn()); const getPendingTaskMock = vi.hoisted(() => vi.fn()); @@ -31,6 +32,9 @@ const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => const hasPendingConfirmedA2ADeliveryForIntegrationTaskMock = vi.hoisted(() => vi.fn(async () => false), ); +const hasOnlyLegacyFailedA2AContinuationsForIntegrationTaskMock = vi.hoisted( + () => vi.fn(async () => false), +); const failA2AContinuationMock = vi.hoisted(() => vi.fn()); const failA2AContinuationsForIntegrationTaskMock = vi.hoisted(() => vi.fn()); const getA2AContinuationMock = vi.hoisted(() => vi.fn()); @@ -52,7 +56,7 @@ const A2AClientMock = vi.hoisted(() => vi.mock("./a2a-continuations-store.js", () => ({ claimA2AContinuation: claimA2AContinuationMock, claimA2AContinuationDelivery: claimA2AContinuationDeliveryMock, - claimDueA2AContinuations: vi.fn(async () => []), + claimDueA2AContinuations: claimDueA2AContinuationsMock, finalizeA2ATerminalHistory: completeA2AContinuationMock, failA2AContinuation: failA2AContinuationMock, failA2AContinuationsForIntegrationTask: @@ -61,6 +65,8 @@ vi.mock("./a2a-continuations-store.js", () => ({ getA2AContinuationTaskOutcome: getA2AContinuationTaskOutcomeMock, hasPendingConfirmedA2ADeliveryForIntegrationTask: hasPendingConfirmedA2ADeliveryForIntegrationTaskMock, + hasOnlyLegacyFailedA2AContinuationsForIntegrationTask: + hasOnlyLegacyFailedA2AContinuationsForIntegrationTaskMock, listRecoverableA2AIntegrationTasks: listRecoverableA2ATasksMock, recoverDueA2AContinuationIds: recoverDueA2AContinuationIdsMock, recordA2ATerminalDeliveryReceipt: recordA2ATerminalDeliveryReceiptMock, @@ -190,6 +196,9 @@ describe("A2A continuation processor", () => { hasPendingConfirmedA2ADeliveryForIntegrationTaskMock.mockResolvedValue( false, ); + hasOnlyLegacyFailedA2AContinuationsForIntegrationTaskMock.mockResolvedValue( + false, + ); failA2AContinuationMock.mockResolvedValue(undefined); recordA2ATerminalDeliveryReceiptMock.mockImplementation( async ( @@ -224,6 +233,7 @@ describe("A2A continuation processor", () => { claimA2AContinuationDeliveryMock.mockImplementation(async (id: string) => continuation({ id, status: "delivering" }), ); + claimDueA2AContinuationsMock.mockResolvedValue([]); recoverDueA2AContinuationIdsMock.mockResolvedValue([]); listRecoverableA2ATasksMock.mockResolvedValue([ { @@ -364,6 +374,36 @@ describe("A2A continuation processor", () => { expect(failA2AContinuationMock).not.toHaveBeenCalled(); }); + it("continues a due batch when one processor and its recovery both fail", async () => { + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + claimDueA2AContinuationsMock.mockResolvedValueOnce([ + continuation({ id: "cont-failing" }), + continuation({ id: "cont-healthy", a2aTaskId: "a2a-task-healthy" }), + ]); + getIntegrationCampaignForTaskMock + .mockRejectedValueOnce(new Error("campaign database unavailable")) + .mockResolvedValue(null); + getA2AContinuationMock.mockRejectedValueOnce( + new Error("recovery database unavailable"), + ); + const { processDueA2AContinuations } = + await import("./a2a-continuation-processor.js"); + + await expect( + processDueA2AContinuations({ + adapters: new Map([["slack", adapter()]]), + }), + ).resolves.toBeUndefined(); + + expect(getTaskMock).toHaveBeenCalledWith("a2a-task-healthy"); + expect(consoleError).toHaveBeenCalledWith( + expect.stringContaining("recovery failed"), + "Error", + ); + }); + it("delivers the durable checkpoint when an escaped processor failure exhausts attempts", async () => { const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); getA2AContinuationMock.mockResolvedValueOnce( @@ -637,6 +677,30 @@ describe("A2A continuation processor", () => { ); }); + it("conservatively fails a waiting parent for ambiguous legacy failed rows", async () => { + getA2AContinuationTaskOutcomeMock.mockResolvedValue( + "terminal-without-delivery", + ); + hasOnlyLegacyFailedA2AContinuationsForIntegrationTaskMock.mockResolvedValue( + true, + ); + durableDispatchEnabledMock.mockReturnValue(true); + const { reconcileTerminalA2AParentIfDisabled } = + await import("./a2a-continuation-processor.js"); + + await expect(reconcileTerminalA2AParentIfDisabled("task-1")).resolves.toBe( + true, + ); + + expect( + failIntegrationCampaignTaskDeliveryContainmentMock, + ).toHaveBeenCalledWith( + "task-1", + expect.stringContaining("Legacy A2A continuation"), + ); + expect(failDisabledIntegrationCampaignTaskMock).not.toHaveBeenCalled(); + }); + it.each(["failed", "completed"] as const)( "does not poll an A2A row after its owning campaign is %s", async (status) => { diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 7ccfdf4a39..ac4d144b86 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -24,6 +24,7 @@ import { finalizeA2ATerminalHistory, getA2AContinuation, getA2AContinuationTaskOutcome, + hasOnlyLegacyFailedA2AContinuationsForIntegrationTask, hasPendingConfirmedA2ADeliveryForIntegrationTask, listRecoverableA2AIntegrationTasks, recoverDueA2AContinuationIds, @@ -261,13 +262,22 @@ export async function processDueA2AContinuations(options: { console.error( `[integrations] A2A continuation ${continuation.id} failed; durable recovery requested`, ); - await recoverA2AContinuationAfterProcessorFailure(continuation.id, { - adapters: options.adapters, - reason: - err instanceof Error - ? err.message.slice(0, 500) - : "continuation processing failed", - }); + try { + await recoverA2AContinuationAfterProcessorFailure(continuation.id, { + adapters: options.adapters, + reason: + err instanceof Error + ? err.message.slice(0, 500) + : "continuation processing failed", + }); + } catch (recoveryError) { + console.error( + `[integrations] A2A continuation ${continuation.id} recovery failed; later continuations will continue`, + recoveryError instanceof Error + ? recoveryError.name + : "recovery_error", + ); + } }, ); } @@ -595,6 +605,17 @@ export async function reconcileTerminalA2AParentIfDisabled( ) { return false; } + if ( + await hasOnlyLegacyFailedA2AContinuationsForIntegrationTask( + integrationTaskId, + ) + ) { + await failIntegrationCampaignTaskDeliveryContainment( + integrationTaskId, + "Legacy A2A continuation ended without durable delivery proof", + ); + return true; + } const task = await getPendingTask(integrationTaskId); if ( !task || diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 0d1bb82534..5d99182dbf 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -232,6 +232,51 @@ describe("A2A continuations store", () => { ); }); + it.each([ + [ + [ + { + status: "failed", + terminal_delivery_kind: null, + terminal_delivery_confirmed_at: null, + terminal_history_payload: null, + }, + ], + true, + ], + [ + [ + { + status: "failed", + terminal_delivery_kind: "failure", + terminal_delivery_confirmed_at: 10, + terminal_history_payload: null, + }, + ], + false, + ], + ])( + "detects ambiguous legacy failed custody as %s", + async (rows, expected) => { + executeMock.mockImplementation( + async (query: string | { sql: string }) => { + if ( + querySql(query).includes("SELECT status, terminal_delivery_kind") + ) { + return { rows, rowsAffected: 0 }; + } + return { rows: [], rowsAffected: 0 }; + }, + ); + const { hasOnlyLegacyFailedA2AContinuationsForIntegrationTask } = + await loadStore(); + + await expect( + hasOnlyLegacyFailedA2AContinuationsForIntegrationTask("task-1"), + ).resolves.toBe(expected); + }, + ); + it("records provider-confirmed terminal delivery without terminalizing or scrubbing history custody", async () => { const persisted = new Map>(); executeMock.mockImplementation( diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 4c7b50b5fc..5f370b6ccf 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -589,6 +589,29 @@ export async function hasPendingConfirmedA2ADeliveryForIntegrationTask( return rows.length > 0; } +export async function hasOnlyLegacyFailedA2AContinuationsForIntegrationTask( + integrationTaskId: string, +): Promise { + await ensureTable(); + const { rows } = await getDbExec().execute({ + sql: `SELECT status, terminal_delivery_kind, + terminal_delivery_confirmed_at, terminal_history_payload + FROM integration_a2a_continuations + WHERE integration_task_id = ?`, + args: [integrationTaskId], + }); + return ( + rows.length > 0 && + rows.every( + (row) => + String(row.status) === "failed" && + row.terminal_delivery_kind == null && + row.terminal_delivery_confirmed_at == null && + row.terminal_history_payload == null, + ) + ); +} + export async function failA2AContinuationsForIntegrationTask( integrationTaskId: string, errorMessage: string, From c9ca9676ac96397cf2072318fd4f6a5de96b76ac Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:16:18 -0400 Subject: [PATCH 08/16] fix(core): retain ambiguous delivery custody --- .../a2a-continuation-processor.spec.ts | 90 +++++++++++++++++++ .../a2a-continuation-processor.ts | 29 ++++++ .../a2a-continuations-store.spec.ts | 21 +++++ .../integrations/a2a-continuations-store.ts | 14 +++ 4 files changed, 154 insertions(+) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index b3173b6680..dad778a7f5 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -26,6 +26,7 @@ const completeIntegrationCampaignTaskAfterA2AMock = vi.hoisted(() => const claimA2AContinuationDeliveryMock = vi.hoisted(() => vi.fn()); const completeA2AContinuationMock = vi.hoisted(() => vi.fn()); const recordA2ATerminalDeliveryReceiptMock = vi.hoisted(() => vi.fn()); +const retainA2AUnconfirmedDeliveryClaimMock = vi.hoisted(() => vi.fn()); const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => vi.fn(async () => "terminal-delivered"), ); @@ -70,6 +71,7 @@ vi.mock("./a2a-continuations-store.js", () => ({ listRecoverableA2AIntegrationTasks: listRecoverableA2ATasksMock, recoverDueA2AContinuationIds: recoverDueA2AContinuationIdsMock, recordA2ATerminalDeliveryReceipt: recordA2ATerminalDeliveryReceiptMock, + retainA2AUnconfirmedDeliveryClaim: retainA2AUnconfirmedDeliveryClaimMock, rescheduleA2AContinuation: rescheduleA2AContinuationMock, saveA2AVerifiedArtifactCheckpoint: saveA2AVerifiedArtifactCheckpointMock, })); @@ -216,6 +218,7 @@ describe("A2A continuation processor", () => { errorMessage: errorMessage ?? null, }), ); + retainA2AUnconfirmedDeliveryClaimMock.mockResolvedValue(undefined); rescheduleA2AContinuationMock.mockResolvedValue(undefined); saveA2AVerifiedArtifactCheckpointMock.mockImplementation( async (_id: string, checkpoint: string) => checkpoint, @@ -458,6 +461,38 @@ describe("A2A continuation processor", () => { expect(fetch).not.toHaveBeenCalled(); }); + it("preserves confirmed sibling custody when adapter exhaustion contains another sibling", async () => { + getA2AContinuationMock.mockResolvedValueOnce( + continuation({ status: "processing", attempts: 30 }), + ); + hasPendingConfirmedA2ADeliveryForIntegrationTaskMock.mockResolvedValueOnce( + true, + ); + const { recoverA2AContinuationAfterProcessorFailure } = + await import("./a2a-continuation-processor.js"); + + await recoverA2AContinuationAfterProcessorFailure("cont-1", { + adapters: new Map(), + reason: "processor failed after its adapter was removed", + }); + + expect(failA2AContinuationsForIntegrationTaskMock).toHaveBeenCalledWith( + "task-1", + "Unknown platform: slack", + ); + expect( + failIntegrationCampaignTaskDeliveryContainmentMock, + ).not.toHaveBeenCalled(); + expect(dispatchPendingIntegrationTaskMock).toHaveBeenCalledWith({ + taskId: "task-1", + task: { + platform: "slack", + externalThreadId: "C123:123.456", + platformContext: { channelId: "C123", threadTs: "123.456" }, + }, + }); + }); + it("allows one durable wake-up dispatch failure without stranding the rest", async () => { recoverDueA2AContinuationIdsMock.mockResolvedValue([ "cont-fail", @@ -775,6 +810,28 @@ describe("A2A continuation processor", () => { expect(fetch).not.toHaveBeenCalled(); }); + it("retains the successful delivery claim when its receipt cannot be recorded", async () => { + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + claimA2AContinuationMock.mockResolvedValueOnce(continuation()); + recordA2ATerminalDeliveryReceiptMock.mockRejectedValue( + new Error("receipt database unavailable"), + ); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledOnce(); + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledTimes(3); + expect(retainA2AUnconfirmedDeliveryClaimMock).toHaveBeenCalledWith( + "cont-1", + ); + expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(completeA2AContinuationMock).not.toHaveBeenCalled(); + }); + it("closes the waiting parent campaign and wakes its successor after the last A2A reply", async () => { const claimed = continuation(); claimA2AContinuationMock.mockResolvedValueOnce(claimed); @@ -1760,6 +1817,39 @@ describe("A2A continuation processor", () => { expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); + it("retains the failure delivery claim when its receipt cannot be recorded", async () => { + const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); + claimA2AContinuationMock.mockResolvedValueOnce(continuation()); + getTaskMock.mockResolvedValueOnce({ + id: "a2a-task-1", + status: { + state: "failed", + message: { + role: "agent", + parts: [{ type: "text", text: "The deck export failed" }], + }, + timestamp: new Date().toISOString(), + }, + }); + recordA2ATerminalDeliveryReceiptMock.mockRejectedValue( + new Error("receipt database unavailable"), + ); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledOnce(); + expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledTimes(3); + expect(retainA2AUnconfirmedDeliveryClaimMock).toHaveBeenCalledWith( + "cont-1", + ); + expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(completeA2AContinuationMock).not.toHaveBeenCalled(); + }); + it("retries a terminal failure notification until delivery is confirmed", async () => { const sendResponse = vi.fn(async () => { throw new Error("Slack delivery unavailable"); diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index ac4d144b86..48eec9769b 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -29,6 +29,7 @@ import { listRecoverableA2AIntegrationTasks, recoverDueA2AContinuationIds, recordA2ATerminalDeliveryReceipt, + retainA2AUnconfirmedDeliveryClaim, rescheduleA2AContinuation, saveA2AVerifiedArtifactCheckpoint, type A2AContinuation, @@ -216,6 +217,26 @@ export async function recoverA2AContinuationAfterProcessorFailure( continuation.integrationTaskId, reason, ); + if ( + await hasPendingConfirmedA2ADeliveryForIntegrationTask( + continuation.integrationTaskId, + ) + ) { + await dispatchPendingIntegrationTask({ + taskId: continuation.integrationTaskId, + task: { + platform: continuation.platform, + externalThreadId: continuation.externalThreadId, + platformContext: continuation.incoming.platformContext, + }, + }).catch((err) => { + console.error( + `[integrations] Failed to wake confirmed sibling history for ${continuation.integrationTaskId}:`, + err, + ); + }); + return; + } await failIntegrationCampaignTaskDeliveryContainment( continuation.integrationTaskId, reason, @@ -965,6 +986,14 @@ async function recordTerminalA2ADelivery( `[integrations] ${continuation.platform} accepted terminal A2A delivery for ${continuation.id}, ` + `but recording its receipt failed after ${COMPLETE_AFTER_DELIVERY_ATTEMPTS} attempts. Leaving it in delivering for stale recovery.`, ); + try { + await retainA2AUnconfirmedDeliveryClaim(continuation.id); + } catch (err) { + console.error( + `[integrations] Failed to retain unconfirmed A2A delivery claim ${continuation.id}:`, + err instanceof Error ? err.name : "receipt_recovery_error", + ); + } return null; } diff --git a/packages/core/src/integrations/a2a-continuations-store.spec.ts b/packages/core/src/integrations/a2a-continuations-store.spec.ts index 5d99182dbf..f7c93c950c 100644 --- a/packages/core/src/integrations/a2a-continuations-store.spec.ts +++ b/packages/core/src/integrations/a2a-continuations-store.spec.ts @@ -192,6 +192,27 @@ describe("A2A continuations store", () => { ).rejects.toThrow("exceeds 16000 characters"); }); + it("retains an unconfirmed delivery claim until stale recovery", async () => { + executeMock.mockResolvedValue({ rows: [], rowsAffected: 1 }); + const { retainA2AUnconfirmedDeliveryClaim } = await loadStore(); + const before = Date.now(); + + await retainA2AUnconfirmedDeliveryClaim("cont-1"); + + const update = executeMock.mock.calls.find(([query]) => + querySql(query).includes("SET next_check_at = ?"), + ); + expect(querySql(update![0])).toContain("status = 'delivering'"); + expect(queryArgs(update![0])).toEqual([ + expect.any(Number), + expect.any(Number), + "cont-1", + ]); + expect(Number(queryArgs(update![0])[0])).toBeGreaterThanOrEqual( + before + 5 * 60_000, + ); + }); + it.each([ [[], "missing"], [ diff --git a/packages/core/src/integrations/a2a-continuations-store.ts b/packages/core/src/integrations/a2a-continuations-store.ts index 5f370b6ccf..71513f8b95 100644 --- a/packages/core/src/integrations/a2a-continuations-store.ts +++ b/packages/core/src/integrations/a2a-continuations-store.ts @@ -924,6 +924,20 @@ export async function rescheduleA2AContinuation( }); } +export async function retainA2AUnconfirmedDeliveryClaim( + id: string, +): Promise { + await ensureTable(); + const now = Date.now(); + await getDbExec().execute({ + sql: `UPDATE integration_a2a_continuations + SET next_check_at = ?, updated_at = ? + WHERE id = ? AND status = 'delivering' + AND terminal_delivery_confirmed_at IS NULL`, + args: [now + PROCESSING_STUCK_AFTER_MS, now, id], + }); +} + export async function saveA2AVerifiedArtifactCheckpoint( id: string, checkpoint: string, From 8314671105489f99e238e6c386899b2e10673071 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:25:08 -0400 Subject: [PATCH 09/16] fix(core): recover finalized A2A parents --- .../a2a-continuation-processor.spec.ts | 4 ++ .../a2a-continuation-processor.ts | 4 ++ .../integration-campaign-recovery.spec.ts | 30 +++++++++++++++ .../integration-campaign-recovery.ts | 5 ++- packages/core/src/integrations/plugin.spec.ts | 37 +++++++++++++++++++ packages/core/src/integrations/plugin.ts | 10 +++-- 6 files changed, 85 insertions(+), 5 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index dad778a7f5..2857fb9693 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -490,6 +490,8 @@ describe("A2A continuation processor", () => { externalThreadId: "C123:123.456", platformContext: { channelId: "C123", threadTs: "123.456" }, }, + campaignContinuation: true, + allowPortableConfirmedReceiptReconciliation: true, }); }); @@ -1617,6 +1619,8 @@ describe("A2A continuation processor", () => { externalThreadId: "C123:123.456", platformContext: { channelId: "C123", threadTs: "123.456" }, }, + campaignContinuation: true, + allowPortableConfirmedReceiptReconciliation: true, }); expect(consoleError).toHaveBeenCalledWith( expect.stringContaining("parent completion remains retryable"), diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 48eec9769b..5618087c03 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -229,6 +229,8 @@ export async function recoverA2AContinuationAfterProcessorFailure( externalThreadId: continuation.externalThreadId, platformContext: continuation.incoming.platformContext, }, + campaignContinuation: true, + allowPortableConfirmedReceiptReconciliation: true, }).catch((err) => { console.error( `[integrations] Failed to wake confirmed sibling history for ${continuation.integrationTaskId}:`, @@ -1043,6 +1045,8 @@ async function persistAndFinalizeConfirmedA2ADelivery( externalThreadId: continuation.externalThreadId, platformContext: continuation.incoming.platformContext, }, + campaignContinuation: true, + allowPortableConfirmedReceiptReconciliation: true, }).catch((err) => { console.error( `[integrations] Failed to wake A2A parent ${continuation.integrationTaskId} after terminal history finalization:`, diff --git a/packages/core/src/integrations/integration-campaign-recovery.spec.ts b/packages/core/src/integrations/integration-campaign-recovery.spec.ts index 01d3a3c1f8..382a37a3fa 100644 --- a/packages/core/src/integrations/integration-campaign-recovery.spec.ts +++ b/packages/core/src/integrations/integration-campaign-recovery.spec.ts @@ -7,6 +7,7 @@ const dispatchMock = vi.hoisted(() => vi.fn()); const durableEnabledMock = vi.hoisted(() => vi.fn()); const failDisabledMock = vi.hoisted(() => vi.fn()); const getNextTaskMock = vi.hoisted(() => vi.fn()); +const getA2AContinuationTaskOutcomeMock = vi.hoisted(() => vi.fn()); vi.mock("./integration-campaigns-store.js", () => ({ listDueIntegrationCampaignIds: listDueMock, @@ -24,6 +25,10 @@ vi.mock("./integration-durable-dispatch.js", () => ({ isIntegrationDurableDispatchEnabledForTask: durableEnabledMock, })); +vi.mock("./a2a-continuations-store.js", () => ({ + getA2AContinuationTaskOutcome: getA2AContinuationTaskOutcomeMock, +})); + describe("integration campaign recovery", () => { beforeEach(() => { vi.clearAllMocks(); @@ -43,6 +48,7 @@ describe("integration campaign recovery", () => { dispatchMock.mockResolvedValue("background-acknowledged"); durableEnabledMock.mockReturnValue(true); getNextTaskMock.mockResolvedValue(null); + getA2AContinuationTaskOutcomeMock.mockResolvedValue("missing"); }); it("wakes due campaigns without claiming or executing them", async () => { @@ -156,4 +162,28 @@ describe("integration campaign recovery", () => { ); expect(failDisabledMock).not.toHaveBeenCalled(); }); + + it("still wakes finalized A2A parent reconciliation after scope is disabled", async () => { + durableEnabledMock.mockReturnValueOnce(false); + getA2AContinuationTaskOutcomeMock.mockResolvedValueOnce( + "terminal-delivered", + ); + const { recoverDueIntegrationCampaigns } = + await import("./integration-campaign-recovery.js"); + + await expect(recoverDueIntegrationCampaigns({})).resolves.toEqual({ + selected: 1, + dispatched: 1, + skipped: 0, + failed: 0, + }); + expect(dispatchMock).toHaveBeenCalledWith( + expect.objectContaining({ + taskId: "task-1", + campaignContinuation: true, + allowPortableConfirmedReceiptReconciliation: true, + }), + ); + expect(failDisabledMock).not.toHaveBeenCalled(); + }); }); diff --git a/packages/core/src/integrations/integration-campaign-recovery.ts b/packages/core/src/integrations/integration-campaign-recovery.ts index 5fbc7a90e8..6523069145 100644 --- a/packages/core/src/integrations/integration-campaign-recovery.ts +++ b/packages/core/src/integrations/integration-campaign-recovery.ts @@ -1,3 +1,4 @@ +import { getA2AContinuationTaskOutcome } from "./a2a-continuations-store.js"; import { getIntegrationCampaign, failDisabledIntegrationCampaignTask, @@ -72,7 +73,9 @@ export async function recoverDueIntegrationCampaigns(options: { : undefined, }, ); - const confirmedReceipt = hasConfirmedDeliveryReceipt(task.payload); + const confirmedReceipt = + hasConfirmedDeliveryReceipt(task.payload) || + (await getA2AContinuationTaskOutcome(task.id)) === "terminal-delivered"; if (!durableDispatchEnabled && !confirmedReceipt) { await failDisabledIntegrationCampaignTask(task.id); const nextTask = await getNextPendingTaskForThread( diff --git a/packages/core/src/integrations/plugin.spec.ts b/packages/core/src/integrations/plugin.spec.ts index 385acd302a..bb02229c7e 100644 --- a/packages/core/src/integrations/plugin.spec.ts +++ b/packages/core/src/integrations/plugin.spec.ts @@ -856,6 +856,43 @@ describe("integrations plugin routes", () => { expect(markTaskCompletedMock).toHaveBeenCalledWith(task.id); }); + it("finishes a history-finalized A2A parent after its rollout scope is disabled", async () => { + process.env.NODE_ENV = "development"; + process.env.NETLIFY = "true"; + process.env.A2A_SECRET = "test-secret"; + delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; + getA2AContinuationTaskOutcomeMock.mockResolvedValueOnce( + "terminal-delivered", + ); + const baseTask = claimedTask(1); + const task = { + ...baseTask, + payload: JSON.stringify({ + kind: "response-delivery", + incoming: JSON.parse(baseTask.payload).incoming, + message: { text: "The parent work is ready", platformContext: {} }, + awaitingA2ACompletion: true, + }), + }; + getPendingTaskMock.mockResolvedValueOnce(task); + const nitroApp = createNitroApp(); + await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + + const result = await dispatch( + nitroApp, + "/_agent-native/integrations/process-task", + "POST", + { taskId: task.id, __integrationCampaignContinuation: true }, + signedTaskHeaders(task.id), + ); + + expect(result.status).toBe(200); + expect(failDisabledIntegrationCampaignTaskMock).not.toHaveBeenCalled(); + expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledWith( + task.id, + ); + }); + it("does not send an unreceipted campaign delivery after scope is disabled", async () => { process.env.NODE_ENV = "development"; delete process.env.AGENT_INTEGRATION_DURABLE_DISPATCH; diff --git a/packages/core/src/integrations/plugin.ts b/packages/core/src/integrations/plugin.ts index af7f576004..2fab9ca5d8 100644 --- a/packages/core/src/integrations/plugin.ts +++ b/packages/core/src/integrations/plugin.ts @@ -1907,13 +1907,18 @@ export function createIntegrationsPlugin( ? { channelId: task.dispatchScope } : undefined, }); + const a2aOutcome = campaignContinuation + ? await getA2AContinuationTaskOutcome(task.id) + : null; const confirmedDeliveryReceipt = taskPayload.kind === "response-delivery" && taskPayload.deliveryReceipt?.status === "delivered"; + const confirmedDeliveryProof = + confirmedDeliveryReceipt || a2aOutcome === "terminal-delivered"; if ( campaignContinuation && !durableCampaignEnabled && - !confirmedDeliveryReceipt + !confirmedDeliveryProof ) { await failDisabledIntegrationCampaignTask(task.id); const nextTask = await getNextPendingTaskForThread( @@ -2112,9 +2117,6 @@ export function createIntegrationsPlugin( ); if (!waiting) return "campaign-active" as const; } - const a2aOutcome = await getA2AContinuationTaskOutcome( - task.id, - ); if (a2aOutcome === "terminal-delivered") { const completed = await completeIntegrationCampaignTaskAfterA2A(task.id); From 435100b06c7641ba82c71546f8d30a2af24aae39 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:49:47 -0400 Subject: [PATCH 10/16] fix(core): complete finalized A2A parents before delivery --- packages/core/src/integrations/plugin.spec.ts | 6 +++++- packages/core/src/integrations/plugin.ts | 18 +++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/core/src/integrations/plugin.spec.ts b/packages/core/src/integrations/plugin.spec.ts index bb02229c7e..1de430ba07 100644 --- a/packages/core/src/integrations/plugin.spec.ts +++ b/packages/core/src/integrations/plugin.spec.ts @@ -875,8 +875,11 @@ describe("integrations plugin routes", () => { }), }; getPendingTaskMock.mockResolvedValueOnce(task); + const sendResponse = vi.fn(adapter.sendResponse); const nitroApp = createNitroApp(); - await createIntegrationsPlugin({ adapters: [adapter] })(nitroApp); + await createIntegrationsPlugin({ + adapters: [{ ...adapter, sendResponse }], + })(nitroApp); const result = await dispatch( nitroApp, @@ -891,6 +894,7 @@ describe("integrations plugin routes", () => { expect(completeIntegrationCampaignTaskAfterA2AMock).toHaveBeenCalledWith( task.id, ); + expect(sendResponse).not.toHaveBeenCalled(); }); it("does not send an unreceipted campaign delivery after scope is disabled", async () => { diff --git a/packages/core/src/integrations/plugin.ts b/packages/core/src/integrations/plugin.ts index 2fab9ca5d8..6930ab97dd 100644 --- a/packages/core/src/integrations/plugin.ts +++ b/packages/core/src/integrations/plugin.ts @@ -1999,6 +1999,17 @@ export function createIntegrationsPlugin( return; } if (taskPayload.kind === "response-delivery") { + if ( + campaignContinuation && + taskPayload.awaitingA2ACompletion && + a2aOutcome === "terminal-delivered" + ) { + const completed = + await completeIntegrationCampaignTaskAfterA2A(task.id); + return completed + ? ("completed" as const) + : ("campaign-active" as const); + } let receipt: void | PlatformDeliveryReceipt = taskPayload.deliveryReceipt; let deliveryLease: @@ -2117,13 +2128,6 @@ export function createIntegrationsPlugin( ); if (!waiting) return "campaign-active" as const; } - if (a2aOutcome === "terminal-delivered") { - const completed = - await completeIntegrationCampaignTaskAfterA2A(task.id); - return completed - ? ("completed" as const) - : ("campaign-active" as const); - } if ( a2aOutcome === "terminal-without-delivery" && (await reconcileTerminalA2AParentIfDisabled(task.id)) From 3e104e7d81f781e36de0f6d2a367405e4353c7d0 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:44:11 -0400 Subject: [PATCH 11/16] fix(core): make Slack continuation delivery idempotent --- .../a2a-continuation-processor.spec.ts | 105 ++++++++++++++---- .../a2a-continuation-processor.ts | 5 +- .../src/integrations/adapters/slack.spec.ts | 96 ++++++++++++++++ .../core/src/integrations/adapters/slack.ts | 51 +++++++-- packages/core/src/integrations/types.ts | 13 ++- 5 files changed, 238 insertions(+), 32 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 2857fb9693..2af6898bee 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -5,7 +5,10 @@ import { buildA2ARecoverableArtifactMessage, } from "../a2a/artifact-response.js"; import type { A2AContinuation } from "./a2a-continuations-store.js"; -import type { PlatformAdapter } from "./types.js"; +import type { + PlatformAdapter, + PlatformDeliveryOptions, +} from "./types.js"; const claimA2AContinuationMock = vi.hoisted(() => vi.fn()); const claimDueA2AContinuationsMock = vi.hoisted(() => vi.fn(async () => [])); @@ -432,7 +435,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("/page/content-1"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); expect(failA2AContinuationMock).not.toHaveBeenCalled(); @@ -806,7 +809,11 @@ describe("A2A continuation processor", () => { text: "https://slides.agent-native.test/deck/deck-qa", }), expect.any(Object), - { placeholderRef: undefined }, + { + idempotencyKey: "a2a-continuation:cont-1", + placeholderRef: undefined, + strictTargetRef: true, + }, ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); expect(fetch).not.toHaveBeenCalled(); @@ -834,6 +841,52 @@ describe("A2A continuation processor", () => { expect(completeA2AContinuationMock).not.toHaveBeenCalled(); }); + it("reuses one provider delivery identity after Slack succeeds but receipt persistence fails", async () => { + const providerDeliveries = new Set(); + let visibleProviderDeliveries = 0; + const sendResponse = vi.fn( + async ( + _message: unknown, + _incoming: unknown, + opts?: PlatformDeliveryOptions, + ) => { + if (!opts?.idempotencyKey) throw new Error("missing idempotency key"); + if (!providerDeliveries.has(opts.idempotencyKey)) { + visibleProviderDeliveries += 1; + } + providerDeliveries.add(opts.idempotencyKey); + return { + status: "delivered" as const, + messageRefs: [opts.idempotencyKey], + }; + }, + ); + claimA2AContinuationMock + .mockResolvedValueOnce(continuation()) + .mockResolvedValueOnce(continuation()); + recordA2ATerminalDeliveryReceiptMock + .mockRejectedValueOnce(new Error("receipt database unavailable")) + .mockRejectedValueOnce(new Error("receipt database unavailable")) + .mockRejectedValueOnce(new Error("receipt database unavailable")); + const { processA2AContinuationById } = + await import("./a2a-continuation-processor.js"); + + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + await processA2AContinuationById("cont-1", { + adapters: new Map([["slack", adapter(sendResponse)]]), + }); + + expect(sendResponse).toHaveBeenCalledTimes(2); + expect(providerDeliveries).toEqual( + new Set(["a2a-continuation:cont-1"]), + ); + expect(visibleProviderDeliveries).toBe(1); + expect(retainA2AUnconfirmedDeliveryClaimMock).toHaveBeenCalledOnce(); + expect(completeA2AContinuationMock).toHaveBeenCalledOnce(); + }); + it("closes the waiting parent campaign and wakes its successor after the last A2A reply", async () => { const claimed = continuation(); claimA2AContinuationMock.mockResolvedValueOnce(claimed); @@ -1304,7 +1357,7 @@ describe("A2A continuation processor", () => { expect(failA2AContinuationMock).not.toHaveBeenCalled(); }); - it("falls back to a thread reply when finalizing a resumed Slack stream fails", async () => { + it("falls back through the stable stream target when finalizing a resumed Slack stream fails", async () => { const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); const complete = vi.fn(async () => { throw new Error("chat.stopStream rejected"); @@ -1313,6 +1366,7 @@ describe("A2A continuation processor", () => { const resumedAdapter = adapter(sendResponse); resumedAdapter.resumeRunProgress = vi.fn(async () => ({ ref: { kind: "slack-stream", streamTs: "1719000000.000001" }, + responseTargetRef: "1719000000.000001", onEvent: vi.fn(async () => ({ status: "delivered" as const })), complete, fail, @@ -1345,7 +1399,11 @@ describe("A2A continuation processor", () => { text: "https://slides.agent-native.test/deck/deck-qa", }), expect.objectContaining({ platform: "slack" }), - { placeholderRef: undefined }, + expect.objectContaining({ + idempotencyKey: "a2a-continuation:cont-1", + placeholderRef: "1719000000.000001", + strictTargetRef: true, + }), ); expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); @@ -1386,13 +1444,13 @@ describe("A2A continuation processor", () => { text: "https://slides.agent-native.test/deck/deck-qa", }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(fail).toHaveBeenCalled(); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); - it("still posts the final answer when closing a failed resumed stream also fails", async () => { + it("still updates the stable target when closing a failed resumed stream also fails", async () => { const sendResponse = vi.fn(async () => ({ status: "delivered" as const })); const complete = vi.fn(async () => { throw new Error("chat.stopStream rejected"); @@ -1403,6 +1461,7 @@ describe("A2A continuation processor", () => { const resumedAdapter = adapter(sendResponse); resumedAdapter.resumeRunProgress = vi.fn(async () => ({ ref: { kind: "slack-stream", streamTs: "1719000000.000001" }, + responseTargetRef: "1719000000.000001", onEvent: vi.fn(async () => ({ status: "delivered" as const })), complete, fail, @@ -1428,7 +1487,11 @@ describe("A2A continuation processor", () => { text: "https://slides.agent-native.test/deck/deck-qa", }), expect.objectContaining({ platform: "slack" }), - { placeholderRef: undefined }, + expect.objectContaining({ + idempotencyKey: "a2a-continuation:cont-1", + placeholderRef: "1719000000.000001", + strictTargetRef: true, + }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); @@ -1465,7 +1528,7 @@ describe("A2A continuation processor", () => { text: "Report: https://agent-workspace.builder.io/analytics/analyses/qa-report", }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); }); @@ -1501,7 +1564,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("could not verify the design URL"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(sendResponse.mock.calls[0][0].text).not.toContain("design_fake"); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); @@ -1545,7 +1608,7 @@ describe("A2A continuation processor", () => { ), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(sendResponse.mock.calls[0][0].text).not.toContain( "could not verify", @@ -1810,7 +1873,7 @@ describe("A2A continuation processor", () => { ), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", @@ -2134,7 +2197,7 @@ describe("A2A continuation processor", () => { ), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", @@ -2290,7 +2353,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("request_final_b"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(sendResponse.mock.calls[0][0].text).not.toContain( "request_checkpoint_a", @@ -2335,7 +2398,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("/page/content-1"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); expect(failA2AContinuationMock).not.toHaveBeenCalled(); @@ -2365,7 +2428,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("/page/content-1"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( @@ -2424,7 +2487,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("request_checkpoint_retry"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(sendResponse.mock.calls[0][0].text).toContain( "did not finish its full response", @@ -2486,7 +2549,7 @@ describe("A2A continuation processor", () => { text: expect.stringContaining("request_org_checkpoint"), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); @@ -2536,7 +2599,7 @@ describe("A2A continuation processor", () => { ), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", @@ -2658,7 +2721,7 @@ describe("A2A continuation processor", () => { text: "https://slides.agent-native.test/deck/deck-qa", }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); }); @@ -2717,7 +2780,7 @@ describe("A2A continuation processor", () => { ), }), expect.any(Object), - { placeholderRef: undefined }, + expect.objectContaining({ placeholderRef: undefined }), ); expect(recordA2ATerminalDeliveryReceiptMock).toHaveBeenCalledWith( "cont-1", diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 5618087c03..b26304e96e 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -872,7 +872,10 @@ async function deliverA2AContinuationResponse( } } const receipt = await adapter.sendResponse(message, continuation.incoming, { - placeholderRef: continuation.placeholderRef ?? undefined, + idempotencyKey: `a2a-continuation:${continuation.id}`, + placeholderRef: + progress?.responseTargetRef ?? continuation.placeholderRef ?? undefined, + strictTargetRef: true, }); if (receipt?.status !== "delivered") { throw new Error("Continuation response completed without delivery proof"); diff --git a/packages/core/src/integrations/adapters/slack.spec.ts b/packages/core/src/integrations/adapters/slack.spec.ts index b548f24e88..0acd17229a 100644 --- a/packages/core/src/integrations/adapters/slack.spec.ts +++ b/packages/core/src/integrations/adapters/slack.spec.ts @@ -957,6 +957,7 @@ describe("slackAdapter", () => { kind: "slack-stream", streamTs: "999.003", }); + expect(progress?.responseTargetRef).toBe("999.003"); expect( requests.find((request) => request.method === "chat.startStream"), ).toBeUndefined(); @@ -1266,6 +1267,101 @@ describe("slackAdapter", () => { }); }); + it("reuses deterministic client message ids across terminal delivery retries", async () => { + process.env.SLACK_BOT_TOKEN = "xoxb-test"; + const deliveryBodies: Array> = []; + vi.stubGlobal( + "fetch", + vi.fn((url: string, init?: RequestInit) => { + if (String(url).includes("chat.postMessage")) { + deliveryBodies.push(JSON.parse(String(init?.body ?? "{}"))); + return Promise.resolve( + new Response( + JSON.stringify({ + ok: true, + ts: `message-${deliveryBodies.length}`, + }), + ), + ); + } + return Promise.resolve(new Response(JSON.stringify({ ok: true }))); + }), + ); + const message = { text: "x".repeat(8_100), platformContext: {} }; + const incoming = { + platform: "slack", + externalThreadId: "C123:123.456", + text: "make a design ask", + timestamp: 1, + platformContext: { channelId: "C123", threadTs: "123.456" }, + }; + const opts = { idempotencyKey: "a2a-continuation:cont-1" }; + + await slackAdapter().sendResponse(message, incoming, opts); + await slackAdapter().sendResponse(message, incoming, opts); + + expect(deliveryBodies).toHaveLength(6); + const firstAttemptIds = deliveryBodies + .slice(0, 3) + .map((body) => body.client_msg_id); + const retryIds = deliveryBodies + .slice(3) + .map((body) => body.client_msg_id); + expect(retryIds).toEqual(firstAttemptIds); + expect(new Set(firstAttemptIds).size).toBe(3); + expect( + firstAttemptIds.every( + (id) => + typeof id === "string" && + /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-a[0-9a-f]{3}-[0-9a-f]{12}$/.test( + id, + ), + ), + ).toBe(true); + }); + + it("does not replace a strict stable target with a fresh terminal post", async () => { + process.env.SLACK_BOT_TOKEN = "xoxb-test"; + const deliveryUrls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn((url: string) => { + deliveryUrls.push(String(url)); + return Promise.resolve( + new Response( + JSON.stringify( + String(url).includes("chat.update") + ? { ok: false, error: "message_not_found" } + : { ok: true }, + ), + ), + ); + }), + ); + + await expect( + slackAdapter().sendResponse( + { text: "done", platformContext: {} }, + { + platform: "slack", + externalThreadId: "C123:123.456", + text: "make a design ask", + timestamp: 1, + platformContext: { channelId: "C123", threadTs: "123.456" }, + }, + { + idempotencyKey: "a2a-continuation:cont-1", + placeholderRef: "1719000000.000001", + strictTargetRef: true, + }, + ), + ).rejects.toThrow("message_not_found"); + + expect(deliveryUrls.some((url) => url.includes("chat.postMessage"))).toBe( + false, + ); + }); + it("fails delivery when no Slack bot token is configured", async () => { await expect( slackAdapter().sendResponse( diff --git a/packages/core/src/integrations/adapters/slack.ts b/packages/core/src/integrations/adapters/slack.ts index d918506bec..10eeee96ed 100644 --- a/packages/core/src/integrations/adapters/slack.ts +++ b/packages/core/src/integrations/adapters/slack.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { H3Event } from "h3"; import { createError, getHeader, readRawBody } from "h3"; @@ -24,6 +25,7 @@ import type { PlatformRunProgress, PlatformRunProgressRef, PlatformDeliveryReceipt, + PlatformDeliveryOptions, IntegrationContextMessage, IntegrationFileReference, } from "../types.js"; @@ -441,7 +443,7 @@ export function slackAdapter( async sendResponse( message: OutgoingMessage, context: IncomingMessage, - opts?: { placeholderRef?: string }, + opts?: PlatformDeliveryOptions, ): Promise { const token = await resolveBotToken(context); if (!token) { @@ -456,6 +458,10 @@ export function slackAdapter( | unknown[] | undefined; const placeholderRef = opts?.placeholderRef; + const clientMessageId = (chunkIndex: number) => + opts?.idempotencyKey + ? slackClientMessageId(opts.idempotencyKey, chunkIndex) + : undefined; // Block-rich path: split text into chunks but render the FIRST chunk as // blocks (so we keep the in-place edit + button) and any overflow as @@ -506,12 +512,16 @@ export function slackAdapter( }; if (!data.ok) { console.error("[slack] chat.update error:", data.error); + if (opts?.strictTargetRef) { + throw new Error(data.error || "chat.update failed"); + } // Fall back to a fresh post so the user still sees a reply const postedTs = await postFresh( token, channelId, threadTs, baseBody, + clientMessageId(0), ); if (postedTs) messageRefs.push(postedTs); } else { @@ -523,6 +533,7 @@ export function slackAdapter( channelId, threadTs, baseBody, + clientMessageId(0), ); if (postedTs) messageRefs.push(postedTs); } @@ -534,14 +545,20 @@ export function slackAdapter( } // Overflow chunks (rare) — post as plain follow-ups in the same thread - for (const chunk of restChunks) { - const postedTs = await postFresh(token, channelId, threadTs, { - channel: channelId, - text: chunk, - unfurl_links: false, - unfurl_media: false, - mrkdwn: true, - }); + for (const [index, chunk] of restChunks.entries()) { + const postedTs = await postFresh( + token, + channelId, + threadTs, + { + channel: channelId, + text: chunk, + unfurl_links: false, + unfurl_media: false, + mrkdwn: true, + }, + clientMessageId(index + 1), + ); if (postedTs) messageRefs.push(postedTs); } return { @@ -1094,6 +1111,7 @@ async function postFresh( channelId: string, threadTs: string | undefined, body: Record, + clientMessageId?: string, ): Promise { const hasBlocks = Array.isArray(body.blocks) && (body.blocks as unknown[]).length > 0; @@ -1108,6 +1126,7 @@ async function postFresh( const payload: Record = { ...body, channel: channelId, + ...(clientMessageId ? { client_msg_id: clientMessageId } : {}), }; if (threadTs && !payload.thread_ts) payload.thread_ts = threadTs; const res = await slackApiFetch("https://slack.com/api/chat.postMessage", { @@ -1130,6 +1149,19 @@ async function postFresh( return data.ts; } +function slackClientMessageId(key: string, chunkIndex: number): string { + const digest = createHash("sha256") + .update(`${key}:${chunkIndex}`) + .digest("hex"); + return [ + digest.slice(0, 8), + digest.slice(8, 12), + `5${digest.slice(13, 16)}`, + `a${digest.slice(17, 20)}`, + digest.slice(20, 32), + ].join("-"); +} + async function slackApiFetch( url: string, init: RequestInit, @@ -1775,6 +1807,7 @@ function createSlackRunProgress( return { ref: { kind: "slack-stream", streamTs }, + responseTargetRef: streamTs, async onEvent(event) { if (!cancelControl) { const context = getIntegrationRequestContext(); diff --git a/packages/core/src/integrations/types.ts b/packages/core/src/integrations/types.ts index b664fe4852..510271e71f 100644 --- a/packages/core/src/integrations/types.ts +++ b/packages/core/src/integrations/types.ts @@ -134,6 +134,15 @@ export interface PlatformDeliveryReceipt { messageRefs?: string[]; } +export interface PlatformDeliveryOptions { + /** Provider-owned message reference that should be updated in place. */ + placeholderRef?: string; + /** Stable logical delivery key for provider-side retry deduplication. */ + idempotencyKey?: string; + /** Do not fall back to a fresh post if the stable target cannot be updated. */ + strictTargetRef?: boolean; +} + /** * Proactive outbound message target for a platform. * Used when the agent needs to send to a saved destination instead of replying @@ -201,6 +210,8 @@ export interface PlatformRunProgress { * credentials, or provider payload. */ ref?: PlatformRunProgressRef; + /** Stable provider message target that can receive the terminal answer. */ + responseTargetRef?: string; /** Receive normalized agent events. Implementations should throttle writes. */ onEvent(event: AgentChatEvent): Promise | void; /** Finalize the provider-native progress surface with the answer. */ @@ -322,7 +333,7 @@ export interface PlatformAdapter { sendResponse( message: OutgoingMessage, context: IncomingMessage, - opts?: { placeholderRef?: string }, + opts?: PlatformDeliveryOptions, ): Promise; /** From 83e4c18fa0c01d8df6b256e996eae923fa981116 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:55:58 -0400 Subject: [PATCH 12/16] fix(core): reconcile Slack continuation replies --- .../a2a-continuation-processor.spec.ts | 3 + .../a2a-continuation-processor.ts | 1 + .../src/integrations/adapters/slack.spec.ts | 114 +++++++++-- .../core/src/integrations/adapters/slack.ts | 193 +++++++++++++++--- packages/core/src/integrations/types.ts | 4 +- 5 files changed, 260 insertions(+), 55 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 2af6898bee..dac256d03b 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -811,6 +811,7 @@ describe("A2A continuation processor", () => { expect.any(Object), { idempotencyKey: "a2a-continuation:cont-1", + reconcileAfter: expect.any(Number), placeholderRef: undefined, strictTargetRef: true, }, @@ -1401,6 +1402,7 @@ describe("A2A continuation processor", () => { expect.objectContaining({ platform: "slack" }), expect.objectContaining({ idempotencyKey: "a2a-continuation:cont-1", + reconcileAfter: expect.any(Number), placeholderRef: "1719000000.000001", strictTargetRef: true, }), @@ -1489,6 +1491,7 @@ describe("A2A continuation processor", () => { expect.objectContaining({ platform: "slack" }), expect.objectContaining({ idempotencyKey: "a2a-continuation:cont-1", + reconcileAfter: expect.any(Number), placeholderRef: "1719000000.000001", strictTargetRef: true, }), diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index b26304e96e..16b2f1eb50 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -873,6 +873,7 @@ async function deliverA2AContinuationResponse( } const receipt = await adapter.sendResponse(message, continuation.incoming, { idempotencyKey: `a2a-continuation:${continuation.id}`, + reconcileAfter: continuation.createdAt, placeholderRef: progress?.responseTargetRef ?? continuation.placeholderRef ?? undefined, strictTargetRef: true, diff --git a/packages/core/src/integrations/adapters/slack.spec.ts b/packages/core/src/integrations/adapters/slack.spec.ts index 0acd17229a..db7397ec56 100644 --- a/packages/core/src/integrations/adapters/slack.spec.ts +++ b/packages/core/src/integrations/adapters/slack.spec.ts @@ -1267,12 +1267,32 @@ describe("slackAdapter", () => { }); }); - it("reuses deterministic client message ids across terminal delivery retries", async () => { + it("reconciles accepted terminal chunks before retrying a fresh post", async () => { process.env.SLACK_BOT_TOKEN = "xoxb-test"; const deliveryBodies: Array> = []; + const reconciliationUrls: string[] = []; + let reconciliationCalls = 0; vi.stubGlobal( "fetch", vi.fn((url: string, init?: RequestInit) => { + if (String(url).includes("conversations.replies")) { + reconciliationUrls.push(String(url)); + reconciliationCalls += 1; + return Promise.resolve( + new Response( + JSON.stringify({ + ok: true, + messages: + reconciliationCalls === 1 + ? [] + : deliveryBodies.map((body, index) => ({ + ts: `message-${index + 1}`, + blocks: body.blocks, + })), + }), + ), + ); + } if (String(url).includes("chat.postMessage")) { deliveryBodies.push(JSON.parse(String(init?.body ?? "{}"))); return Promise.resolve( @@ -1295,29 +1315,81 @@ describe("slackAdapter", () => { timestamp: 1, platformContext: { channelId: "C123", threadTs: "123.456" }, }; - const opts = { idempotencyKey: "a2a-continuation:cont-1" }; - - await slackAdapter().sendResponse(message, incoming, opts); - await slackAdapter().sendResponse(message, incoming, opts); - - expect(deliveryBodies).toHaveLength(6); - const firstAttemptIds = deliveryBodies - .slice(0, 3) - .map((body) => body.client_msg_id); - const retryIds = deliveryBodies - .slice(3) - .map((body) => body.client_msg_id); - expect(retryIds).toEqual(firstAttemptIds); - expect(new Set(firstAttemptIds).size).toBe(3); + const opts = { + idempotencyKey: "a2a-continuation:cont-1", + reconcileAfter: 1_783_979_263_000, + }; + + const firstReceipt = await slackAdapter().sendResponse( + message, + incoming, + opts, + ); + const retryReceipt = await slackAdapter().sendResponse( + message, + incoming, + opts, + ); + + expect(deliveryBodies).toHaveLength(3); + expect(reconciliationCalls).toBe(2); + const markers = deliveryBodies.map( + (body) => + (body.blocks as Array<{ block_id?: string }> | undefined)?.[0] + ?.block_id, + ); + expect(new Set(markers).size).toBe(3); expect( - firstAttemptIds.every( - (id) => - typeof id === "string" && - /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-a[0-9a-f]{3}-[0-9a-f]{12}$/.test( - id, - ), + markers.every( + (marker) => + typeof marker === "string" && + /^agent_native_terminal_[0-9a-f]{32}$/.test(marker), ), ).toBe(true); + expect(firstReceipt).toEqual({ + status: "delivered", + messageRefs: ["message-1", "message-2", "message-3"], + }); + expect(retryReceipt).toEqual(firstReceipt); + expect(reconciliationUrls[0]).toContain("oldest=1783979263"); + }); + + it("fails closed when terminal delivery reconciliation is unavailable", async () => { + process.env.SLACK_BOT_TOKEN = "xoxb-test"; + const deliveryUrls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn((url: string) => { + deliveryUrls.push(String(url)); + return Promise.resolve( + new Response( + JSON.stringify( + String(url).includes("conversations.replies") + ? { ok: false, error: "ratelimited" } + : { ok: true, ts: "unexpected" }, + ), + ), + ); + }), + ); + + await expect( + slackAdapter().sendResponse( + { text: "done", platformContext: {} }, + { + platform: "slack", + externalThreadId: "C123:123.456", + text: "make a design ask", + timestamp: 1, + platformContext: { channelId: "C123", threadTs: "123.456" }, + }, + { idempotencyKey: "a2a-continuation:cont-1" }, + ), + ).rejects.toThrow("ratelimited"); + + expect(deliveryUrls.some((url) => url.includes("chat.postMessage"))).toBe( + false, + ); }); it("does not replace a strict stable target with a fresh terminal post", async () => { diff --git a/packages/core/src/integrations/adapters/slack.ts b/packages/core/src/integrations/adapters/slack.ts index 10eeee96ed..8073817083 100644 --- a/packages/core/src/integrations/adapters/slack.ts +++ b/packages/core/src/integrations/adapters/slack.ts @@ -49,6 +49,9 @@ const SLACK_IDENTITY_NEGATIVE_CACHE_TTL_MS = 30 * 1_000; const SLACK_IDENTITY_CACHE_MAX_ENTRIES = 1_000; const SLACK_TOKEN_IDENTITY_CACHE_TTL_MS = 10 * 60 * 1_000; const SLACK_TOKEN_IDENTITY_NEGATIVE_CACHE_TTL_MS = 30 * 1_000; +const SLACK_DELIVERY_RECONCILIATION_PAGE_LIMIT = 200; +const SLACK_DELIVERY_RECONCILIATION_MAX_PAGES = 3; +const SLACK_DELIVERY_MARKER_PREFIX = "agent_native_terminal_"; type SlackTokenIdentity = { teamId: string | null; @@ -458,10 +461,6 @@ export function slackAdapter( | unknown[] | undefined; const placeholderRef = opts?.placeholderRef; - const clientMessageId = (chunkIndex: number) => - opts?.idempotencyKey - ? slackClientMessageId(opts.idempotencyKey, chunkIndex) - : undefined; // Block-rich path: split text into chunks but render the FIRST chunk as // blocks (so we keep the in-place edit + button) and any overflow as @@ -477,6 +476,21 @@ export function slackAdapter( } const restChunks = chunks.slice(1); const messageRefs: string[] = []; + const freshChunkIndexes = [ + ...(!placeholderRef ? [0] : []), + ...restChunks.map((_, index) => index + 1), + ]; + const reconciledRefs = + opts?.idempotencyKey && freshChunkIndexes.length > 0 + ? await reconcileSlackDeliveryChunks( + token, + channelId, + threadTs, + opts.idempotencyKey, + freshChunkIndexes, + opts.reconcileAfter, + ) + : new Map(); const finalBlocks = blocks ?? @@ -520,22 +534,29 @@ export function slackAdapter( token, channelId, threadTs, - baseBody, - clientMessageId(0), + opts?.idempotencyKey + ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0) + : baseBody, ); if (postedTs) messageRefs.push(postedTs); } else { messageRefs.push(data.ts || placeholderRef); } } else { - const postedTs = await postFresh( - token, - channelId, - threadTs, - baseBody, - clientMessageId(0), - ); - if (postedTs) messageRefs.push(postedTs); + const reconciledRef = reconciledRefs.get(0); + if (reconciledRef) { + messageRefs.push(reconciledRef); + } else { + const postedTs = await postFresh( + token, + channelId, + threadTs, + opts?.idempotencyKey + ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0) + : baseBody, + ); + if (postedTs) messageRefs.push(postedTs); + } } // Clear the AI-assistant "is thinking…" status now that we've @@ -546,18 +567,30 @@ export function slackAdapter( // Overflow chunks (rare) — post as plain follow-ups in the same thread for (const [index, chunk] of restChunks.entries()) { + const chunkIndex = index + 1; + const reconciledRef = reconciledRefs.get(chunkIndex); + if (reconciledRef) { + messageRefs.push(reconciledRef); + continue; + } + const overflowBody: Record = { + channel: channelId, + text: chunk, + unfurl_links: false, + unfurl_media: false, + mrkdwn: true, + }; const postedTs = await postFresh( token, channelId, threadTs, - { - channel: channelId, - text: chunk, - unfurl_links: false, - unfurl_media: false, - mrkdwn: true, - }, - clientMessageId(index + 1), + opts?.idempotencyKey + ? withSlackDeliveryMarker( + overflowBody, + opts.idempotencyKey, + chunkIndex, + ) + : overflowBody, ); if (postedTs) messageRefs.push(postedTs); } @@ -1111,7 +1144,6 @@ async function postFresh( channelId: string, threadTs: string | undefined, body: Record, - clientMessageId?: string, ): Promise { const hasBlocks = Array.isArray(body.blocks) && (body.blocks as unknown[]).length > 0; @@ -1126,7 +1158,6 @@ async function postFresh( const payload: Record = { ...body, channel: channelId, - ...(clientMessageId ? { client_msg_id: clientMessageId } : {}), }; if (threadTs && !payload.thread_ts) payload.thread_ts = threadTs; const res = await slackApiFetch("https://slack.com/api/chat.postMessage", { @@ -1149,17 +1180,113 @@ async function postFresh( return data.ts; } -function slackClientMessageId(key: string, chunkIndex: number): string { +function slackDeliveryMarker(key: string, chunkIndex: number): string { const digest = createHash("sha256") .update(`${key}:${chunkIndex}`) - .digest("hex"); - return [ - digest.slice(0, 8), - digest.slice(8, 12), - `5${digest.slice(13, 16)}`, - `a${digest.slice(17, 20)}`, - digest.slice(20, 32), - ].join("-"); + .digest("hex") + .slice(0, 32); + return `${SLACK_DELIVERY_MARKER_PREFIX}${digest}`; +} + +function withSlackDeliveryMarker( + body: Record, + key: string, + chunkIndex: number, +): Record { + const marker = slackDeliveryMarker(key, chunkIndex); + const blocks = Array.isArray(body.blocks) ? body.blocks : []; + if (blocks.length > 0) { + const [first, ...rest] = blocks; + if (first && typeof first === "object" && !Array.isArray(first)) { + return { + ...body, + blocks: [ + { ...(first as Record), block_id: marker }, + ...rest, + ], + }; + } + } + return { + ...body, + blocks: [ + { + type: "section", + block_id: marker, + text: { type: "mrkdwn", text: String(body.text ?? "") }, + }, + ], + }; +} + +async function reconcileSlackDeliveryChunks( + token: string, + channelId: string, + threadTs: string | undefined, + key: string, + chunkIndexes: number[], + reconcileAfter?: number, +): Promise> { + if (!threadTs) { + throw new Error( + "Cannot reconcile an idempotent Slack delivery without a thread timestamp", + ); + } + const refs = new Map(); + const expectedMarkers = new Map( + chunkIndexes.map((chunkIndex) => [ + slackDeliveryMarker(key, chunkIndex), + chunkIndex, + ]), + ); + let cursor = ""; + for ( + let page = 0; + page < SLACK_DELIVERY_RECONCILIATION_MAX_PAGES; + page += 1 + ) { + const url = new URL("https://slack.com/api/conversations.replies"); + url.searchParams.set("channel", channelId); + url.searchParams.set("ts", threadTs); + url.searchParams.set( + "limit", + String(SLACK_DELIVERY_RECONCILIATION_PAGE_LIMIT), + ); + if (cursor) url.searchParams.set("cursor", cursor); + if (typeof reconcileAfter === "number" && reconcileAfter > 0) { + url.searchParams.set("oldest", String(Math.floor(reconcileAfter / 1000))); + url.searchParams.set("inclusive", "true"); + } + const response = await slackApiFetch(url.toString(), { + headers: { Authorization: `Bearer ${token}` }, + }); + const body = (await response.json()) as { + ok: boolean; + error?: string; + messages?: Array<{ + ts?: string; + blocks?: Array<{ block_id?: string }>; + }>; + response_metadata?: { next_cursor?: string }; + }; + if (!body.ok) { + throw new Error(body.error || "conversations.replies failed"); + } + for (const message of body.messages ?? []) { + if (typeof message.ts !== "string") continue; + for (const block of message.blocks ?? []) { + if (typeof block.block_id !== "string") continue; + const chunkIndex = expectedMarkers.get(block.block_id); + if (chunkIndex !== undefined) refs.set(chunkIndex, message.ts); + } + } + if (refs.size === expectedMarkers.size) return refs; + cursor = body.response_metadata?.next_cursor?.trim() ?? ""; + if (!cursor) return refs; + } + throw new Error( + "Slack delivery reconciliation exceeded the bounded thread scan", + ); } async function slackApiFetch( diff --git a/packages/core/src/integrations/types.ts b/packages/core/src/integrations/types.ts index 510271e71f..5a75308e78 100644 --- a/packages/core/src/integrations/types.ts +++ b/packages/core/src/integrations/types.ts @@ -137,8 +137,10 @@ export interface PlatformDeliveryReceipt { export interface PlatformDeliveryOptions { /** Provider-owned message reference that should be updated in place. */ placeholderRef?: string; - /** Stable logical delivery key for provider-side retry deduplication. */ + /** Stable logical delivery key for provider-side retry reconciliation. */ idempotencyKey?: string; + /** Earliest provider timestamp (epoch ms) that can contain this delivery. */ + reconcileAfter?: number; /** Do not fall back to a fresh post if the stable target cannot be updated. */ strictTargetRef?: boolean; } From ab0bdb827c772f1f0512bf04b4fa7aeb9c58a90e Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:59:57 -0400 Subject: [PATCH 13/16] style(core): format Slack delivery repair --- .../src/integrations/a2a-continuation-processor.spec.ts | 9 ++------- packages/core/src/integrations/adapters/slack.ts | 1 + 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index dac256d03b..b1ccb5452f 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -5,10 +5,7 @@ import { buildA2ARecoverableArtifactMessage, } from "../a2a/artifact-response.js"; import type { A2AContinuation } from "./a2a-continuations-store.js"; -import type { - PlatformAdapter, - PlatformDeliveryOptions, -} from "./types.js"; +import type { PlatformAdapter, PlatformDeliveryOptions } from "./types.js"; const claimA2AContinuationMock = vi.hoisted(() => vi.fn()); const claimDueA2AContinuationsMock = vi.hoisted(() => vi.fn(async () => [])); @@ -880,9 +877,7 @@ describe("A2A continuation processor", () => { }); expect(sendResponse).toHaveBeenCalledTimes(2); - expect(providerDeliveries).toEqual( - new Set(["a2a-continuation:cont-1"]), - ); + expect(providerDeliveries).toEqual(new Set(["a2a-continuation:cont-1"])); expect(visibleProviderDeliveries).toBe(1); expect(retainA2AUnconfirmedDeliveryClaimMock).toHaveBeenCalledOnce(); expect(completeA2AContinuationMock).toHaveBeenCalledOnce(); diff --git a/packages/core/src/integrations/adapters/slack.ts b/packages/core/src/integrations/adapters/slack.ts index 8073817083..bca2a419be 100644 --- a/packages/core/src/integrations/adapters/slack.ts +++ b/packages/core/src/integrations/adapters/slack.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; + import type { H3Event } from "h3"; import { createError, getHeader, readRawBody } from "h3"; From cd249196acc7856d4c768a51adae9a7ddde9c1d2 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:06:33 -0400 Subject: [PATCH 14/16] fix(core): cancel Slack delivery before claim release --- .../a2a-continuation-processor.spec.ts | 24 +++++- .../a2a-continuation-processor.ts | 80 ++++++++++++------- .../src/integrations/adapters/slack.spec.ts | 41 ++++++++++ .../core/src/integrations/adapters/slack.ts | 64 +++++++++++---- packages/core/src/integrations/types.ts | 14 +++- 5 files changed, 175 insertions(+), 48 deletions(-) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index b1ccb5452f..294aad3b9f 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -1270,11 +1270,13 @@ describe("A2A continuation processor", () => { ); expect(onEvent).toHaveBeenCalledWith( expect.objectContaining({ type: "agent_call", status: "done" }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ); expect(complete).toHaveBeenCalledWith( expect.objectContaining({ text: "https://slides.agent-native.test/deck/deck-qa", }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ); expect(sendResponse).not.toHaveBeenCalled(); expect(completeA2AContinuationMock).toHaveBeenCalledWith("cont-1"); @@ -1344,6 +1346,7 @@ describe("A2A continuation processor", () => { "https://content.agent-native.com/page/design_ask_123", ), }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ); expect(sendResponse).not.toHaveBeenCalled(); expect(claimA2AContinuationDeliveryMock).toHaveBeenCalledTimes(1); @@ -1386,9 +1389,11 @@ describe("A2A continuation processor", () => { expect.objectContaining({ text: "https://slides.agent-native.test/deck/deck-qa", }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), ); expect(fail).toHaveBeenCalledWith( "I couldn't update the live response, but I posted the final result in this thread.", + expect.objectContaining({ signal: expect.any(AbortSignal) }), ); expect(sendResponse).toHaveBeenCalledWith( expect.objectContaining({ @@ -2810,9 +2815,21 @@ describe("A2A continuation processor", () => { expect(completeA2AContinuationMock).not.toHaveBeenCalled(); }); - it("reschedules and redispatches when the platform send hangs", async () => { + it("aborts and settles a hung platform send before releasing its claim", async () => { vi.useFakeTimers(); - const sendResponse = vi.fn(() => new Promise(() => {})); + let settleAbortedSend: (() => void) | undefined; + const sendResponse = vi.fn( + (_message: unknown, _incoming: unknown, opts?: PlatformDeliveryOptions) => + new Promise((_resolve, reject) => { + opts?.signal?.addEventListener( + "abort", + () => { + settleAbortedSend = () => reject(opts.signal?.reason); + }, + { once: true }, + ); + }), + ); claimA2AContinuationMock.mockResolvedValueOnce(continuation()); const { processA2AContinuationById } = await import("./a2a-continuation-processor.js"); @@ -2822,6 +2839,9 @@ describe("A2A continuation processor", () => { }); await vi.advanceTimersByTimeAsync(12_000); + expect(rescheduleA2AContinuationMock).not.toHaveBeenCalled(); + expect(settleAbortedSend).toBeTypeOf("function"); + settleAbortedSend?.(); await processing; expect(rescheduleA2AContinuationMock).toHaveBeenCalledWith( diff --git a/packages/core/src/integrations/a2a-continuation-processor.ts b/packages/core/src/integrations/a2a-continuation-processor.ts index 16b2f1eb50..4a79b8b59f 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.ts @@ -753,13 +753,15 @@ async function notifyAndFailA2AContinuation( try { outgoing = adapter.formatAgentResponse(message); deliveryReceipt = await withTimeout( - deliverA2AContinuationResponse( - adapter, - deliveryContinuation, - outgoing, - progress, - "error", - ), + (signal) => + deliverA2AContinuationResponse( + adapter, + deliveryContinuation, + outgoing, + progress, + "error", + signal, + ), PLATFORM_SEND_TIMEOUT_MS, `${deliveryContinuation.platform} failure notification timed out`, ); @@ -813,13 +815,15 @@ async function deliverAndCompleteA2AContinuation( stripA2APersistedArtifactMarkers(text), ); deliveryReceipt = await withTimeout( - deliverA2AContinuationResponse( - adapter, - deliveryContinuation, - outgoing, - progress, - "done", - ), + (signal) => + deliverA2AContinuationResponse( + adapter, + deliveryContinuation, + outgoing, + progress, + "done", + signal, + ), PLATFORM_SEND_TIMEOUT_MS, `${deliveryContinuation.platform} response delivery timed out`, ); @@ -844,18 +848,24 @@ async function deliverA2AContinuationResponse( message: OutgoingMessage, progress: PlatformRunProgress | null, status: "done" | "error", + signal: AbortSignal, ): Promise { if (progress) { try { - await progress.onEvent({ - type: "agent_call", - agent: continuation.agentName, - status, - }); - const receipt = await progress.complete(message); + await progress.onEvent( + { + type: "agent_call", + agent: continuation.agentName, + status, + }, + { signal }, + ); + throwIfAborted(signal); + const receipt = await progress.complete(message, { signal }); if (receipt?.status === "delivered") return receipt; throw new Error("Continuation progress completed without delivery proof"); } catch { + throwIfAborted(signal); // A resumed Slack stream can no longer be finalized (for example when // chat.stopStream rejects). Preserve the final answer with the same // thread reply fallback used by the initial webhook run. Also ask the @@ -864,6 +874,7 @@ async function deliverA2AContinuationResponse( try { await progress.fail?.( "I couldn't update the live response, but I posted the final result in this thread.", + { signal }, ); } catch { // The thread reply below is still the authoritative final answer. @@ -871,9 +882,11 @@ async function deliverA2AContinuationResponse( logA2AContinuationTransition("native_progress_fallback", continuation); } } + throwIfAborted(signal); const receipt = await adapter.sendResponse(message, continuation.incoming, { idempotencyKey: `a2a-continuation:${continuation.id}`, reconcileAfter: continuation.createdAt, + signal, placeholderRef: progress?.responseTargetRef ?? continuation.placeholderRef ?? undefined, strictTargetRef: true, @@ -1194,23 +1207,36 @@ function sleep(ms: number): Promise { } async function withTimeout( - promise: Promise, + operation: (signal: AbortSignal) => Promise, timeoutMs: number, message: string, ): Promise { + const controller = new AbortController(); let timer: ReturnType | undefined; + let timedOut = false; try { - return await Promise.race([ - promise, - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(message)), timeoutMs); - }), - ]); + timer = setTimeout(() => { + timedOut = true; + controller.abort(new Error(message)); + }, timeoutMs); + const result = await operation(controller.signal); + if (timedOut) throw new Error(message); + return result; + } catch (error) { + if (timedOut) throw new Error(message); + throw error; } finally { if (timer) clearTimeout(timer); } } +function throwIfAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + throw signal.reason instanceof Error + ? signal.reason + : new Error("Platform delivery was aborted"); +} + function sanitizeFailureReason(reason: string): string { const oneLine = reason.replace(/\s+/g, " ").trim(); const withoutEnvNames = oneLine.replace( diff --git a/packages/core/src/integrations/adapters/slack.spec.ts b/packages/core/src/integrations/adapters/slack.spec.ts index db7397ec56..cd4a5d13ea 100644 --- a/packages/core/src/integrations/adapters/slack.spec.ts +++ b/packages/core/src/integrations/adapters/slack.spec.ts @@ -1392,6 +1392,47 @@ describe("slackAdapter", () => { ); }); + it("aborts reconciliation before a fresh terminal post can outlive its claim", async () => { + process.env.SLACK_BOT_TOKEN = "xoxb-test"; + const controller = new AbortController(); + const deliveryUrls: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn((url: string, init?: RequestInit) => { + deliveryUrls.push(String(url)); + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { once: true }, + ); + }); + }), + ); + + const delivery = slackAdapter().sendResponse( + { text: "done", platformContext: {} }, + { + platform: "slack", + externalThreadId: "C123:123.456", + text: "make a design ask", + timestamp: 1, + platformContext: { channelId: "C123", threadTs: "123.456" }, + }, + { + idempotencyKey: "a2a-continuation:cont-1", + signal: controller.signal, + }, + ); + await vi.waitFor(() => expect(deliveryUrls).toHaveLength(1)); + controller.abort(new Error("delivery claim expired")); + + await expect(delivery).rejects.toThrow("delivery claim expired"); + expect(deliveryUrls).toEqual([ + expect.stringContaining("conversations.replies"), + ]); + }); + it("does not replace a strict stable target with a fresh terminal post", async () => { process.env.SLACK_BOT_TOKEN = "xoxb-test"; const deliveryUrls: string[] = []; diff --git a/packages/core/src/integrations/adapters/slack.ts b/packages/core/src/integrations/adapters/slack.ts index bca2a419be..3d69061a6c 100644 --- a/packages/core/src/integrations/adapters/slack.ts +++ b/packages/core/src/integrations/adapters/slack.ts @@ -490,6 +490,7 @@ export function slackAdapter( opts.idempotencyKey, freshChunkIndexes, opts.reconcileAfter, + opts.signal, ) : new Map(); @@ -519,6 +520,7 @@ export function slackAdapter( "Content-Type": "application/json", }, body: JSON.stringify({ ...baseBody, ts: placeholderRef }), + signal: opts?.signal, }); const data = (await res.json()) as { ok: boolean; @@ -538,6 +540,7 @@ export function slackAdapter( opts?.idempotencyKey ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0) : baseBody, + opts?.signal, ); if (postedTs) messageRefs.push(postedTs); } else { @@ -555,6 +558,7 @@ export function slackAdapter( opts?.idempotencyKey ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0) : baseBody, + opts?.signal, ); if (postedTs) messageRefs.push(postedTs); } @@ -592,6 +596,7 @@ export function slackAdapter( chunkIndex, ) : overflowBody, + opts?.signal, ); if (postedTs) messageRefs.push(postedTs); } @@ -1145,6 +1150,7 @@ async function postFresh( channelId: string, threadTs: string | undefined, body: Record, + signal?: AbortSignal, ): Promise { const hasBlocks = Array.isArray(body.blocks) && (body.blocks as unknown[]).length > 0; @@ -1168,6 +1174,7 @@ async function postFresh( "Content-Type": "application/json", }, body: JSON.stringify(payload), + signal, }); const data = (await res.json()) as { ok: boolean; @@ -1227,6 +1234,7 @@ async function reconcileSlackDeliveryChunks( key: string, chunkIndexes: number[], reconcileAfter?: number, + signal?: AbortSignal, ): Promise> { if (!threadTs) { throw new Error( @@ -1260,6 +1268,7 @@ async function reconcileSlackDeliveryChunks( } const response = await slackApiFetch(url.toString(), { headers: { Authorization: `Bearer ${token}` }, + signal, }); const body = (await response.json()) as { ok: boolean; @@ -1297,6 +1306,16 @@ async function slackApiFetch( ): Promise { const controller = typeof AbortController !== "undefined" ? new AbortController() : undefined; + const externalSignal = init.signal; + const abortFromExternal = () => controller?.abort(externalSignal?.reason); + if (controller && externalSignal) { + if (externalSignal.aborted) abortFromExternal(); + else { + externalSignal.addEventListener("abort", abortFromExternal, { + once: true, + }); + } + } const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : undefined; @@ -1307,6 +1326,7 @@ async function slackApiFetch( }); } finally { if (timer) clearTimeout(timer); + externalSignal?.removeEventListener("abort", abortFromExternal); } } @@ -1778,6 +1798,7 @@ async function postSlackJson( token: string, method: string, body: Record, + signal?: AbortSignal, ): Promise> { const response = await slackApiFetch(`https://slack.com/api/${method}`, { method: "POST", @@ -1786,6 +1807,7 @@ async function postSlackJson( "Content-Type": "application/json", }, body: JSON.stringify(body), + signal, }); const data = (await response.json()) as Record; if (!data.ok) throw new Error(data.error || `${method} failed`); @@ -2094,7 +2116,7 @@ function createSlackRunProgress( }); } }, - async complete(message) { + async complete(message, opts) { if (pendingTimer) clearTimeout(pendingTimer); const finalChunks = [...tasks.entries()].map(([id, task]) => ({ type: "task_update", @@ -2129,25 +2151,35 @@ function createSlackRunProgress( }, ] : []; - await postSlackJson(token, "chat.stopStream", { - channel, - ts: streamTs, - markdown_text: message.text || "Done.", - ...(finalChunks.length ? { chunks: finalChunks } : {}), - ...(messageBlocks.length || controlBlocks.length - ? { blocks: [...messageBlocks, ...controlBlocks].slice(0, 50) } - : {}), - }); + await postSlackJson( + token, + "chat.stopStream", + { + channel, + ts: streamTs, + markdown_text: message.text || "Done.", + ...(finalChunks.length ? { chunks: finalChunks } : {}), + ...(messageBlocks.length || controlBlocks.length + ? { blocks: [...messageBlocks, ...controlBlocks].slice(0, 50) } + : {}), + }, + opts?.signal, + ); setSlackAssistantStatus(token, channel, threadTs, ""); return { status: "delivered", messageRefs: [streamTs] }; }, - async fail(message) { + async fail(message, opts) { if (pendingTimer) clearTimeout(pendingTimer); - await postSlackJson(token, "chat.stopStream", { - channel, - ts: streamTs, - markdown_text: message.slice(0, SLACK_MAX_LENGTH), - }).catch(() => {}); + await postSlackJson( + token, + "chat.stopStream", + { + channel, + ts: streamTs, + markdown_text: message.slice(0, SLACK_MAX_LENGTH), + }, + opts?.signal, + ).catch(() => {}); setSlackAssistantStatus(token, channel, threadTs, ""); }, }; diff --git a/packages/core/src/integrations/types.ts b/packages/core/src/integrations/types.ts index 5a75308e78..32a3c6076b 100644 --- a/packages/core/src/integrations/types.ts +++ b/packages/core/src/integrations/types.ts @@ -141,6 +141,8 @@ export interface PlatformDeliveryOptions { idempotencyKey?: string; /** Earliest provider timestamp (epoch ms) that can contain this delivery. */ reconcileAfter?: number; + /** Abort provider I/O before the caller releases its durable delivery claim. */ + signal?: AbortSignal; /** Do not fall back to a fresh post if the stable target cannot be updated. */ strictTargetRef?: boolean; } @@ -215,11 +217,17 @@ export interface PlatformRunProgress { /** Stable provider message target that can receive the terminal answer. */ responseTargetRef?: string; /** Receive normalized agent events. Implementations should throttle writes. */ - onEvent(event: AgentChatEvent): Promise | void; + onEvent( + event: AgentChatEvent, + opts?: { signal?: AbortSignal }, + ): Promise | void; /** Finalize the provider-native progress surface with the answer. */ - complete(message: OutgoingMessage): Promise; + complete( + message: OutgoingMessage, + opts?: { signal?: AbortSignal }, + ): Promise; /** Mark the provider-native surface failed and leave a retryable explanation. */ - fail?(message: string): Promise; + fail?(message: string, opts?: { signal?: AbortSignal }): Promise; } /** From a74eaa8e1773109e3c1dd8d8646177195618541f Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:10:18 -0400 Subject: [PATCH 15/16] fix(core): keep Slack body reads abortable --- .../src/integrations/adapters/slack.spec.ts | 19 ++++--- .../core/src/integrations/adapters/slack.ts | 55 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/packages/core/src/integrations/adapters/slack.spec.ts b/packages/core/src/integrations/adapters/slack.spec.ts index cd4a5d13ea..ca740df7a6 100644 --- a/packages/core/src/integrations/adapters/slack.spec.ts +++ b/packages/core/src/integrations/adapters/slack.spec.ts @@ -1392,7 +1392,7 @@ describe("slackAdapter", () => { ); }); - it("aborts reconciliation before a fresh terminal post can outlive its claim", async () => { + it("aborts a stalled reconciliation body before a fresh post outlives its claim", async () => { process.env.SLACK_BOT_TOKEN = "xoxb-test"; const controller = new AbortController(); const deliveryUrls: string[] = []; @@ -1400,13 +1400,16 @@ describe("slackAdapter", () => { "fetch", vi.fn((url: string, init?: RequestInit) => { deliveryUrls.push(String(url)); - return new Promise((_resolve, reject) => { - init?.signal?.addEventListener( - "abort", - () => reject(init.signal?.reason), - { once: true }, - ); - }); + return Promise.resolve({ + json: () => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { once: true }, + ); + }), + } as Response); }), ); diff --git a/packages/core/src/integrations/adapters/slack.ts b/packages/core/src/integrations/adapters/slack.ts index 3d69061a6c..cb6e520260 100644 --- a/packages/core/src/integrations/adapters/slack.ts +++ b/packages/core/src/integrations/adapters/slack.ts @@ -513,16 +513,18 @@ export function slackAdapter( try { if (placeholderRef) { // Replace the "thinking…" placeholder in place. - const res = await slackApiFetch("https://slack.com/api/chat.update", { - method: "POST", - headers: { - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", + const data = (await slackApiJson( + "https://slack.com/api/chat.update", + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ ...baseBody, ts: placeholderRef }), + signal: opts?.signal, }, - body: JSON.stringify({ ...baseBody, ts: placeholderRef }), - signal: opts?.signal, - }); - const data = (await res.json()) as { + )) as { ok: boolean; error?: string; ts?: string; @@ -684,7 +686,7 @@ export function slackAdapter( if (target.threadRef) body.thread_ts = target.threadRef; try { - const res = await slackApiFetch( + const data = (await slackApiJson( "https://slack.com/api/chat.postMessage", { method: "POST", @@ -694,8 +696,7 @@ export function slackAdapter( }, body: JSON.stringify(body), }, - ); - const data = (await res.json()) as { ok: boolean; error?: string }; + )) as { ok: boolean; error?: string }; if (!data.ok) { throw new Error(data.error || "chat.postMessage failed"); } @@ -1093,7 +1094,7 @@ function setSlackAssistantStatus( threadTs: string, status: string, ): void { - slackApiFetch("https://slack.com/api/assistant.threads.setStatus", { + slackApiJson("https://slack.com/api/assistant.threads.setStatus", { method: "POST", headers: { Authorization: `Bearer ${token}`, @@ -1167,7 +1168,7 @@ async function postFresh( channel: channelId, }; if (threadTs && !payload.thread_ts) payload.thread_ts = threadTs; - const res = await slackApiFetch("https://slack.com/api/chat.postMessage", { + const data = (await slackApiJson("https://slack.com/api/chat.postMessage", { method: "POST", headers: { Authorization: `Bearer ${token}`, @@ -1175,8 +1176,7 @@ async function postFresh( }, body: JSON.stringify(payload), signal, - }); - const data = (await res.json()) as { + })) as { ok: boolean; error?: string; ts?: string; @@ -1266,11 +1266,10 @@ async function reconcileSlackDeliveryChunks( url.searchParams.set("oldest", String(Math.floor(reconcileAfter / 1000))); url.searchParams.set("inclusive", "true"); } - const response = await slackApiFetch(url.toString(), { + const body = (await slackApiJson(url.toString(), { headers: { Authorization: `Bearer ${token}` }, signal, - }); - const body = (await response.json()) as { + })) as { ok: boolean; error?: string; messages?: Array<{ @@ -1299,11 +1298,11 @@ async function reconcileSlackDeliveryChunks( ); } -async function slackApiFetch( +async function slackApiJson( url: string, init: RequestInit, timeoutMs = SLACK_API_TIMEOUT_MS, -): Promise { +): Promise> { const controller = typeof AbortController !== "undefined" ? new AbortController() : undefined; const externalSignal = init.signal; @@ -1320,10 +1319,11 @@ async function slackApiFetch( ? setTimeout(() => controller.abort(), timeoutMs) : undefined; try { - return await fetch(url, { + const response = await fetch(url, { ...init, signal: controller?.signal ?? init.signal, }); + return (await response.json()) as Record; } finally { if (timer) clearTimeout(timer); externalSignal?.removeEventListener("abort", abortFromExternal); @@ -1345,12 +1345,11 @@ async function resolveSlackThreadPermalink( const url = new URL("https://slack.com/api/chat.getPermalink"); url.searchParams.set("channel", channelId); url.searchParams.set("message_ts", threadTs); - const res = await slackApiFetch( + const data = (await slackApiJson( url.toString(), { headers: { Authorization: `Bearer ${token}` } }, SLACK_PERMALINK_TIMEOUT_MS, - ); - const data = (await res.json()) as { + )) as { ok?: boolean; permalink?: unknown; }; @@ -1440,12 +1439,11 @@ async function slackJson( for (const [key, value] of Object.entries(params)) { url.searchParams.set(key, value); } - const response = await slackApiFetch( + const body = await slackApiJson( url.toString(), { headers: { Authorization: `Bearer ${token}` } }, timeoutMs, ); - const body = (await response.json()) as Record; return body.ok ? body : null; } catch { return null; @@ -1800,7 +1798,7 @@ async function postSlackJson( body: Record, signal?: AbortSignal, ): Promise> { - const response = await slackApiFetch(`https://slack.com/api/${method}`, { + const data = await slackApiJson(`https://slack.com/api/${method}`, { method: "POST", headers: { Authorization: `Bearer ${token}`, @@ -1809,7 +1807,6 @@ async function postSlackJson( body: JSON.stringify(body), signal, }); - const data = (await response.json()) as Record; if (!data.ok) throw new Error(data.error || `${method} failed`); return data; } From 9e8a5ba71fc0961a64bf4ccd0bd993f06b97ef87 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:16:07 -0400 Subject: [PATCH 16/16] test(core): expect delivery cancellation signal --- .../core/src/integrations/a2a-continuation-processor.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/integrations/a2a-continuation-processor.spec.ts b/packages/core/src/integrations/a2a-continuation-processor.spec.ts index 294aad3b9f..4194006e81 100644 --- a/packages/core/src/integrations/a2a-continuation-processor.spec.ts +++ b/packages/core/src/integrations/a2a-continuation-processor.spec.ts @@ -809,6 +809,7 @@ describe("A2A continuation processor", () => { { idempotencyKey: "a2a-continuation:cont-1", reconcileAfter: expect.any(Number), + signal: expect.any(AbortSignal), placeholderRef: undefined, strictTargetRef: true, },