|
| 1 | +import { afterEach, beforeEach, expect, test } from "bun:test"; |
| 2 | +import { Database } from "bun:sqlite"; |
| 3 | +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 4 | +import { tmpdir } from "node:os"; |
| 5 | +import { join } from "node:path"; |
| 6 | +import { DreamTokenBudgetExceeded } from "../src/features/magic-context/dreamer/token-budget"; |
| 7 | +import { dreamBudgetChildSessionId, normalizeError, runFailureSentinel } from "./subagent-failure-sentinel"; |
| 8 | + |
| 9 | +const NOW = 2_000_000_000_000; |
| 10 | +const HOUR = 3_600_000; |
| 11 | +const PROJECT = "/fixture/project"; |
| 12 | +const CHILDREN = ["ses_e3247cff8ffex5Zghx4fDip24A", "ses_e324513faffeXtFIkdMrhuJCGL"]; |
| 13 | +const PARENTS = ["ses_12f72c654ffe94Fkze4RXRpXng", "ses_0d9602e50ffe0yxsEU1JrCXRVx"]; |
| 14 | +let root: string; |
| 15 | +let db: Database; |
| 16 | + |
| 17 | +beforeEach(() => { |
| 18 | + mkdirSync(join(tmpdir(), "magic-context"), { recursive: true }); |
| 19 | + root = mkdtempSync(join(tmpdir(), "magic-context", "subagent-failure-budget-")); |
| 20 | + db = new Database(join(root, "context.db")); |
| 21 | + db.exec(readFileSync(join(import.meta.dir, "../../../crates/mc-module/tests/fixtures/context-db-schema.sql"), "utf8")); |
| 22 | +}); |
| 23 | +afterEach(() => { db.close(false); rmSync(root, { recursive: true, force: true }); }); |
| 24 | + |
| 25 | +function budgetError(child = CHILDREN[0]!, spent = 18_891) { |
| 26 | + const error = new DreamTokenBudgetExceeded(child, spent); |
| 27 | + return `${error.name} message="${error.message}"`; |
| 28 | +} |
| 29 | + |
| 30 | +function invocation(error = budgetError(), time = NOW - 1_000, parent = PARENTS[0]!, subagent = "dreamer") { |
| 31 | + const row = db.query("INSERT INTO subagent_invocations (session_id, harness, subagent, task, started_at, ended_at, status, error) VALUES (?, 'opencode', ?, 'verify', ?, ?, 'failed', ?) RETURNING id") |
| 32 | + .get(parent, subagent, time - 100, time, error) as { id: number }; |
| 33 | + db.query("INSERT OR REPLACE INTO session_projects (session_id, harness, project_path, updated_at) VALUES (?, 'opencode', ?, ?)") |
| 34 | + .run(parent, PROJECT, time); |
| 35 | + return row.id; |
| 36 | +} |
| 37 | + |
| 38 | +function dreamRun(status: "completed" | "failed", child = CHILDREN[0]!, finished = NOW - 500, project = PROJECT) { |
| 39 | + const second = child === CHILDREN[1]; |
| 40 | + const tasks = [{ |
| 41 | + name: "verify", status, durationMs: 10_000, resultChars: 0, |
| 42 | + ...(status === "failed" ? { error: budgetError(child, 1_502_666), failure: { failure_class: "token_budget" } } : {}), |
| 43 | + progress: second ? "verify: processed 36 (verified 15, archived 21); 723 remain" : "verify: processed 50 (verified 45, refused 5); 75 remain", |
| 44 | + children: [{ sessionId: child, promptTokens: second ? 32_486 : 18_891 }], |
| 45 | + tokenBudget: { budget: status === "failed" ? 1_500_000 : 2_500_000, spent: status === "failed" ? 1_502_666 : second ? 2_502_431 : 2_500_109, finalizeFired: status === "completed", banked: second ? 36 : 45 }, |
| 46 | + }]; |
| 47 | + db.query("INSERT INTO dream_runs (project_path, started_at, finished_at, holder_id, tasks_json, tasks_succeeded, tasks_failed, smart_notes_surfaced, smart_notes_pending, memory_changes_json, parent_session_id) VALUES (?, ?, ?, 'fixture-holder', ?, ?, ?, 0, 0, NULL, ?)") |
| 48 | + .run(project, finished - 10_000, finished, JSON.stringify(tasks), status === "completed" ? 1 : 0, status === "failed" ? 1 : 0, PARENTS[second ? 1 : 0]!); |
| 49 | +} |
| 50 | + |
| 51 | +async function run(now = NOW) { |
| 52 | + const logs: string[] = []; |
| 53 | + const result = await runFailureSentinel({ db: join(root, "context.db"), stateFile: join(root, "state.json"), connectionFile: join(root, "none"), send: false, settleMs: 0, now: () => now, stdout: () => {}, stderr: (line) => logs.push(line) }); |
| 54 | + return { ...result, summary: JSON.parse(logs.at(-1)!), state: JSON.parse(readFileSync(join(root, "state.json"), "utf8")) }; |
| 55 | +} |
| 56 | + |
| 57 | +test("budget regression: class text folds the two wake child ids and message ids", () => { |
| 58 | + const expected = 'DreamTokenBudgetExceeded message="Dreamer child <id> spent <number> prompt tokens (token_budget)"'; |
| 59 | + expect(normalizeError(budgetError())).toBe(expected); |
| 60 | + expect(normalizeError(budgetError(CHILDREN[1], 32_486))).toBe(expected); |
| 61 | + expect(dreamBudgetChildSessionId(budgetError())).toBe(CHILDREN[0]); |
| 62 | + expect(normalizeError("error in msg_abc123 request 6ccd58a4-fd7d-4695-ae30-b3cbd8c5b633")) |
| 63 | + .toBe(normalizeError("error in msg_def456 request 3f5266e0-127a-42d4-9a78-8f5c32e6bc8c")); |
| 64 | +}); |
| 65 | + |
| 66 | +test("budget regression: old state classes are normalised on load without a new-class wake", async () => { |
| 67 | + const oldClass = `DreamTokenBudgetExceeded message="Dreamer child ${CHILDREN[0]} spent <number> prompt tokens (token_budget)"`; |
| 68 | + writeFileSync(join(root, "state.json"), JSON.stringify({ version: 1, cursor: { ended_at: NOW - 300_000, id: 0 }, sent: { |
| 69 | + [JSON.stringify(["new_class", "dreamer", "verify", oldClass])]: { at: NOW - 60_000, count: 1 }, |
| 70 | + } })); |
| 71 | + invocation(budgetError(CHILDREN[1], 32_486), NOW - 1_000, PARENTS[1]); |
| 72 | + dreamRun("failed", CHILDREN[1]); |
| 73 | + const result = await run(); |
| 74 | + expect(result.alerts).toEqual([]); |
| 75 | + expect(Object.keys(result.state.sent)).toEqual([JSON.stringify(["new_class", "dreamer", "verify", normalizeError(budgetError())])]); |
| 76 | +}); |
| 77 | + |
| 78 | +test("budget regression: completed verify runs exclude their child stops from every rule and log the count", async () => { |
| 79 | + invocation(); |
| 80 | + invocation(budgetError(CHILDREN[1], 32_486), NOW - 900, PARENTS[1]); |
| 81 | + dreamRun("completed"); |
| 82 | + dreamRun("completed", CHILDREN[1]); |
| 83 | + const result = await run(); |
| 84 | + expect(result.alerts).toEqual([]); |
| 85 | + expect(result.summary.budget_stops_not_counted).toBe(2); |
| 86 | +}); |
| 87 | + |
| 88 | +test("colliding old class reservations combine counts instead of re-alerting on upgrade", async () => { |
| 89 | + const sent = Object.fromEntries(CHILDREN.map((child) => [JSON.stringify(["new_class", "dreamer", "verify", |
| 90 | + `DreamTokenBudgetExceeded message="Dreamer child ${child} spent <number> prompt tokens (token_budget)"`]), { at: NOW - 60_000, count: 1 }])); |
| 91 | + writeFileSync(join(root, "state.json"), JSON.stringify({ version: 1, cursor: { ended_at: NOW - 300_000, id: 0 }, sent })); |
| 92 | + for (const child of CHILDREN) { invocation(budgetError(child)); dreamRun("failed", child); } |
| 93 | + const result = await run(); |
| 94 | + expect(result.alerts).toEqual([]); |
| 95 | + expect(Object.values(result.state.sent)).toEqual([{ at: NOW - 60_000, count: 2 }]); |
| 96 | +}); |
| 97 | + |
| 98 | +test("a failed task with the budget class still wakes with the existing new-class rule", async () => { |
| 99 | + invocation(budgetError(CHILDREN[0], 1_502_666)); |
| 100 | + dreamRun("failed"); |
| 101 | + const result = await run(); |
| 102 | + expect(result.alerts.map((alert) => alert.rule)).toEqual(["new_class"]); |
| 103 | + expect(result.alerts[0]).toMatchObject({ count: 1, share: 1, class: normalizeError(budgetError()) }); |
| 104 | + expect(result.summary.budget_stops_not_counted).toBe(0); |
| 105 | +}); |
| 106 | + |
| 107 | +test("a pending budget stop survives restart and is retried without a new invocation", async () => { |
| 108 | + const id = invocation(); |
| 109 | + const first = await run(); |
| 110 | + expect(first.alerts).toEqual([]); |
| 111 | + expect(first.state.pending).toEqual([id]); |
| 112 | + expect(first.state.cursor.id).toBe(id); |
| 113 | + dreamRun("completed", CHILDREN[0], NOW + 30_000); |
| 114 | + const second = await run(NOW + 60_000); |
| 115 | + expect(second.alerts).toEqual([]); |
| 116 | + expect(second.summary.budget_stops_not_counted).toBe(1); |
| 117 | + expect(second.state.pending).toEqual([]); |
| 118 | +}); |
| 119 | + |
| 120 | +test("an unresolved budget stop counts as a failure after sixty minutes", async () => { |
| 121 | + invocation(); |
| 122 | + expect((await run()).alerts).toEqual([]); |
| 123 | + const result = await run(NOW + HOUR); |
| 124 | + expect(result.alerts.map((alert) => alert.rule)).toEqual(["new_class"]); |
| 125 | + expect(result.summary.budget_stops_not_counted).toBe(0); |
| 126 | + expect(result.state.pending).toEqual([]); |
| 127 | +}); |
| 128 | + |
| 129 | +test("a late failed task row wakes on the retry", async () => { |
| 130 | + invocation(); |
| 131 | + expect((await run()).alerts).toEqual([]); |
| 132 | + dreamRun("failed", CHILDREN[0], NOW + 30_000); |
| 133 | + expect((await run(NOW + 60_000)).alerts.map((alert) => alert.rule)).toEqual(["new_class"]); |
| 134 | +}); |
| 135 | + |
| 136 | +test("completed budget stops are absent from failure counts and share denominators", async () => { |
| 137 | + for (let i = 0; i < 6; i++) { |
| 138 | + db.query("INSERT INTO subagent_invocations (session_id, harness, subagent, task, started_at, ended_at, status) VALUES ('fixture', 'opencode', 'dreamer', 'verify', ?, ?, 'completed')") |
| 139 | + .run(NOW - 2_100, NOW - 2_000); |
| 140 | + } |
| 141 | + invocation(); |
| 142 | + dreamRun("completed"); |
| 143 | + invocation("actual failure A", NOW - 800); |
| 144 | + invocation("actual failure B", NOW - 700); |
| 145 | + const result = await run(); |
| 146 | + const share = result.alerts.find((alert) => alert.rule === "failure_share"); |
| 147 | + expect(share).toMatchObject({ count: 2, share: 0.25 }); |
| 148 | + expect(result.alerts.some((alert) => alert.example_error.includes("token_budget"))).toBe(false); |
| 149 | +}); |
| 150 | + |
| 151 | +test("budget lookup requires the child, project, and sixty-minute run window", async () => { |
| 152 | + invocation(); |
| 153 | + dreamRun("completed", CHILDREN[1]); |
| 154 | + dreamRun("completed", CHILDREN[0], NOW - 500, "/another/project"); |
| 155 | + dreamRun("completed", CHILDREN[0], NOW - 2_000); |
| 156 | + dreamRun("completed", CHILDREN[0], NOW + HOUR); |
| 157 | + expect((await run()).state.pending).toHaveLength(1); |
| 158 | + expect((await run(NOW + HOUR)).alerts.map((alert) => alert.rule)).toEqual(["new_class"]); |
| 159 | +}); |
| 160 | + |
| 161 | +test("malformed task JSON is ignored and a missing project falls back to the time window", async () => { |
| 162 | + invocation(); |
| 163 | + db.exec("DELETE FROM session_projects"); |
| 164 | + for (const tasks of ["{bad", "{}", '[null, {"children":{}}, {"children":[null]}]']) { |
| 165 | + db.query("INSERT INTO dream_runs (project_path, started_at, finished_at, holder_id, tasks_json) VALUES (?, ?, ?, 'fixture', ?)") |
| 166 | + .run(PROJECT, NOW - 10_000, NOW - 500, tasks); |
| 167 | + } |
| 168 | + dreamRun("completed"); |
| 169 | + const result = await run(); |
| 170 | + expect(result.alerts).toEqual([]); |
| 171 | + expect(result.summary.budget_stops_not_counted).toBe(1); |
| 172 | +}); |
| 173 | + |
| 174 | +test("an unextractable budget error is judged as a failure rather than silenced", async () => { |
| 175 | + invocation("DreamTokenBudgetExceeded message=truncated token_budget"); |
| 176 | + expect((await run()).alerts.map((alert) => alert.rule)).toEqual(["new_class"]); |
| 177 | +}); |
| 178 | + |
| 179 | +test("non-budget errors and non-dreamer failures need no dream run lookup", async () => { |
| 180 | + invocation("ordinary failure"); |
| 181 | + invocation(budgetError(), NOW - 900, PARENTS[1], "historian"); |
| 182 | + db.exec("DROP TABLE dream_runs; DROP TABLE session_projects"); |
| 183 | + expect((await run()).alerts.map((alert) => alert.rule)).toEqual(["new_class", "new_class"]); |
| 184 | +}); |
| 185 | + |
| 186 | +test("pending budget stops are bounded without losing the unprocessed cursor tail", async () => { |
| 187 | + for (let i = 0; i < 501; i++) invocation(budgetError(`ses_child${i}`), NOW - 1_000 + i); |
| 188 | + const first = await run(); |
| 189 | + expect(first.state.pending).toHaveLength(500); |
| 190 | + expect(first.bounded).toBe(true); |
| 191 | + const second = await run(NOW + 1_000); |
| 192 | + expect(second.state.pending).toHaveLength(500); |
| 193 | + expect(second.state.cursor.id).toBe(500); |
| 194 | + expect(second.bounded).toBe(true); |
| 195 | + const expired = await run(NOW + HOUR); |
| 196 | + expect(expired.state.pending).toEqual([]); |
| 197 | + expect(expired.state.cursor.id).toBe(501); |
| 198 | + expect(expired.alerts.map((alert) => alert.rule)).toContain("failure_share"); |
| 199 | +}); |
0 commit comments