Skip to content

Commit 30f6c8b

Browse files
Background-agent failure watcher: a dreamer budget stop that kept its progress is not a failure
A dreamer task has a prompt-token budget shared by its child sessions. The child running when the budget is spent reports DreamTokenBudgetExceeded even when the task then ends completed with its progress kept. The watcher counted that as a failure, and because the error text kept the child's session id, every occurrence was a new error class and woke at urgent level. - Session ids in an error text are folded like other ids, and classes saved under the old text are normalised on load, so the change sends no wake for classes already known. - For a dreamer budget error the watcher looks up the run that names the child (dream_runs, same project, within 60 minutes). A completed task is left out of every rule and counted in the run summary; a failed task wakes as before. No run row yet: held and judged on a later run, and counted as a failure if none appears within 60 minutes. - An error text the child id cannot be read from is judged as before. Co-authored-by: Alfonso <alfonso-magic-context@users.noreply.github.com>
1 parent 5933477 commit 30f6c8b

2 files changed

Lines changed: 289 additions & 9 deletions

File tree

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import { afterEach, beforeEach, expect, test } from "bun:test";
2+
import { Database } from "bun:sqlite";
3+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
4+
import { tmpdir } from "node:os";
5+
import { join } from "node:path";
6+
import { DreamTokenBudgetExceeded } from "../src/features/magic-context/dreamer/token-budget";
7+
import { dreamBudgetChildSessionId, normalizeError, runFailureSentinel } from "./subagent-failure-sentinel";
8+
9+
const NOW = 2_000_000_000_000;
10+
const HOUR = 3_600_000;
11+
const PROJECT = "/fixture/project";
12+
const CHILDREN = ["ses_e3247cff8ffex5Zghx4fDip24A", "ses_e324513faffeXtFIkdMrhuJCGL"];
13+
const PARENTS = ["ses_12f72c654ffe94Fkze4RXRpXng", "ses_0d9602e50ffe0yxsEU1JrCXRVx"];
14+
let root: string;
15+
let db: Database;
16+
17+
beforeEach(() => {
18+
mkdirSync(join(tmpdir(), "magic-context"), { recursive: true });
19+
root = mkdtempSync(join(tmpdir(), "magic-context", "subagent-failure-budget-"));
20+
db = new Database(join(root, "context.db"));
21+
db.exec(readFileSync(join(import.meta.dir, "../../../crates/mc-module/tests/fixtures/context-db-schema.sql"), "utf8"));
22+
});
23+
afterEach(() => { db.close(false); rmSync(root, { recursive: true, force: true }); });
24+
25+
function budgetError(child = CHILDREN[0]!, spent = 18_891) {
26+
const error = new DreamTokenBudgetExceeded(child, spent);
27+
return `${error.name} message="${error.message}"`;
28+
}
29+
30+
function invocation(error = budgetError(), time = NOW - 1_000, parent = PARENTS[0]!, subagent = "dreamer") {
31+
const row = db.query("INSERT INTO subagent_invocations (session_id, harness, subagent, task, started_at, ended_at, status, error) VALUES (?, 'opencode', ?, 'verify', ?, ?, 'failed', ?) RETURNING id")
32+
.get(parent, subagent, time - 100, time, error) as { id: number };
33+
db.query("INSERT OR REPLACE INTO session_projects (session_id, harness, project_path, updated_at) VALUES (?, 'opencode', ?, ?)")
34+
.run(parent, PROJECT, time);
35+
return row.id;
36+
}
37+
38+
function dreamRun(status: "completed" | "failed", child = CHILDREN[0]!, finished = NOW - 500, project = PROJECT) {
39+
const second = child === CHILDREN[1];
40+
const tasks = [{
41+
name: "verify", status, durationMs: 10_000, resultChars: 0,
42+
...(status === "failed" ? { error: budgetError(child, 1_502_666), failure: { failure_class: "token_budget" } } : {}),
43+
progress: second ? "verify: processed 36 (verified 15, archived 21); 723 remain" : "verify: processed 50 (verified 45, refused 5); 75 remain",
44+
children: [{ sessionId: child, promptTokens: second ? 32_486 : 18_891 }],
45+
tokenBudget: { budget: status === "failed" ? 1_500_000 : 2_500_000, spent: status === "failed" ? 1_502_666 : second ? 2_502_431 : 2_500_109, finalizeFired: status === "completed", banked: second ? 36 : 45 },
46+
}];
47+
db.query("INSERT INTO dream_runs (project_path, started_at, finished_at, holder_id, tasks_json, tasks_succeeded, tasks_failed, smart_notes_surfaced, smart_notes_pending, memory_changes_json, parent_session_id) VALUES (?, ?, ?, 'fixture-holder', ?, ?, ?, 0, 0, NULL, ?)")
48+
.run(project, finished - 10_000, finished, JSON.stringify(tasks), status === "completed" ? 1 : 0, status === "failed" ? 1 : 0, PARENTS[second ? 1 : 0]!);
49+
}
50+
51+
async function run(now = NOW) {
52+
const logs: string[] = [];
53+
const result = await runFailureSentinel({ db: join(root, "context.db"), stateFile: join(root, "state.json"), connectionFile: join(root, "none"), send: false, settleMs: 0, now: () => now, stdout: () => {}, stderr: (line) => logs.push(line) });
54+
return { ...result, summary: JSON.parse(logs.at(-1)!), state: JSON.parse(readFileSync(join(root, "state.json"), "utf8")) };
55+
}
56+
57+
test("budget regression: class text folds the two wake child ids and message ids", () => {
58+
const expected = 'DreamTokenBudgetExceeded message="Dreamer child <id> spent <number> prompt tokens (token_budget)"';
59+
expect(normalizeError(budgetError())).toBe(expected);
60+
expect(normalizeError(budgetError(CHILDREN[1], 32_486))).toBe(expected);
61+
expect(dreamBudgetChildSessionId(budgetError())).toBe(CHILDREN[0]);
62+
expect(normalizeError("error in msg_abc123 request 6ccd58a4-fd7d-4695-ae30-b3cbd8c5b633"))
63+
.toBe(normalizeError("error in msg_def456 request 3f5266e0-127a-42d4-9a78-8f5c32e6bc8c"));
64+
});
65+
66+
test("budget regression: old state classes are normalised on load without a new-class wake", async () => {
67+
const oldClass = `DreamTokenBudgetExceeded message="Dreamer child ${CHILDREN[0]} spent <number> prompt tokens (token_budget)"`;
68+
writeFileSync(join(root, "state.json"), JSON.stringify({ version: 1, cursor: { ended_at: NOW - 300_000, id: 0 }, sent: {
69+
[JSON.stringify(["new_class", "dreamer", "verify", oldClass])]: { at: NOW - 60_000, count: 1 },
70+
} }));
71+
invocation(budgetError(CHILDREN[1], 32_486), NOW - 1_000, PARENTS[1]);
72+
dreamRun("failed", CHILDREN[1]);
73+
const result = await run();
74+
expect(result.alerts).toEqual([]);
75+
expect(Object.keys(result.state.sent)).toEqual([JSON.stringify(["new_class", "dreamer", "verify", normalizeError(budgetError())])]);
76+
});
77+
78+
test("budget regression: completed verify runs exclude their child stops from every rule and log the count", async () => {
79+
invocation();
80+
invocation(budgetError(CHILDREN[1], 32_486), NOW - 900, PARENTS[1]);
81+
dreamRun("completed");
82+
dreamRun("completed", CHILDREN[1]);
83+
const result = await run();
84+
expect(result.alerts).toEqual([]);
85+
expect(result.summary.budget_stops_not_counted).toBe(2);
86+
});
87+
88+
test("colliding old class reservations combine counts instead of re-alerting on upgrade", async () => {
89+
const sent = Object.fromEntries(CHILDREN.map((child) => [JSON.stringify(["new_class", "dreamer", "verify",
90+
`DreamTokenBudgetExceeded message="Dreamer child ${child} spent <number> prompt tokens (token_budget)"`]), { at: NOW - 60_000, count: 1 }]));
91+
writeFileSync(join(root, "state.json"), JSON.stringify({ version: 1, cursor: { ended_at: NOW - 300_000, id: 0 }, sent }));
92+
for (const child of CHILDREN) { invocation(budgetError(child)); dreamRun("failed", child); }
93+
const result = await run();
94+
expect(result.alerts).toEqual([]);
95+
expect(Object.values(result.state.sent)).toEqual([{ at: NOW - 60_000, count: 2 }]);
96+
});
97+
98+
test("a failed task with the budget class still wakes with the existing new-class rule", async () => {
99+
invocation(budgetError(CHILDREN[0], 1_502_666));
100+
dreamRun("failed");
101+
const result = await run();
102+
expect(result.alerts.map((alert) => alert.rule)).toEqual(["new_class"]);
103+
expect(result.alerts[0]).toMatchObject({ count: 1, share: 1, class: normalizeError(budgetError()) });
104+
expect(result.summary.budget_stops_not_counted).toBe(0);
105+
});
106+
107+
test("a pending budget stop survives restart and is retried without a new invocation", async () => {
108+
const id = invocation();
109+
const first = await run();
110+
expect(first.alerts).toEqual([]);
111+
expect(first.state.pending).toEqual([id]);
112+
expect(first.state.cursor.id).toBe(id);
113+
dreamRun("completed", CHILDREN[0], NOW + 30_000);
114+
const second = await run(NOW + 60_000);
115+
expect(second.alerts).toEqual([]);
116+
expect(second.summary.budget_stops_not_counted).toBe(1);
117+
expect(second.state.pending).toEqual([]);
118+
});
119+
120+
test("an unresolved budget stop counts as a failure after sixty minutes", async () => {
121+
invocation();
122+
expect((await run()).alerts).toEqual([]);
123+
const result = await run(NOW + HOUR);
124+
expect(result.alerts.map((alert) => alert.rule)).toEqual(["new_class"]);
125+
expect(result.summary.budget_stops_not_counted).toBe(0);
126+
expect(result.state.pending).toEqual([]);
127+
});
128+
129+
test("a late failed task row wakes on the retry", async () => {
130+
invocation();
131+
expect((await run()).alerts).toEqual([]);
132+
dreamRun("failed", CHILDREN[0], NOW + 30_000);
133+
expect((await run(NOW + 60_000)).alerts.map((alert) => alert.rule)).toEqual(["new_class"]);
134+
});
135+
136+
test("completed budget stops are absent from failure counts and share denominators", async () => {
137+
for (let i = 0; i < 6; i++) {
138+
db.query("INSERT INTO subagent_invocations (session_id, harness, subagent, task, started_at, ended_at, status) VALUES ('fixture', 'opencode', 'dreamer', 'verify', ?, ?, 'completed')")
139+
.run(NOW - 2_100, NOW - 2_000);
140+
}
141+
invocation();
142+
dreamRun("completed");
143+
invocation("actual failure A", NOW - 800);
144+
invocation("actual failure B", NOW - 700);
145+
const result = await run();
146+
const share = result.alerts.find((alert) => alert.rule === "failure_share");
147+
expect(share).toMatchObject({ count: 2, share: 0.25 });
148+
expect(result.alerts.some((alert) => alert.example_error.includes("token_budget"))).toBe(false);
149+
});
150+
151+
test("budget lookup requires the child, project, and sixty-minute run window", async () => {
152+
invocation();
153+
dreamRun("completed", CHILDREN[1]);
154+
dreamRun("completed", CHILDREN[0], NOW - 500, "/another/project");
155+
dreamRun("completed", CHILDREN[0], NOW - 2_000);
156+
dreamRun("completed", CHILDREN[0], NOW + HOUR);
157+
expect((await run()).state.pending).toHaveLength(1);
158+
expect((await run(NOW + HOUR)).alerts.map((alert) => alert.rule)).toEqual(["new_class"]);
159+
});
160+
161+
test("malformed task JSON is ignored and a missing project falls back to the time window", async () => {
162+
invocation();
163+
db.exec("DELETE FROM session_projects");
164+
for (const tasks of ["{bad", "{}", '[null, {"children":{}}, {"children":[null]}]']) {
165+
db.query("INSERT INTO dream_runs (project_path, started_at, finished_at, holder_id, tasks_json) VALUES (?, ?, ?, 'fixture', ?)")
166+
.run(PROJECT, NOW - 10_000, NOW - 500, tasks);
167+
}
168+
dreamRun("completed");
169+
const result = await run();
170+
expect(result.alerts).toEqual([]);
171+
expect(result.summary.budget_stops_not_counted).toBe(1);
172+
});
173+
174+
test("an unextractable budget error is judged as a failure rather than silenced", async () => {
175+
invocation("DreamTokenBudgetExceeded message=truncated token_budget");
176+
expect((await run()).alerts.map((alert) => alert.rule)).toEqual(["new_class"]);
177+
});
178+
179+
test("non-budget errors and non-dreamer failures need no dream run lookup", async () => {
180+
invocation("ordinary failure");
181+
invocation(budgetError(), NOW - 900, PARENTS[1], "historian");
182+
db.exec("DROP TABLE dream_runs; DROP TABLE session_projects");
183+
expect((await run()).alerts.map((alert) => alert.rule)).toEqual(["new_class", "new_class"]);
184+
});
185+
186+
test("pending budget stops are bounded without losing the unprocessed cursor tail", async () => {
187+
for (let i = 0; i < 501; i++) invocation(budgetError(`ses_child${i}`), NOW - 1_000 + i);
188+
const first = await run();
189+
expect(first.state.pending).toHaveLength(500);
190+
expect(first.bounded).toBe(true);
191+
const second = await run(NOW + 1_000);
192+
expect(second.state.pending).toHaveLength(500);
193+
expect(second.state.cursor.id).toBe(500);
194+
expect(second.bounded).toBe(true);
195+
const expired = await run(NOW + HOUR);
196+
expect(expired.state.pending).toEqual([]);
197+
expect(expired.state.cursor.id).toBe(501);
198+
expect(expired.alerts.map((alert) => alert.rule)).toContain("failure_share");
199+
});

0 commit comments

Comments
 (0)