From a3999dbf68755c17350a4dc2440e9ae872402ab4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 11 Jun 2026 17:59:37 +0000 Subject: [PATCH 1/2] fix(gateway): ignore NODE_OPTIONS in the SEA binary to keep the V8 code cache valid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standalone `lore` binary emitted `Warning: Code cache data rejected` at startup (and silently recompiled) for any user with V8 flags in NODE_OPTIONS (e.g. `NODE_OPTIONS=--max-old-space-size=8192`, common for Claude Code). Those flags change V8's FlagList::Hash() at runtime, so V8 rejected the embedded code cache that was generated on CI with default flags. Bump fossilize to ^0.10.0 and pass `ignoreNodeOptions: true`, which patches the binary to ignore NODE_OPTIONS (equivalent to `--without-node-options`). The flag-hash then matches the build-time default and the cache is accepted — no warning, cache kept. process.env.NODE_OPTIONS is untouched, so the agent lore launches (e.g. claude) still inherits the user's flags. See BYK/fossilize#31. --- packages/gateway/package.json | 2 +- packages/gateway/script/build-binary-sea.ts | 6 ++++++ pnpm-lock.yaml | 10 +++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 869eec3d..013c02af 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -64,7 +64,7 @@ "@sentry/bun": "^10.52.0", "@types/bun": "^1.2.0", "@types/semver": "^7.7.1", - "fossilize": "^0.9.2", + "fossilize": "^0.10.0", "undici": "^7.27.2" } } diff --git a/packages/gateway/script/build-binary-sea.ts b/packages/gateway/script/build-binary-sea.ts index 13d1dce4..fc8e4069 100644 --- a/packages/gateway/script/build-binary-sea.ts +++ b/packages/gateway/script/build-binary-sea.ts @@ -367,6 +367,12 @@ async function runFossilize( platforms: targets.map(fossilizeTarget), noBundle: true, holePunch: true, + // Make the binary ignore NODE_OPTIONS so user V8 flags (e.g. + // `NODE_OPTIONS=--max-old-space-size=8192`, common for Claude Code) + // don't change V8's flag-hash and reject our embedded code cache + // ("Code cache data rejected"). process.env is untouched, so the + // user's flags still reach the agent lore launches. + ignoreNodeOptions: true, outputName: "lore", outDir: distBinDir, cacheDir: join(packageDir, ".node-cache"), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f32a37ae..4f548238 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,8 +98,8 @@ importers: specifier: ^7.7.1 version: 7.7.1 fossilize: - specifier: ^0.9.2 - version: 0.9.2 + specifier: ^0.10.0 + version: 0.10.0 undici: specifier: ^7.27.2 version: 7.27.2 @@ -2326,8 +2326,8 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} - fossilize@0.9.2: - resolution: {integrity: sha512-tiCkn2JtQJNV3kQ8IcETb8lGZuXqzL3lrOKL6x0kNt+xLNBumPa9mEY3Gm2TtZWwTSW5VlrWVb8ovmJJgiuo2Q==} + fossilize@0.10.0: + resolution: {integrity: sha512-LUrHif+3eFO67YY6pm9fiSznelIL47ut+Q9jV3a6gF0XhNX4aeJEepGPM/cWBndr+3yTPd4QEE7jfG4zfuKWCA==} engines: {node: '>=18'} hasBin: true @@ -6203,7 +6203,7 @@ snapshots: dependencies: fetch-blob: 3.2.0 - fossilize@0.9.2: + fossilize@0.10.0: dependencies: '@stricli/auto-complete': 1.2.7 '@stricli/core': 1.2.7 From d4c4428612f9f5e2816b5ccd8cfcf0c6d1addec3 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 11 Jun 2026 19:48:45 +0000 Subject: [PATCH 2/2] fix(gateway): bump fossilize to ^0.10.1 (win-x64/node24 NODE_OPTIONS fix) --- .lore.md | 42 ++++++++++++++++++++--------------- packages/gateway/package.json | 2 +- pnpm-lock.yaml | 10 ++++----- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/.lore.md b/.lore.md index 02881659..de475c02 100644 --- a/.lore.md +++ b/.lore.md @@ -8,19 +8,19 @@ * **craft NpmTarget workspace expansion and OIDC auth strategy**: craft NpmTarget: \`expand()\` discovers workspaces via pnpm-workspace.yaml or package.json, topologically sorts, validates public packages don't depend on private ones. Auth priority: \`oidc=true\` config → force OIDC; \`NPM\_TOKEN\` set → token auth; no token + OIDC env detected → auto OIDC; else error. OIDC env: \`ACTIONS\_ID\_TOKEN\_REQUEST\_URL\`+\`ACTIONS\_ID\_TOKEN\_REQUEST\_TOKEN\` (GitHub) OR \`NPM\_ID\_TOKEN\` (GitLab). \`publishPackage()\`: path arg ALWAYS pushed last. \`getPublishTag()\`: pre-release → 'next'; new version < current latest → 'old'; else undefined. \`publishMain()\`: artifact download → workspace expand → state resume → publish loop (state written after each target) → merge release branch (non-fatal) → post-release command. State file uses \`XDG\_STATE\_HOME\`. Legacy cwd state file no longer read (PR #797). -* **llm-adapter.ts: retry logic, backoff schedule, and circuit breaker integration**: \`llm-adapter.ts\` retry/backoff: \`TRANSIENT\_CODES={429,500,502,503,529}\`; \`AUTH\_ERROR\_CODES={401,403}\`. \`DEFAULT\_MAX\_RETRIES=8\` (overridable via \`LORE\_MAX\_RETRIES\`). \`LORE\_MAX\_RETRIES\` falls back to default 8 for values \`"0"\`, \`"-1"\`, \`""\`, \`"abc"\`, \`"Infinity"\` — never disables retries. \`backoffMs()\`: Retry-After present → \`min(retryAfterMs, 32000)\`; exponential fallback \`min(500×2^attempt, 32000)\` + 0–25% jitter. On 429 → \`tripCircuitBreaker(pauseSec)\`. Auth errors: \`recordWorkerFailure('auth-rejected')\` ALWAYS called first, then \`markAuthStale(sessionID, providerID)\`, re-resolves, retries once if \`credentialChanged && attempt===0\`. Session-less 401/403 → \`markGlobalAuthStale()\`. \`opts.thinking\` intentionally NOT forwarded — prevents thinking tokens. \`recordWorkerSuccess()\` intentionally NOT called at transport layer. \`AbortError\` downgraded to \`log.info\`. +* **llm-adapter.ts: retry logic, backoff schedule, and circuit breaker integration**: \`llm-adapter.ts\` retry/backoff: \`TRANSIENT\_CODES={429,500,502,503,529}\`; \`AUTH\_ERROR\_CODES={401,403}\`. \`DEFAULT\_MAX\_RETRIES=8\` (overridable via \`LORE\_MAX\_RETRIES\`). \`LORE\_MAX\_RETRIES\` falls back to default 8 for values \`"0"\`, \`"-1"\`, \`""\`, \`"abc"\`, \`"Infinity"\`. \`backoffMs()\`: Retry-After present → \`min(retryAfterMs, 32000)\`; exponential fallback \`min(500×2^attempt, 32000)\` + 0–25% jitter. On 429 → \`tripCircuitBreaker(pauseSec)\`. Auth errors: \`recordWorkerFailure('auth-rejected')\` ALWAYS called first, then \`markAuthStale(sessionID, providerID)\`, re-resolves, retries once if \`credentialChanged && attempt===0\`. Session-less 401/403 → \`markGlobalAuthStale()\`. \`opts.thinking\` intentionally NOT forwarded. \`recordWorkerSuccess()\` intentionally NOT called at transport layer. \`AbortError\` downgraded to \`log.info\`. * **log.ts: sink bridge, file rotation, and error visibility rules**: \`packages/core/src/log.ts\`: (1) \`log.error()\` always calls \`console.error()\` — always visible. \`log.info()\`/\`log.warn()\` only print if \`LORE\_DEBUG=1\`. (2) \`sink?.captureException(err)\` called when \`Error\` instance found via \`findError()\` — gateway bridges to Sentry via \`registerSink()\`. (3) File logging: all levels written to \`~/.local/share/lore/lore.log\`; rotated at 5 MB, check every 1000 writes, single \`.log.1\` backup; disabled in \`NODE\_ENV=test\`. (4) Only one \`LogSink\` supported. TRAP: any \`log.error(..., errorObj)\` auto-forwards to Sentry — including expected shutdown/auth errors. Use typed error subclasses or \`TRANSIENT\_ERROR\_PATTERNS\` filter. \`AbortError\` (DOMException) in pipeline.ts and llm-adapter.ts downgraded to \`log.info\`. -* **LTM forSession() budget, phases, and safety-net constants**: LTM forSession() architecture: budgets \`budget.ltm\`=0.05, \`budget.preferenceLtm\`=0.02. Phases: (1) architecture entries up to ARCH\_BUDGET\_FRACTION=0.2; (2) remaining by score desc; (3) .lore.md sections. HEADER\_OVERHEAD\_TOKENS=15 subtracted upfront. PROJECT\_SAFETY\_NET=5: top-confidence project entries ALWAYS included even with zero relevance. NO\_CONTEXT\_FALLBACK\_CAP=10. Global entries (pid=null) must be cross\_project=1. Pool SQL: project pool uses \`cross\_project=0\`; cross pool uses \`project\_id IS NULL OR cross\_project=1\`. Promoted entries lose PROJECT\_SAFETY\_NET in home project. GOTCHA: \`vectorSearch()\` queries \`knowledge WHERE embedding IS NOT NULL AND confidence>0.2\` — tests must \`DELETE FROM knowledge WHERE embedding IS NOT NULL\` in beforeEach. Dedup threshold for Nomic v1.5 is 0.935. +* **LTM forSession() budget, phases, and safety-net constants**: LTM forSession() architecture: budgets \`budget.ltm\`=0.05, \`budget.preferenceLtm\`=0.02. Phases: (1) architecture entries up to ARCH\_BUDGET\_FRACTION=0.2; (2) remaining by score desc; (3) .lore.md sections. HEADER\_OVERHEAD\_TOKENS=15 subtracted upfront. PROJECT\_SAFETY\_NET=5: top-confidence project entries ALWAYS included even with zero relevance. NO\_CONTEXT\_FALLBACK\_CAP=10. Global entries (pid=null) must be cross\_project=1. Pool SQL: project pool uses \`cross\_project=0\`; cross pool uses \`project\_id IS NULL OR cross\_project=1\`. \`searchCrossProjectRepos()\` uses \`AND cross\_project = 0\` — strictly complementary to \`search()\` which handles cross\_project=1 repos. This prevents RRF double-boost for repos appearing in both paths. Promoted entries lose PROJECT\_SAFETY\_NET in home project. GOTCHA: \`vectorSearch()\` queries \`knowledge WHERE embedding IS NOT NULL AND confidence>0.2\` — tests must \`DELETE FROM knowledge WHERE embedding IS NOT NULL\` in beforeEach. Dedup threshold for Nomic v1.5 is 0.935. * **Multilingual support: FTS tokenizer, Unicode tokenization, and non-English fallbacks**: Lore supports non-English (e.g. Turkish) conversations. Key design: (1) FTS5 uses \`tokenize='unicode61 remove\_diacritics 0'\` (NOT \`porter\` — English-only stemmer; NOT diacritic-folding — Turkish ç/ğ/ı/ö/ş/ü are distinct letters). Migration v32 rebuilds all 6 FTS tables. (2) \`filterTerms()\`/\`extractTopTerms()\` in \`search.ts\` strip via \`/\[^\p{L}\p{N}\_\s]/gu\` (Unicode-aware) so Turkish words don't split at special letters. Underscore kept for snake\_case. (3) \`pattern-extract.ts\` regexes need NO translation — they run on the LLM observer's \*English-normalized\* output. (4) Two raw-user-text regex sites have a structural fallback: \`hasNonAsciiLetters()\` (exported from \`instruction-detect.ts\`) gates fallbacks in \`extractInstructionCandidates\` (emit whole message as candidate) and \`detectAssertions\` (pin first sentence) — requires ≥3 non-ASCII letters to avoid false positives on English loanwords (café, naïve). (5) \`ltm.rerankPreferences()\` no longer demotes entries lacking English directive words to 0.8. KNOWN LIMITATIONS (deferred): English \`STOPWORDS\` not filtered for other languages (vector+RRF compensates); ~3-4 chars/token estimate is English-tuned (fine for Turkish, off for CJK). -* **Remote sync backend: Supabase Postgres + RLS (chosen over Turso/Neon)**: Remote sync backend: Supabase Postgres + RLS (chosen over Turso/Neon). Supabase bundles Auth (GitHub OAuth, SAML 2.0 SSO on Pro+), S3-compatible Storage, and RLS. Neon rejected: brand-new Auth, no integrated object storage. Single shared multi-tenant Postgres DB with \`owner\_user\_id = auth.uid()\` RLS on every synced table. CRITICAL: missing RLS policy = cross-user data leak. Phase 2 adds \`team\_id\` + team-sharing policies. RLS accepts external JWTs from WorkOS/Auth0/Clerk via \`auth.jwt()\`. Migration path: Supabase Auth + GitHub OAuth now → front with WorkOS for enterprise SAML/SCIM later, no DB re-platforming. Phase 1 sync: outbox pattern with monotonic seq, push-then-pull, server-time authority. Dependency: \`@supabase/supabase-js\`. +* **Remote sync backend: Supabase Postgres + RLS (chosen over Turso/Neon)**: Remote sync backend: Supabase Postgres + RLS (chosen over Turso/Neon). Bundles Auth (GitHub OAuth, SAML 2.0 SSO on Pro+), S3-compatible Storage, RLS. Single shared multi-tenant Postgres DB with \`owner\_user\_id = auth.uid()\` RLS on every synced table. CRITICAL: missing RLS policy = cross-user data leak. Phase 2 adds \`team\_id\` + team-sharing policies. RLS accepts external JWTs via \`auth.jwt()\`. Migration path: Supabase Auth + GitHub OAuth now → WorkOS for enterprise SAML/SCIM later. Phase 1 sync: outbox pattern with monotonic seq, push-then-pull, server-time authority. Dependency: \`@supabase/supabase-js\`. * **Sub-agent session detection, isolation, and differential treatment**: Sub-agent session detection via \`x-parent-session-id\` header (pipeline.ts). Each sub-agent gets independent session: temporal storage, gradient state, distillation pipeline, LTM injection, cost tracking. Key differential treatment: (1) cache warming ALWAYS skipped; (2) \`findRotationPredecessor()\` skips sub-agents from Tier 1b rotation; (3) idle distillation/curation runs normally. \`isSubagent\`/\`parentSessionId\` persisted to DB (migration v26). Dashboard: collapsible cost-rollup tree via \`loadParentChildMap()\` + \`buildLiveSessionRows()\`; \`rollUp()\` propagates grandchildren costs bottom-up. GAP: OpenCode plugin \`chat.headers\` hook does NOT set \`x-parent-session-id\`. \`x-session-id\` added to \`KNOWN\_SESSION\_HEADERS\` (position 3); \`ClientType\`/\`detectClientType()\` replaced with \`isClaudeCodeClient()\` boolean. @@ -34,19 +34,19 @@ * **distillations table schema: NOT NULL columns narrative, facts, source\_ids (not content)**: Trap: \`distillations\` table has no \`content\` column — looks like it should because other tables use \`content\`. Actual NOT NULL columns: \`narrative\`, \`facts\`, \`source\_ids\`, \`generation\`, \`token\_count\`, \`created\_at\`. Also: \`tool\_calls\` table uses \`tool\` (not \`tool\_name\`) and requires \`message\_id\`. Test helpers that insert into these tables must include all NOT NULL columns or get constraint errors. Discovered while writing \`packages/core/test/data-move.test.ts\`. -* **LOREAI-GATEWAY-1E: resp.usage undefined crash in postResponse() for vLLM/OpenAI path**: Trap: \`GatewayUsage\` typed as non-optional on \`GatewayResponse\`, so \`resp.usage.inputTokens\` looks safe. But \`resp.usage\` can be \`undefined\` at runtime (e.g., vLLM returning partial response). Crash sites: pipeline.ts lines 2069, 2101, 2137-2140, 2150-2153, 2204-2206, 2282-2285, 2341, 2363, 2402; also \`buildOpenAINonStreamResponse\` (openai.ts:296-300). Fix: add \`const usage = resp.usage ?? { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }\` guard at top of \`postResponse()\` and \`nonStreamHttpResponse()\`, then use \`usage.\*\` throughout. +* **LOREAI-GATEWAY-1E: resp.usage undefined crash in postResponse() for vLLM/OpenAI path**: Trap: \`GatewayUsage\` typed as non-optional on \`GatewayResponse\`, so \`resp.usage.inputTokens\` looks safe. But \`resp.usage\` can be \`undefined\` at runtime (e.g., vLLM partial response). Crash sites: pipeline.ts postResponse() and openai.ts buildOpenAINonStreamResponse. Fix: add \`const usage = resp.usage ?? { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }\` guard at top of \`postResponse()\` and \`nonStreamHttpResponse()\`, then use \`usage.\*\` throughout. -* **LOREAI-GATEWAY-Z: OAuth token expiry causes 401 storm — resolveAuth returns same stale token**: LOREAI-GATEWAY-Z: OAuth token expiry causes 401 storm. \`resolveAuth(sessionID)\` marks session stale and falls back to \`getLastSeenAuth()\` — same expired token. \`credentialChanged=false\` so retry-once path never taken. Fix: (1) \`isAuthStale(sessionID) && !resolveAuth(sessionID)\` guard in \`scheduleBackgroundWork()\` and \`idle.ts:144\`; (2) \`resolveAuth()\` returns null when global fallback matches stale credential; (3) add \`/Worker upstream auth error/\` to \`TRANSIENT\_ERROR\_PATTERNS\`. PR #691: \`setSessionAuth()\` no longer dual-writes to \`\_default\` for named providerID. \`getSessionAuth()\` with explicit providerID no longer falls back to \`\_default\`. \`globalAuthStale\` boolean: \`markGlobalAuthStale()\`, \`isGlobalAuthStale()\`, \`setLastSeenAuth()\` clears staleness only when new credential differs. +* **LOREAI-GATEWAY-Z: OAuth token expiry causes 401 storm — resolveAuth returns same stale token**: LOREAI-GATEWAY-Z: OAuth token expiry causes 401 storm. \`resolveAuth(sessionID)\` marks session stale and falls back to \`getLastSeenAuth()\` — same expired token. \`credentialChanged=false\` so retry-once path never taken. Fix: (1) \`isAuthStale(sessionID) && !resolveAuth(sessionID)\` guard in \`scheduleBackgroundWork()\` and \`idle.ts:144\`; (2) \`resolveAuth()\` returns null when global fallback matches stale credential; (3) add \`/Worker upstream auth error/\` to \`TRANSIENT\_ERROR\_PATTERNS\`. PR #691: \`setSessionAuth()\` no longer dual-writes to \`\_default\`; \`globalAuthStale\` boolean via \`markGlobalAuthStale()\`/\`isGlobalAuthStale()\`/\`setLastSeenAuth()\` clears staleness only when new credential differs. -* **openai.ts: array-format content in system/developer messages silently dropped**: openai.ts / openai-responses.ts parsing traps: (1) Array-format system/developer content: extract text parts (not coerce to ""). (2) \`buildOpenAIMessages\`: \`tool\_result\` blocks pushed mid-loop before \`textParts\`/\`toolUses\` flushed — tool\_result appears before text/tool\_use of same message. (3) Always re-emits system as \`role:'system'\`, never \`'developer'\`. (4) \`parseOpenAIResponsesRequest\` reads system from top-level \`instructions\`. (5) \`item\_reference\` and \`reasoning\` items silently dropped; \`store\` field not in extras allow-list — silently dropped. (6) LTM injection for Responses API: \`\[req.system, cache?.stableLtmSystem, cache?.ltmSystem].filter(Boolean).join(' ')\`. (7) Consecutive role:"tool" messages coalesced into one user message. (8) \`translateAnthropicStreamToResponses\`: on \`output\_text.done\`, replaces delta-accumulated text with final authoritative value. (9) \`nonStreamHttpResponse()\` scales usage via \`scaleUsage()\`. +* **openai.ts: array-format content in system/developer messages silently dropped**: openai.ts / openai-responses.ts traps: (1) Array-format system/developer content: extract text parts, don't coerce to "". (2) \`buildOpenAIMessages\`: \`tool\_result\` blocks pushed mid-loop before \`textParts\`/\`toolUses\` flushed — tool\_result appears before text/tool\_use of same message. (3) Always re-emits system as \`role:'system'\`, never \`'developer'\`. (4) \`parseOpenAIResponsesRequest\` reads system from top-level \`instructions\`. (5) \`item\_reference\` and \`reasoning\` items silently dropped; \`store\` field not in extras allow-list — silently dropped. (6) LTM injection for Responses API: \`\[req.system, cache?.stableLtmSystem, cache?.ltmSystem].filter(Boolean).join(' ')\`. (7) Consecutive role:"tool" messages coalesced into one user message. (8) \`nonStreamHttpResponse()\` scales usage via \`scaleUsage()\`. -* **recall.ts: unescaped quotes/ellipsis in marker strings cause store miss and raw marker leak**: Trap: \`buildRecallMarker()\` embeds raw \`query\` between double-quotes and raw \`id\` before \`…\` — looks safe because queries are usually plain text. But if \`query\` contains \`"\`, \`MARKER\_REGEX\` (\`/📚 Searching (.+?) for "(.+?)"…/\`) truncates the captured group at the first inner quote; \`recallStoreKey()\` produces a mismatched key; \`store.get(key)\` returns undefined; the marker is left as raw text in the conversation visible to the upstream LLM. Same issue for \`id\` containing \`…\`. Fix: escape \`"\` → \`\\"\` in \`buildRecallMarker()\` and unescape in \`parseRecallMarker()\`; for IDs, percent-encode or strip \`…\` before embedding. +* **recall.ts: unescaped quotes/ellipsis in marker strings cause store miss and raw marker leak**: Trap: \`buildRecallMarker()\` embeds raw \`query\` between double-quotes and raw \`id\` before \`…\` in marker strings. If \`query\` contains \`"\`, \`MARKER\_REGEX\` truncates the captured group; \`recallStoreKey()\` produces a mismatched key; \`store.get(key)\` returns undefined; marker leaks as raw text visible to the upstream LLM. Same issue for \`id\` containing \`…\`. Fix: escape \`"\` → \`\\"\` in \`buildRecallMarker()\` and unescape in \`parseRecallMarker()\`; for IDs, percent-encode or strip \`…\` before embedding. -* **splitSegments() infinite recursion on oversized single messages**: \`splitSegments()\` infinite recursion on oversized single messages in \`packages/core/src/distillation.ts\`: recurses infinitely when a single message exceeds \`maxSegmentTokens\` (16384). \`findSplitIndex()\` returns \`messages.length\` (=1), so \`left = messages.slice(0, 1)\` produces an identical recursive call. Triggered on large tool outputs (~49KB+). Fix: add base case after the \`totalTokens <= maxTokens\` guard — \`if (messages.length <= 1) return \[messages]\`. The oversized message becomes an indivisible segment. Also: \`distillTokenBudget()\` uses \`10×√N\` formula clamped to \[256, 4096]. \`storeDistillation()\` stores no file path column — paths only in \`temporal\_messages.content\` via \`toolStripAnnotation()\`. +* **splitSegments() infinite recursion on oversized single messages**: \`splitSegments()\` infinite recursion in \`packages/core/src/distillation.ts\`: recurses infinitely when a single message exceeds \`maxSegmentTokens\` (16384). \`findSplitIndex()\` returns \`messages.length\` (=1), so \`left = messages.slice(0, 1)\` produces identical recursive call. Triggered on large tool outputs (~49KB+). Fix: add base case — \`if (messages.length <= 1) return \[messages]\`. Also: \`distillTokenBudget()\` uses \`10×√N\` clamped to \[256, 4096]. \`storeDistillation()\` stores no file path column. * **suggest.ts: RegExp flag deduplication required before appending 'g'**: Trap: constructing \`new RegExp(pattern.source, ${pattern.flags}g)\` produces \`'gg'\` if the source pattern already has the \`g\` flag, causing \`Invalid flags supplied to RegExp constructor\`. Looks safe because most patterns won't have \`g\`. Fix: strip \`g\` from \`pattern.flags\` before appending: \`pattern.flags.replace(/g/g, '') + 'g'\`. Applies anywhere a RegExp is reconstructed with forced flags in \`packages/gateway/src/suggest.ts\`. @@ -54,7 +54,7 @@ ### Pattern -* **api.ts resolveProject() priority and parseBody zstd support**: \`packages/gateway/src/api.ts\` helpers: \`jsonResponse()\` (line 59), \`errorResponse()\` (line 66), \`parseBody\()\` (line 79) — supports optional zstd decompression via \`Content-Encoding: zstd\`. \`resolveProject()\` priority: (1) \`:id\` route param, (2) \`?git\_remote=\`, (3) \`?path=\`. POST dispatch: literal routes first, then parameterized. Mutation handlers: \`handleDeleteSession\`, \`handleDeleteDistillation\`, \`handleDeleteProject\`, \`handleClearProject\`, \`handleMergeProjects\`, \`handleReindex\`, \`handleMoveSessions\`, \`handleMoveKnowledge\`. TRAP: \`moveSessions()\`/\`reassignKnowledge()\` update \`knowledge SET project\_id=?\` but leave \`knowledge\_transfers\` rows pointing to old project — also update \`knowledge\_transfers\`. \`moveSessions()\` BFS-expands children before moving; \`sessions\_moved\` return = \`allIds.length\` (overcounts if BFS includes children from other projects — use pre-counted WHERE queries). \`rebindActiveSession\` in api.ts/ui.ts iterates only user-provided IDs, not BFS-expanded children — child sessions' in-memory state not rebound. +* **api.ts resolveProject() priority and parseBody zstd support**: \`packages/gateway/src/api.ts\`: \`resolveProject()\` priority: (1) \`:id\` route param, (2) \`?git\_remote=\`, (3) \`?path=\`. POST dispatch: literal routes first, then parameterized. TRAP: \`moveSessions()\`/\`reassignKnowledge()\` update \`knowledge SET project\_id=?\` but leave \`knowledge\_transfers\` rows pointing to old project — also update \`knowledge\_transfers\`. \`moveSessions()\` BFS-expands children before moving; \`sessions\_moved\` return overcounts if BFS includes children from other projects — use pre-counted WHERE queries. \`rebindActiveSession\` iterates only user-provided IDs, not BFS-expanded children. \`parseBody\()\` supports optional zstd decompression via \`Content-Encoding: zstd\`. * **Gateway test patterns: unit vs integration vs subprocess isolation**: Gateway/OpenCode test patterns — three isolation levels: (1) Unit: import functions directly, build fixtures via \`makeRequest()\`/\`makeUserMsg()\` factories. (2) Integration: \`createHarness()\` starts real server on random port with SSE-aware replay interceptor (\`test/helpers/replay.ts\`), provides \`chat()\` + \`queryDB()\`. (3) Subprocess isolation: \`mock.module()\` calls isolated in subprocess files to avoid global cache pollution. \`harness.ts\` sends \`x-lore-project: process.cwd()\` by default — suppress with \`extraHeaders\` empty string. \`queryDB()\` opens fresh read-only \`DatabaseSync\` per call. Each harness gets isolated temp DB + random port + reset pipeline state. TRAP: streaming turns need SSE-format fixtures — JSON-only interceptor returns empty body for \`stream:true\`. Fix: use \`test/helpers/replay.ts\` streaming-aware interceptor. Never modify \`src/recorder.ts\` for test-only SSE needs. @@ -64,23 +64,29 @@ ### Preference - -* **Always create a PR and watch CI after committing a fix**: After implementing and committing a fix (especially for gateway/infrastructure issues), the user consistently follows up by creating a pull request and monitoring CI results. This includes: creating a bookmark/branch, pushing it, opening a PR with a conventional commit-style title, then checking each CI job's status. If CI fails, the user investigates whether the failure is pre-existing on main or caused by the PR's changes. The assistant should proactively offer or execute these steps (jj bookmark, push, gh pr create, CI watch) immediately after a successful commit without waiting to be asked. + +* **Always assert domain knowledge to correct or redirect investigation hypotheses**: User corrections about codebase behavior are ground truth — immediately accept and update mental model without re-arguing. Always assert corrections to factual inaccuracies immediately. Treat user assertions about codebase behavior as authoritative. When proposing root causes or theories, be prepared to pivot immediately on correction. + + +* **Always create a PR after completing a fix or feature implementation**: After finishing a bug fix or feature: (1) create a git/jj bookmark/branch, push to origin, open a PR on GitHub (BYK/loreai repo) with a conventional commit-style title. (2) Monitor CI after PR creation, distinguish pre-existing failures from regressions. (3) Self-review PRs before merging — categorize issues by severity (BLOCKING/MEDIUM/LOW), fix immediately, amend into existing commit. + + +* **Always document architectural invariants as code comments at the point of enforcement**: Always annotate non-obvious code decisions and architectural invariants as inline comments at the exact enforcement point — not in external docs. Use declarative language ('always', 'never', 'must') explaining the \*why\*. Treat these comments as ground truth during review. Also: embed behavioral directives as code comments in production files (prompt.ts, entities.ts, pipeline.ts, README.md) — scan for and surface them as 🔴 observations during review. -* **Always enforce specific CI invariants as non-negotiable directives**: CI invariants — never violate: (1) Full CI always runs on main/release: \`code=true\` forced unconditionally; dorny/paths-filter only applies to PRs. (2) \`actionlint\` runs unconditionally, \`-shellcheck=""\` always set. (3) Coverage/artifacts upload even on failure (\`if: '!cancelled()'\`). (4) Placeholder files ensure upload-artifact always creates an artifact. (5) \`ci-status\` job exits 1 on \`failure\` or \`cancelled\`; new CI jobs MUST appear in its \`needs:\` array. (6) Path filters must include all relevant files. (7) vitest.config.ts \`resolve.alias\` is top-level, NOT nested under \`test:\`. (8) Delete stale \`.tsbuildinfo\` files to clear phantom type errors. (9) \`check-docs\` CI job gates on both config-docs and env-docs drift. (10) Glob \`\*\*/\*.config.ts\` accidentally matches \`packages/core/src/config.ts\` and \`packages/gateway/src/config.ts\` — use \`\*.config.ts\` (root-only) for tooling configs only. (11) Biome \`noNonNullAssertion\` is CI-blocking — replace \`r!.property\` with \`r?.property\` after null checks in tests. +* **Always enforce specific CI invariants as non-negotiable directives**: CI invariants — never violate: (1) Full CI always runs on main/release: \`code=true\` forced unconditionally; dorny/paths-filter only applies to PRs. (2) \`actionlint\` runs unconditionally, \`-shellcheck=""\` always set. (3) Coverage/artifacts upload even on failure (\`if: '!cancelled()'\`). (4) Placeholder files ensure upload-artifact always creates an artifact. (5) \`ci-status\` job exits 1 on \`failure\` or \`cancelled\`; new CI jobs MUST appear in its \`needs:\` array. (6) Path filters must include all relevant files. (7) vitest.config.ts \`resolve.alias\` is top-level, NOT nested under \`test:\`. (8) Delete stale \`.tsbuildinfo\` files to clear phantom type errors. (9) \`check-docs\` CI job gates on both config-docs and env-docs drift — after any refactor touching env-var use sites, run \`pnpm run generate:env-docs\` and commit updated \`environment.md\` BEFORE pushing. (10) Glob \`\*\*/\*.config.ts\` accidentally matches \`packages/core/src/config.ts\` and \`packages/gateway/src/config.ts\` — use \`\*.config.ts\` (root-only) for tooling configs only. (11) Biome \`noNonNullAssertion\` is CI-blocking — replace \`r!.property\` with \`r?.property\` after null checks in tests. * **Always read and document db.ts schema migrations in full before making changes**: DB schema change discipline: Before any schema change, read the full MIGRATIONS array to catalog every migration (version, index, table, columns, purpose). Document current highest version and next available index. New migrations: NOT NULL DEFAULT for safe defaults, nullable for optional/backfilled columns, \`recoverMissingObjects()\` for idempotent recovery, \`schema\_version\` normalized to \`MIGRATIONS.length\`. Always verify \`MIGRATIONS.length\` before proposing a new migration. Reconnaissance first — no code until schema is fully understood. - -* **Always reaffirm existing architectural decisions after evaluating alternatives**: When evaluating alternative tools/frameworks/services: provide a structured scorecard comparison against locked requirements, then give a decisive verdict. If the alternative doesn't clearly win, explicitly reaffirm the existing decision. Name what the alternative does better, where it falls short, and whether it changes the plan. Don't leave settled decisions open-ended without strong justification. - -* **Always remove all Bun-specific code and build targets**: Project uses Node.js >= 22.15 + pnpm + Vitest. Never add Bun-specific code. Replace Bun APIs with Node.js equivalents: \`node:worker\_threads\`, \`node:fs\`, \`node:zlib\`, \`node:https\`. Sentry: \`@sentry/node\` via \`sentryBunToNodePlugin\`; polyfill \`getSystemErrorMap()\` BEFORE \`Sentry.init()\` in \`packages/gateway/instrument.ts\`. Always use \`vitest run\` (never \`bun test\`). Eval run commands use \`npx tsx\`. Install examples use \`pnpm add\`. TRAP: \`DecompressionStream('zstd')\` is Bun-only — Node.js throws at construction; replace with \`createZstdDecompress\` + \`Readable.toWeb\`. TRAP: \`timeout: false\` in fetch init requires \`as RequestInit\` cast. TRAP: undici@7 hangs on incremental reads under Bun — use native fetch under Bun, keep undici for Node. TRAP: Bun native fetch has a hard 5-min inactivity timeout — replaced with \`node:https\` streaming in \`packages/gateway/src/fetch.ts\` to bypass it entirely. Node path uses undici with \`{ bodyTimeout: 0, headersTimeout: 0 }\` dispatcher (memoized singleton). +* **Always remove all Bun-specific code and build targets**: Project uses Node.js >= 22.15 + pnpm + Vitest. Never add Bun-specific code. Replace Bun APIs with Node.js equivalents: \`node:worker\_threads\`, \`node:fs\`, \`node:zlib\`, \`node:https\`. Sentry: \`@sentry/node\`; polyfill \`getSystemErrorMap()\` BEFORE \`Sentry.init()\`. Always use \`vitest run\`. TRAPS: \`DecompressionStream('zstd')\` is Bun-only — use \`createZstdDecompress\` + \`Readable.toWeb\`. \`timeout: false\` in fetch init requires \`as RequestInit\` cast. undici@7 hangs on incremental reads under Bun. Bun native fetch has hard 5-min inactivity timeout — replaced with \`node:https\` streaming; Node path uses undici with \`{ bodyTimeout: 0, headersTimeout: 0 }\` (memoized singleton). \`NODE\_OPTIONS\` V8 flags cause SEA code cache rejection — use fossilize \`ignoreNodeOptions: true\`. + + +* **Always run typecheck across all workspace packages before considering work complete**: Mandatory pre-completion gates: (1) Full typecheck across all workspace packages (packages/core, packages/gateway, packages/opencode, packages/pi) — zero errors. (2) Run full vitest suite. (3) Run \`pnpm biome check --write\`. (4) Verify patch application end-to-end. (5) Restore env vars in tests: save previous value, restore in cleanup — never delete. Pattern: \`const saved = process.env\[VAR]\`; cleanup: \`if (saved !== undefined) process.env\[VAR] = saved; else delete process.env\[VAR]\`. Provide commands as bare copy-pasteable text. - -* **Always request harsh multi-pass code review covering logic flaws, edge cases, and invariant violations**: Code review and commit discipline: (1) Harsh multi-pass self-review — logic flaws, edge cases, invariant violations. (2) Enumerate 5–10 distinct targets with exact file:line refs before implementing. (3) Fix all failing tests; run full typecheck (zero errors) and full vitest suite before committing. (4) Use Biome (never ESLint/Prettier); run \`pnpm biome check --write\` before pushing. (5) Annotate behavioral constraints as code comments at exact implementation site. (6) CLI flags used in subcommands MUST be declared in global OPTIONS map in main.ts. (7) Always push branch and create PR as last step; watch CI to completion. (8) Stash/exclude \`.lore.md\` from PR commits; resolve .lore.md merge conflicts by taking upstream (theirs). (9) Check: flags set in memory but never persisted; flag lifecycle; dead code; missing tests. (10) All packages must be devDependencies. (11) Real production bugs found in tests: fix in same PR with \`fix(scope):\`. (12) Never modify \`src/recorder.ts\` for test-only SSE needs. (13) New test files go on a dedicated \`test/\\` branch. (14) Before any \`git commit\`, always run \`git add .lore.md\`. + +* **Always update tests to match implementation changes, removing overly-strict assertions**: When implementation changes affect output shape or scope, update tests to reflect new behavior: remove overly-strict \`toEqual\` assertions checking exact object shape, replace with targeted checks for specific properties that matter. Update all affected test files together with implementation changes. Avoid assertions that will break as the feature grows (e.g., provider count). * **OpenCode plugin: per-project state map for project path header injection**: OpenCode plugin per-project state: \`projectState\` Map keyed by \`ctx.project.id\` → \`{ projectPath, gitRemote, lastSeenAt }\`; TTL \`SESSION\_STATE\_TTL\_MS = 24h\`. \`reapStaleProjectState()\` called on every new entry. \`loreInitPromise\` memoizes concurrent plugin calls racing on probe→spawn. Gateway discovery order: (0) \`LORE\_REMOTE\_URL\`; (1) \`LORE\_GATEWAY\_URL\`; (2) port file; (3) known ports \`\[3207, 5673]\`; (4) in-process \`startGateway({ quiet:true, local:true })\`. \`remoteGateway\` auto-detection: (1) \`LORE\_REMOTE\_GATEWAY\` env; (2) \`LORE\_HOSTED\_MODE\`; (3) non-loopback bind; (4) default=true. \`--local\` CLI flag ALWAYS wins. In-process callers MUST pass \`local:true\`. GAP: \`chat.headers\` injects \`x-lore-session-id\`/\`x-lore-agent\` but NOT \`x-parent-session-id\`. diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 013c02af..d2ef30bb 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -64,7 +64,7 @@ "@sentry/bun": "^10.52.0", "@types/bun": "^1.2.0", "@types/semver": "^7.7.1", - "fossilize": "^0.10.0", + "fossilize": "^0.10.1", "undici": "^7.27.2" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4f548238..a962e55e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -98,8 +98,8 @@ importers: specifier: ^7.7.1 version: 7.7.1 fossilize: - specifier: ^0.10.0 - version: 0.10.0 + specifier: ^0.10.1 + version: 0.10.1 undici: specifier: ^7.27.2 version: 7.27.2 @@ -2326,8 +2326,8 @@ packages: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} - fossilize@0.10.0: - resolution: {integrity: sha512-LUrHif+3eFO67YY6pm9fiSznelIL47ut+Q9jV3a6gF0XhNX4aeJEepGPM/cWBndr+3yTPd4QEE7jfG4zfuKWCA==} + fossilize@0.10.1: + resolution: {integrity: sha512-+7rb/72OJmlgwDGnqmtzwcZ3A4sOut/BTu4kzUo2jzsa8k0F401TXFkHSg1y2U/M442BqaYtARgWv508gjdw+g==} engines: {node: '>=18'} hasBin: true @@ -6203,7 +6203,7 @@ snapshots: dependencies: fetch-blob: 3.2.0 - fossilize@0.10.0: + fossilize@0.10.1: dependencies: '@stricli/auto-complete': 1.2.7 '@stricli/core': 1.2.7