From 48c1da0afbe53a0444f123adda8816591a3f8700 Mon Sep 17 00:00:00 2001 From: Miranda Limonczenko Date: Fri, 14 Aug 2026 12:58:03 -0700 Subject: [PATCH] feat(framework): score committed solutions without an agent run `pnpm eval:solution -- --eval ` boots the eval's environment, copies a directory from the eval's `solutions/` over the workspace, and runs the scorer. Deterministic, so a verdict that moves is the scorer changing rather than the agent. The overlay copy lands between `localDir` and `supabase start`, so a solution's migrations apply on boot with no per-eval apply step. Eval discovery and run setup move to `eval-discovery.ts` so the agent and solution entry points can't drift on what an eval is. Local-stack only. A tools eval's starting state is a seeded hosted project, not a workspace to copy files into. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 + CONTRIBUTING.md | 19 +- README.md | 12 ++ apps/framework/harness/eval-discovery.ts | 116 ++++++++++++ apps/framework/harness/run-eval.ts | 111 +---------- apps/framework/harness/run-solution.ts | 174 ++++++++++++++++++ apps/framework/package.json | 1 + .../solutions/README.md | 9 + ...0260102000000_add_products_description.sql | 1 + .../green/supabase/schemas/products.sql | 6 + ...0260102000000_add_products_description.sql | 1 + package.json | 1 + packages/core/src/index.ts | 6 + packages/sandbox/src/agent-environment.ts | 3 + packages/sandbox/src/local-stack-runtime.ts | 2 + packages/sandbox/src/supabase.ts | 12 ++ 16 files changed, 370 insertions(+), 105 deletions(-) create mode 100644 apps/framework/harness/eval-discovery.ts create mode 100644 apps/framework/harness/run-solution.ts create mode 100644 evals/build-cli-002-declarative-schema/solutions/README.md create mode 100644 evals/build-cli-002-declarative-schema/solutions/green/supabase/migrations/20260102000000_add_products_description.sql create mode 100644 evals/build-cli-002-declarative-schema/solutions/green/supabase/schemas/products.sql create mode 100644 evals/build-cli-002-declarative-schema/solutions/migration-only/supabase/migrations/20260102000000_add_products_description.sql diff --git a/.gitignore b/.gitignore index 5dc75e68..1b60a018 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ evals/*/local/supabase/.branches/ .DS_Store results/*/ .sync-tmp/ +.solution-runs/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7ae6949d..d54f2b7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ Then add a folder under `evals/` containing: 1. `PROMPT.md` with frontmatter metadata and the task the agent sees. 2. `EVAL.ts` with the scorer. -3. `solutions/` with the example solutions you scored the eval against. See [Checking your scorer](#checking-your-scorer). +3. `solutions/` with the example solutions you scored the eval against, one directory per solution. See [Checking your scorer](#checking-your-scorer). 4. Optional `remote/` data when the scenario needs to seed hosted project state, such as database, logs, or functions. 5. Optional `local/` files when the scenario needs to seed a local filesystem, such as a local `supabase/` project. @@ -77,6 +77,8 @@ Write each solution in the shape of the thing your eval measures. | Project state | SQL and functions, shaped like the eval's `remote/` seed. | | A report | The answer text, for evals scored on what the agent said rather than what it built. | +`pnpm eval:solution` scores workspace-state solutions. Project-state and report solutions are still worth committing as the record of what you checked, but you score them by hand for now. + ### What to write Write two kinds of solution: @@ -88,16 +90,25 @@ Write two kinds of solution: ### How to score them +`pnpm eval:solution` runs the eval's scorer against a committed solution with no agent, so the same input gives the same verdicts every time. + 1. Write down the checks you expect each bad solution to fail. Do this before you run anything. -2. Score the green solution. Every check should pass. -3. Score each bad solution. The failures should match your list. +2. Score every solution in the eval's `solutions/` directory: + + ```bash + pnpm eval:solution -- --eval build-cli-002-declarative-schema + ``` + + Add `--solution ` to score one. The run needs Docker and the local stack's default ports, same as any local-stack eval, and writes nothing to `results/`. + +3. Read the output. The green solution should pass every check, and each bad solution should fail the ones you listed. 4. Commit the list next to the solution it describes. ### Reading the results - **The green solution fails a check.** Look at the solution and the check. Either could be the thing that's wrong. - **A bad solution fails several checks.** Expected. One flaw usually trips more than one check, so matching your list is the bar, not one failure per solution. -- **A check reads the agent's transcript.** It has nothing to read on a solution you wrote yourself. Expect it to fail, and note that in the list. +- **A check reads the agent's transcript.** It has nothing to read on a solution you wrote yourself, since no agent ran. Expect it to fail, and note that in the list. ## Adding an experiment diff --git a/README.md b/README.md index 31527982..822d8835 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,18 @@ Run all benchmark and no-skills experiments across all benchmark evals: pnpm eval -- --suite benchmark --experiment-suite benchmark,no-skills ``` +### Score an example solution + +Run an eval's committed solutions through its scorer with no agent: + +```bash +pnpm eval:solution -- --eval build-cli-002-declarative-schema +``` + +Each directory under the eval's `solutions/` is copied over the workspace after `local/` and before the stack starts, so migrations it ships apply on `supabase start`. Pass `--solution ` to score one instead of all of them. + +The run prints every check and writes nothing to `results/`; the exported workspace lands in `.solution-runs///`. Checks that read the agent's transcript fail here by construction, because no agent ran. Local-stack evals only — a tools eval's starting state is a seeded hosted project, not a workspace to copy files into. + ### View results in the web app After running evals locally, export their results to `eval-results.json` for the web app: diff --git a/apps/framework/harness/eval-discovery.ts b/apps/framework/harness/eval-discovery.ts new file mode 100644 index 00000000..eb886941 --- /dev/null +++ b/apps/framework/harness/eval-discovery.ts @@ -0,0 +1,116 @@ +/** + * Eval discovery and the small pieces of run setup that both entry points + * share: `run-eval.ts` (agent runs) and `run-solution.ts` (scoring a committed + * solution). Keeping them here means the two can't drift on what an eval is or + * how its seed data is found. + */ + +import { + cpSync, + existsSync, + readdirSync, + readFileSync, + statSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; + +import type { EvalInterface, EvalManifest, EvalMode } from './types.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const ROOT = join(__dirname, '..', '..', '..'); + +// Fixed identifiers for the mocked hosted project a local-stack eval links to. +// Both must satisfy the CLI's format checks: ref is `^[a-z]{20}$`, token is +// `^sbp_[a-f0-9]{40}$`. platform-lite accepts whatever token it's booted with. +export const HOSTED_PROJECT_REF = 'evalshostedprojectxy'; +export const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40); + +/** + * Resolve the run mode. The sandbox (local-stack) is needed when the agent + * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that + * start from an empty workspace — or when the eval ships a `local/` workspace + * of starting files. Everything else runs against the in-memory tools runtime. + * + * `interface` is otherwise a benchmark dimension (KPI), not a runtime switch. + */ +export function resolveEvalMode( + interfaceKind: EvalInterface | undefined, + hasLocal: boolean +): EvalMode { + if (interfaceKind === 'cli' || hasLocal) return 'local-stack'; + return 'tools'; +} + +export function discoverEvals(): EvalManifest[] { + const dir = join(ROOT, 'evals'); + if (!existsSync(dir)) return []; + const out: EvalManifest[] = []; + for (const id of readdirSync(dir)) { + const evalDir = join(dir, id); + if (!statSync(evalDir).isDirectory()) continue; + const localDir = join(evalDir, 'local'); + const promptPath = join(evalDir, 'PROMPT.md'); + const evalPath = join(evalDir, 'EVAL.ts'); + const metadata = parseEvalMarkdown( + readFileSync(promptPath, 'utf8'), + `evals/${id}/PROMPT.md` + ).metadata; + const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory(); + const mode = resolveEvalMode(metadata.interface, hasLocal); + out.push({ + id, + mode, + metadata, + stage: metadata.stage, + product: metadata.product, + suite: metadata.suite, + topic: metadata.topic, + dir: evalDir, + localDir: hasLocal ? localDir : undefined, + promptPath, + evalPath, + remoteDir: join(evalDir, 'remote'), + }); + } + return out; +} + +export function readSessionSeedArgs(ev: EvalManifest) { + const projectSeedSql = join(ev.remoteDir, 'project.sql'); + const logsSeedJsonl = join(ev.remoteDir, 'logs.jsonl'); + const functionsSeedDir = join(ev.remoteDir, 'functions'); + + return { + projectSeedSql: existsSync(projectSeedSql) ? projectSeedSql : undefined, + logsSeedJsonl: existsSync(logsSeedJsonl) ? logsSeedJsonl : undefined, + functionsSeedDir: existsSync(functionsSeedDir) + ? functionsSeedDir + : undefined, + pgvector: ev.metadata.product.includes('vectors'), + }; +} + +export function copyWithheldTests(ev: EvalManifest, workspace: string) { + const testsDir = join(ev.dir, 'tests'); + if (existsSync(testsDir)) { + cpSync(testsDir, join(workspace, 'tests'), { recursive: true }); + } +} + +/** + * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with + * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`, + * `return`, or a throw), including when a *later* resource created in the same + * scope throws before its own `try`/`finally` is reached. + */ +export function disposable }>( + resource: T +): T & AsyncDisposable { + return Object.assign(resource, { + [Symbol.asyncDispose]: async () => { + await resource.close(); + }, + }); +} diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index c9b4e255..e9eef504 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -1,17 +1,15 @@ #!/usr/bin/env tsx import { - cpSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, - statSync, writeFileSync, } from 'node:fs'; import { join, dirname, relative } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { pathToFileURL } from 'node:url'; import { jsonSchema, tool, type ToolSet } from 'ai'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { @@ -26,6 +24,15 @@ import { readSuiteFilters, } from '../lib/cli-args.js'; import { bootPlatformBackend } from './platform-backend.js'; +import { + HOSTED_ACCESS_TOKEN, + HOSTED_PROJECT_REF, + ROOT, + copyWithheldTests, + discoverEvals, + disposable, + readSessionSeedArgs, +} from './eval-discovery.js'; import { viteBuild, vitestRun } from './project-runner.js'; import { buildDocsResult, @@ -35,7 +42,6 @@ import { } from '@supabase-evals/core'; import type { ExperimentConfig, - EvalInterface, EvalManifest, EvalMode, EvalSuite, @@ -48,15 +54,6 @@ import type { TranscriptPart, } from './types.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, '..', '..', '..'); - -// Fixed identifiers for the mocked hosted project a local-stack eval links to. -// Both must satisfy the CLI's format checks: ref is `^[a-z]{20}$`, token is -// `^sbp_[a-f0-9]{40}$`. platform-lite accepts whatever token it's booted with. -const HOSTED_PROJECT_REF = 'evalshostedprojectxy'; -const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40); - const rawArgs = process.argv.slice(2); const args = new Set(rawArgs); const FORCE = !args.has('--skip-existing'); @@ -107,56 +104,6 @@ function readFlag(name: string): string | undefined { return undefined; } -/** - * Resolve the run mode. The sandbox (local-stack) is needed when the agent - * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that - * start from an empty workspace — or when the eval ships a `local/` workspace - * of starting files. Everything else runs against the in-memory tools runtime. - * - * `interface` is otherwise a benchmark dimension (KPI), not a runtime switch. - */ -function resolveEvalMode( - interfaceKind: EvalInterface | undefined, - hasLocal: boolean -): EvalMode { - if (interfaceKind === 'cli' || hasLocal) return 'local-stack'; - return 'tools'; -} - -function discoverEvals(): EvalManifest[] { - const dir = join(ROOT, 'evals'); - if (!existsSync(dir)) return []; - const out: EvalManifest[] = []; - for (const id of readdirSync(dir)) { - const evalDir = join(dir, id); - if (!statSync(evalDir).isDirectory()) continue; - const localDir = join(evalDir, 'local'); - const promptPath = join(evalDir, 'PROMPT.md'); - const evalPath = join(evalDir, 'EVAL.ts'); - const metadata = parseEvalMarkdown( - readFileSync(promptPath, 'utf8'), - `evals/${id}/PROMPT.md` - ).metadata; - const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory(); - const mode = resolveEvalMode(metadata.interface, hasLocal); - out.push({ - id, - mode, - metadata, - stage: metadata.stage, - product: metadata.product, - suite: metadata.suite, - topic: metadata.topic, - dir: evalDir, - localDir: hasLocal ? localDir : undefined, - promptPath, - evalPath, - remoteDir: join(evalDir, 'remote'), - }); - } - return out; -} - type ToolsSkill = { name: string; description: string; body: string }; /** @@ -284,28 +231,6 @@ function workspacePath(modelName: string, evalId: string, attempt: number) { ); } -function copyWithheldTests(ev: EvalManifest, workspace: string) { - const testsDir = join(ev.dir, 'tests'); - if (existsSync(testsDir)) { - cpSync(testsDir, join(workspace, 'tests'), { recursive: true }); - } -} - -function readSessionSeedArgs(ev: EvalManifest) { - const projectSeedSql = join(ev.remoteDir, 'project.sql'); - const logsSeedJsonl = join(ev.remoteDir, 'logs.jsonl'); - const functionsSeedDir = join(ev.remoteDir, 'functions'); - - return { - projectSeedSql: existsSync(projectSeedSql) ? projectSeedSql : undefined, - logsSeedJsonl: existsSync(logsSeedJsonl) ? logsSeedJsonl : undefined, - functionsSeedDir: existsSync(functionsSeedDir) - ? functionsSeedDir - : undefined, - pgvector: ev.metadata.product.includes('vectors'), - }; -} - function basePromptFor(mode: EvalMode): string { if (mode === 'local-stack') { return ( @@ -331,22 +256,6 @@ function buildSystemPrompt( return blocks.join('\n\n'); } -/** - * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with - * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`, - * `return`, or a throw), including when a *later* resource created in the same - * scope throws before its own `try`/`finally` is reached. - */ -function disposable }>( - resource: T -): T & AsyncDisposable { - return Object.assign(resource, { - [Symbol.asyncDispose]: async () => { - await resource.close(); - }, - }); -} - async function runOne( expName: string, exp: ExperimentConfig, diff --git a/apps/framework/harness/run-solution.ts b/apps/framework/harness/run-solution.ts new file mode 100644 index 00000000..eaa32df6 --- /dev/null +++ b/apps/framework/harness/run-solution.ts @@ -0,0 +1,174 @@ +#!/usr/bin/env tsx +/** + * Score an eval's committed example solutions with no agent in the loop. + * + * The eval's environment boots exactly as it does for an agent run, the + * solution is copied over the workspace before the stack starts, and the + * scorer runs against the result. Deterministic, so a verdict that moves is + * the scorer changing rather than the agent. + * + * pnpm eval:solution -- --eval build-docs-002-rls-guide + * pnpm eval:solution -- --eval build-docs-002-rls-guide --solution green + */ +import { existsSync, readdirSync, rmSync, statSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { localStackRuntime } from '@supabase-evals/sandbox'; +import { readRepeatedFlag } from '../lib/cli-args.js'; +import { + HOSTED_ACCESS_TOKEN, + HOSTED_PROJECT_REF, + ROOT, + copyWithheldTests, + discoverEvals, + disposable, + readSessionSeedArgs, +} from './eval-discovery.js'; +import { bootPlatformBackend } from './platform-backend.js'; +import { viteBuild, vitestRun } from './project-runner.js'; +import type { EvalManifest, LocalStackScorer, ScoreResult } from './types.js'; + +const rawArgs = process.argv.slice(2); +const EVAL_FILTERS = readRepeatedFlag(rawArgs, 'eval'); +const SOLUTION_FILTERS = readRepeatedFlag(rawArgs, 'solution'); + +// Exported workspaces land outside `results/`, which is reserved for scored +// agent runs the web app reads. +const OUTPUT_DIR = join(ROOT, '.solution-runs'); + +function solutionsDir(ev: EvalManifest) { + return join(ev.dir, 'solutions'); +} + +function listSolutions(ev: EvalManifest): string[] { + const dir = solutionsDir(ev); + if (!existsSync(dir)) return []; + return readdirSync(dir).filter((name) => + statSync(join(dir, name)).isDirectory() + ); +} + +async function scoreSolution( + ev: EvalManifest, + solution: string +): Promise { + const scorer = (await import(pathToFileURL(ev.evalPath).href)) + .default as LocalStackScorer; + + await using hostedBackend = ev.metadata.hostedProject + ? disposable( + await bootPlatformBackend({ + ...readSessionSeedArgs(ev), + ref: HOSTED_PROJECT_REF, + accessToken: HOSTED_ACCESS_TOKEN, + hostname: '0.0.0.0', + pgWire: true, + }) + ) + : undefined; + + // No experiment: a solution run has no agent, so skills and the MCP surface + // an experiment would configure have nothing to act on. + await using session = disposable( + await localStackRuntime().startSession({ + cliVersion: ev.metadata.cliVersion, + localDir: ev.localDir, + solutionDir: join(solutionsDir(ev), solution), + includeServices: ev.metadata.services, + projectRunning: ev.metadata.projectRunning, + hosted: hostedBackend + ? { + port: Number(new URL(hostedBackend.url).port), + pgPort: hostedBackend.pgPort, + ref: hostedBackend.ref, + accessToken: hostedBackend.accessToken, + mgmt: hostedBackend.mgmt, + query: hostedBackend.query, + invokeFunction: hostedBackend.invokeFunction, + } + : undefined, + skills: [], + skipCliInstall: ev.metadata.skipCliInstall, + }) + ); + + const hostWorkspace = join(OUTPUT_DIR, ev.id, solution, 'workspace'); + rmSync(hostWorkspace, { recursive: true, force: true }); + await session.exportWorkspace(hostWorkspace); + let copiedWithheldTests = false; + + return scorer({ + ...session.scoringContext, + // A solution nobody ran an agent against has no transcript, so checks + // reading one fail by construction. + toolCalls: [], + transcript: [], + agentReport: '', + hostWorkspace, + runViteBuild: () => viteBuild(hostWorkspace), + runVitest: () => { + if (!copiedWithheldTests) { + copyWithheldTests(ev, hostWorkspace); + copiedWithheldTests = true; + } + return vitestRun(hostWorkspace); + }, + }); +} + +function report(solution: string, result: ScoreResult) { + const checks = result.checks ?? []; + const failed = checks.filter((check) => !check.passed); + console.log(`\n${solution}`); + for (const check of checks) { + console.log(` ${check.passed ? '✅' : '❌'} ${check.name}`); + } + console.log( + ` ${checks.length - failed.length}/${checks.length} checks passed` + ); +} + +async function main() { + if (EVAL_FILTERS.length !== 1) { + throw new Error('pass exactly one --eval'); + } + const evalId = EVAL_FILTERS[0]; + const ev = discoverEvals().find((candidate) => candidate.id === evalId); + if (!ev) throw new Error(`no eval matched: ${evalId}`); + + // Tools-mode evals seed a hosted project rather than a workspace, so a + // solution for one is SQL applied to platform-lite, not files copied in. + if (ev.mode !== 'local-stack') { + throw new Error( + `${ev.id} is a tools eval; scoring solutions is local-stack only` + ); + } + + const available = listSolutions(ev); + if (available.length === 0) { + throw new Error( + `${ev.id} has no solutions — add one under ${relative(ROOT, solutionsDir(ev))}/` + ); + } + const missing = SOLUTION_FILTERS.filter((name) => !available.includes(name)); + if (missing.length > 0) { + throw new Error( + `no solution matched: ${missing.join(',')} (have ${available.join(', ')})` + ); + } + const solutions = + SOLUTION_FILTERS.length > 0 ? SOLUTION_FILTERS : available.sort(); + + console.log( + `${ev.id}: scoring ${solutions.length} solution(s) with no agent run` + ); + // Serial: each solution boots its own stack on the same host ports. + for (const solution of solutions) { + report(solution, await scoreSolution(ev, solution)); + } + console.log( + `\nCompare these against the failures you expected. A bad solution failing checks is the point.` + ); +} + +await main(); diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..c5e7d2d0 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -8,6 +8,7 @@ "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", + "eval:solution": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-solution.ts", "typecheck": "tsc --noEmit", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", diff --git a/evals/build-cli-002-declarative-schema/solutions/README.md b/evals/build-cli-002-declarative-schema/solutions/README.md new file mode 100644 index 00000000..acd063a3 --- /dev/null +++ b/evals/build-cli-002-declarative-schema/solutions/README.md @@ -0,0 +1,9 @@ +# Example solutions + +What the scorer should say about each. `supabase db diff used to generate the +migration` reads the agent's tool calls, so it fails on every solution here. + +| Solution | Expected result | +| ---------------- | ------------------------------------------------------------------------ | +| `green` | Everything but the tool-call check passes. | +| `migration-only` | Also fails `schema file updated to include description column`. The column reaches the database, but the declarative schema no longer describes it, so the next `db diff` would try to drop it. | diff --git a/evals/build-cli-002-declarative-schema/solutions/green/supabase/migrations/20260102000000_add_products_description.sql b/evals/build-cli-002-declarative-schema/solutions/green/supabase/migrations/20260102000000_add_products_description.sql new file mode 100644 index 00000000..79870c2c --- /dev/null +++ b/evals/build-cli-002-declarative-schema/solutions/green/supabase/migrations/20260102000000_add_products_description.sql @@ -0,0 +1 @@ +alter table "public"."products" add column "description" text; diff --git a/evals/build-cli-002-declarative-schema/solutions/green/supabase/schemas/products.sql b/evals/build-cli-002-declarative-schema/solutions/green/supabase/schemas/products.sql new file mode 100644 index 00000000..ee4ddb41 --- /dev/null +++ b/evals/build-cli-002-declarative-schema/solutions/green/supabase/schemas/products.sql @@ -0,0 +1,6 @@ +create table public.products ( + id serial primary key, + name text not null, + price numeric not null, + description text +); diff --git a/evals/build-cli-002-declarative-schema/solutions/migration-only/supabase/migrations/20260102000000_add_products_description.sql b/evals/build-cli-002-declarative-schema/solutions/migration-only/supabase/migrations/20260102000000_add_products_description.sql new file mode 100644 index 00000000..79870c2c --- /dev/null +++ b/evals/build-cli-002-declarative-schema/solutions/migration-only/supabase/migrations/20260102000000_add_products_description.sql @@ -0,0 +1 @@ +alter table "public"."products" add column "description" text; diff --git a/package.json b/package.json index ab7f505e..9b0de333 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "eval": "pnpm --filter @supabase-evals/framework eval", "eval:dry": "pnpm --filter @supabase-evals/framework eval:dry", "eval:smoke": "pnpm --filter @supabase-evals/framework eval:smoke", + "eval:solution": "pnpm --filter @supabase-evals/framework eval:solution", "eval:force": "pnpm --filter @supabase-evals/framework eval:force", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 347b26ef..b17a93bc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -417,6 +417,12 @@ export type LocalStackSessionArgs = { * workspace — the developer's working directory. */ localDir?: string; + /** + * Host directory copied over the workspace after `localDir`, before the + * stack starts. Scoring a committed example solution passes one here, so + * migrations the solution ships apply on `supabase start` with no agent run. + */ + solutionDir?: string; /** * Local-stack services this eval needs (from `services:` frontmatter). * Everything else is excluded from `supabase start` to keep boots fast; diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index d17bc08d..52c4eaa1 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -38,6 +38,8 @@ export interface AgentEnvironmentOptions { cliVersion?: string; /** Host directory whose contents seed the workspace. */ localDir?: string; + /** Host directory copied over the workspace after `localDir`, before the stack starts. */ + solutionDir?: string; /** Skills to install into the sandbox (the agent reads them with its file tools). */ skills?: readonly SkillSource[]; /** @@ -78,6 +80,7 @@ export async function createAgentEnvironment( cliVersion: options.cliVersion, includeServices: options.localStack.includeServices, localDir: options.localDir, + solutionDir: options.solutionDir, projectRunning: options.localStack.projectRunning, hosted: options.localStack.hosted, skipCliInstall: options.localStack.skipCliInstall, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 67243bf3..9ee0f571 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -75,6 +75,7 @@ export function localStackRuntime( async startSession({ cliVersion, localDir, + solutionDir, includeServices, projectRunning, hosted, @@ -84,6 +85,7 @@ export function localStackRuntime( const env = await createAgentEnvironment({ cliVersion: cliVersion ?? options.cliVersion, localDir, + solutionDir, skills, localStack: { includeServices, diff --git a/packages/sandbox/src/supabase.ts b/packages/sandbox/src/supabase.ts index 0f2c3857..ae33aec5 100644 --- a/packages/sandbox/src/supabase.ts +++ b/packages/sandbox/src/supabase.ts @@ -169,6 +169,12 @@ export interface SetupSupabaseSandboxOptions { includeServices?: readonly string[]; /** Host directory whose contents seed the sandbox workspace. */ localDir?: string; + /** + * Host directory copied over the workspace after `localDir`, before the + * stack starts. Scoring a committed example solution passes one here, so + * migrations the solution ships apply on `supabase start`. + */ + solutionDir?: string; /** * Whether the local stack should already be running when the agent starts * (default true): the workspace must then contain supabase/config.toml and @@ -257,6 +263,12 @@ export async function setupSupabaseSandbox( await sandbox.copyToContainer(options.localDir, sandbox.workdir); } + // After localDir so a solution's files win, before the stack starts so its + // migrations apply on `supabase start`. + if (options.solutionDir) { + await sandbox.copyToContainer(options.solutionDir, sandbox.workdir); + } + // When linked to a hosted project with a wire endpoint, the CLI wrapper routes // linked DB commands (`db push`/`migration repair`/…) at it via --db-url (the // CLI otherwise hardcodes the linked port to 5432). This is needed whether or