feat(framework): score committed solutions without an agent run

.gitignore

@@ -11,4 +11,5 @@ evals/*/local/supabase/.branches/
 .DS_Store
 results/*/
 .sync-tmp/
+.solution-runs/
 

CONTRIBUTING.md

@@ -13,7 +13,7 @@ Then add a folder under `evals/` containing:
 
 1. `PROMPT.md` with frontmatter metadata and the task the agent sees.
 2. `EVAL.ts` with the scorer.
-3. `solutions/` with the example solutions you scored the eval against. See [Checking your scorer](#checking-your-scorer).
+3. `solutions/` with the example solutions you scored the eval against, one directory per solution. See [Checking your scorer](#checking-your-scorer).
 4. Optional `remote/` data when the scenario needs to seed hosted project state, such as database, logs, or functions.
 5. Optional `local/` files when the scenario needs to seed a local filesystem, such as a local `supabase/` project.
 
@@ -77,6 +77,8 @@ Write each solution in the shape of the thing your eval measures.
 | Project state     | SQL and functions, shaped like the eval's `remote/` seed.                            |
 | A report          | The answer text, for evals scored on what the agent said rather than what it built.  |
 
+`pnpm eval:solution` scores workspace-state solutions. Project-state and report solutions are still worth committing as the record of what you checked, but you score them by hand for now.
+
 ### What to write
 
 Write two kinds of solution:
@@ -88,16 +90,25 @@ Write two kinds of solution:
 
 ### How to score them
 
+`pnpm eval:solution` runs the eval's scorer against a committed solution with no agent, so the same input gives the same verdicts every time.
+
 1. Write down the checks you expect each bad solution to fail. Do this before you run anything.
-2. Score the green solution. Every check should pass.
-3. Score each bad solution. The failures should match your list.
+2. Score every solution in the eval's `solutions/` directory:
+
+   ```bash
+   pnpm eval:solution -- --eval build-cli-002-declarative-schema
+   ```
+
+   Add `--solution <name>` to score one. The run needs Docker and the local stack's default ports, same as any local-stack eval, and writes nothing to `results/`.
+
+3. Read the output. The green solution should pass every check, and each bad solution should fail the ones you listed.
 4. Commit the list next to the solution it describes.
 
 ### Reading the results
 
 - **The green solution fails a check.** Look at the solution and the check. Either could be the thing that's wrong.
 - **A bad solution fails several checks.** Expected. One flaw usually trips more than one check, so matching your list is the bar, not one failure per solution.
-- **A check reads the agent's transcript.** It has nothing to read on a solution you wrote yourself. Expect it to fail, and note that in the list.
+- **A check reads the agent's transcript.** It has nothing to read on a solution you wrote yourself, since no agent ran. Expect it to fail, and note that in the list.
 
 ## Adding an experiment
 

README.md

@@ -64,6 +64,18 @@ Run all benchmark and no-skills experiments across all benchmark evals:
 pnpm eval -- --suite benchmark --experiment-suite benchmark,no-skills
 ```
 
+### Score an example solution
+
+Run an eval's committed solutions through its scorer with no agent:
+
+```bash
+pnpm eval:solution -- --eval build-cli-002-declarative-schema
+```
+
+Each directory under the eval's `solutions/` is copied over the workspace after `local/` and before the stack starts, so migrations it ships apply on `supabase start`. Pass `--solution <name>` to score one instead of all of them.
+
+The run prints every check and writes nothing to `results/`; the exported workspace lands in `.solution-runs/<eval>/<solution>/`. Checks that read the agent's transcript fail here by construction, because no agent ran. Local-stack evals only — a tools eval's starting state is a seeded hosted project, not a workspace to copy files into.
+
 ### View results in the web app
 
 After running evals locally, export their results to `eval-results.json` for the web app:

apps/framework/harness/eval-discovery.ts

@@ -0,0 +1,116 @@
+/**
+ * Eval discovery and the small pieces of run setup that both entry points
+ * share: `run-eval.ts` (agent runs) and `run-solution.ts` (scoring a committed
+ * solution). Keeping them here means the two can't drift on what an eval is or
+ * how its seed data is found.
+ */
+
+import {
+  cpSync,
+  existsSync,
+  readdirSync,
+  readFileSync,
+  statSync,
+} from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
+
+import type { EvalInterface, EvalManifest, EvalMode } from './types.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const ROOT = join(__dirname, '..', '..', '..');
+
+// Fixed identifiers for the mocked hosted project a local-stack eval links to.
+// Both must satisfy the CLI's format checks: ref is `^[a-z]{20}$`, token is
+// `^sbp_[a-f0-9]{40}$`. platform-lite accepts whatever token it's booted with.
+export const HOSTED_PROJECT_REF = 'evalshostedprojectxy';
+export const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40);
+
+/**
+ * Resolve the run mode. The sandbox (local-stack) is needed when the agent
+ * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that
+ * start from an empty workspace — or when the eval ships a `local/` workspace
+ * of starting files. Everything else runs against the in-memory tools runtime.
+ *
+ * `interface` is otherwise a benchmark dimension (KPI), not a runtime switch.
+ */
+export function resolveEvalMode(
+  interfaceKind: EvalInterface | undefined,
+  hasLocal: boolean
+): EvalMode {
+  if (interfaceKind === 'cli' || hasLocal) return 'local-stack';
+  return 'tools';
+}
+
+export function discoverEvals(): EvalManifest[] {
+  const dir = join(ROOT, 'evals');
+  if (!existsSync(dir)) return [];
+  const out: EvalManifest[] = [];
+  for (const id of readdirSync(dir)) {
+    const evalDir = join(dir, id);
+    if (!statSync(evalDir).isDirectory()) continue;
+    const localDir = join(evalDir, 'local');
+    const promptPath = join(evalDir, 'PROMPT.md');
+    const evalPath = join(evalDir, 'EVAL.ts');
+    const metadata = parseEvalMarkdown(
+      readFileSync(promptPath, 'utf8'),
+      `evals/${id}/PROMPT.md`
+    ).metadata;
+    const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory();
+    const mode = resolveEvalMode(metadata.interface, hasLocal);
+    out.push({
+      id,
+      mode,
+      metadata,
+      stage: metadata.stage,
+      product: metadata.product,
+      suite: metadata.suite,
+      topic: metadata.topic,
+      dir: evalDir,
+      localDir: hasLocal ? localDir : undefined,
+      promptPath,
+      evalPath,
+      remoteDir: join(evalDir, 'remote'),
+    });
+  }
+  return out;
+}
+
+export function readSessionSeedArgs(ev: EvalManifest) {
+  const projectSeedSql = join(ev.remoteDir, 'project.sql');
+  const logsSeedJsonl = join(ev.remoteDir, 'logs.jsonl');
+  const functionsSeedDir = join(ev.remoteDir, 'functions');
+
+  return {
+    projectSeedSql: existsSync(projectSeedSql) ? projectSeedSql : undefined,
+    logsSeedJsonl: existsSync(logsSeedJsonl) ? logsSeedJsonl : undefined,
+    functionsSeedDir: existsSync(functionsSeedDir)
+      ? functionsSeedDir
+      : undefined,
+    pgvector: ev.metadata.product.includes('vectors'),
+  };
+}
+
+export function copyWithheldTests(ev: EvalManifest, workspace: string) {
+  const testsDir = join(ev.dir, 'tests');
+  if (existsSync(testsDir)) {
+    cpSync(testsDir, join(workspace, 'tests'), { recursive: true });
+  }
+}
+
+/**
+ * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
+ * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
+ * `return`, or a throw), including when a *later* resource created in the same
+ * scope throws before its own `try`/`finally` is reached.
+ */
+export function disposable<T extends { close(): Promise<unknown> }>(
+  resource: T
+): T & AsyncDisposable {
+  return Object.assign(resource, {
+    [Symbol.asyncDispose]: async () => {
+      await resource.close();
+    },
+  });
+}

apps/framework/harness/run-eval.ts

@@ -1,17 +1,15 @@
 #!/usr/bin/env tsx
 import {
-  cpSync,
   existsSync,
   mkdirSync,
   readdirSync,
   readFileSync,
   realpathSync,
   rmSync,
-  statSync,
   writeFileSync,
 } from 'node:fs';
 import { join, dirname, relative } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
+import { pathToFileURL } from 'node:url';
 import { jsonSchema, tool, type ToolSet } from 'ai';
 import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
 import {
@@ -26,6 +24,15 @@ import {
   readSuiteFilters,
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
+import {
+  HOSTED_ACCESS_TOKEN,
+  HOSTED_PROJECT_REF,
+  ROOT,
+  copyWithheldTests,
+  discoverEvals,
+  disposable,
+  readSessionSeedArgs,
+} from './eval-discovery.js';
 import { viteBuild, vitestRun } from './project-runner.js';
 import {
   buildDocsResult,
@@ -35,7 +42,6 @@ import {
 } from '@supabase-evals/core';
 import type {
   ExperimentConfig,
-  EvalInterface,
   EvalManifest,
   EvalMode,
   EvalSuite,
@@ -48,15 +54,6 @@ import type {
   TranscriptPart,
 } from './types.js';
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, '..', '..', '..');
-
-// Fixed identifiers for the mocked hosted project a local-stack eval links to.
-// Both must satisfy the CLI's format checks: ref is `^[a-z]{20}$`, token is
-// `^sbp_[a-f0-9]{40}$`. platform-lite accepts whatever token it's booted with.
-const HOSTED_PROJECT_REF = 'evalshostedprojectxy';
-const HOSTED_ACCESS_TOKEN = 'sbp_' + '0'.repeat(40);
-
 const rawArgs = process.argv.slice(2);
 const args = new Set(rawArgs);
 const FORCE = !args.has('--skip-existing');
@@ -107,56 +104,6 @@ function readFlag(name: string): string | undefined {
   return undefined;
 }
 
-/**
- * Resolve the run mode. The sandbox (local-stack) is needed when the agent
- * uses the Supabase CLI (`interface: cli`) — including bootstrap scenarios that
- * start from an empty workspace — or when the eval ships a `local/` workspace
- * of starting files. Everything else runs against the in-memory tools runtime.
- *
- * `interface` is otherwise a benchmark dimension (KPI), not a runtime switch.
- */
-function resolveEvalMode(
-  interfaceKind: EvalInterface | undefined,
-  hasLocal: boolean
-): EvalMode {
-  if (interfaceKind === 'cli' || hasLocal) return 'local-stack';
-  return 'tools';
-}
-
-function discoverEvals(): EvalManifest[] {
-  const dir = join(ROOT, 'evals');
-  if (!existsSync(dir)) return [];
-  const out: EvalManifest[] = [];
-  for (const id of readdirSync(dir)) {
-    const evalDir = join(dir, id);
-    if (!statSync(evalDir).isDirectory()) continue;
-    const localDir = join(evalDir, 'local');
-    const promptPath = join(evalDir, 'PROMPT.md');
-    const evalPath = join(evalDir, 'EVAL.ts');
-    const metadata = parseEvalMarkdown(
-      readFileSync(promptPath, 'utf8'),
-      `evals/${id}/PROMPT.md`
-    ).metadata;
-    const hasLocal = existsSync(localDir) && statSync(localDir).isDirectory();
-    const mode = resolveEvalMode(metadata.interface, hasLocal);
-    out.push({
-      id,
-      mode,
-      metadata,
-      stage: metadata.stage,
-      product: metadata.product,
-      suite: metadata.suite,
-      topic: metadata.topic,
-      dir: evalDir,
-      localDir: hasLocal ? localDir : undefined,
-      promptPath,
-      evalPath,
-      remoteDir: join(evalDir, 'remote'),
-    });
-  }
-  return out;
-}
-
 type ToolsSkill = { name: string; description: string; body: string };
 
 /**
@@ -284,28 +231,6 @@ function workspacePath(modelName: string, evalId: string, attempt: number) {
   );
 }
 
-function copyWithheldTests(ev: EvalManifest, workspace: string) {
-  const testsDir = join(ev.dir, 'tests');
-  if (existsSync(testsDir)) {
-    cpSync(testsDir, join(workspace, 'tests'), { recursive: true });
-  }
-}
-
-function readSessionSeedArgs(ev: EvalManifest) {
-  const projectSeedSql = join(ev.remoteDir, 'project.sql');
-  const logsSeedJsonl = join(ev.remoteDir, 'logs.jsonl');
-  const functionsSeedDir = join(ev.remoteDir, 'functions');
-
-  return {
-    projectSeedSql: existsSync(projectSeedSql) ? projectSeedSql : undefined,
-    logsSeedJsonl: existsSync(logsSeedJsonl) ? logsSeedJsonl : undefined,
-    functionsSeedDir: existsSync(functionsSeedDir)
-      ? functionsSeedDir
-      : undefined,
-    pgvector: ev.metadata.product.includes('vectors'),
-  };
-}
-
 function basePromptFor(mode: EvalMode): string {
   if (mode === 'local-stack') {
     return (
@@ -331,22 +256,6 @@ function buildSystemPrompt(
   return blocks.join('\n\n');
 }
 
-/**
- * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
- * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
- * `return`, or a throw), including when a *later* resource created in the same
- * scope throws before its own `try`/`finally` is reached.
- */
-function disposable<T extends { close(): Promise<unknown> }>(
-  resource: T
-): T & AsyncDisposable {
-  return Object.assign(resource, {
-    [Symbol.asyncDispose]: async () => {
-      await resource.close();
-    },
-  });
-}
-
 async function runOne(
   expName: string,
   exp: ExperimentConfig,