diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ef82568d41c9f..e12ca635d7fb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -984,6 +984,43 @@ repos: ^scripts/ci/prek/generate_agent_skills\.py$ pass_filenames: false additional_dependencies: ['pyyaml', 'rich>=13.6.0'] + - id: run-skill-eval + name: Run skill-eval against AGENTS.md and cases + entry: ./dev/skill-evals/eval.py + language: node + language_version: '22.22.0' + additional_dependencies: ['promptfoo@0.121.17', '@anthropic-ai/claude-agent-sdk@0.3.185'] + stages: ['manual'] + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/ + pass_filenames: false + require_serial: true + verbose: true + - id: view-skill-eval + name: View skill-eval results in browser (manual) + entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view + language: node + language_version: '22.22.0' + additional_dependencies: ['promptfoo@0.121.17', '@anthropic-ai/claude-agent-sdk@0.3.185'] + stages: ['manual'] + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/ + pass_filenames: false + require_serial: true + - id: check-eval-hash + name: Verify skill-eval ran on current AGENTS.md and cases + entry: ./scripts/ci/prek/check_eval_hash.py + language: python + files: > + (?x) + ^AGENTS\.md$| + ^dev/skill-evals/cases/| + ^dev/skill-evals/last-eval-hash\.txt$ + pass_filenames: false - id: update-pyproject-toml name: Update Airflow's meta-package pyproject.toml language: python diff --git a/.rat-excludes b/.rat-excludes index f0cd2910f5b27..b84b09160e658 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -158,6 +158,7 @@ PKG-INFO # checksum files .*\.md5sum +last-eval-hash.txt # Openapi files .openapi-generator-ignore diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md new file mode 100644 index 0000000000000..57748b96c2e5c --- /dev/null +++ b/dev/skill-evals/README.md @@ -0,0 +1,151 @@ + + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Skill-Eval Harness](#skill-eval-harness) + - [Prerequisites](#prerequisites) + - [Usage](#usage) + - [Cleanup](#cleanup) + - [Adding cases](#adding-cases) + - [How it works](#how-it-works) + - [Eval-run hash gate](#eval-run-hash-gate) + + + +# Skill-Eval Harness + +Test whether changes to `AGENTS.md` break or improve agent decisions. +The harness compares the `main` branch version against your working +tree, running the same cases against both and reporting the diff. + +Each arm is a **git worktree** of the real repo — the agent sees +actual source files (`pyproject.toml`, directory structure, etc.). +The only difference between arms is which `AGENTS.md` is present. + +The agent reads guidance through the `CLAUDE.md → AGENTS.md` symlink, +so the harness verifies that symlink exists (in the working tree and +on the base branch) before running and aborts if it is broken — a +regular-file CLAUDE.md would make every arm read identical guidance. + +## Prerequisites + +- **Authentication** (one of): + - Claude Code session (`claude /login`) — Pro/Max subscription + - `ANTHROPIC_API_KEY` environment variable — API credits + +That's it — prek provisions Node, promptfoo, and the Claude Agent SDK +automatically. + +## Usage + +```bash +# Run the eval (single pass over all cases — satisfies the hash gate). +# Stage your changes first — prek stashes unstaged edits: +prek run run-skill-eval --hook-stage manual --all-files + +# Repeat each case to reduce nondeterminism: +EVAL_REPEAT=3 prek run run-skill-eval --hook-stage manual --all-files + +# Add a baseline arm (no AGENTS.md) to measure raw model capability: +EVAL_FULL=1 prek run run-skill-eval --hook-stage manual --all-files + +# Use a cheaper model for fast iteration: +MODEL=claude-haiku-4-5-20251001 prek run run-skill-eval --hook-stage manual --all-files + +# Test a skill alongside AGENTS.md (not combinable with EVAL_FULL — the +# baseline arm has no skill, so the skill-used assertion would always fail): +SKILL_NAME=airflow-contribution prek run run-skill-eval --hook-stage manual --all-files + +# View results in browser (Ctrl-C to stop the server): +prek run view-skill-eval --hook-stage manual --all-files +``` + +Other promptfoo flags (`--filter*`, `--no-cache`) are argv-only — +`prek run` can't forward arguments, so wire them as fixed entry args +on a hook variant if a preset is needed. + +A run with `--filter*` flags covers only a subset of cases, so it does +not update the hash file. + +Each run also writes a JSON report to `files/skill-evals/results.json` +(per the `files/` output convention) — handy for pasting results into +a PR. + +## Cleanup + +Everything the harness stores lives inside the repo — nothing is left +in your home directory: + +```bash +rm -rf .build/promptfoo # eval history and cache +prek clean # prek-provisioned node envs +``` + +## Adding cases + +Cases live in `cases/*.yaml`. Add entries to an existing file or +create a new one — no config changes needed. + +```yaml +- description: "Scheduler bugfix (#64322): no newsfragment" + vars: + request: | + I fixed the scheduler to skip asset-triggered Dags that don't + have a SerializedDagModel yet. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' +``` + +The agent returns structured JSON (`{should_create, type, rationale}`). +Use `output.should_create` directly in assertions. + +## How it works + +1. Creates git worktrees — one with `main`'s AGENTS.md, one with + your working tree version. Both are full repo checkouts. +2. Generates a [promptfoo](https://github.com/promptfoo/promptfoo) + config with `anthropic:claude-agent-sdk` provider and + `output_format: json_schema` for structured output. +3. Runs each case against all arms in parallel. +4. Reports pass/fail diff. Worktrees cleaned up on exit. + +## Eval-run hash gate + +After every completed run, `eval.py` records a hash of its inputs +(`AGENTS.md` + `cases/*.yaml`) in `last-eval-hash.txt`. The +`check-eval-hash` prek hook — enforced locally and in CI — recomputes +the hash and fails when guidance changed without a re-run. Commit the +updated hash file together with your change. + +- The hash proves the eval **ran** on this exact content, not that all + cases passed (some cases fail even on `main` — that is signal, not a + defect). +- WIP commits: `SKIP=check-eval-hash git commit ...` — CI stays red + until the eval is re-run. +- Can't run the eval (no Claude subscription or API key)? Ask a + maintainer to run it and push the updated hash file to your PR branch. +- `SKILL.md` files are not covered by the gate yet — there are no skill + cases, so a hash over them would prove nothing. Extend + `compute_guidance_hash` in `scripts/ci/prek/check_eval_hash.py` when + per-skill cases land. diff --git a/dev/skill-evals/cases/newsfragment.yaml b/dev/skill-evals/cases/newsfragment.yaml new file mode 100644 index 0000000000000..b769afa54864a --- /dev/null +++ b/dev/skill-evals/cases/newsfragment.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Cases from real PRs where agents created newsfragments that +# reviewers asked to remove. These are the ambiguous boundary +# cases — the changes look like they could be user-facing. + +- description: "Scheduler bugfix (#64322): no newsfragment" + vars: + request: | + I fixed the scheduler to skip asset-triggered Dags that don't + have a SerializedDagModel yet. Without this fix, premature + asset-triggered DagRuns could be created during parsing. + The fix is in airflow-core/src/airflow/models/dag.py. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "i18n cache busting (#65720): no newsfragment" + vars: + request: | + I fixed stale translation files after Airflow upgrades by + appending the version number to the i18n file load path. + The fix is in airflow-core/src/airflow/ui/src/i18n/config.ts. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "API query optimization (#66696): no newsfragment" + vars: + request: | + I replaced COALESCE with index-friendly OR conditions in the + datetime range filters in airflow-core API to improve query + performance. No behavior change for API consumers. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "Provider bug fix (#67333): no newsfragment" + vars: + request: | + I fixed a monitoring-pod leak in KubernetesJobOperator. + The fix is in providers/cncf/kubernetes/. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === false' + +- description: "Worker connection testing (#62343): feature newsfragment" + vars: + request: | + I added asynchronous connection testing: connection tests can now + run on a worker instead of in-process on the API server, submitted + via a new POST /connections/enqueue-test endpoint and configured + under a new [connection_test] config section. The changes span + airflow-core and task-sdk. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === true' + +- description: "Coordinator layer (#65958): feature newsfragment" + vars: + request: | + I added a coordinator layer to the task execution supervisor so + coordinators written in other languages can drive task execution, + along with a first Java coordinator and a new config option. Most + of the changes are in task-sdk, with some in airflow-core. + Should I create a newsfragment? + assert: + - type: javascript + value: 'output.should_create === true' diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py new file mode 100755 index 0000000000000..8505f29374d78 --- /dev/null +++ b/dev/skill-evals/eval.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.9" +# /// +"""Airflow skill-eval harness. + +Test whether AGENTS.md guidance affects agent decisions by comparing +arms with and without the guidance. Each arm is a git worktree of the +real repo — the agent sees actual source files. + +Usage: + prek run run-skill-eval --hook-stage manual --all-files + +Env knobs: MODEL, SKILL_NAME, EVAL_REPEAT, EVAL_FULL (baseline arm). +Promptfoo flags like --filter* are argv-only — wire them as fixed entry +args on a hook variant when needed. + +Authentication: Claude Code session (claude /login) or ANTHROPIC_API_KEY. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +PROMPTFOO_VERSION = "0.121.17" + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +SCRIPT_DIR = Path(__file__).resolve().parent +AGENTS_SRC = REPO_ROOT / "AGENTS.md" +CASES_DIR = SCRIPT_DIR / "cases" +HASH_FILE = SCRIPT_DIR / "last-eval-hash.txt" + +# Tool state lives under .build (repo-scoped, established cleanup culture); +# per-run reports go to files/ per the AGENTS.md output convention. +PROMPTFOO_STATE_DIR = REPO_ROOT / ".build" / "promptfoo" +RESULTS_FILE = REPO_ROOT / "files" / "skill-evals" / "results.json" + +# Shared with the check-eval-hash prek hook so recorded and verified hashes can't drift. +sys.path.insert(0, str(REPO_ROOT / "scripts" / "ci" / "prek")) +from check_eval_hash import compute_guidance_hash, write_recorded_hash # noqa: E402 + +OUTPUT_FORMAT = { + "type": "json_schema", + "schema": { + "type": "object", + "required": ["should_create", "rationale"], + "additionalProperties": False, + "properties": { + "should_create": {"type": "boolean"}, + "type": { + "type": "string", + "enum": ["bugfix", "feature", "improvement", "doc", "misc", "significant"], + }, + "rationale": {"type": "string"}, + }, + }, +} + + +def run(cmd: list[str], **kwargs) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, check=False, **kwargs) + + +def find_sdk_modules() -> Path: + """Locate the node_modules dir (in the prek env) that contains the Claude Agent SDK. + + promptfoo resolves the SDK from the eval config's directory, not from its + own install tree — the caller symlinks this dir next to the config. + """ + promptfoo_bin = shutil.which("promptfoo") + if promptfoo_bin: + version = run(["promptfoo", "--version"]).stdout.strip() + if version == PROMPTFOO_VERSION: + pf_pkg = Path(promptfoo_bin).resolve() + while pf_pkg.name != "promptfoo" or not (pf_pkg / "package.json").is_file(): + if pf_pkg.parent == pf_pkg: + break + pf_pkg = pf_pkg.parent + for candidate in (pf_pkg / "node_modules", pf_pkg.parent): + if (candidate / "@anthropic-ai" / "claude-agent-sdk").is_dir(): + return candidate + print( + "Error: promptfoo with the Claude Agent SDK not found. Run the eval via:\n" + " prek run run-skill-eval --hook-stage manual --all-files", + file=sys.stderr, + ) + sys.exit(1) + + +def check_claude_md_symlink(base_branch: str) -> None: + """Verify CLAUDE.md is a symlink to AGENTS.md, in the working tree and on the base branch. + + The Claude Agent SDK reads CLAUDE.md. Swapping AGENTS.md between arms only + affects the agent while CLAUDE.md resolves to it — if CLAUDE.md ever becomes + a regular file, every arm would read identical guidance and the eval would + silently measure nothing. + """ + link = REPO_ROOT / "CLAUDE.md" + working_ok = link.is_symlink() and os.readlink(link) == "AGENTS.md" + tree_entry = run(["git", "-C", str(REPO_ROOT), "ls-tree", base_branch, "CLAUDE.md"]).stdout + target = run(["git", "-C", str(REPO_ROOT), "show", f"{base_branch}:CLAUDE.md"]).stdout + branch_ok = tree_entry.startswith("120000") and target == "AGENTS.md" + if not (working_ok and branch_ok): + print( + "Error: CLAUDE.md must be a symlink to AGENTS.md (in the working tree" + f" and on '{base_branch}'). The eval swaps AGENTS.md between arms and" + " relies on the agent reading it through that symlink.", + file=sys.stderr, + ) + sys.exit(1) + + +def resolve_base_branch() -> str: + result = run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "main"]) + if result.returncode == 0: + return "main" + branch = run(["git", "-C", str(REPO_ROOT), "rev-parse", "--abbrev-ref", "HEAD"]).stdout.strip() + print(f"Warning: 'main' not found, using current branch '{branch}' as base") + return branch + + +def git_show_file(base_branch: str, path: str, dest: Path) -> bool: + result = run(["git", "-C", str(REPO_ROOT), "show", f"{base_branch}:{path}"]) + if result.returncode != 0: + return False + dest.write_text(result.stdout) + return True + + +def create_worktree( + work_dir: Path, + name: str, + base_branch: str, + agents_file: Path | None, + worktrees: list[Path], + skill_name: str | None = None, + skill_file: Path | None = None, +) -> Path: + """Create a git worktree arm with the specified AGENTS.md and optional SKILL.md.""" + wt_dir = work_dir / name + result = run( + [ + "git", + "-C", + str(REPO_ROOT), + "worktree", + "add", + "--quiet", + "--detach", + str(wt_dir), + base_branch, + ] + ) + if result.returncode != 0: + print( + f"Error: 'git worktree add' failed for arm '{name}':\n{result.stderr.strip()}", + file=sys.stderr, + ) + sys.exit(1) + # Register before mutating the checkout so cleanup covers this worktree + # even if a later step in this function fails. + worktrees.append(wt_dir) + + if agents_file is None: + # Baseline arm: remove all guidance. CLAUDE.md is a symlink to + # AGENTS.md — drop it too so the SDK finds no dangling link. + (wt_dir / "AGENTS.md").unlink(missing_ok=True) + (wt_dir / "CLAUDE.md").unlink(missing_ok=True) + else: + shutil.copy2(agents_file, wt_dir / "AGENTS.md") + + if skill_name and skill_file: + skill_dir = wt_dir / ".agents" / "skills" / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(skill_file, skill_dir / "SKILL.md") + + return wt_dir + + +def remove_worktree(wt_dir: Path) -> None: + run(["git", "-C", str(REPO_ROOT), "worktree", "remove", "--force", str(wt_dir)]) + + +def build_provider(label: str, working_dir: Path, model: str, skill_name: str | None = None) -> dict: + config: dict = { + "model": model, + "apiKeyRequired": False, + "setting_sources": ["project"], + "append_allowed_tools": ["Read", "Grep", "Glob"], + "working_dir": str(working_dir), + "output_format": OUTPUT_FORMAT, + } + if skill_name: + config["skills"] = [skill_name] + return {"id": "anthropic:claude-agent-sdk", "label": label, "config": config} + + +def count_provider_errors(results_file: Path) -> int: + """Count results whose provider call errored (as opposed to failing an assertion).""" + try: + results = json.loads(results_file.read_text())["results"]["results"] + except (OSError, KeyError, ValueError): + return 0 + return sum(1 for r in results if (r.get("response") or {}).get("error")) + + +def main() -> int: + sdk_modules = find_sdk_modules() + + model = os.environ.get("MODEL", "claude-sonnet-4-6") + skill_name = os.environ.get("SKILL_NAME") + skill_src = None + if skill_name: + skill_src = REPO_ROOT / ".agents" / "skills" / skill_name / "SKILL.md" + if not skill_src.is_file(): + print(f"Error: {skill_src} not found", file=sys.stderr) + return 1 + + # Parse flags — argv (hook entry args) plus single-value env knobs, + # since `prek run` can't forward arguments. No filter knob on purpose: + # a lingering export must not be able to change what the proof claims. + full_mode = "--full" in sys.argv or os.environ.get("EVAL_FULL", "").lower() in ("1", "true") + promptfoo_args = [a for a in sys.argv[1:] if a != "--full"] + repeat = os.environ.get("EVAL_REPEAT") + if repeat: + if not repeat.isdigit() or int(repeat) < 1: + print(f"Error: EVAL_REPEAT must be a positive integer, got {repeat!r}", file=sys.stderr) + return 1 + promptfoo_args += ["--repeat", repeat] + + if full_mode and skill_name: + print( + "Error: EVAL_FULL/--full cannot be combined with SKILL_NAME — the baseline" + " arm has no skill, so the 'skill-used' assertion would always fail on it.", + file=sys.stderr, + ) + return 1 + + base_branch = resolve_base_branch() + check_claude_md_symlink(base_branch) + + # Hash before building arms so edits made mid-run aren't recorded as tested. + guidance_hash = compute_guidance_hash(AGENTS_SRC, CASES_DIR) + + # Temp dir for config and worktrees + work_dir = Path(tempfile.mkdtemp()) + worktrees: list[Path] = [] + + try: + # promptfoo resolves the Claude Agent SDK from the config directory + (work_dir / "node_modules").symlink_to(sdk_modules) + + # Extract main-branch AGENTS.md + main_agents = work_dir / "main-agents.md" + if not git_show_file(base_branch, "AGENTS.md", main_agents): + shutil.copy2(AGENTS_SRC, main_agents) + print(f"Warning: AGENTS.md not found on {base_branch} — main arm uses working tree copy") + + main_skill = None + if skill_name and skill_src: + main_skill = work_dir / "main-skill.md" + skill_git_path = f".agents/skills/{skill_name}/SKILL.md" + if not git_show_file(base_branch, skill_git_path, main_skill): + shutil.copy2(skill_src, main_skill) + print(f"Warning: SKILL.md not found on {base_branch} — main arm uses working tree copy") + + # Detect changes to decide which arms to build + agents_changed = run(["diff", "-q", str(main_agents), str(AGENTS_SRC)]).returncode != 0 + skill_changed = False + if skill_name and main_skill and skill_src: + skill_changed = run(["diff", "-q", str(main_skill), str(skill_src)]).returncode != 0 + need_working = agents_changed or skill_changed + + # Assemble arms. Prune first to self-heal stale worktree registrations + # left behind by a previously interrupted run. + print("Assembling arms (git worktrees) ...") + run(["git", "-C", str(REPO_ROOT), "worktree", "prune"]) + + arm_main = create_worktree( + work_dir, "main", base_branch, main_agents, worktrees, skill_name, main_skill + ) + + arm_working = None + if need_working: + arm_working = create_worktree( + work_dir, "working", base_branch, AGENTS_SRC, worktrees, skill_name, skill_src + ) + else: + print(" AGENTS.md unchanged — skipping working arm") + + arm_baseline = None + if full_mode: + arm_baseline = create_worktree(work_dir, "baseline", base_branch, None, worktrees) + + # Generate config (JSON — valid promptfoo config, keeps the script stdlib-only) + providers = [build_provider("main", arm_main, model, skill_name)] + if arm_working: + providers.append(build_provider("working", arm_working, model, skill_name)) + if full_mode and arm_baseline: + providers.append(build_provider("baseline", arm_baseline, model)) + + default_test: dict = {"options": {"disableVarExpansion": True}} + if skill_name: + default_test["assert"] = [{"type": "skill-used", "value": skill_name}] + + config = { + "prompts": ["{{request}}"], + "providers": providers, + "defaultTest": default_test, + "tests": f"file://{SCRIPT_DIR}/cases/*.yaml", + } + config_path = work_dir / "promptfooconfig.json" + config_path.write_text(json.dumps(config, indent=2)) + + # Report + if skill_name: + print(f"Mode: {len(providers)} arms, skill '{skill_name}', model: {model}") + else: + print(f"Mode: {len(providers)} arms, AGENTS.md only, model: {model}") + + print() + print(f"Changes detected (vs {base_branch}):") + print(f" AGENTS.md — {'modified' if agents_changed else 'unchanged'}") + if skill_name: + print(f" SKILL.md — {'modified' if skill_changed else 'unchanged'} ({skill_name})") + print() + + # Run promptfoo — state under .build, per-run report under files/ + PROMPTFOO_STATE_DIR.mkdir(parents=True, exist_ok=True) + RESULTS_FILE.parent.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ["promptfoo", "eval", "-c", str(config_path), "--output", str(RESULTS_FILE), *promptfoo_args], + check=False, + env={**os.environ, "PROMPTFOO_CONFIG_DIR": str(PROMPTFOO_STATE_DIR)}, + ) + + # 0 = all passed, 100 = some assertions failed — both mean the eval + # completed and the results were seen, which is what the hash proves. + # Don't record partial runs (--filter* covers a subset of cases) or + # runs with provider errors (nothing was actually evaluated). + partial_run = any(arg.startswith("--filter") for arg in promptfoo_args) + provider_errors = count_provider_errors(RESULTS_FILE) + if result.returncode in (0, 100) and not partial_run and not provider_errors: + write_recorded_hash(guidance_hash, HASH_FILE) + print() + print(f"Recorded eval run in {HASH_FILE.relative_to(REPO_ROOT)} — commit it with your change.") + elif partial_run: + print() + print("Partial run (--filter*) — hash not recorded; run the full case set to update it.") + elif provider_errors: + print() + print(f"{provider_errors} provider error(s) — hash not recorded; fix the setup and rerun.") + + print() + print(f"Results report: {RESULTS_FILE.relative_to(REPO_ROOT)}") + print( + f"View results: PROMPTFOO_CONFIG_DIR={PROMPTFOO_STATE_DIR.relative_to(REPO_ROOT)}" + f" npx promptfoo@{PROMPTFOO_VERSION} view" + ) + return result.returncode + + finally: + for wt in worktrees: + remove_worktree(wt) + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dev/skill-evals/last-eval-hash.txt b/dev/skill-evals/last-eval-hash.txt new file mode 100644 index 0000000000000..d0863d6365794 --- /dev/null +++ b/dev/skill-evals/last-eval-hash.txt @@ -0,0 +1,3 @@ +# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. +# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate. +f4a71898d35b99e6c42ce756a71fa24ba243ada0611ed55966fce25711ac8c02 diff --git a/scripts/ci/prek/check_eval_hash.py b/scripts/ci/prek/check_eval_hash.py new file mode 100755 index 0000000000000..540338807e994 --- /dev/null +++ b/scripts/ci/prek/check_eval_hash.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Verify the skill-eval ran against the current guidance files. + +``dev/skill-evals/eval.py`` records a hash of its inputs in +``dev/skill-evals/last-eval-hash.txt`` after every completed run. This hook +recomputes the hash from the current file contents and fails when AGENTS.md +or the eval cases changed without a re-run. The hash proves the eval *ran* +on this exact content — not that all cases passed. + +The comparison is pure file content — no git state — so the hook behaves +identically at commit time, at push time, and in CI (``prek --all-files``). +""" + +from __future__ import annotations + +import hashlib +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +AGENTS_FILE = REPO_ROOT / "AGENTS.md" +CASES_DIR = REPO_ROOT / "dev" / "skill-evals" / "cases" +HASH_FILE = REPO_ROOT / "dev" / "skill-evals" / "last-eval-hash.txt" + +HASH_FILE_HEADER = """\ +# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand. +# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate. +""" + + +def compute_guidance_hash(agents_file: Path, cases_dir: Path) -> str: + """Hash the skill-eval inputs: AGENTS.md plus every case file. + + Labels are hashed alongside contents so renaming a case file changes + the hash. SKILL.md files are intentionally excluded — no skill cases + exist yet; extend the inputs when per-skill cases land (runs with + ``SKILL_NAME=...``). + """ + digest = hashlib.sha256() + entries = [("AGENTS.md", agents_file)] + entries += [(f"cases/{path.name}", path) for path in sorted(cases_dir.glob("*.yaml"))] + for label, path in entries: + digest.update(label.encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def read_recorded_hash(hash_file: Path) -> str | None: + """Return the hash recorded in ``hash_file``, or None if absent.""" + if not hash_file.is_file(): + return None + for line in hash_file.read_text().splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + return stripped + return None + + +def write_recorded_hash(value: str, hash_file: Path) -> None: + """Record ``value`` with a regeneration note, like breeze's output-commands-hash.txt.""" + hash_file.write_text(f"{HASH_FILE_HEADER}{value}\n") + + +def main() -> int: + current = compute_guidance_hash(AGENTS_FILE, CASES_DIR) + recorded = read_recorded_hash(HASH_FILE) + if current == recorded: + return 0 + print( + "AGENTS.md or dev/skill-evals/cases/ changed without re-running the" + " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n" + "\n" + " Run: prek run run-skill-eval --hook-stage manual --all-files\n" + " then commit the updated dev/skill-evals/last-eval-hash.txt\n" + "\n" + " WIP commit: SKIP=check-eval-hash git commit ...\n" + " Can't run the eval? Ask a maintainer to run it and push the updated\n" + " hash file to your PR branch (see dev/skill-evals/README.md).", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/ci/prek/test_check_eval_hash.py b/scripts/tests/ci/prek/test_check_eval_hash.py new file mode 100644 index 0000000000000..d3aeec4129eaa --- /dev/null +++ b/scripts/tests/ci/prek/test_check_eval_hash.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest +from ci.prek.check_eval_hash import ( + compute_guidance_hash, + main, + read_recorded_hash, + write_recorded_hash, +) + + +@pytest.fixture +def guidance(tmp_path): + """A tmp AGENTS.md and cases dir mirroring the eval input layout.""" + agents_file = tmp_path / "AGENTS.md" + agents_file.write_text("golden rule\n") + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "newsfragment.yaml").write_text("- description: case\n") + return agents_file, cases_dir + + +def test_compute_guidance_hash_is_deterministic(guidance): + agents_file, cases_dir = guidance + + assert compute_guidance_hash(agents_file, cases_dir) == compute_guidance_hash(agents_file, cases_dir) + + +@pytest.mark.parametrize( + "mutate", + [ + pytest.param( + lambda agents_file, cases_dir: agents_file.write_text("golden rule, revised\n"), + id="agents-content", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").write_text( + "- description: revised case\n" + ), + id="case-content", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "routing.yaml").write_text( + "- description: another case\n" + ), + id="case-added", + ), + pytest.param( + lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").rename( + cases_dir / "renamed.yaml" + ), + id="case-renamed", + ), + ], +) +def test_compute_guidance_hash_changes_when_inputs_change(guidance, mutate): + agents_file, cases_dir = guidance + before = compute_guidance_hash(agents_file, cases_dir) + + mutate(agents_file, cases_dir) + + assert compute_guidance_hash(agents_file, cases_dir) != before + + +def test_read_recorded_hash_returns_none_when_missing(tmp_path): + assert read_recorded_hash(tmp_path / "absent.txt") is None + + +def test_write_then_read_roundtrip(tmp_path): + hash_file = tmp_path / "last-eval-hash.txt" + + write_recorded_hash("abc123", hash_file) + + assert read_recorded_hash(hash_file) == "abc123" + assert hash_file.read_text().startswith("# Generated by dev/skill-evals/eval.py") + assert hash_file.read_text().endswith("abc123\n") + + +def test_main_passes_when_hash_matches(guidance, tmp_path, capsys): + agents_file, cases_dir = guidance + hash_file = tmp_path / "last-eval-hash.txt" + write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file) + + with ( + mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file), + mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir), + mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file), + ): + assert main() == 0 + + assert capsys.readouterr().err == "" + + +@pytest.mark.parametrize("record_stale_hash", [True, False], ids=["stale-hash", "missing-file"]) +def test_main_fails_on_mismatch_or_missing_record(guidance, tmp_path, capsys, record_stale_hash): + agents_file, cases_dir = guidance + hash_file = tmp_path / "last-eval-hash.txt" + if record_stale_hash: + write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file) + agents_file.write_text("changed after the recorded run\n") + + with ( + mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file), + mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir), + mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file), + ): + assert main() == 1 + + err = capsys.readouterr().err + assert "prek run run-skill-eval" in err + assert "SKIP=check-eval-hash" in err