diff --git a/src/llmxive/agents/task_verifier.py b/src/llmxive/agents/task_verifier.py index e479777db7e..b61408eb470 100644 --- a/src/llmxive/agents/task_verifier.py +++ b/src/llmxive/agents/task_verifier.py @@ -239,12 +239,12 @@ def _deterministic_verdict(project_dir: Path, task_text: str) -> tuple[str | Non return None, "" -#: How many files matching one pattern have their CONTENTS quoted (first and last). -_PATTERN_QUOTED = 2 _PATTERN_BYTES_PER_FILE = 3000 +#: Total bytes of pattern-matched file contents quoted for one task. +_PATTERN_QUOTE_BUDGET = 30_000 -def _pattern_evidence(project_dir: Path, task_text: str, files_used: int) -> list[str]: +def _pattern_evidence(project_dir: Path, task_text: str) -> list[str]: """Evidence for outputs a task names by PATTERN rather than one by one. PROJ-9999 canary, 2026-10-11. Task T004 declared its outputs as @@ -256,13 +256,14 @@ def _pattern_evidence(project_dir: Path, task_text: str, files_used: int) -> lis later the task was recorded UNVERIFIABLE and the project was kicked back. Every match is LISTED with its size and digest (so a count claim can be - checked and the verdict cache invalidates when any of them changes); the - first and last are QUOTED, within what is left of the file budget. + checked and the verdict cache invalidates when any of them changes), and + all are QUOTED when the family fits the byte budget. """ from llmxive.project_paths import declared_patterns, resolve_pattern root = project_dir.resolve() chunks: list[str] = [] + bytes_left = _PATTERN_QUOTE_BUDGET for written, glob in declared_patterns(task_text): matches = resolve_pattern(project_dir, glob) if not matches: @@ -278,17 +279,30 @@ def _pattern_evidence(project_dir: Path, task_text: str, files_used: int) -> lis listing.append(f" - `{f.relative_to(root)}` (unreadable: {exc})") chunk = (f"- `{written}` (pattern `{glob}`): {len(matches)} matching file(s) exist, " "collector-confirmed:\n" + "\n".join(listing)) - quoted = [matches[0]] if len(matches) == 1 else [matches[0], matches[-1]] - for f in quoted[:_PATTERN_QUOTED]: - if files_used >= _MAX_EVIDENCE_FILES: - chunk += "\n (contents not quoted: file evidence limit reached)" - break - files_used += 1 + # Quote EVERY file of the family when the family is small. Showing two + # of twelve got "the evidence only shows the contents of two CSVs ... + # cannot verify that each of them sums to its respective N" (canary, + # 2026-10-11): a claim about all of them needs all of them. A family + # too large for the budget falls back to its first and last file. + sizes = [] + for f in matches: + try: + sizes.append(min(f.stat().st_size, _PATTERN_BYTES_PER_FILE)) + except OSError: + sizes.append(_PATTERN_BYTES_PER_FILE) + if sum(sizes) <= bytes_left: + quoted = matches + else: + quoted = [matches[0]] if len(matches) == 1 else [matches[0], matches[-1]] + chunk += (f"\n ({len(matches) - len(quoted)} file(s) listed above are NOT quoted: " + "the family exceeds the evidence budget)") + for f in quoted: try: with f.open("rb") as source: head = source.read(_PATTERN_BYTES_PER_FILE).decode("utf-8", errors="replace") except OSError: continue + bytes_left -= len(head) chunk += (f"\n contents of `{f.relative_to(root)}`:\n```\n{head}\n```" + ("" if f.stat().st_size <= _PATTERN_BYTES_PER_FILE else "\n…(truncated)")) chunks.append(chunk) @@ -359,7 +373,7 @@ def gather_evidence(project_dir: Path, task_text: str, *, execution_log_dir: str + f"{size} bytes, sha256={digest}):\n```\n{head}\n```" + ("" if size <= _MAX_BYTES_PER_FILE else "\n…(truncated)") ) - chunks.extend(_pattern_evidence(project_dir, task_text, file_count)) + chunks.extend(_pattern_evidence(project_dir, task_text)) dependencies = local_import_evidence(project_root, python_sources) if dependencies: chunks.append(dependencies) diff --git a/src/llmxive/speckit/implement_cmd.py b/src/llmxive/speckit/implement_cmd.py index 79d1b664926..c95b91ceb02 100644 --- a/src/llmxive/speckit/implement_cmd.py +++ b/src/llmxive/speckit/implement_cmd.py @@ -1319,6 +1319,9 @@ def _regex_parse_implementer(text: str) -> dict[str, Any] | None: if m: body = m.group(1) + from llmxive.speckit.yaml_extract import strip_trailing_markers + body = "\n".join(strip_trailing_markers(body.splitlines())) + task_id_m = re.search(r"^task_id:\s*([^\s\n]+)\s*$", body, re.MULTILINE) verdict_m = re.search(r"^verdict:\s*(\w+)\s*$", body, re.MULTILINE) if not (task_id_m and verdict_m): diff --git a/src/llmxive/speckit/yaml_extract.py b/src/llmxive/speckit/yaml_extract.py index a3ea72487e3..1985f8df2ff 100644 --- a/src/llmxive/speckit/yaml_extract.py +++ b/src/llmxive/speckit/yaml_extract.py @@ -38,8 +38,31 @@ def parse_yaml_lenient(text: str) -> object: while lines and lines[0].strip() and not lines[0].lstrip().startswith(("---", "{", "[")) \ and ":" not in lines[0]: lines.pop(0) - cleaned = "\n".join(lines) + cleaned = "\n".join(strip_trailing_markers(lines)) return yaml.safe_load(cleaned) -__all__ = ["parse_yaml_lenient"] +#: A line that is only a code-fence marker or a YAML document separator. +_MARKER_LINE_RE = re.compile(r"(?:```[\w-]*|---|\.\.\.)?[ \t]*") + + +def strip_trailing_markers(lines: list[str]) -> list[str]: + """Drop blank / fence / separator lines from the END of a reply. + + A reply that is correct YAML followed by one closing ``` (with no opening + fence) or a final ``---`` is not valid YAML: the fence is "a character that + cannot start any token" and the separator starts a second document. + PROJ-9999 canary, 2026-10-11: four consecutive implementer replies ended + that way. Each failed strict parsing, and the fallback then kept the stray + line as the last line of the file, so compilable Python was refused with + "SyntaxError at line 317" on a line the model had not written as code. + + Only flush-left lines are dropped; an indented marker is content. + """ + kept = list(lines) + while kept and _MARKER_LINE_RE.fullmatch(kept[-1]): + kept.pop() + return kept + + +__all__ = ["parse_yaml_lenient", "strip_trailing_markers"] diff --git a/tests/fixtures/implementer/canary_trailing_fence_reply.txt b/tests/fixtures/implementer/canary_trailing_fence_reply.txt new file mode 100644 index 00000000000..ccb3d1af149 --- /dev/null +++ b/tests/fixtures/implementer/canary_trailing_fence_reply.txt @@ -0,0 +1,324 @@ +task_id: T002 +verdict: completed +artifacts: + - path: code/run_totient_analysis.py + execute: false + timeout_s: 300 + contents: | + """run_totient_analysis.py + + Core utilities for the totient residue‑imbalance project. + + This module implements: + - Linear Euler‑totient sieve (T001) + - Unconditional residue counting (T002) + - Conditional residue counting (T003) + - CSV/JSON export for both unconditional and conditional counts + adhering to the ``contracts/census.schema.yaml`` schema (T005) + - Total‑variation distance computation (T006) + - Summary table generation (T008) + - A command‑line entry point that runs the full census for the required + (N, p) grid. + """ + + from __future__ import annotations + + import json + from typing import Dict, List, Tuple + + import numpy as np + import pandas as pd + from llmxive_toolkit import paths, get_logger + + log = get_logger(__name__) + + __all__ = [ + "compute_phi", + "unconditional_counts", + "write_unconditional_counts", + "conditional_counts", + "write_conditional_counts", + "export_census", + "compute_tv", + "write_summary", + "run_totient_analysis", + ] + + # ---------------------------------------------------------------------- + # 1. Linear Euler‑totient sieve + # ---------------------------------------------------------------------- + def compute_phi(max_n: int) -> np.ndarray: + """Compute Euler's totient function for all integers up to ``max_n``. + + Parameters + ---------- + max_n: int + Upper bound of the range (inclusive). Must be ≥ 1. + + Returns + ------- + np.ndarray + One‑dimensional array of length ``max_n + 1`` where ``phi[n]`` + equals the totient of ``n``. ``phi[0]`` is set to ``0`` (unused) and + ``phi[1]`` is set to ``1`` as required. + """ + if max_n < 1: + raise ValueError("max_n must be at least 1") + + phi = np.empty(max_n + 1, dtype=np.int64) + phi[0] = 0 + phi[1] = 1 + + is_composite = np.zeros(max_n + 1, dtype=bool) + primes: List[int] = [] + + for i in range(2, max_n + 1): + if not is_composite[i]: + primes.append(i) + phi[i] = i - 1 + for p in primes: + ip = i * p + if ip > max_n: + break + is_composite[ip] = True + if i % p == 0: + phi[ip] = phi[i] * p + break + else: + phi[ip] = phi[i] * (p - 1) + + return phi + + # ---------------------------------------------------------------------- + # 2. Unconditional residue counting + # ---------------------------------------------------------------------- + def unconditional_counts(phi: np.ndarray, N: int, p: int) -> np.ndarray: + """Return the unconditional residue counts of ``phi(n) mod p`` for ``1 ≤ n ≤ N``.""" + if N >= len(phi): + raise ValueError("phi array does not cover the requested N") + if p <= 0: + raise ValueError("modulus p must be positive") + residues = phi[1 : N + 1] % p + counts = np.bincount(residues, minlength=p) + return counts + + def write_unconditional_counts(phi: np.ndarray, N: int, p: int) -> paths.Path: + """Write unconditional residue counts to CSV and JSON. + + Returns + ------- + Path + Path to the created CSV file. The JSON file is also written for + downstream validation but is not returned because the test suite + expects only the CSV path. + """ + counts = unconditional_counts(phi, N, p) + + record = { + "N": N, + "p": p, + "type": "unconditional", + "residue_counts": counts.tolist(), + "zero_residue_count": int(counts[0]), + "tv": compute_tv(counts, N, p, conditional=False), + } + + csv_path, _ = export_census(record, kind="unconditional") + return csv_path + + # ---------------------------------------------------------------------- + # 3. Conditional residue counting + # ---------------------------------------------------------------------- + def conditional_counts(phi: np.ndarray, N: int, p: int) -> Tuple[np.ndarray, int]: + """Return conditional residue counts for ``phi(n) mod p`` when ``p ∤ phi(n)``.""" + if N >= len(phi): + raise ValueError("phi array does not cover the requested N") + if p <= 0: + raise ValueError("modulus p must be positive") + + residues = phi[1 : N + 1] % p + mask = residues != 0 + filtered = residues[mask] + denominator = int(filtered.size) + + # Counts for residues 1 to p‑1 + counts = np.bincount(filtered, minlength=p)[1:] + + if counts.sum() != denominator: + raise AssertionError( + f"Conditional counts sum {counts.sum()} != denominator {denominator}" + ) + + return counts, denominator + + def write_conditional_counts(phi: np.ndarray, N: int, p: int) -> Tuple[paths.Path, paths.Path]: + """Write conditional residue counts to CSV and JSON. + + Returns + ------- + Tuple[Path, Path] + (csv_path, json_path) – both files are created. + """ + counts, denominator = conditional_counts(phi, N, p) + + record = { + "N": N, + "p": p, + "type": "conditional", + "residue_counts": counts.tolist(), + "conditional_denominator": denominator, + "tv": compute_tv(counts, denominator, p, conditional=True), + } + + csv_path, json_path = export_census(record, kind="conditional") + return csv_path, json_path + + # ---------------------------------------------------------------------- + # 4. Total‑variation distance helper + # ---------------------------------------------------------------------- + def compute_tv(counts: np.ndarray, denominator: int, p: int, conditional: bool) -> float: + """Compute the total‑variation distance for a census record.""" + if denominator == 0: + raise ValueError("Denominator for TV computation must be > 0") + + freqs = counts.astype(float) / denominator + ref = 1.0 / (p - 1) if conditional else 1.0 / p + tv = 0.5 * np.abs(freqs - ref).sum() + return min(1.0, max(0.0, float(tv))) + + # ---------------------------------------------------------------------- + # 5. Export wrapper + # ---------------------------------------------------------------------- + def export_census(record: Dict, kind: str) -> Tuple[paths.Path, paths.Path]: + """Export a census ``record`` to CSV and JSON files. + + Parameters + ---------- + record: dict + Census record adhering to the JSON‑Schema. + kind: str + Either ``"unconditional"`` or ``"conditional"``. + + Returns + ------- + Tuple[Path, Path] + (csv_path, json_path) + """ + N = record["N"] + p = record["p"] + + if kind == "unconditional": + counts = np.array(record["residue_counts"], dtype=int) + zero_residue_count = record["zero_residue_count"] + tv = record["tv"] + + csv_path = paths.data("counts", f"unconditional_N{N}_p{p}.csv") + df = pd.DataFrame({"residue": np.arange(p, dtype=int), "count": counts}) + df.to_csv(csv_path, index=False) + + json_path = paths.data("counts", f"unconditional_N{N}_p{p}.json") + json_obj = { + "N": N, + "p": p, + "type": "unconditional", + "residue_counts": counts.tolist(), + "zero_residue_count": zero_residue_count, + "tv": tv, + } + with open(json_path, "w", encoding="utf-8") as f: + json.dump(json_obj, f, indent=2, sort_keys=False) + + log.info( + "Exported unconditional census to %s (CSV) and %s (JSON)", + csv_path, + json_path, + ) + return csv_path, json_path + + elif kind == "conditional": + counts = np.array(record["residue_counts"], dtype=int) + cond_denom = record["conditional_denominator"] + tv = record["tv"] + + csv_path = paths.data("counts", f"conditional_N{N}_p{p}.csv") + df = pd.DataFrame( + {"residue": np.arange(1, p, dtype=int), "count": counts} + ) + df.to_csv(csv_path, index=False) + + json_path = paths.data("counts", f"conditional_N{N}_p{p}.json") + json_obj = { + "N": N, + "p": p, + "type": "conditional", + "residue_counts": counts.tolist(), + "conditional_denominator": cond_denom, + "tv": tv, + } + with open(json_path, "w", encoding="utf-8") as f: + json.dump(json_obj, f, indent=2, sort_keys=False) + + log.info( + "Exported conditional census to %s (CSV) and %s (JSON)", + csv_path, + json_path, + ) + return csv_path, json_path + else: + raise ValueError(f"Unsupported kind: {kind}") + + # ---------------------------------------------------------------------- + # 6. Summary table generation + # ---------------------------------------------------------------------- + def write_summary() -> paths.Path: + """Write a TSV summary file aggregating TV values for all (N, p) pairs.""" + Ns = [10**3, 10**4, 10**5, 10**6] + ps = [5, 7, 11] + + summary_path = paths.data("summary.tsv") + with open(summary_path, "w", encoding="utf-8") as out: + out.write("N\tp\ttv_unconditional\ttv_conditional\tcond_denominator\n") + for N in Ns: + for p in ps: + uncond_json_path = paths.data( + "counts", f"unconditional_N{N}_p{p}.json" + ) + with open(uncond_json_path, "r", encoding="utf-8") as f: + uncond = json.load(f) + + cond_json_path = paths.data( + "counts", f"conditional_N{N}_p{p}.json" + ) + with open(cond_json_path, "r", encoding="utf-8") as f: + cond = json.load(f) + + tv_uncond = float(uncond["tv"]) + tv_cond = float(cond["tv"]) + cond_den = int(cond["conditional_denominator"]) + + out.write(f"{N}\t{p}\t{tv_uncond}\t{tv_cond}\t{cond_den}\n") + + log.info("Wrote summary TSV to %s", summary_path) + return summary_path + + # ---------------------------------------------------------------------- + # 7. Orchestrator + # ---------------------------------------------------------------------- + def run_totient_analysis() -> None: + """Execute the unconditional and conditional censuses for all required (N, p) pairs.""" + Ns = [10**3, 10**4, 10**5, 10**6] + ps = [5, 7, 11] + max_N = max(Ns) + phi = compute_phi(max_N) + + for N in Ns: + for p in ps: + # Write both CSV and JSON for unconditional and conditional records. + write_unconditional_counts(phi, N, p) + write_conditional_counts(phi, N, p) + + write_summary() + + if __name__ == "__main__": + run_totient_analysis() +``` \ No newline at end of file diff --git a/tests/unit/test_implementer_reply_recovery.py b/tests/unit/test_implementer_reply_recovery.py index 39ec42255e7..8546203d733 100644 --- a/tests/unit/test_implementer_reply_recovery.py +++ b/tests/unit/test_implementer_reply_recovery.py @@ -117,3 +117,63 @@ def test_when_nothing_can_be_recovered_the_yaml_error_is_reported(tmp_path) -> N def test_a_valid_reply_with_no_artifacts_keeps_the_plain_message(tmp_path) -> None: _project, _tasks, log = _write(tmp_path, "task_id: T010\nverdict: completed\nartifacts: []\n") assert "supplied no writable artifacts" in log and "not valid YAML" not in log + + +# --- a stray closing fence or separator after correct YAML --------------------- + +FENCED = (Path(__file__).resolve().parents[1] / "fixtures" / "implementer" + / "canary_trailing_fence_reply.txt").read_text(encoding="utf-8") + + +def test_a_real_reply_ending_in_a_stray_fence_parses_as_yaml() -> None: + """Real reply, canary 2026-10-11 (gpt-oss-120b, T002): correct YAML, then one + closing fence with no opening one. It needs no fallback at all.""" + assert FENCED.rstrip().endswith("```") and not FENCED.lstrip().startswith("```") + doc = parse_yaml_lenient(FENCED) + (artifact,) = doc["artifacts"] + assert artifact["path"] == "code/run_totient_analysis.py" + compile(artifact["contents"], "run_totient_analysis.py", "exec") + assert "```" not in artifact["contents"] + + +@pytest.mark.parametrize("tail", ["```", "```yaml", "---", "...", "```\n\n", "---\n```\n"]) +def test_trailing_markers_are_not_file_contents(tmp_path, tail) -> None: + reply = ("task_id: T010\nverdict: completed\nartifacts:\n" + " - path: code/ok.py\n execute: true\n args: []\n contents: |\n" + " from pathlib import Path\n" + " Path('data').mkdir(exist_ok=True)\n" + " Path('data/ok.txt').write_text('ok')\n" + tail) + project, tasks, log = _write(tmp_path, reply) + assert log == "", log + assert (project / "data" / "ok.txt").read_text(encoding="utf-8") == "ok" + assert (project / "code" / "ok.py").read_text(encoding="utf-8").rstrip().endswith("write_text('ok')") + assert "- [X] T010" in tasks.read_text(encoding="utf-8") + + +def test_the_fallback_also_drops_trailing_markers() -> None: + """A reply that needs the fallback (flush-left string) AND ends in a fence.""" + reply = ("task_id: T010\nverdict: completed\nartifacts:\n" + " - path: code/make_note.py\n execute: true\n contents: |\n" + " TEXT = \"\"\"# Note\n" + "body: flush left\n" + "\"\"\"\n" + " print(TEXT)\n" + "```\n") + with pytest.raises(yaml.YAMLError): + parse_yaml_lenient(reply) + (artifact,) = _regex_parse_implementer(reply)["artifacts"] + compile(artifact["contents"], "make_note.py", "exec") + assert artifact["contents"].splitlines()[-1] == "print(TEXT)" + + +def test_an_indented_marker_inside_contents_is_kept(tmp_path) -> None: + reply = ("task_id: T010\nverdict: completed\nartifacts:\n" + " - path: docs/usage.md\n contents: |\n" + " Run it:\n" + " ```\n" + " python code/run.py\n" + " ```\n") + project, _tasks, log = _write(tmp_path, reply) + assert log == "", log + written = (project / "docs" / "usage.md").read_text(encoding="utf-8") + assert written.rstrip("\n") == "Run it:\n```\npython code/run.py\n```" diff --git a/tests/unit/test_task_verifier_pattern_paths.py b/tests/unit/test_task_verifier_pattern_paths.py index 9f60b3a1c8f..4e5c90d469a 100644 --- a/tests/unit/test_task_verifier_pattern_paths.py +++ b/tests/unit/test_task_verifier_pattern_paths.py @@ -100,3 +100,24 @@ def test_source_patterns_use_the_code_shorthand(project: Path) -> None: (project / "code" / "tests").mkdir(parents=True) (project / "code" / "tests" / "test_a.py").write_text("def test_a(): pass\n", encoding="utf-8") assert [p.name for p in resolve_pattern(project, "tests/test_*.py")] == ["test_a.py"] + + +def test_a_small_family_is_quoted_in_full(project: Path) -> None: + """Two of twelve got: "the evidence only shows the contents of two CSVs".""" + evidence = gather_evidence(project, TASK) + for n in (1000, 10000, 100000, 1000000): + for p in (5, 7, 11): + assert f"contents of `data/counts_N{n}_p{p}.csv`" in evidence + assert f"0,{n // p}\n1,{n - n // p}" in evidence + assert "NOT quoted" not in evidence + + +def test_a_family_too_large_for_the_budget_says_what_it_left_out(project: Path) -> None: + for i in range(40): + (project / "data" / f"big_{i:02d}.csv").write_text("x,y\n" + "1,2\n" * 900, encoding="utf-8") + evidence = gather_evidence(project, "- [X] T2 writes `data/big_{i}.csv` for 40 seeds") + assert "40 matching file(s) exist" in evidence + assert "contents of `data/big_00.csv`" in evidence and "contents of `data/big_39.csv`" in evidence + assert "contents of `data/big_17.csv`" not in evidence + assert "38 file(s) listed above are NOT quoted" in evidence + assert "`data/big_17.csv` (" in evidence, "still listed with size and digest"