From d6a7a918010a8b28d5092fd87a392ca27001e7b1 Mon Sep 17 00:00:00 2001 From: Joel Stenberg Date: Thu, 9 Jul 2026 10:44:35 +0200 Subject: [PATCH] bench(coverage): figure-coverage metric + baseline (73% on the sample) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A deterministic, extraction-independent yardstick for "did figmark capture the figures the document declares?". Ground truth = the document's own numbered captions (Chart/Figure/Diagram N); a number is covered if a non-skipped figure lands on a page where that caption appears. Page-level, so it is a conservative lower bound on misses — it never cries wolf. Baseline on the sample (current main extraction): BoC 58%, BoJ 79%, aggregate 73%. Misses are not one chart type (BoC Chart 4 is a line chart dropped by the MIN_SOLID_DRAWINGS gate). This anchors the coming extraction redesign (drop the geometric pre-classification). Quality/relevance is a separate track (LLM judge). Co-Authored-By: Claude Opus 4.8 --- scripts/coverage_bench/BASELINE.md | 42 ++++++++++ scripts/coverage_bench/coverage.py | 120 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 scripts/coverage_bench/BASELINE.md create mode 100755 scripts/coverage_bench/coverage.py diff --git a/scripts/coverage_bench/BASELINE.md b/scripts/coverage_bench/BASELINE.md new file mode 100644 index 0000000..ace493b --- /dev/null +++ b/scripts/coverage_bench/BASELINE.md @@ -0,0 +1,42 @@ +# Figure-coverage baseline + +**What this measures.** Did figmark capture the figures the document itself +declares? Ground truth = the document's own numbered figure captions +(`Chart N` / `Figure N` / `Diagram N` / …). A figure number is *covered* if +figmark produced a non-skipped figure on a page where that caption appears. + +**Deliberately a lower bound on the problem.** The match is page-level, so a page +with two captioned charts where figmark caught one counts *both* as covered — and +a cross-reference ("see Chart 5") on another page can over-credit. So true +figure-level coverage is **≤ these numbers**; the metric never cries wolf. + +**Scope.** Coverage only — *did we get the figure at all*. It says nothing about +description quality/relevance (that needs an LLM judge; see the session analysis). +Extraction-independent: the same yardstick compares the current geometric +detector against any future extraction approach. + +**Run it:** +```bash +python scripts/coverage_bench/coverage.py DOC.pdf OUTPUT_DIR [DOC2.pdf OUTDIR2 ...] +``` +(`OUTPUT_DIR` = a figmark run dir containing `.figures.json`.) + +## Baseline — current `main` extraction (2026-07-09, gemma-4-31B via Berget) + +| Document | Caption word | Captions | Covered | Coverage | Figures captured | +|---|---|---|---|---|---| +| boc-mpr-202410.pdf (Bank of Canada MPR) | Chart | 26 | 15 | **58 %** | 16 | +| boj-outlook-2410.pdf (Bank of Japan Outlook) | Chart | 58 | 46 | **79 %** | 55 | +| **Aggregate** | | **84** | **61** | **73 %** | | + +Uncaptioned documents (e.g. `govuk-social-care-consultation.docx`) have no +caption ground truth → **N/A** (measure those with the quality/judge track). + +**Missed figures (caption present, no captured figure on its page):** +- BoC: Chart 4, 5, 6, 11, 16, 18, 19, 21, 23, 24, 26 (11 of 26) +- BoJ: Chart 1, 4, 5, 6, 7, 13, 14, 17, 18, 19, 52, 56 (12 of 58) + +The misses are not one chart type — BoC Chart 4 is a plain **line chart** dropped +by the detector's `MIN_SOLID_DRAWINGS_PER_CLUSTER` gate, others are bars. This is +the yardstick to beat when the geometric pre-classification is replaced with a +simpler "render every visual region, let the vision model decide" approach. diff --git a/scripts/coverage_bench/coverage.py b/scripts/coverage_bench/coverage.py new file mode 100755 index 0000000..4187a36 --- /dev/null +++ b/scripts/coverage_bench/coverage.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Figure-coverage metric: did figmark capture the figures the document declares? + +Extraction-independent yardstick. Ground truth = the document's own numbered +figure captions ("Chart N", "Figure N", "Diagram N", ...). Captured = the +non-skipped figures in figmark's ``figures.json``. A figure number is *covered* +if figmark captured a figure on any page where that caption appears. + +This is a deliberate LOWER BOUND on misses (page-level, and a cross-reference to +"Chart 5" on another page can over-credit) — so it never cries wolf. It measures +coverage, not description quality (that needs an LLM judge). + +Usage: + python scripts/coverage_bench/coverage.py DOC.pdf OUTPUT_DIR [DOC2.pdf OUTDIR2 ...] + +Each OUTPUT_DIR is the figmark run dir containing .figures.json. +Prints a per-document table + an aggregate line. Docs with < 2 figure captions +report N/A (no caption ground truth — e.g. an uncaptioned Word file). +""" + +from __future__ import annotations + +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +import fitz + +# Caption words that denote an interpretable figure. "Table" is excluded on +# purpose — tables go through a different pipeline path, not the figure path. +FIGURE_WORDS = ["Chart", "Figure", "Diagram", "Graph", "Exhibit", "Figur"] + + +def caption_pages(pdf_path: Path) -> tuple[str, dict[int, set[int]]]: + """Return (chosen caption word, {figure_number: set_of_1based_pages}). + + Picks the caption word with the most distinct numbers — the document's own + convention (Chart / Diagram / ...). A number maps to every page its caption + text appears on. + """ + doc = fitz.open(pdf_path) + per_word: dict[str, dict[int, set[int]]] = {w: defaultdict(set) for w in FIGURE_WORDS} + for i in range(doc.page_count): + text = doc[i].get_text() + for w in FIGURE_WORDS: + for m in re.finditer(rf"\b{w}[  ]?(\d{{1,3}})\b", text): + per_word[w][int(m.group(1))].add(i + 1) + doc.close() + word = max(per_word, key=lambda w: len(per_word[w])) + return word, dict(per_word[word]) + + +def captured(outdir: Path) -> tuple[set[int], int]: + """(1-based pages with a non-skipped figure, total non-skipped figure count).""" + jsons = list(outdir.glob("*.figures.json")) + if not jsons: + raise FileNotFoundError(f"no *.figures.json in {outdir}") + figs = json.loads(jsons[0].read_text()) + if not isinstance(figs, list): + figs = figs.get("figures", figs.get("items", [])) + kept = [f for f in figs if not f.get("skipped", False) and "page" in f] + return {f["page"] for f in kept}, len(kept) + + +def score(pdf_path: Path, outdir: Path) -> dict: + word, caps = caption_pages(pdf_path) + got, n_figs = captured(outdir) + total = len(caps) + covered = sorted(n for n, pages in caps.items() if pages & got) + missed = sorted((n, sorted(caps[n])) for n in caps if not (caps[n] & got)) + return { + "doc": pdf_path.name, + "word": word, + "total": total, + "covered": len(covered), + "missed": missed, + "captured_figures": n_figs, + "pct": (len(covered) / total * 100) if total else None, + } + + +def main(argv: list[str]) -> int: + if len(argv) < 2 or len(argv) % 2 != 0: + print(__doc__) + return 2 + pairs = [(Path(argv[i]), Path(argv[i + 1])) for i in range(0, len(argv), 2)] + results = [score(p, o) for p, o in pairs] + + print( + f"\n{'Document':34} {'Word':9} {'Caps':>8} {'Covered':>8} {'Coverage':>9} {'Figs':>5}" + ) + print("-" * 84) + agg_t = agg_c = 0 + for r in results: + if r["total"] < 2: + print(f"{r['doc']:34} {r['word']:9} {'N/A (no caption ground truth)':>36}") + continue + agg_t += r["total"] + agg_c += r["covered"] + print( + f"{r['doc']:34} {r['word']:9} {r['total']:8} {r['covered']:8} " + f"{r['pct']:8.0f}% {r['captured_figures']:5}" + ) + if agg_t: + print("-" * 84) + print(f"{'AGGREGATE':34} {'':9} {agg_t:8} {agg_c:8} {agg_c / agg_t * 100:8.0f}%") + + print("\nMissed figure numbers (caption present, no captured figure on its page):") + for r in results: + if r["total"] < 2 or not r["missed"]: + continue + items = ", ".join(f"{r['word']} {n} (p{pages[0]})" for n, pages in r["missed"]) + print(f" {r['doc']}: {items}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))