diff --git a/.github/workflows/github-actions-cron-update-yosys.yml b/.github/workflows/github-actions-cron-update-yosys.yml index 3fd4a9e708..8e356bc117 100644 --- a/.github/workflows/github-actions-cron-update-yosys.yml +++ b/.github/workflows/github-actions-cron-update-yosys.yml @@ -1,6 +1,8 @@ name: Create draft PR for updated YOSYS submodule on: push: + branches: + - master schedule: - cron: "0 8 * * MON" # Allows you to run this workflow manually from the Actions tab @@ -9,6 +11,9 @@ on: jobs: update: runs-on: ${{ vars.USE_SELF_HOSTED == 'true' && 'self-hosted' || 'ubuntu-latest' }} + permissions: + contents: write + pull-requests: write steps: - name: Check out repository code recursively uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/PR_EXTENSION_DEV_LOG.md b/PR_EXTENSION_DEV_LOG.md index 93ffc410f4..5d31d88651 100644 --- a/PR_EXTENSION_DEV_LOG.md +++ b/PR_EXTENSION_DEV_LOG.md @@ -664,6 +664,62 @@ Branch `pr-extension` → `master`. --- +### 2026-08-27 — Regression / benchmark dashboard + +**Goal:** turn the single-run `pr_metrics.py` snapshot into a history that can catch +regressions across runs/commits, usable both interactively and as a CI gate. + +**`flow/util/benchmark_dashboard.py`:** new module, two subcommands, argparse styled +after `pr_metrics.py` (`--platform`/`--design`/`--tag` or `--reports-dir`/`--logs-dir`, +same `--flow-dir` default). Imports `collect()` from `pr_metrics.py` rather than +re-parsing reports/logs — that duplication (pr_metrics.py/triage_agent.py/loop_agent.py/ +compare_hook.sh all independently extracting the same metrics) was already flagged as a +review issue, so this is strictly a history/regression layer on top of the existing +parser. `pr_metrics.py` itself is untouched. + +- `record`: runs `collect()`, appends one JSON object (timestamp, `git rev-parse HEAD`, + platform/design/tag, per-stage metrics dict) as a line to + `flow/util/benchmark_history/____.jsonl`. JSONL + open-append + (`"a"` mode) was chosen specifically so a crash or concurrent writer can never + corrupt or rewrite prior history — each record is independent and the file is safe to + tail/grep. +- `report`: reads the history file for a stage (default `Finish`), prints a table with + per-metric deltas vs. the previous record and `worse-than-best-*` flags vs. the + best-ever value across history. Regression detection compares only the latest record + against its immediate predecessor (not best-ever) against three configurable + thresholds — WNS worsening (`--wns-threshold`, default 0.01 ns), Fmax percentage drop + (`--fmax-threshold-pct`, default 1.0), GRT/GP overflow increase (`--overflow-threshold`, + default 0.001) — and exits 1 if any fire, 0 otherwise, so it drops straight into a CI + pipeline as a gate. Fewer than 2 records just prints the single row and exits 0. + `--html` additionally emits one self-contained HTML file (inline `` line charts + for WNS/Fmax/HPWL, inline ``, and has no +`http(s)://` references (confirms it's genuinely offline-renderable), and +`resolve_dirs()` tag-derivation tests (`--reports-dir` derives the tag from the path's +last component when `--tag` isn't passed; an explicit `--tag` overrides derivation). +Ran together with the existing suite: + +```bash +cd flow/util && python3 -m pytest test_benchmark_dashboard.py test_loop_agent.py -v +``` +62 passed (34 new + 28 existing), confirming no regression to `loop_agent.py`. Formatted +both new files with `black` (26.5.1). + +**Left out of scope:** no Makefile/CI wiring to auto-invoke `record` after every flow +run (roadmap says infra-only for this pass; wiring belongs with whichever CI workflow +task consumes it), no retention/pruning policy for history files (JSONL is cheap and +append-only; pruning can be a follow-up if files get large), no cross-design aggregate +dashboard (each `____` gets its own file/report, matching how +`pr_metrics.py` is already scoped to one run at a time). + ## Planned Next Steps 1. ~~Implement `pr_metrics.py`~~ ✓ @@ -686,3 +742,162 @@ Branch `pr-extension` → `master`. (currently unit-tested only) 16. **Second design**: run triage + loop on ibex or another design to validate generalization 17. (Blocked on ML data) Congestion-feedback parameter tuner +18. ~~Regression/benchmark dashboard (`benchmark_dashboard.py`)~~ ✓ + +--- + +### 2026-09-11 — Independent validator review: 7 fixes to benchmark_dashboard.py + +An independent validator agent re-ran the CI-gate scenarios end-to-end against +`flow/util/benchmark_dashboard.py` and found seven ways the "gate" could report +green (or crash) on a genuinely broken run. All seven are fixed on this branch, +`benchmark_dashboard.py` only: + +- **HIGH — torn/corrupt newest line silently gated green.** `load_records` now + returns `(records, dropped_last_line)`; if the *most recent* physical line in + the history file was corrupt/malformed, `cmd_report` prints a clear + `stderr` error ("history file has a corrupt/truncated record and cannot be + safely compared") and exits 1, instead of silently comparing record N-2 vs + N-1 and reporting success. +- **HIGH — empty latest-stage metrics gated green.** `detect_regressions` now + flags `{}`/missing metrics on the current record (when the previous record + had non-empty metrics for the same stage) as its own regression + ("stage produced no metrics — design may have failed to reach this stage"), + so `cmd_report` exits 1 instead of reporting a clean run when a design + stopped producing timing numbers for the requested stage. +- **HIGH — `resolve_dirs` accepted a one-level-too-high `--reports-dir`.** + Previously any path with ≥3 components was silently sliced into + platform/design/tag, so pointing `--reports-dir` at a *design* directory + (missing the tag level) produced `platform='reports'` and a garbage history + file. `resolve_dirs` now checks that the component 4 levels above the + presumed tag is literally `"reports"`; if not, it raises a clear + `SystemExit` ("does not look like .../reports///") + instead of proceeding. +- **MEDIUM — non-dict/null JSON lines crashed with a raw traceback.** + `load_records` now validates each parsed line is a JSON object with the + expected shape (top-level dict; `stages`, if present, a dict whose values + are each a dict or `null`) and treats anything else as corrupt using the + same skip+warn+last-line-tracking path as a `JSONDecodeError`. `timestamp` + is now read defensively (`rec.get("timestamp") or "—"`) like `git_sha` + already was, and `build_report_rows`/`best_ever` guard against a `null` + nested stage value (`.get(stage) or {}`) instead of crashing on + `None.get(...)`. +- **MEDIUM — non-numeric metric value crashed formatting.** `fmt`/`fmt_delta` + now render anything that isn't `int`/`float` (not just `None`) as `"—"` + instead of raising `ValueError` out of `str.format`. +- **MEDIUM — `cmd_record` died with an unhandled traceback on a malformed + report.** The `collect()` call in `cmd_record` is now wrapped in + `try`/`except Exception`, printing a clear message naming the reports dir + and the underlying exception to `stderr` and exiting 1, rather than letting + a raw traceback surface. `pr_metrics.py` itself was not touched (shared + file, out of scope for this branch). +- **MEDIUM — reader took no lock.** `load_records` now takes a shared lock + (`fcntl.flock(..., LOCK_SH)`) around the read, matching the exclusive lock + `append_record` already takes, so a reader can no longer observe a + partially-written record from a concurrent `record` invocation. + +**Tests (`flow/util/test_benchmark_dashboard.py`):** extended to 50 (from 43), +covering all seven fixes above, plus updated three pre-existing tests that +exercised the old (buggy) `resolve_dirs`/`load_records` behavior directly — +`test_reports_dir_derives_tag_from_path_when_not_passed` and +`test_reports_dir_explicit_tag_overrides_path_derivation` now use a +`--reports-dir` that actually has `reports/` in the right position, and all +`bd.load_records(...)` call sites were updated to unpack the new +`(records, dropped_last_line)` return. + +```bash +cd flow/util && python3 -m pytest test_benchmark_dashboard.py -v +``` +50 passed. + +### 2026-09-11 — Round-2 independent validator review: 5 remaining fixes to benchmark_dashboard.py + +A second, independent validator agent re-tested the fixes above end-to-end +with real CLI runs and found five more real issues, all now fixed on this +branch, `benchmark_dashboard.py` only: + +- **HIGH — `resolve_dirs`'s "4 levels up must be literally `reports`" check + was too strict, and its own error message's suggested workaround was + impossible.** `--platform` and `--reports-dir` are in a mutually-exclusive, + required argparse group, so telling a user hitting the error to "pass + `--platform`/`--design`/`--tag` explicitly" was a dead end for `--platform`. + Worse, it newly rejected previously-working inputs: a relative + `nangate45/ibex/base` path (no `reports` ancestor) or a bare CI artifact + dir like `/tmp/artifacts/nangate45/ibex/base` (no `reports` component at + all) now hard-failed. Fixed by locating the *last* literal `reports` (or + `logs`, mirroring whichever kind of dir is being resolved) path component + via search instead of a fixed offset. If found, exactly 3 components + (platform/design/tag) must follow it — this still catches the original + "one level too high" bug. If no `reports`/`logs` component exists anywhere + in the path, fall back to the prior permissive behavior (last 3 path + components) instead of hard-erroring. (At the time, there was no + argparse-valid way to explicitly override the check in the + `--reports-dir` case — see the follow-up fix below.) +- **MEDIUM — `compute_delta` still crashed on two non-numeric metric + values.** The `fmt`/`fmt_delta` hardening from round 1 didn't cover the + subtraction in `compute_delta` itself, so a history file with `"wns": + "n/a"` in two consecutive records raised an unhandled `TypeError` — + exiting 1 for the same reason a real regression exits 1, making corruption + indistinguishable from a genuine quality regression. `compute_delta` now + returns `None` unless both operands are real numbers. Audited and fixed + the same exposure in `detect_regressions`'s `prev_fmax > 0` comparison and + `render_html`'s point-series filtering (feeding `y_span = y_max - y_min`). +- **MEDIUM — `dropped_last_line` detection was defeated by a trailing blank + line.** It keyed on `lineno == total_lines` (the last *physical* line), but + blank lines are skipped before that check runs, so a corrupt record + immediately followed by a blank line silently escaped detection — exactly + the gap round-1's fix #1 was meant to close. `load_records` now tracks the + last *non-blank* line number and compares against that instead. +- **MEDIUM-LOW — `dropped_last_line`'s exit-1 check in `cmd_report` never + ran when history had zero valid records left after dropping corrupt + lines**, because the `if not records: ... sys.exit(0)` short-circuit ran + first — an all-garbage history file reported exit 0 ("No history found") + instead of flagging corruption. `cmd_report` now checks + `dropped_last_line` before the empty-records short-circuit. +- **LOW — `fmt`/`fmt_delta` accepted `bool`** (since `bool` is an `int` + subclass in Python), rendering a stray JSON `true`/`false` as `1.000`/ + `0.000` instead of `"—"`. Added a shared `is_number()` helper + (`isinstance(val, (int, float)) and not isinstance(val, bool)`) used by + `fmt`, `fmt_delta`, `compute_delta`, `detect_regressions`, and + `render_html`'s series filter. + +**Tests (`flow/util/test_benchmark_dashboard.py`):** extended to 58 (from +50), adding: a `--reports-dir` with no `reports` component and a relative +3-component path both still resolving correctly (item 1); a two-record +history where both records have non-numeric metrics not crashing +`build_report_rows`/`print_report` (item 2); a corrupt-record-followed-by- +blank-line history correctly flagged as `dropped_last_line` (item 3); an +all-garbage history file exiting non-zero via the CLI (item 4); and a bool +JSON value rendering as `"—"` in both `fmt` and `fmt_delta` (item 5). + +```bash +cd flow/util && python3 -m pytest test_benchmark_dashboard.py -v +``` +58 passed. + +### 2026-09-11 — Follow-up: make the strict-shape override actually reachable + +Round-2's fix still left a real usability gap: when the strict path-shape +check does fire, its own suggested remediation ("pass `--platform`, +`--design`, and `--tag` explicitly") was unreachable via the CLI, since +`--platform` and `--reports-dir` lived in the same mutually-exclusive, +required argparse group. Fixed by dropping that group — `--platform` and +`--reports-dir` can now both be passed. `resolve_dirs` now checks +`args.platform` (not just `args.reports_dir`) first: when `--platform` is +given (with or without `--reports-dir`), it derives the path from +platform/design/tag as before, bypassing path-shape validation entirely. +Passing `--reports-dir` alone still goes through the shape check unchanged. +Error messages were updated to point at this override instead of the +now-fixed advice. + +**Tests:** added +`test_record_cli_reports_dir_with_explicit_overrides_bypasses_shape_check`, +exercising the previously-impossible override end-to-end via the CLI +(not just `resolve_dirs()` in isolation): confirms plain `--reports-dir` +one level too high still fails the shape check, and that adding +`--platform`/`--design`/`--tag` alongside it now succeeds. + +```bash +cd flow/util && python3 -m pytest test_benchmark_dashboard.py -v +``` +59 passed. diff --git a/flow/util/benchmark_dashboard.py b/flow/util/benchmark_dashboard.py new file mode 100644 index 0000000000..5e8d09616e --- /dev/null +++ b/flow/util/benchmark_dashboard.py @@ -0,0 +1,637 @@ +#!/usr/bin/env python3 +""" +Regression / benchmark dashboard for P&R quality history. + +Builds a history/regression layer on top of `pr_metrics.collect()` — it does +not re-parse ORFS reports or logs itself. Each `record` invocation appends one +JSON line to a per-design/tag history file; `report` reads that history back +and flags regressions against thresholds so it can gate a CI pipeline. + +Usage: + python3 flow/util/benchmark_dashboard.py record --platform nangate45 --design ibex --tag base + python3 flow/util/benchmark_dashboard.py report --platform nangate45 --design ibex --tag base + python3 flow/util/benchmark_dashboard.py report --platform nangate45 --design ibex --tag base \ + --stage "Global route" --last 10 --html out.html +""" + +import argparse +import fcntl +import html +import json +import os +import subprocess +import sys +from datetime import datetime, timezone + +from pr_metrics import collect + +HISTORY_DIRNAME = "benchmark_history" + +DEFAULT_WNS_THRESHOLD_NS = 0.01 +DEFAULT_FMAX_THRESHOLD_PCT = 1.0 +DEFAULT_OVERFLOW_THRESHOLD = 0.001 + + +def history_dir(flow_util_dir): + return os.path.join(flow_util_dir, HISTORY_DIRNAME) + + +def history_path(flow_util_dir, platform, design, tag): + fname = f"{platform}__{design}__{tag}.jsonl" + return os.path.join(history_dir(flow_util_dir), fname) + + +def git_sha(repo_dir): + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_dir, + capture_output=True, + text=True, + check=True, + ) + return out.stdout.strip() + except (subprocess.CalledProcessError, OSError, FileNotFoundError): + return None + + +def rows_to_stage_dict(rows): + return {name: metrics for name, metrics in rows} + + +def append_record(path, record): + os.makedirs(os.path.dirname(path), exist_ok=True) + line = json.dumps(record) + "\n" + # flock rather than relying on OS atomic-append: a full multi-stage record + # can exceed PIPE_BUF (4096 bytes), so plain O_APPEND no longer guarantees + # writes from concurrent `record` invocations won't interleave. + with open(path, "a") as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + try: + f.write(line) + f.flush() + os.fsync(f.fileno()) + finally: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + + +def load_records(path): + """Return (records, dropped_last_line). + + dropped_last_line is True if the most recent physical line in the file + was corrupt/malformed and had to be skipped — callers that compare the + latest record against history must treat that as unsafe to report on, + since the "latest" record would silently become a stale one. + """ + records = [] + dropped_last_line = False + if not os.path.isfile(path): + return records, dropped_last_line + + with open(path) as f: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) + try: + lines = f.readlines() + finally: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + + last_nonblank_lineno = 0 + for lineno, raw_line in enumerate(lines, start=1): + if raw_line.strip(): + last_nonblank_lineno = lineno + + for lineno, raw_line in enumerate(lines, start=1): + line = raw_line.strip() + if not line: + continue + try: + rec = json.loads(line) + except json.JSONDecodeError as e: + print( + f"WARNING: skipping corrupt history line {lineno} in {path}: {e}", + file=sys.stderr, + ) + if lineno == last_nonblank_lineno: + dropped_last_line = True + continue + + stages = rec.get("stages") if isinstance(rec, dict) else None + valid_shape = isinstance(rec, dict) and ( + "stages" not in rec + or ( + isinstance(stages, dict) + and all(v is None or isinstance(v, dict) for v in stages.values()) + ) + ) + if not valid_shape: + print( + f"WARNING: skipping malformed history line {lineno} in {path}: " + "record is not a valid JSON object with the expected shape", + file=sys.stderr, + ) + if lineno == last_nonblank_lineno: + dropped_last_line = True + continue + + records.append(rec) + return records, dropped_last_line + + +def overflow_of(stage_metrics): + if "gp_overflow" in stage_metrics: + return stage_metrics["gp_overflow"] + return stage_metrics.get("grt_overflow") + + +def compute_delta(prev_metrics, cur_metrics, key): + prev = prev_metrics.get(key) if prev_metrics else None + cur = cur_metrics.get(key) + if not is_number(prev) or not is_number(cur): + return None + return cur - prev + + +def detect_regressions( + prev_metrics, cur_metrics, wns_threshold, fmax_threshold_pct, overflow_threshold +): + regressions = [] + + if prev_metrics and not cur_metrics: + regressions.append( + "REGRESSION: stage produced no metrics — design may have failed " + "to reach this stage" + ) + + wns_delta = compute_delta(prev_metrics, cur_metrics, "wns") + if wns_delta is not None and wns_delta < -wns_threshold: + regressions.append( + f"REGRESSION: WNS worsened by {wns_delta:.4f} ns " + f"(threshold {wns_threshold} ns)" + ) + + prev_fmax = prev_metrics.get("fmax_mhz") if prev_metrics else None + cur_fmax = cur_metrics.get("fmax_mhz") + if is_number(prev_fmax) and is_number(cur_fmax) and prev_fmax > 0: + pct_drop = (prev_fmax - cur_fmax) / prev_fmax * 100.0 + if pct_drop > fmax_threshold_pct: + regressions.append( + f"REGRESSION: Fmax dropped {pct_drop:.2f}% " + f"(threshold {fmax_threshold_pct}%)" + ) + + prev_overflow = overflow_of(prev_metrics) if prev_metrics else None + cur_overflow = overflow_of(cur_metrics) + if prev_overflow is not None and cur_overflow is not None: + overflow_delta = cur_overflow - prev_overflow + if overflow_delta > overflow_threshold: + regressions.append( + f"REGRESSION: routing overflow increased by {overflow_delta:.5f} " + f"(threshold {overflow_threshold})" + ) + + return regressions + + +def best_ever(records, stage, key, better="lower"): + values = [] + for r in records: + v = (r.get("stages", {}).get(stage) or {}).get(key) + if v is not None: + values.append(v) + if not values: + return None + return min(values) if better == "lower" else max(values) + + +def is_number(val): + return isinstance(val, (int, float)) and not isinstance(val, bool) + + +def fmt(val, fmt_str, missing="—"): + if not is_number(val): + return missing + return fmt_str.format(val) + + +def fmt_delta(val, fmt_str, missing="—"): + if not is_number(val): + return missing + return fmt_str.format(val) + + +def build_report_rows( + records, stage, wns_threshold, fmax_threshold_pct, overflow_threshold +): + """Return (table_rows, regressions_against_latest).""" + table_rows = [] + prev_metrics = None + + best_wns = best_ever(records, stage, "wns", "higher") + best_fmax = best_ever(records, stage, "fmax_mhz", "higher") + best_hpwl = best_ever(records, stage, "hpwl", "lower") + + for idx, rec in enumerate(records): + metrics = rec.get("stages", {}).get(stage) or {} + + wns_delta = compute_delta(prev_metrics, metrics, "wns") + tns_delta = compute_delta(prev_metrics, metrics, "tns") + fmax_delta = compute_delta(prev_metrics, metrics, "fmax_mhz") + hpwl_delta = compute_delta(prev_metrics, metrics, "hpwl") + + flags = [] + if ( + best_wns is not None + and metrics.get("wns") is not None + and metrics["wns"] < best_wns + ): + flags.append("worse-than-best-WNS") + if ( + best_fmax is not None + and metrics.get("fmax_mhz") is not None + and metrics["fmax_mhz"] < best_fmax + ): + flags.append("worse-than-best-Fmax") + if ( + best_hpwl is not None + and metrics.get("hpwl") is not None + and metrics["hpwl"] > best_hpwl + ): + flags.append("worse-than-best-HPWL") + + regressions = [] + if idx > 0: + regressions = detect_regressions( + prev_metrics, + metrics, + wns_threshold, + fmax_threshold_pct, + overflow_threshold, + ) + + table_rows.append( + { + "record": rec, + "metrics": metrics, + "wns_delta": wns_delta, + "tns_delta": tns_delta, + "fmax_delta": fmax_delta, + "hpwl_delta": hpwl_delta, + "flags": flags, + "regressions": regressions, + } + ) + prev_metrics = metrics + + latest_regressions = table_rows[-1]["regressions"] if table_rows else [] + return table_rows, latest_regressions + + +def print_report(stage, table_rows, label): + print(f"\nBenchmark history — {label} — stage: {stage}") + print("=" * 110) + header = ( + f"{'Timestamp':<21} {'SHA':<9} {'WNS (ns)':>10} {'dWNS':>8} " + f"{'Fmax(MHz)':>10} {'dFmax':>8} {'HPWL':>12} {'dHPWL':>10} {'Flags':<24}" + ) + print(header) + print("-" * 110) + + for row in table_rows: + rec = row["record"] + m = row["metrics"] + ts = (rec.get("timestamp") or "—")[:19] + sha = (rec.get("git_sha") or "—")[:8] + wns = fmt(m.get("wns"), "{:+.3f}") + dwns = fmt_delta(row["wns_delta"], "{:+.3f}") + fmax = fmt(m.get("fmax_mhz"), "{:.1f}") + dfmax = fmt_delta(row["fmax_delta"], "{:+.1f}") + hpwl = fmt(m.get("hpwl"), "{:,.0f}") + dhpwl = fmt_delta(row["hpwl_delta"], "{:+,.0f}") + flags = ",".join(row["flags"]) if row["flags"] else "" + + print( + f"{ts:<21} {sha:<9} {wns:>10} {dwns:>8} {fmax:>10} {dfmax:>8} " + f"{hpwl:>12} {dhpwl:>10} {flags:<24}" + ) + + print("-" * 110) + + latest_regressions = table_rows[-1]["regressions"] if table_rows else [] + if latest_regressions: + print() + for r in latest_regressions: + print(r) + elif len(table_rows) >= 2: + print("\nNo regressions detected against previous record.") + else: + print("\nOnly one record present — nothing to compare against yet.") + print() + + +def render_html(records, stage, table_rows, label, out_path): + width, height = 760, 260 + pad_left, pad_right, pad_top, pad_bottom = 60, 20, 20, 30 + + def series(key): + return [ + (i, row["metrics"].get(key)) + for i, row in enumerate(table_rows) + if is_number(row["metrics"].get(key)) + ] + + def svg_for(key, title, color): + pts = series(key) + if len(pts) < 2: + return f"

Not enough data to chart {title}.

" + xs = [p[0] for p in pts] + ys = [p[1] for p in pts] + x_min, x_max = min(xs), max(xs) + y_min, y_max = min(ys), max(ys) + if y_min == y_max: + y_min -= 1 + y_max += 1 + x_span = max(x_max - x_min, 1) + y_span = y_max - y_min + + def sx(x): + return pad_left + (x - x_min) / x_span * (width - pad_left - pad_right) + + def sy(y): + return ( + height + - pad_bottom + - (y - y_min) / y_span * (height - pad_top - pad_bottom) + ) + + poly = " ".join(f"{sx(x):.1f},{sy(y):.1f}" for x, y in pts) + circles = "".join( + f'' + for x, y in pts + ) + return f""" +
+

{html.escape(title)}

+ + + + + {circles} + max {y_max:.4g} + min {y_min:.4g} + +
+ """ + + charts = ( + svg_for("wns", "WNS (ns)", "#c0392b") + + svg_for("fmax_mhz", "Fmax (MHz)", "#2471a3") + + svg_for("hpwl", "HPWL", "#27ae60") + ) + + rows_html = "" + for row in table_rows: + rec = row["record"] + m = row["metrics"] + flags = ", ".join(row["flags"]) if row["flags"] else "" + regressions = "
".join(html.escape(r) for r in row["regressions"]) + rows_html += ( + "" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + "\n" + ) + + safe_label = html.escape(label) + safe_stage = html.escape(stage) + + doc = f""" + + + +Benchmark dashboard — {safe_label} + + + +

Benchmark dashboard — {safe_label} — stage: {safe_stage}

+{charts} +
{html.escape(str(rec.get('timestamp', ''))[:19])}{html.escape(str(rec.get('git_sha') or '')[:8])}{fmt(m.get('wns'), '{:+.3f}')}{fmt(m.get('fmax_mhz'), '{:.1f}')}{fmt(m.get('hpwl'), '{:,.0f}')}{html.escape(flags)}{regressions}
+ + +{rows_html} + +
TimestampSHAWNSFmaxHPWLFlagsRegressions
+ + +""" + with open(out_path, "w") as f: + f.write(doc) + + +def cmd_record(args, flow_dir, flow_util_dir, reports_dir, logs_dir, label): + if not os.path.isdir(reports_dir): + print(f"ERROR: reports directory not found: {reports_dir}", file=sys.stderr) + sys.exit(1) + + try: + rows = collect(reports_dir, logs_dir) + except Exception as e: + print( + f"ERROR: failed to parse reports in {reports_dir}: {e}", + file=sys.stderr, + ) + sys.exit(1) + + repo_dir = os.path.dirname(flow_dir) + record = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "git_sha": git_sha(repo_dir), + "platform": args.platform, + "design": args.design, + "tag": args.tag, + "stages": rows_to_stage_dict(rows), + } + + path = history_path(flow_util_dir, args.platform, args.design, args.tag) + append_record(path, record) + print(f"Recorded benchmark for {label} -> {path}") + + +def cmd_report(args, flow_dir, flow_util_dir, reports_dir, logs_dir, label): + path = history_path(flow_util_dir, args.platform, args.design, args.tag) + records, dropped_last_line = load_records(path) + + if dropped_last_line: + print( + f"ERROR: history file {path} has a corrupt/truncated record and " + "cannot be safely compared", + file=sys.stderr, + ) + sys.exit(1) + + if not records: + print(f"No history found at {path}") + sys.exit(0) + + stage = args.stage + # best-ever and deltas are computed over the FULL history so that --last + # only narrows what's displayed, never what "worse than all-time best" + # or the regression check against the immediately-previous record means. + table_rows, latest_regressions = build_report_rows( + records, + stage, + args.wns_threshold, + args.fmax_threshold_pct, + args.overflow_threshold, + ) + + display_rows = table_rows[-args.last :] if args.last else table_rows + + print_report(stage, display_rows, label) + + if args.html: + render_html(records, stage, display_rows, label, args.html) + print(f"Wrote HTML dashboard: {args.html}") + + sys.exit(1 if latest_regressions else 0) + + +def add_common_args(parser, flow_dir_default): + parser.add_argument("--platform", help="Platform name (e.g. nangate45)") + parser.add_argument( + "--reports-dir", + help="Direct path to reports directory. May be combined with " + "--platform/--design/--tag to override path-derived values and " + "skip path-shape validation.", + ) + + parser.add_argument("--design", help="Design name (required with --platform)") + parser.add_argument("--tag", help="Tag / variant (default: base)", default=None) + parser.add_argument("--logs-dir", help="Direct path to logs directory") + parser.add_argument( + "--flow-dir", + default=flow_dir_default, + help=f"Path to flow/ directory (default: {flow_dir_default})", + ) + + +def resolve_dirs(args): + if not args.platform and not args.reports_dir: + raise SystemExit("error: one of --platform or --reports-dir is required") + + if args.reports_dir: + reports_dir = args.reports_dir + logs_dir = args.logs_dir or reports_dir.replace("/reports/", "/logs/") + label = reports_dir + if not args.platform or not args.design or not args.tag: + parts = os.path.normpath(reports_dir).split(os.sep) + dir_kind = "reports" if "reports" in parts else "logs" + if dir_kind in parts: + anchor = len(parts) - 1 - parts[::-1].index(dir_kind) + remainder = parts[anchor + 1 :] + if len(remainder) != 3: + raise SystemExit( + "error: --reports-dir " + f"{reports_dir!r} does not look like " + f".../{dir_kind}/// (expected " + "exactly platform/design/tag after the " + f"'{dir_kind}' directory); pass --platform, --design, " + "and --tag explicitly alongside --reports-dir to " + "override path derivation" + ) + args.platform = args.platform or remainder[0] + args.design = args.design or remainder[1] + args.tag = args.tag or remainder[2] + elif len(parts) >= 3: + args.platform = args.platform or parts[-3] + args.design = args.design or parts[-2] + args.tag = args.tag or parts[-1] + + if not args.platform or not args.design: + raise SystemExit( + "error: could not determine --platform/--design from " + f"--reports-dir {reports_dir!r} (need at least " + "// path components); pass " + "--platform and --design explicitly alongside --reports-dir" + ) + args.tag = args.tag or "base" + else: + if not args.design: + raise SystemExit("--design is required when using --platform") + args.tag = args.tag or "base" + reports_dir = os.path.join( + args.flow_dir, "reports", args.platform, args.design, args.tag + ) + logs_dir = os.path.join( + args.flow_dir, "logs", args.platform, args.design, args.tag + ) + label = f"{args.platform}/{args.design}/{args.tag}" + return reports_dir, logs_dir, label + + +def main(): + script_dir = os.path.dirname(os.path.abspath(__file__)) + flow_dir_default = os.path.dirname(script_dir) + + parser = argparse.ArgumentParser(description="Regression / benchmark dashboard") + sub = parser.add_subparsers(dest="command", required=True) + + p_record = sub.add_parser("record", help="Record one run's metrics into history") + add_common_args(p_record, flow_dir_default) + + p_report = sub.add_parser( + "report", help="Print history report and detect regressions" + ) + add_common_args(p_report, flow_dir_default) + p_report.add_argument( + "--stage", default="Finish", help="Stage name to report on (default: Finish)" + ) + p_report.add_argument( + "--last", type=int, default=None, help="Limit to N most recent records" + ) + p_report.add_argument( + "--html", help="Write a self-contained HTML dashboard to this path" + ) + p_report.add_argument( + "--wns-threshold", + type=float, + default=DEFAULT_WNS_THRESHOLD_NS, + help=f"WNS regression threshold in ns (default: {DEFAULT_WNS_THRESHOLD_NS})", + ) + p_report.add_argument( + "--fmax-threshold-pct", + type=float, + default=DEFAULT_FMAX_THRESHOLD_PCT, + help=f"Fmax regression threshold in %% (default: {DEFAULT_FMAX_THRESHOLD_PCT})", + ) + p_report.add_argument( + "--overflow-threshold", + type=float, + default=DEFAULT_OVERFLOW_THRESHOLD, + help=f"Routing overflow regression threshold (default: {DEFAULT_OVERFLOW_THRESHOLD})", + ) + + args = parser.parse_args() + flow_dir = args.flow_dir + flow_util_dir = os.path.dirname(os.path.abspath(__file__)) + reports_dir, logs_dir, label = resolve_dirs(args) + + if args.command == "record": + cmd_record(args, flow_dir, flow_util_dir, reports_dir, logs_dir, label) + elif args.command == "report": + cmd_report(args, flow_dir, flow_util_dir, reports_dir, logs_dir, label) + + +if __name__ == "__main__": + main() diff --git a/flow/util/test_benchmark_dashboard.py b/flow/util/test_benchmark_dashboard.py new file mode 100644 index 0000000000..2457590271 --- /dev/null +++ b/flow/util/test_benchmark_dashboard.py @@ -0,0 +1,858 @@ +#!/usr/bin/env python3 +"""Unit tests for benchmark_dashboard.py — no Docker, no API, no live ORFS run.""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import benchmark_dashboard as bd + + +def make_record(timestamp, sha, wns, fmax_mhz, hpwl, grt_overflow=0.0): + return { + "timestamp": timestamp, + "git_sha": sha, + "platform": "nangate45", + "design": "ibex", + "tag": "base", + "stages": { + "Finish": { + "wns": wns, + "tns": wns * 10 if wns is not None else None, + "fmax_mhz": fmax_mhz, + "hpwl": hpwl, + "grt_overflow": grt_overflow, + } + }, + } + + +class TestAppendRecord(unittest.TestCase): + def test_append_creates_dir_and_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "sub", "history.jsonl") + bd.append_record(path, make_record("t0", "sha0", -0.1, 500.0, 100000)) + self.assertTrue(os.path.isfile(path)) + records, dropped_last_line = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["git_sha"], "sha0") + self.assertFalse(dropped_last_line) + + def test_append_is_append_only_across_multiple_calls(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + bd.append_record(path, make_record("t0", "sha0", -0.1, 500.0, 100000)) + bd.append_record(path, make_record("t1", "sha1", -0.2, 490.0, 105000)) + bd.append_record(path, make_record("t2", "sha2", -0.05, 510.0, 98000)) + + records, _ = bd.load_records(path) + self.assertEqual(len(records), 3) + self.assertEqual([r["git_sha"] for r in records], ["sha0", "sha1", "sha2"]) + + with open(path) as f: + lines = f.readlines() + self.assertEqual(len(lines), 3) + for line in lines: + json.loads(line) + + def test_prior_lines_unmodified_after_new_append(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + bd.append_record(path, make_record("t0", "sha0", -0.1, 500.0, 100000)) + with open(path) as f: + first_line_before = f.readlines()[0] + bd.append_record(path, make_record("t1", "sha1", -0.2, 490.0, 105000)) + with open(path) as f: + first_line_after = f.readlines()[0] + self.assertEqual(first_line_before, first_line_after) + + +class TestGitSha(unittest.TestCase): + def test_git_sha_returns_string_in_real_repo(self): + repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + sha = bd.git_sha(repo_dir) + self.assertIsInstance(sha, str) + self.assertEqual(len(sha), 40) + + def test_git_sha_returns_none_on_failure(self): + with mock.patch( + "benchmark_dashboard.subprocess.run", + side_effect=FileNotFoundError, + ): + self.assertIsNone(bd.git_sha("/nonexistent")) + + +class TestComputeDelta(unittest.TestCase): + def test_delta_none_when_missing(self): + self.assertIsNone(bd.compute_delta(None, {"wns": -0.1}, "wns")) + self.assertIsNone(bd.compute_delta({"wns": -0.1}, {}, "wns")) + + def test_delta_computed(self): + self.assertAlmostEqual( + bd.compute_delta({"wns": -0.2}, {"wns": -0.1}, "wns"), 0.1 + ) + + +class TestComputeDeltaNonNumeric(unittest.TestCase): + def test_delta_none_when_both_non_numeric(self): + self.assertIsNone(bd.compute_delta({"wns": "n/a"}, {"wns": "n/a"}, "wns")) + + def test_report_survives_both_non_numeric_history_records(self): + records = [ + { + "timestamp": "t0", + "git_sha": "s0", + "platform": "nangate45", + "design": "ibex", + "tag": "base", + "stages": {"Finish": {"wns": "n/a", "fmax_mhz": "n/a", "hpwl": "n/a"}}, + }, + { + "timestamp": "t1", + "git_sha": "s1", + "platform": "nangate45", + "design": "ibex", + "tag": "base", + "stages": {"Finish": {"wns": "n/a", "fmax_mhz": "n/a", "hpwl": "n/a"}}, + }, + ] + table_rows, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertIsNone(table_rows[1]["wns_delta"]) + self.assertEqual(latest, []) + bd.print_report("Finish", table_rows, "nangate45/ibex/base") + + +class TestDetectRegressions(unittest.TestCase): + def test_wns_regression_flagged(self): + prev = {"wns": -0.10, "fmax_mhz": 500.0, "grt_overflow": 0.0} + cur = {"wns": -0.15, "fmax_mhz": 500.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertTrue(any("WNS" in r for r in regs)) + + def test_wns_within_threshold_not_flagged(self): + prev = {"wns": -0.10, "fmax_mhz": 500.0, "grt_overflow": 0.0} + cur = {"wns": -0.105, "fmax_mhz": 500.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertFalse(any("WNS" in r for r in regs)) + + def test_fmax_regression_flagged(self): + prev = {"wns": 0.0, "fmax_mhz": 500.0, "grt_overflow": 0.0} + cur = {"wns": 0.0, "fmax_mhz": 480.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertTrue(any("Fmax" in r for r in regs)) + + def test_fmax_within_threshold_not_flagged(self): + prev = {"wns": 0.0, "fmax_mhz": 500.0, "grt_overflow": 0.0} + cur = {"wns": 0.0, "fmax_mhz": 498.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertFalse(any("Fmax" in r for r in regs)) + + def test_overflow_regression_flagged(self): + prev = {"wns": 0.0, "fmax_mhz": 500.0, "grt_overflow": 0.0} + cur = {"wns": 0.0, "fmax_mhz": 500.0, "grt_overflow": 0.01} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertTrue(any("overflow" in r for r in regs)) + + def test_improvement_not_flagged(self): + prev = {"wns": -0.2, "fmax_mhz": 480.0, "grt_overflow": 0.01} + cur = {"wns": -0.1, "fmax_mhz": 500.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(prev, cur, 0.01, 1.0, 0.001) + self.assertEqual(regs, []) + + def test_no_prev_no_regressions(self): + cur = {"wns": -0.1, "fmax_mhz": 500.0, "grt_overflow": 0.0} + regs = bd.detect_regressions(None, cur, 0.01, 1.0, 0.001) + self.assertEqual(regs, []) + + +class TestBestEver(unittest.TestCase): + def test_best_ever_lower(self): + records = [ + make_record("t0", "s0", -0.1, 500, 100), + make_record("t1", "s1", -0.1, 500, 90), + make_record("t2", "s2", -0.1, 500, 120), + ] + self.assertEqual(bd.best_ever(records, "Finish", "hpwl", "lower"), 90) + + def test_best_ever_higher(self): + records = [ + make_record("t0", "s0", -0.2, 480, 100), + make_record("t1", "s1", -0.05, 510, 100), + ] + self.assertEqual(bd.best_ever(records, "Finish", "wns", "higher"), -0.05) + + def test_best_ever_empty(self): + self.assertIsNone(bd.best_ever([], "Finish", "wns", "higher")) + + +class TestBuildReportRows(unittest.TestCase): + def test_single_record_no_regressions(self): + records = [make_record("t0", "s0", -0.1, 500, 100000)] + table_rows, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertEqual(len(table_rows), 1) + self.assertEqual(latest, []) + self.assertIsNone(table_rows[0]["wns_delta"]) + + def test_regression_detected_on_latest_record(self): + records = [ + make_record("t0", "s0", -0.10, 500.0, 100000), + make_record("t1", "s1", -0.30, 500.0, 100000), + ] + table_rows, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertEqual(len(table_rows), 2) + self.assertTrue(any("WNS" in r for r in latest)) + self.assertAlmostEqual(table_rows[1]["wns_delta"], -0.20) + + def test_flags_worse_than_best(self): + records = [ + make_record("t0", "s0", -0.05, 510.0, 90000), + make_record("t1", "s1", -0.05, 510.0, 90000), + make_record("t2", "s2", -0.20, 480.0, 150000), + ] + table_rows, _ = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + flags = table_rows[2]["flags"] + self.assertIn("worse-than-best-WNS", flags) + self.assertIn("worse-than-best-Fmax", flags) + self.assertIn("worse-than-best-HPWL", flags) + + def test_no_regression_when_improving(self): + records = [ + make_record("t0", "s0", -0.30, 480.0, 150000), + make_record("t1", "s1", -0.05, 510.0, 90000), + ] + _, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertEqual(latest, []) + + def test_empty_latest_stage_metrics_flagged_as_regression(self): + records = [ + make_record("t0", "s0", -0.05, 510.0, 90000), + { + "timestamp": "t1", + "git_sha": "s1", + "platform": "nangate45", + "design": "ibex", + "tag": "base", + "stages": {}, + }, + ] + table_rows, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertTrue(any("no metrics" in r for r in latest)) + + def test_empty_latest_stage_metrics_after_previous_success(self): + regs = bd.detect_regressions( + {"wns": -0.05, "fmax_mhz": 510.0}, {}, 0.01, 1.0, 0.001 + ) + self.assertTrue(any("no metrics" in r for r in regs)) + + def test_no_regression_when_both_prev_and_cur_empty(self): + regs = bd.detect_regressions(None, {}, 0.01, 1.0, 0.001) + self.assertEqual(regs, []) + + +class TestCliRecordAndReport(unittest.TestCase): + def _run(self, args, cwd): + return subprocess.run( + [sys.executable, "benchmark_dashboard.py"] + args, + cwd=cwd, + capture_output=True, + text=True, + ) + + def _util_dir(self): + return os.path.dirname(os.path.abspath(__file__)) + + def _make_fake_flow(self, tmp): + reports = os.path.join(tmp, "flow", "reports", "nangate45", "ibex", "base") + logs = os.path.join(tmp, "flow", "logs", "nangate45", "ibex", "base") + os.makedirs(reports) + os.makedirs(logs) + with open(os.path.join(reports, "6_finish.rpt"), "w") as f: + f.write( + "tns max -1.0\nwns max -0.10\nworst slack max -0.10\nfmax = 500.0\n" + ) + return os.path.join(tmp, "flow") + + def test_record_cli_writes_history_file(self): + util_dir = self._util_dir() + with tempfile.TemporaryDirectory() as tmp: + flow_dir = self._make_fake_flow(tmp) + history_file = os.path.join( + util_dir, "benchmark_history", "nangate45__ibex__base.jsonl" + ) + if os.path.isfile(history_file): + os.remove(history_file) + try: + proc = self._run( + [ + "record", + "--platform", + "nangate45", + "--design", + "ibex", + "--tag", + "base", + "--flow-dir", + flow_dir, + ], + util_dir, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue(os.path.isfile(history_file)) + records, _ = bd.load_records(history_file) + self.assertEqual(len(records), 1) + self.assertAlmostEqual(records[0]["stages"]["Finish"]["wns"], -0.10) + finally: + if os.path.isfile(history_file): + os.remove(history_file) + + def test_record_cli_reports_dir_with_explicit_overrides_bypasses_shape_check(self): + util_dir = self._util_dir() + with tempfile.TemporaryDirectory() as tmp: + reports_dir = os.path.join(tmp, "flow", "reports", "nangate45", "ibex") + logs_dir = os.path.join(tmp, "flow", "logs", "nangate45", "ibex") + os.makedirs(reports_dir) + os.makedirs(logs_dir) + with open(os.path.join(reports_dir, "6_finish.rpt"), "w") as f: + f.write( + "tns max -1.0\nwns max -0.10\nworst slack max -0.10\n" + "fmax = 500.0\n" + ) + history_file = os.path.join( + util_dir, "benchmark_history", "nangate45__ibex__override-tag.jsonl" + ) + if os.path.isfile(history_file): + os.remove(history_file) + try: + # Plain --reports-dir is one directory level too high (no + # tag component) and is rejected by the strict shape check. + proc_fail = self._run( + ["report", "--reports-dir", reports_dir], + util_dir, + ) + self.assertNotEqual(proc_fail.returncode, 0) + self.assertIn("does not look like", proc_fail.stderr) + + # Explicit --platform/--design/--tag can now be combined + # with --reports-dir (argparse no longer rejects the + # combination) to override the derivation and bypass the + # shape check entirely. + proc = self._run( + [ + "record", + "--reports-dir", + reports_dir, + "--logs-dir", + logs_dir, + "--platform", + "nangate45", + "--design", + "ibex", + "--tag", + "override-tag", + ], + util_dir, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue(os.path.isfile(history_file)) + records, _ = bd.load_records(history_file) + self.assertEqual(len(records), 1) + self.assertAlmostEqual(records[0]["stages"]["Finish"]["wns"], -0.10) + finally: + if os.path.isfile(history_file): + os.remove(history_file) + + def test_report_exit_code_regression_vs_clean(self): + util_dir = self._util_dir() + with tempfile.TemporaryDirectory() as tmp: + history_file = os.path.join(tmp, "clean.jsonl") + bd.append_record(history_file, make_record("t0", "s0", -0.05, 510.0, 90000)) + bd.append_record(history_file, make_record("t1", "s1", -0.06, 508.0, 91000)) + + records, _ = bd.load_records(history_file) + _, latest = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + self.assertEqual(latest, []) + + history_file2 = os.path.join(tmp, "regressed.jsonl") + bd.append_record( + history_file2, make_record("t0", "s0", -0.05, 510.0, 90000) + ) + bd.append_record( + history_file2, make_record("t1", "s1", -0.30, 480.0, 90000) + ) + records2, _ = bd.load_records(history_file2) + _, latest2 = bd.build_report_rows(records2, "Finish", 0.01, 1.0, 0.001) + self.assertTrue(len(latest2) > 0) + + +class TestLoadRecordsCorruptLines(unittest.TestCase): + def test_corrupt_line_skipped_not_fatal(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + good0 = json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + good1 = json.dumps(make_record("t1", "s1", -0.06, 508.0, 91000)) + with open(path, "w") as f: + f.write(good0 + "\n") + f.write("{not valid json truncated mid-rec\n") + f.write(good1 + "\n") + + with mock.patch("sys.stderr") as mock_stderr: + records, dropped_last_line = bd.load_records(path) + + self.assertEqual(len(records), 2) + self.assertEqual([r["git_sha"] for r in records], ["s0", "s1"]) + self.assertFalse(dropped_last_line) + written = "".join(c.args[0] for c in mock_stderr.write.call_args_list) + self.assertIn("line 2", written) + + def test_all_corrupt_lines_returns_empty_not_raises(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + with open(path, "w") as f: + f.write("{{{not json\n") + f.write("also not json\n") + with mock.patch("sys.stderr"): + records, dropped_last_line = bd.load_records(path) + self.assertEqual(records, []) + self.assertTrue(dropped_last_line) + + def test_report_cli_survives_corrupt_history_line(self): + util_dir = os.path.dirname(os.path.abspath(__file__)) + tag = "corrupt-line-test" + history_file = os.path.join( + util_dir, "benchmark_history", f"nangate45__ibex__{tag}.jsonl" + ) + try: + os.makedirs(os.path.dirname(history_file), exist_ok=True) + with open(history_file, "w") as f: + f.write(json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + "\n") + f.write("not valid json\n") + f.write(json.dumps(make_record("t1", "s1", -0.06, 508.0, 91000)) + "\n") + + proc = subprocess.run( + [ + sys.executable, + "benchmark_dashboard.py", + "report", + "--platform", + "nangate45", + "--design", + "ibex", + "--tag", + tag, + ], + cwd=util_dir, + capture_output=True, + text=True, + ) + self.assertIn(proc.returncode, (0, 1)) + self.assertIn("WARNING", proc.stderr) + self.assertIn("s0", proc.stdout) + self.assertIn("s1", proc.stdout) + finally: + if os.path.isfile(history_file): + os.remove(history_file) + + def test_report_cli_fails_when_last_line_is_corrupt(self): + util_dir = os.path.dirname(os.path.abspath(__file__)) + tag = "corrupt-last-line-test" + history_file = os.path.join( + util_dir, "benchmark_history", f"nangate45__ibex__{tag}.jsonl" + ) + try: + os.makedirs(os.path.dirname(history_file), exist_ok=True) + with open(history_file, "w") as f: + f.write(json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + "\n") + f.write(json.dumps(make_record("t1", "s1", -0.06, 508.0, 91000)) + "\n") + f.write('{"truncated": tr\n') + + proc = subprocess.run( + [ + sys.executable, + "benchmark_dashboard.py", + "report", + "--platform", + "nangate45", + "--design", + "ibex", + "--tag", + tag, + ], + cwd=util_dir, + capture_output=True, + text=True, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("corrupt/truncated record", proc.stderr) + finally: + if os.path.isfile(history_file): + os.remove(history_file) + + def test_load_records_flags_dropped_last_line(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + with open(path, "w") as f: + f.write(json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + "\n") + f.write("not valid json at all\n") + with mock.patch("sys.stderr"): + records, dropped_last_line = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertTrue(dropped_last_line) + + def test_non_dict_json_line_skipped(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + with open(path, "w") as f: + f.write("null\n") + f.write(json.dumps([1, 2, 3]) + "\n") + f.write(json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + "\n") + with mock.patch("sys.stderr"): + records, dropped_last_line = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["git_sha"], "s0") + self.assertFalse(dropped_last_line) + + def test_null_nested_stage_value_treated_as_valid_empty_stage(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + rec = make_record("t0", "s0", -0.05, 510.0, 90000) + rec["stages"]["Global route"] = None + with open(path, "w") as f: + f.write(json.dumps(rec) + "\n") + records, dropped_last_line = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertFalse(dropped_last_line) + table_rows, _ = bd.build_report_rows( + records, "Global route", 0.01, 1.0, 0.001 + ) + self.assertEqual(table_rows[0]["metrics"], {}) + + def test_load_records_flags_dropped_last_line_with_trailing_blank(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + with open(path, "w") as f: + f.write(json.dumps(make_record("t0", "s0", -0.05, 510.0, 90000)) + "\n") + f.write("not valid json at all\n") + f.write("\n") + with mock.patch("sys.stderr"): + records, dropped_last_line = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertTrue(dropped_last_line) + + def test_report_cli_fails_on_all_garbage_history(self): + util_dir = os.path.dirname(os.path.abspath(__file__)) + tag = "all-garbage-test" + history_file = os.path.join( + util_dir, "benchmark_history", f"nangate45__ibex__{tag}.jsonl" + ) + try: + os.makedirs(os.path.dirname(history_file), exist_ok=True) + with open(history_file, "w") as f: + f.write("not valid json\n") + f.write("also not valid json\n") + + proc = subprocess.run( + [ + sys.executable, + "benchmark_dashboard.py", + "report", + "--platform", + "nangate45", + "--design", + "ibex", + "--tag", + tag, + ], + cwd=util_dir, + capture_output=True, + text=True, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("corrupt/truncated record", proc.stderr) + finally: + if os.path.isfile(history_file): + os.remove(history_file) + + def test_load_records_takes_shared_lock(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "history.jsonl") + bd.append_record(path, make_record("t0", "s0", -0.05, 510.0, 90000)) + + flock_calls = [] + real_flock = bd.fcntl.flock + + def spy_flock(fd, op): + flock_calls.append(op) + return real_flock(fd, op) + + with mock.patch("benchmark_dashboard.fcntl.flock", side_effect=spy_flock): + records, _ = bd.load_records(path) + self.assertEqual(len(records), 1) + self.assertIn(bd.fcntl.LOCK_SH, flock_calls) + self.assertIn(bd.fcntl.LOCK_UN, flock_calls) + + +class TestHtmlEscaping(unittest.TestCase): + def test_malicious_label_and_stage_are_escaped(self): + malicious = "" + records = [ + make_record("t0", "s0", -0.10, 500.0, 100000), + make_record("t1", "s1", -0.05, 510.0, 90000), + ] + table_rows, _ = bd.build_report_rows(records, "Finish", 0.01, 1.0, 0.001) + with tempfile.TemporaryDirectory() as tmp: + out_path = os.path.join(tmp, "out.html") + bd.render_html(records, malicious, table_rows, malicious, out_path) + with open(out_path) as f: + content = f.read() + self.assertNotIn("