Refuse to publish releases with skipped out-of-sample reforms

packages/populace-data/src/populace/data/publish_cli.py

@@ -4,12 +4,31 @@
 
 import argparse
 import json
+import sys
 from pathlib import Path
 
 from populace.data.release import publish_release
 from populace.data.slack import notify_release
 
 
+def _reform_validation_skipped(release_dir: Path) -> bool:
+    """True if the release carries a reform_validation.json that was built with
+    out-of-sample reforms skipped (``out_of_sample_simulated`` false).
+
+    Such a release publishes blank out-of-sample (OBBBA) rows on the dashboard,
+    indistinguishable from a real result — so publishing one is refused unless
+    explicitly allowed. Absent/unreadable file or a true flag → not skipped.
+    """
+    path = release_dir / "reform_validation.json"
+    if not path.exists():
+        return False
+    try:
+        payload = json.loads(path.read_text())
+    except (OSError, ValueError):
+        return False
+    return payload.get("out_of_sample_simulated") is False
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("release_dir", help="Local releases/<release_id> directory.")
@@ -54,8 +73,30 @@ def main(argv: list[str] | None = None) -> int:
         "--updated-at",
         help="Optional latest.json timestamp override for reproducible tests.",
     )
+    parser.add_argument(
+        "--allow-incomplete-reform-validation",
+        action="store_true",
+        help=(
+            "Publish even if reform_validation.json was built with out-of-sample "
+            "reforms skipped (out_of_sample_simulated false). Off by default so a "
+            "release never silently ships blank OBBBA validation."
+        ),
+    )
     args = parser.parse_args(argv)
 
+    if not args.allow_incomplete_reform_validation and _reform_validation_skipped(
+        Path(args.release_dir)
+    ):
+        print(
+            "refusing to publish: reform_validation.json has "
+            "out_of_sample_simulated=false (built with "
+            "--skip-out-of-sample-reforms), so the dashboard would show blank "
+            "out-of-sample reforms. Rebuild without skipping, or pass "
+            "--allow-incomplete-reform-validation to publish anyway.",
+            file=sys.stderr,
+        )
+        return 1
+
     pointer = publish_release(
         Path(args.release_dir),
         args.repo_id,

packages/populace-data/tests/test_publish_guard.py

@@ -0,0 +1,34 @@
+import json
+from pathlib import Path
+
+from populace.data.publish_cli import _reform_validation_skipped, main
+
+
+def _write_rv(release_dir: Path, *, out_of_sample_simulated: bool | None) -> None:
+    payload: dict = {"schema_version": 1, "reforms": []}
+    if out_of_sample_simulated is not None:
+        payload["out_of_sample_simulated"] = out_of_sample_simulated
+    (release_dir / "reform_validation.json").write_text(json.dumps(payload))
+
+
+def test_skipped_detects_false_flag(tmp_path):
+    _write_rv(tmp_path, out_of_sample_simulated=False)
+    assert _reform_validation_skipped(tmp_path) is True
+
+
+def test_not_skipped_when_simulated(tmp_path):
+    _write_rv(tmp_path, out_of_sample_simulated=True)
+    assert _reform_validation_skipped(tmp_path) is False
+
+
+def test_not_skipped_when_no_reform_validation(tmp_path):
+    # e.g. a UK release, which carries no reform_validation.json
+    assert _reform_validation_skipped(tmp_path) is False
+
+
+def test_publish_refused_when_out_of_sample_skipped(tmp_path, capsys):
+    _write_rv(tmp_path, out_of_sample_simulated=False)
+    # The guard returns before publish_release is ever called (no HF upload).
+    rc = main([str(tmp_path), "--repo-id", "policyengine/populace-us"])
+    assert rc == 1
+    assert "refusing to publish" in capsys.readouterr().err