From c3b4f51793f1a2e75fa30d5566323d30fb2ad8be Mon Sep 17 00:00:00 2001 From: Daphne Hansell <128793799+daphnehanse11@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:29:12 -0400 Subject: [PATCH] Add ACA metal-level release diagnostics --- .../tests/test_us_fiscal_refresh_builder.py | 127 +++++++++ tools/build_us_fiscal_refresh_release.py | 242 ++++++++++++++++++ 2 files changed, 369 insertions(+) diff --git a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py index 35c1a92..d5f518c 100644 --- a/packages/populace-build/tests/test_us_fiscal_refresh_builder.py +++ b/packages/populace-build/tests/test_us_fiscal_refresh_builder.py @@ -968,6 +968,132 @@ def test_fiscal_target_value_basis_uses_only_amount_and_count() -> None: assert builder._fiscal_target_value_basis(bronze_count) == "count" +def test_aca_metal_level_diagnostics_report_state_and_national_shares() -> None: + builder = _load_builder_module() + + def aca_spec(name, role, state_fips, value): + return TargetSpec( + name=name, + entity="household", + measure="selected_marketplace_plan_benchmark_ratio", + value=value, + source="CMS Marketplace OEP", + family="cms_aca", + period=builder.PERIOD, + metadata={ + "target_role": role, + "state_fips": state_fips, + "source_measure_id": name.rsplit(".", maxsplit=1)[-1], + }, + ) + + ca_aptc = aca_spec( + "cms_aca.oep2024.state_marketplace.ca.aptc_recipients", + "aca_ptc_recipients", + "06", + 100.0, + ) + ny_aptc = aca_spec( + "cms_aca.oep2024.state_marketplace.ny.aptc_recipients", + "aca_ptc_recipients", + "36", + 50.0, + ) + ca_bronze = aca_spec( + "cms_aca.oep2024.state_metal.ca.bronze_aptc_consumers", + "aca_bronze_aptc_consumers", + "06", + 40.0, + ) + ny_bronze = aca_spec( + "cms_aca.oep2024.state_metal.ny.bronze_aptc_consumers", + "aca_bronze_aptc_consumers", + "36", + 10.0, + ) + + def diagnostic(spec, initial_estimate, final_estimate): + return SimpleNamespace( + name=f"{spec.name}@{spec.period}", + target=spec.value, + initial_estimate=initial_estimate, + final_estimate=final_estimate, + relative_error=(final_estimate - spec.value) / spec.value, + ) + + result = SimpleNamespace( + diagnostics=( + diagnostic(ca_aptc, 100.0, 80.0), + diagnostic(ny_aptc, 50.0, 50.0), + diagnostic(ca_bronze, 40.0, 32.0), + diagnostic(ny_bronze, 10.0, 20.0), + ) + ) + + payload = builder._aca_metal_level_diagnostics( + result, + (ca_aptc, ny_aptc, ca_bronze, ny_bronze), + ) + + assert payload["available"] is True + assert payload["declared_target_count"] == 2 + assert payload["target_count"] == 2 + assert payload["state_count"] == 2 + assert payload["missing_diagnostics"] == [] + assert payload["missing_denominator_states"] == [] + + national = payload["national"] + assert national["aptc_recipients"]["target"] == 150.0 + assert national["aptc_recipients"]["final_estimate"] == 130.0 + assert national["below_benchmark_ptc_consumers"]["target"] == 50.0 + assert national["below_benchmark_ptc_consumers"]["final_estimate"] == 52.0 + assert np.isclose(national["below_benchmark_share"]["target"], 50.0 / 150.0) + assert np.isclose(national["below_benchmark_share"]["final_estimate"], 0.4) + assert np.isclose(national["below_benchmark_share"]["relative_error"], 0.2) + + states = {row["state_fips"]: row for row in payload["states"]} + assert np.isclose(states["06"]["below_benchmark_share"]["final_estimate"], 0.4) + assert np.isclose(states["36"]["below_benchmark_share"]["target"], 0.2) + assert np.isclose(states["36"]["below_benchmark_share"]["final_estimate"], 0.4) + assert payload["largest_abs_share_errors"][0]["state_fips"] == "36" + + +def test_aca_metal_level_diagnostics_reports_unavailable_without_metal_targets() -> ( + None +): + builder = _load_builder_module() + aptc = TargetSpec( + name="cms_aca.oep2024.state_marketplace.ca.aptc_recipients", + entity="household", + measure="takes_up_aca_if_eligible", + value=100.0, + source="CMS Marketplace OEP", + family="cms_aca", + period=builder.PERIOD, + metadata={"target_role": "aca_ptc_recipients", "state_fips": "06"}, + ) + result = SimpleNamespace( + diagnostics=( + SimpleNamespace( + name=f"{aptc.name}@{aptc.period}", + target=100.0, + initial_estimate=100.0, + final_estimate=100.0, + relative_error=0.0, + ), + ) + ) + + payload = builder._aca_metal_level_diagnostics(result, (aptc,)) + + assert payload["available"] is False + assert payload["declared_target_count"] == 0 + assert payload["target_count"] == 0 + assert payload["state_count"] == 0 + assert payload["national"] is None + assert payload["states"] == [] + + def test_release_calibration_diagnostics_include_gate_failures( monkeypatch, tmp_path ) -> None: @@ -1027,6 +1153,7 @@ def fake_write_calibration_diagnostics(result, path, *, target_registry, build): "failures": [], "details": {"population": 334_200_000.0}, } + assert build["aca_metal_level_diagnostics"]["available"] is False def test_main_writes_diagnostics_before_post_calibration_gate_failure( diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 4bb9309..23df231 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -271,6 +271,14 @@ "aca_bronze_ptc_consumers": "cms_aca_bronze_aptc_consumers_by_state", "aca_below_benchmark_ptc_consumers": "cms_aca_bronze_aptc_consumers_by_state", } +US_ACA_PLAN_CHOICE_DENOMINATOR_ROLE = "aca_ptc_recipients" +US_ACA_METAL_LEVEL_TARGET_ROLES = frozenset( + { + "aca_bronze_aptc_consumers", + "aca_bronze_ptc_consumers", + "aca_below_benchmark_ptc_consumers", + } +) US_ACA_PERSON_COUNT_TARGET_TABLES = frozenset( { US_ACA_APTC_TARGET_TABLE, @@ -1956,6 +1964,236 @@ def _fiscal_target_value_basis(spec) -> str: return "amount" +def _finite_diagnostic_number(value: object) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(number): + return None + return number + + +def _safe_ratio(numerator: float | None, denominator: float | None) -> float | None: + if numerator is None or denominator is None or denominator == 0.0: + return None + return numerator / denominator + + +def _relative_error_from_values( + *, + target: float | None, + final_estimate: float | None, +) -> float | None: + if target is None or final_estimate is None: + return None + if target == 0.0: + return final_estimate - target + return (final_estimate - target) / target + + +def _sum_diagnostic_records( + records: Iterable[Mapping[str, object]], + field: str, +) -> float | None: + total = 0.0 + for record in records: + value = _finite_diagnostic_number(record.get(field)) + if value is None: + return None + total += value + return total + + +def _aca_target_count_summary( + records: Iterable[Mapping[str, object]], +) -> dict[str, object]: + materialized = tuple(records) + target = _sum_diagnostic_records(materialized, "target") + initial_estimate = _sum_diagnostic_records(materialized, "initial_estimate") + final_estimate = _sum_diagnostic_records(materialized, "final_estimate") + return { + "target_count": len(materialized), + "target": target, + "initial_estimate": initial_estimate, + "final_estimate": final_estimate, + "relative_error": _relative_error_from_values( + target=target, + final_estimate=final_estimate, + ), + "row_names": sorted(str(record["row_name"]) for record in materialized), + "source_record_ids": sorted( + str(record["source_record_id"]) for record in materialized + ), + } + + +def _aca_plan_choice_share_summary( + numerator: Mapping[str, object], + denominator: Mapping[str, object], +) -> dict[str, float | None]: + target_share = _safe_ratio( + _finite_diagnostic_number(numerator.get("target")), + _finite_diagnostic_number(denominator.get("target")), + ) + initial_share = _safe_ratio( + _finite_diagnostic_number(numerator.get("initial_estimate")), + _finite_diagnostic_number(denominator.get("initial_estimate")), + ) + final_share = _safe_ratio( + _finite_diagnostic_number(numerator.get("final_estimate")), + _finite_diagnostic_number(denominator.get("final_estimate")), + ) + difference = ( + None + if target_share is None or final_share is None + else final_share - target_share + ) + return { + "target": target_share, + "initial_estimate": initial_share, + "final_estimate": final_share, + "difference": difference, + "relative_error": _relative_error_from_values( + target=target_share, + final_estimate=final_share, + ), + } + + +def _aca_metal_level_diagnostics( + result, + target_specs: Iterable[object], +) -> dict[str, object]: + diagnostics_by_name = { + str(diagnostic.name): diagnostic + for diagnostic in getattr(result, "diagnostics", ()) + if getattr(diagnostic, "name", None) is not None + } + grouped: dict[str, dict[str, list[dict[str, object]]]] = {} + missing_diagnostics: list[str] = [] + declared_metal_states: set[str] = set() + declared_metal_target_count = 0 + + for spec in target_specs: + if getattr(spec, "family", None) != "cms_aca": + continue + metadata = dict(getattr(spec, "metadata", {}) or {}) + target_role = metadata.get("target_role") + if target_role == US_ACA_PLAN_CHOICE_DENOMINATOR_ROLE: + bucket = "aptc_recipients" + elif target_role in US_ACA_METAL_LEVEL_TARGET_ROLES: + bucket = "below_benchmark_ptc_consumers" + else: + continue + state_fips = metadata.get("state_fips") + if not state_fips: + continue + state_key = str(state_fips).zfill(2) + if bucket == "below_benchmark_ptc_consumers": + declared_metal_states.add(state_key) + declared_metal_target_count += 1 + + row_name = f"{spec.name}@{spec.period}" + diagnostic = diagnostics_by_name.get(row_name) + if diagnostic is None: + missing_diagnostics.append(row_name) + continue + + grouped.setdefault( + state_key, + { + "aptc_recipients": [], + "below_benchmark_ptc_consumers": [], + }, + )[bucket].append( + { + "row_name": row_name, + "source_record_id": spec.name, + "target": getattr(diagnostic, "target", None), + "initial_estimate": getattr(diagnostic, "initial_estimate", None), + "final_estimate": getattr(diagnostic, "final_estimate", None), + } + ) + + state_rows: list[dict[str, object]] = [] + all_aptc_records: list[dict[str, object]] = [] + all_metal_records: list[dict[str, object]] = [] + for state_fips in sorted(grouped): + records = grouped[state_fips] + aptc = _aca_target_count_summary(records["aptc_recipients"]) + below_benchmark = _aca_target_count_summary( + records["below_benchmark_ptc_consumers"] + ) + share = _aca_plan_choice_share_summary(below_benchmark, aptc) + state_rows.append( + { + "state_fips": state_fips, + "aptc_recipients": aptc, + "below_benchmark_ptc_consumers": below_benchmark, + "below_benchmark_share": share, + } + ) + if records["below_benchmark_ptc_consumers"]: + all_aptc_records.extend(records["aptc_recipients"]) + all_metal_records.extend(records["below_benchmark_ptc_consumers"]) + + national_aptc = _aca_target_count_summary(all_aptc_records) + national_below_benchmark = _aca_target_count_summary(all_metal_records) + national = { + "aptc_recipients": national_aptc, + "below_benchmark_ptc_consumers": national_below_benchmark, + "below_benchmark_share": _aca_plan_choice_share_summary( + national_below_benchmark, + national_aptc, + ), + } + share_errors = [ + { + "state_fips": row["state_fips"], + "target": row["below_benchmark_share"]["target"], + "final_estimate": row["below_benchmark_share"]["final_estimate"], + "difference": row["below_benchmark_share"]["difference"], + "relative_error": row["below_benchmark_share"]["relative_error"], + } + for row in state_rows + if row["below_benchmark_share"]["difference"] is not None + ] + share_errors.sort( + key=lambda row: abs(float(row["difference"])), + reverse=True, + ) + missing_denominator_states = [ + str(row["state_fips"]) + for row in state_rows + if row["below_benchmark_ptc_consumers"]["target_count"] + and not row["aptc_recipients"]["target_count"] + ] + return { + "schema_version": 1, + "available": bool(all_metal_records), + "metal_target_roles": sorted(US_ACA_METAL_LEVEL_TARGET_ROLES), + "denominator_role": US_ACA_PLAN_CHOICE_DENOMINATOR_ROLE, + "declared_target_count": declared_metal_target_count, + "declared_state_count": len(declared_metal_states), + "target_count": len(all_metal_records), + "state_count": sum( + 1 + for row in state_rows + if row["below_benchmark_ptc_consumers"]["target_count"] + ), + "missing_diagnostics": sorted(missing_diagnostics), + "missing_denominator_states": missing_denominator_states, + "national": national if all_metal_records else None, + "states": [ + row + for row in state_rows + if row["below_benchmark_ptc_consumers"]["target_count"] + ], + "largest_abs_share_errors": share_errors[:10], + } + + def _release_gate_failures( result, compilation: Mapping[str, object], @@ -2257,6 +2495,10 @@ def _write_release_calibration_diagnostics( if base_population_gate is not None else None ), + "aca_metal_level_diagnostics": _aca_metal_level_diagnostics( + result, + registry.specs, + ), "release_gates": { "passed": not failures, "failures": failures,