diff --git a/bedrock/transform/eeio/derived_cornerstone.py b/bedrock/transform/eeio/derived_cornerstone.py index d082134d..b87dcf85 100644 --- a/bedrock/transform/eeio/derived_cornerstone.py +++ b/bedrock/transform/eeio/derived_cornerstone.py @@ -46,7 +46,6 @@ load_2017_V_usa, load_2017_value_added_usa, load_2017_Ytot_usa, - load_summary_Uimp_usa, ) from bedrock.transform.allocation.derived import derive_E_usa from bedrock.transform.eeio.cornerstone_bea_intermediates import ( @@ -94,12 +93,11 @@ ) from bedrock.utils.math.disaggregation import disaggregate_vector from bedrock.utils.math.formulas import ( + backcompute_y_from_A_and_q, compute_q, compute_Unorm_matrix, compute_Vnorm_matrix, compute_x, - compute_y_for_national_accounting_balance, - compute_y_imp, ) from bedrock.utils.math.handle_negatives import ( handle_negative_matrix_values, @@ -127,9 +125,6 @@ USA_2017_FINAL_DEMAND_IMPORT_CODE, USA_2017_FINAL_DEMAND_PERSONAL_CONSUMPTION_EXPENDITURE_CODE, ) -from bedrock.utils.taxonomy.bea.v2017_industry_summary import ( - USA_2017_SUMMARY_INDUSTRY_CODES, -) from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( get_bea_v2017_summary_to_cornerstone_corresp_df, ) @@ -904,40 +899,17 @@ def derive_cornerstone_Y_and_trade_scaled() -> SingleRegionYtotAndTradeVectorSet @functools.cache def derive_cornerstone_y_nab() -> pd.Series[float]: - """Y for national accounting balance, year-scaled.""" - cfg = get_usa_config() - detail_2017 = derive_cornerstone_Ytot_matrix_set() - - y_nab_2017 = compute_y_for_national_accounting_balance( - y_tot=detail_2017.ytot, - y_imp=compute_y_imp( - imports=detail_2017.imports, - Uimp=derive_cornerstone_U_set().Uimp, - ), - exports=detail_2017.exports, - ) - - summary_Y = derive_summary_Ytot_usa_matrix_set(cfg.usa_io_data_year) - y_nab_summary = compute_y_for_national_accounting_balance( - y_tot=summary_Y.ytot, - y_imp=compute_y_imp( - imports=summary_Y.imports, - Uimp=load_summary_Uimp_usa(cfg.usa_io_data_year).loc[ - USA_2017_SUMMARY_INDUSTRY_CODES, USA_2017_SUMMARY_INDUSTRY_CODES - ], - ), - exports=summary_Y.exports, - ) + """National-accounting final demand consistent with scaled ``Adom`` and ``q``. - y_nab_scaled = _disaggregate_and_inflate_vector( - base=y_nab_summary, - weight=y_nab_2017, - corresp_df=get_bea_v2017_summary_to_cornerstone_corresp_df(), - original_year=cfg.usa_io_data_year, - target_year=cfg.model_base_year, - ) + Enforces row balance ``q = Adom @ diag(q) + y_nab`` using the same + ``derive_cornerstone_Aq_scaled`` object whose ``scaled_q`` is snapshotted. + Any future change to ``scaled_q`` in that path propagates to ``y_nab``. - return handle_negative_vector_values(y_nab_scaled) + Negative values are retained so ``q ≈ L_dom @ y_nab`` holds numerically; + clipping would break the domestic Leontief identity. + """ + aq = derive_cornerstone_Aq_scaled() + return backcompute_y_from_A_and_q(A=aq.Adom, q=aq.scaled_q) def derive_cornerstone_ydom_and_yimp() -> SingleRegionYVectorSet: diff --git a/bedrock/utils/math/formulas.py b/bedrock/utils/math/formulas.py index 8d1d44c7..deee4370 100644 --- a/bedrock/utils/math/formulas.py +++ b/bedrock/utils/math/formulas.py @@ -254,7 +254,7 @@ def backcompute_q_from_L_and_y( return (L @ np.diag(y)).sum(axis=1) -def backcompute_y_from_q_and_Aq( +def backcompute_y_from_A_and_q( *, A: pd.DataFrame, q: pd.Series[float] ) -> pd.Series[float]: return q - A.multiply(q, axis=1).sum(axis=1) diff --git a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py index 6d16e241..6726c644 100644 --- a/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_eeio_diagnostics.py @@ -353,28 +353,27 @@ def test_compare_Uset_y_dom_and_q_usa( @pytest.mark.eeio_integration @pytest.mark.parametrize( - "modelType, use_domestic", - [ - ("Commodity", False), - ], -) # TODO: add ("Commodity", True) test and industry parameters when Industry models become available [("Industry", False), ("Industry", True)] -@pytest.mark.parametrize( - "pipeline", + "modelType, use_domestic, pipeline", [ + ("Commodity", True, "cornerstone"), pytest.param( - "ceda", + "Commodity", + False, + "cornerstone", marks=pytest.mark.xfail( - reason="CEDA: scaled q≠L_total·y_total for ~298 commodity sectors (total Leontief identity).", + reason="Cornerstone total L·y still uses ytot/trade, not y_nab.", ), ), pytest.param( - "cornerstone", + "Commodity", + False, + "ceda", marks=pytest.mark.xfail( - reason="Cornerstone: scaled q≠L_total·y_total for 323 sectors; A-matrix vs Y-inflation path mismatch.", + reason="CEDA: scaled q≠L_total·y_total for ~298 commodity sectors (total Leontief identity).", ), ), ], -) +) # TODO: add industry parameters when Industry models become available def test_compare_output_and_L_y( modelType: str, use_domestic: bool, @@ -396,23 +395,17 @@ def test_compare_output_and_L_y( y = y_set.ytot + y_set.exports - y_set.imports L = formulas.compute_L_matrix(A=Aq.Adom + Aq.Aimp) else: - # Cornerstone total L·y uses year-scaled Adom+Aimp and scaled_q from - # derive_cornerstone_Aq_scaled, compared to L @ y where y = y_tot + - # exports − imports from derive_cornerstone_Y_and_trade_scaled (summary- - # disaggregated and inflated to model year). 323 of 392 commodities fail - # at 1% tolerance—a systemic scale mismatch between the A-matrix scaling - # pipeline and the Y/trade inflation path (see zero-weight disaggregation - # warning during Y scaling), not an isolated wiring error in this test. # Cornerstone scales A and q to model year; CEDA branch stays in 2017 detail. Aq = derive_cornerstone_Aq_scaled() # Output must match Aq scaling (scaled_q), not derive_cornerstone_q() from V. output = Aq.scaled_q if modelType == "Commodity" else derive_cornerstone_x() if use_domestic: - # Precomputed scaled y_nab (same path as derive_y_for_national_accounting_balance_usa). + # y_nab from backcompute_y_from_A_and_q(Adom, scaled_q); unclipped. y = derive_cornerstone_y_nab() L = formulas.compute_L_matrix(A=Aq.Adom) else: - # Total y from summary-disaggregated inflated trade, not raw Ytot_matrix_set. + # Total L·y still uses y from derive_cornerstone_Y_and_trade_scaled + # (summary-disaggregated BEA Y/trade), not IO-balanced y_nab. y_trade = derive_cornerstone_Y_and_trade_scaled() y = y_trade.ytot + y_trade.exports - y_trade.imports L = formulas.compute_L_matrix(A=Aq.Adom + Aq.Aimp)