diff --git a/packages/populace-build/src/populace/build/source_manifest.py b/packages/populace-build/src/populace/build/source_manifest.py index e46abca..63f746f 100644 --- a/packages/populace-build/src/populace/build/source_manifest.py +++ b/packages/populace-build/src/populace/build/source_manifest.py @@ -64,6 +64,7 @@ "read_table", "read_tables", "replace_sentinels", + "split_component_by_share", "support_clip", "uprate", "zero_when_false", diff --git a/packages/populace-build/src/populace/build/us/country_package.json b/packages/populace-build/src/populace/build/us/country_package.json index 7fed91f..6ddd138 100644 --- a/packages/populace-build/src/populace/build/us/country_package.json +++ b/packages/populace-build/src/populace/build/us/country_package.json @@ -6,6 +6,7 @@ "fiscal_target_references.json", "obbba_reforms.json", "puf_aggregate_record_disaggregation.json", + "soca_capital_gain_distribution_shares.json", "soi_baseline_levels.json", "source_stages.json", "support_spine.json", diff --git a/packages/populace-build/src/populace/build/us/soca_capital_gain_distribution_shares.json b/packages/populace-build/src/populace/build/us/soca_capital_gain_distribution_shares.json new file mode 100644 index 0000000..4150078 --- /dev/null +++ b/packages/populace-build/src/populace/build/us/soca_capital_gain_distribution_shares.json @@ -0,0 +1,478 @@ +{ + "version": 1, + "country": "us", + "description": "Schedule D capital gain distributions split parameters. The IRS SOI Sales of Capital Assets study (latest: TY2015) reports capital gain distributions only nationally; no by-AGI CGD split exists in published SOI data (SOCA Table 2 merges CGD into 'Other asset types'). The v1 imputation therefore applies the national Schedule-D route share of long-term net gains proportionally within eligible records. The by-AGI tables below (the direct-1040 route and the 'other assets' envelope) are shipped for a future banded refinement.", + "units": "source amounts in thousands of USD; shares are ratios", + "sources": { + "soca_index": "https://www.irs.gov/statistics/soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns", + "soca_table1_ty2015": "https://www.irs.gov/pub/irs-soi/15in01soca.xlsx", + "soca_table2_ty2015": "https://www.irs.gov/pub/irs-soi/15in02soca.xlsx", + "pub1304_t14_index": "https://www.irs.gov/statistics/soi-tax-stats-individual-income-tax-returns-complete-report-publication-1304-basic-tables-part-1", + "pub1304_t14_ty2015": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "pub1304_t14_ty2022": "https://www.irs.gov/pub/irs-soi/22in14ar.xls", + "pub1304_t14_ty2023": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "soca_data_release_pdf_2013_2015": "https://www.irs.gov/pub/irs-soi/soi-a-soca-id2205.pdf" + }, + "national_anchor_ty2015": { + "soca_all_route_capital_gain_distributions_k": 79615698.086, + "soca_long_term_total_net_gain_k": 702271858.85, + "pub1304_direct_1040_route_k": 11563203.0, + "schedule_d_route_residual_k": 68052495.086, + "schedule_d_cgd_share_of_lt_net_gains": 0.09852561497474391, + "note": "residual = SOCA all-route CGD minus Pub 1304 Table 1.4 direct-1040 CGD; share = residual / (LT net gains minus the direct route, which is not on Schedule D). CGD is entirely long-term (s. 852(b)(3): capital gain dividends)." + }, + "direct_1040_route_by_agi": { + "note": "Line = 'Capital gain distributions reported on Form 1040' (E01100, the NON-Schedule-D direct route). Units: amounts in thousands of dollars. AGI bands per Pub 1304 Table 1.4 (All returns panel).", + "TY2015": { + "file": "https://www.irs.gov/pub/irs-soi/15in14ar.xls", + "col_index": 23, + "bands": [ + { + "band": "All returns, total", + "cgd_returns": 4323250.0, + "cgd_amount_k": 11563203.0, + "share_of_1040_cgd_total": 1.0 + }, + { + "band": "No adjusted gross income", + "cgd_returns": 29097.0, + "cgd_amount_k": 46969.0, + "share_of_1040_cgd_total": 0.0041 + }, + { + "band": "$1 under $5,000", + "cgd_returns": 222771.0, + "cgd_amount_k": 195181.0, + "share_of_1040_cgd_total": 0.0169 + }, + { + "band": "$5,000 under $10,000", + "cgd_returns": 179305.0, + "cgd_amount_k": 224309.0, + "share_of_1040_cgd_total": 0.0194 + }, + { + "band": "$10,000 under $15,000", + "cgd_returns": 159924.0, + "cgd_amount_k": 234373.0, + "share_of_1040_cgd_total": 0.0203 + }, + { + "band": "$15,000 under $20,000", + "cgd_returns": 169070.0, + "cgd_amount_k": 251117.0, + "share_of_1040_cgd_total": 0.0217 + }, + { + "band": "$20,000 under $25,000", + "cgd_returns": 147995.0, + "cgd_amount_k": 259026.0, + "share_of_1040_cgd_total": 0.0224 + }, + { + "band": "$25,000 under $30,000", + "cgd_returns": 128487.0, + "cgd_amount_k": 245274.0, + "share_of_1040_cgd_total": 0.0212 + }, + { + "band": "$30,000 under $40,000", + "cgd_returns": 287469.0, + "cgd_amount_k": 462119.0, + "share_of_1040_cgd_total": 0.04 + }, + { + "band": "$40,000 under $50,000", + "cgd_returns": 274422.0, + "cgd_amount_k": 479156.0, + "share_of_1040_cgd_total": 0.0414 + }, + { + "band": "$50,000 under $75,000", + "cgd_returns": 662186.0, + "cgd_amount_k": 1542572.0, + "share_of_1040_cgd_total": 0.1334 + }, + { + "band": "$75,000 under $100,000", + "cgd_returns": 564896.0, + "cgd_amount_k": 1370439.0, + "share_of_1040_cgd_total": 0.1185 + }, + { + "band": "$100,000 under $200,000", + "cgd_returns": 1078572.0, + "cgd_amount_k": 3639073.0, + "share_of_1040_cgd_total": 0.3147 + }, + { + "band": "$200,000 under $500,000", + "cgd_returns": 369651.0, + "cgd_amount_k": 1989633.0, + "share_of_1040_cgd_total": 0.1721 + }, + { + "band": "$500,000 under $1,000,000", + "cgd_returns": 39917.0, + "cgd_amount_k": 397268.0, + "share_of_1040_cgd_total": 0.0344 + }, + { + "band": "$1,000,000 under $1,500,000", + "cgd_returns": 5619.0, + "cgd_amount_k": 82981.0, + "share_of_1040_cgd_total": 0.0072 + }, + { + "band": "$1,500,000 under $2,000,000", + "cgd_returns": 1759.0, + "cgd_amount_k": 35912.0, + "share_of_1040_cgd_total": 0.0031 + }, + { + "band": "$2,000,000 under $5,000,000", + "cgd_returns": 1846.0, + "cgd_amount_k": 90952.0, + "share_of_1040_cgd_total": 0.0079 + }, + { + "band": "$5,000,000 under $10,000,000", + "cgd_returns": 206.0, + "cgd_amount_k": 10409.0, + "share_of_1040_cgd_total": 0.0009 + }, + { + "band": "$10,000,000 or more", + "cgd_returns": 58.0, + "cgd_amount_k": 6438.0, + "share_of_1040_cgd_total": 0.0006 + } + ] + }, + "TY2022": { + "file": "https://www.irs.gov/pub/irs-soi/22in14ar.xls", + "col_index": 35, + "bands": [ + { + "band": "All returns, total", + "cgd_returns": 3980047.0, + "cgd_amount_k": 12863423.0, + "share_of_1040_cgd_total": 1.0 + }, + { + "band": "No adjusted gross income", + "cgd_returns": 27012.0, + "cgd_amount_k": 88131.0, + "share_of_1040_cgd_total": 0.0069 + }, + { + "band": "$1 under $5,000", + "cgd_returns": 132582.0, + "cgd_amount_k": 86558.0, + "share_of_1040_cgd_total": 0.0067 + }, + { + "band": "$5,000 under $10,000", + "cgd_returns": 130890.0, + "cgd_amount_k": 182403.0, + "share_of_1040_cgd_total": 0.0142 + }, + { + "band": "$10,000 under $15,000", + "cgd_returns": 106473.0, + "cgd_amount_k": 188178.0, + "share_of_1040_cgd_total": 0.0146 + }, + { + "band": "$15,000 under $20,000", + "cgd_returns": 111615.0, + "cgd_amount_k": 215886.0, + "share_of_1040_cgd_total": 0.0168 + }, + { + "band": "$20,000 under $25,000", + "cgd_returns": 99575.0, + "cgd_amount_k": 154072.0, + "share_of_1040_cgd_total": 0.012 + }, + { + "band": "$25,000 under $30,000", + "cgd_returns": 94978.0, + "cgd_amount_k": 185113.0, + "share_of_1040_cgd_total": 0.0144 + }, + { + "band": "$30,000 under $40,000", + "cgd_returns": 222677.0, + "cgd_amount_k": 421788.0, + "share_of_1040_cgd_total": 0.0328 + }, + { + "band": "$40,000 under $50,000", + "cgd_returns": 191959.0, + "cgd_amount_k": 308187.0, + "share_of_1040_cgd_total": 0.024 + }, + { + "band": "$50,000 under $75,000", + "cgd_returns": 546979.0, + "cgd_amount_k": 1186628.0, + "share_of_1040_cgd_total": 0.0922 + }, + { + "band": "$75,000 under $100,000", + "cgd_returns": 523797.0, + "cgd_amount_k": 1359330.0, + "share_of_1040_cgd_total": 0.1057 + }, + { + "band": "$100,000 under $200,000", + "cgd_returns": 1191091.0, + "cgd_amount_k": 4073791.0, + "share_of_1040_cgd_total": 0.3167 + }, + { + "band": "$200,000 under $500,000", + "cgd_returns": 522256.0, + "cgd_amount_k": 3461697.0, + "share_of_1040_cgd_total": 0.2691 + }, + { + "band": "$500,000 under $1,000,000", + "cgd_returns": 78163.0, + "cgd_amount_k": 951661.0, + "share_of_1040_cgd_total": 0.074 + }, + { + "band": "$1,000,000 under $1,500,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$1,500,000 under $2,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$2,000,000 under $5,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$5,000,000 under $10,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$10,000,000 or more", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + } + ] + }, + "TY2023": { + "file": "https://www.irs.gov/pub/irs-soi/23in14ar.xls", + "col_index": 35, + "bands": [ + { + "band": "All returns, total", + "cgd_returns": 3209131.0, + "cgd_amount_k": 9340820.0, + "share_of_1040_cgd_total": 1.0 + }, + { + "band": "No adjusted gross income", + "cgd_returns": 14040.0, + "cgd_amount_k": 28143.0, + "share_of_1040_cgd_total": 0.003 + }, + { + "band": "$1 under $5,000", + "cgd_returns": 96972.0, + "cgd_amount_k": 88282.0, + "share_of_1040_cgd_total": 0.0095 + }, + { + "band": "$5,000 under $10,000", + "cgd_returns": 92567.0, + "cgd_amount_k": 119014.0, + "share_of_1040_cgd_total": 0.0127 + }, + { + "band": "$10,000 under $15,000", + "cgd_returns": 93705.0, + "cgd_amount_k": 116098.0, + "share_of_1040_cgd_total": 0.0124 + }, + { + "band": "$15,000 under $20,000", + "cgd_returns": 66376.0, + "cgd_amount_k": 88994.0, + "share_of_1040_cgd_total": 0.0095 + }, + { + "band": "$20,000 under $25,000", + "cgd_returns": 79275.0, + "cgd_amount_k": 171826.0, + "share_of_1040_cgd_total": 0.0184 + }, + { + "band": "$25,000 under $30,000", + "cgd_returns": 80695.0, + "cgd_amount_k": 85630.0, + "share_of_1040_cgd_total": 0.0092 + }, + { + "band": "$30,000 under $40,000", + "cgd_returns": 173756.0, + "cgd_amount_k": 227844.0, + "share_of_1040_cgd_total": 0.0244 + }, + { + "band": "$40,000 under $50,000", + "cgd_returns": 175969.0, + "cgd_amount_k": 257041.0, + "share_of_1040_cgd_total": 0.0275 + }, + { + "band": "$50,000 under $75,000", + "cgd_returns": 432289.0, + "cgd_amount_k": 892702.0, + "share_of_1040_cgd_total": 0.0956 + }, + { + "band": "$75,000 under $100,000", + "cgd_returns": 414537.0, + "cgd_amount_k": 895073.0, + "share_of_1040_cgd_total": 0.0958 + }, + { + "band": "$100,000 under $200,000", + "cgd_returns": 980903.0, + "cgd_amount_k": 3353841.0, + "share_of_1040_cgd_total": 0.3591 + }, + { + "band": "$200,000 under $500,000", + "cgd_returns": 441988.0, + "cgd_amount_k": 2315496.0, + "share_of_1040_cgd_total": 0.2479 + }, + { + "band": "$500,000 under $1,000,000", + "cgd_returns": 66063.0, + "cgd_amount_k": 700837.0, + "share_of_1040_cgd_total": 0.075 + }, + { + "band": "$1,000,000 under $1,500,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$1,500,000 under $2,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$2,000,000 under $5,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$5,000,000 under $10,000,000", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + }, + { + "band": "$10,000,000 or more", + "cgd_returns": 0.0, + "cgd_amount_k": 0.0, + "share_of_1040_cgd_total": 0.0 + } + ] + } + }, + "lt_gain_other_assets_envelope_by_agi_ty2015": { + "note": "By-AGI long-term gain from SOCA Table 2 TY2015. 'Other asset types' is the bucket CONTAINING capital gain distributions (also contains options, futures, all mutual funds' realized gains, partnership/S-corp/trust interests, residences, etc). CGD is ~12.9% of this bucket nationally, so these shares are an UPPER BOUND envelope for CGD, not CGD itself. lt_gain columns are gross gain from LT gain transactions (col E), in $thousands. 7 AGI bands only.", + "bands": [ + { + "band": "All returns", + "lt_gain_all_assets_k": 882733542.22, + "lt_gain_other_assets_k": 619193352.06, + "other_over_all_lt_gain_share": 0.7014 + }, + { + "band": "Adjusted gross deficit", + "lt_gain_all_assets_k": 36341641.229, + "lt_gain_other_assets_k": 26291407.805, + "other_over_all_lt_gain_share": 0.7235 + }, + { + "band": "Under $20,000", + "lt_gain_all_assets_k": 8753282.1697, + "lt_gain_other_assets_k": 5802365.9917, + "other_over_all_lt_gain_share": 0.6629 + }, + { + "band": "$20,000 under $50,000", + "lt_gain_all_assets_k": 19470227.656, + "lt_gain_other_assets_k": 11740551.36, + "other_over_all_lt_gain_share": 0.603 + }, + { + "band": "$50,000 under $100,000", + "lt_gain_all_assets_k": 42702648.788, + "lt_gain_other_assets_k": 27250438.314, + "other_over_all_lt_gain_share": 0.6381 + }, + { + "band": "$100,000 under $200,000", + "lt_gain_all_assets_k": 81533406.973, + "lt_gain_other_assets_k": 49121148.672, + "other_over_all_lt_gain_share": 0.6025 + }, + { + "band": "$200,000 under $500,000", + "lt_gain_all_assets_k": 116538323.01, + "lt_gain_other_assets_k": 69445019.585, + "other_over_all_lt_gain_share": 0.5959 + }, + { + "band": "$500,000 under $1,000,000", + "lt_gain_all_assets_k": 82692308.357, + "lt_gain_other_assets_k": 54950324.231, + "other_over_all_lt_gain_share": 0.6645 + }, + { + "band": "$1,000,000 or more", + "lt_gain_all_assets_k": 494701704.04, + "lt_gain_other_assets_k": 374592096.1, + "other_over_all_lt_gain_share": 0.7572 + } + ] + }, + "caveats": [ + "ALL SOI DOLLAR FIGURES ARE IN THOUSANDS OF DOLLARS.", + "SOCA latest year is TY2015; the study was discontinued/not-updated for later years (last release Feb 2022 covering TY2013-2015).", + "SOCA Table 2 does NOT isolate capital gain distributions by AGI - CGD is merged into 'Other asset types'. The only AGI-resolved CGD-specific IRS figure is Pub 1304 Table 1.4's 'Capital gain distributions reported on Form 1040' line, but that captures ONLY the non-Schedule-D direct route (E01100), which is ~15% of all CGD.", + "SOCA Table 1 national CGD = $79.6B (TY2015) captures ALL capital gain distributions regardless of route (line 7 direct OR Schedule D line 13). CGD is entirely long-term.", + "Pub 1304 Table 1.4 E01100 line = $11.56B (TY2015), $12.86B (TY2022), $9.34B (TY2023): this is ONLY the direct-1040 route used by filers with no other Schedule D activity. The remaining ~$68B (TY2015) of CGD is reported on Schedule D line 13 by filers who file Schedule D for other reasons.", + "Pub 1304 Table 1.4 collapses the $1M+ AGI bands to 0 for the direct-1040 CGD line in TY2022 and TY2023 (disclosure/rounding); TY2015 shows small but nonzero $1M+ values (~$227M total across all $1M+ bands). High-AGI filers overwhelmingly use Schedule D, so the direct route is genuinely tiny at the top - but not exactly zero.", + "SOCA AGI bands (7) are coarser and differently cut than Pub 1304 Table 1.4 bands (19). SOCA has an 'Adjusted gross deficit' (negative AGI) band; Table 1.4 has 'No adjusted gross income'.", + "SOCA gain amounts in Table 2 are GROSS gains from gain transactions (col E), reconciling to Table 1C col I (gain), not net of losses. Net LT gain nationally (Table 1C col E) = $702.3B.", + "Counts are numbers of returns/transactions from SOI samples, NOT population weights.", + "The v1 split is deterministic and proportional: every eligible record (positive long-term gains, no direct-route distributions) receives the national share. Participation margins and AGI-band tilt are documented follow-ups; no by-AGI CGD-specific series exists to pin them.", + "The share is a TY2015 ratio applied to later-year gains; fund distribution intensity varies across years (ICI industry series is the follow-up anchor for year-specific levels).", + "Schedule D line 13 includes REIT capital gain dividends as well as RIC distributions; reforms restricted to RICs (subchapter M part I) need a further split." + ] +} diff --git a/packages/populace-build/src/populace/build/us/source_stages.json b/packages/populace-build/src/populace/build/us/source_stages.json index 6c2a520..f98d06b 100644 --- a/packages/populace-build/src/populace/build/us/source_stages.json +++ b/packages/populace-build/src/populace/build/us/source_stages.json @@ -33,7 +33,12 @@ "kind": "disaggregate_aggregate_records", "method": "donor_template_calibration", "spec": "puf_aggregate_record_disaggregation", - "replace_records": [999996, 999997, 999998, 999999], + "replace_records": [ + 999996, + 999997, + 999998, + 999999 + ], "weight": "s006", "amount_columns": "irs_puf_amount_columns", "seed_from_build_config": true @@ -123,6 +128,50 @@ ], "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. IRS disclosure aggregate rows are replaced from raw PUF aggregate totals before uprating; Forbes top-tail synthesis is not enabled." }, + { + "stage": "capital_gain_distributions", + "survey": "IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", + "source": "https://www.irs.gov/statistics/soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns", + "grain": "tax_unit", + "artifacts": [ + { + "kind": "administrative_table", + "format": "published_estimate", + "locator": "SOI Sales of Capital Assets Table 1 (1A/1C), capital gain distributions line", + "vintage": "2015" + }, + { + "kind": "administrative_table", + "format": "published_estimate", + "locator": "SOI Pub 1304 Table 1.4, capital gain distributions reported on Form 1040", + "vintage": "2015-2023" + } + ], + "notes": "Splits the Schedule D line 13 route of capital gain distributions out of long-term gains as a memo component. No by-AGI CGD series exists in published SOI data (SOCA Table 2 merges CGD into 'Other asset types'), so the v1 split applies the national Schedule-D share of LT net gains proportionally within eligible records; the two reporting routes are mutually exclusive on a real return, so records with direct-route distributions are excluded. Anchor arithmetic, the by-AGI tables that do exist, and follow-ups live in soca_capital_gain_distribution_shares.json.", + "operations": [ + { + "kind": "read_table", + "table": "tax_unit", + "weight": "tax_unit_weight" + }, + { + "kind": "split_component_by_share", + "resource": "soca_capital_gain_distribution_shares", + "share_field": "schedule_d_cgd_share_of_lt_net_gains", + "source_column": "long_term_capital_gains_before_response", + "output": "schedule_d_capital_gain_distributions", + "exclusive_with": [ + "non_sch_d_capital_gains" + ] + } + ], + "outputs": [ + "schedule_d_capital_gain_distributions" + ], + "nonnegative_outputs": [ + "schedule_d_capital_gain_distributions" + ] + }, { "stage": "scf_wealth", "survey": "Fed SCF 2022", @@ -429,7 +478,7 @@ "ssn_card_type", "immigration_status_str" ], - "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic seeded order until the remaining undocumented worker count matches Pew's 8.3M and the undocumented student count matches the Higher Ed Immigration Portal's ~408k — the only forced margins. The total undocumented population is emergent (≈13M on the 2024 ASEC, within the range of published 2023-24 estimates) and is gated against Pew's 11.0M 2022 anchor with a coarse plausibility band rather than forced; reconciling the level is the calibration lane's job. Immigration-status tags carry only statutory tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status — blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions and over-grant refugee-class benefit exemptions." + "notes": "Citizenship is measured (PRCITSHP), not imputed. Non-citizens with any ASEC-UA legal-status indicator (Van Hook et al., SSRN 4662801) become OTHER_NON_CITIZEN. The CPS carries no work-authorization variable, so among residual non-citizens, workers and students spill to NON_CITIZEN_VALID_EAD in deterministic seeded order until the remaining undocumented worker count matches Pew's 8.3M and the undocumented student count matches the Higher Ed Immigration Portal's ~408k \u2014 the only forced margins. The total undocumented population is emergent (\u224813M on the 2024 ASEC, within the range of published 2023-24 estimates) and is gated against Pew's 11.0M 2022 anchor with a coarse plausibility band rather than forced; reconciling the level is the calibration lane's job. Immigration-status tags carry only statutory tests the data supports: DACA (arrival cohort among EAD holders) and CUBAN_HAITIAN_ENTRANT (nativity plus post-1980 arrival); other documented non-citizens stay LEGAL_PERMANENT_RESIDENT, the modal true status \u2014 blanket REFUGEE/TPS labels are deliberately not fabricated because they would mislabel millions and over-grant refugee-class benefit exemptions." }, { "stage": "mortgage_conversion", diff --git a/packages/populace-build/src/populace/build/us_runtime/__init__.py b/packages/populace-build/src/populace/build/us_runtime/__init__.py index 8f01be0..c0cb385 100644 --- a/packages/populace-build/src/populace/build/us_runtime/__init__.py +++ b/packages/populace-build/src/populace/build/us_runtime/__init__.py @@ -363,6 +363,19 @@ def to_manifest(self) -> dict[str, object]: "realized ranges." ), ), + "capital_gain_distributions": DonorSpec( + survey="IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4", + source=( + "https://www.irs.gov/statistics/" + "soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns" + ), + notes=( + "Schedule D line 13 capital gain distributions split out of " + "long-term gains as a memo component at the national SOCA-derived " + "share; the direct-1040 route is already a PUF-stage output and " + "the two routes are mutually exclusive on a real return." + ), + ), "acs_rent": DonorSpec( survey="Census ACS 2022", source="https://www.census.gov/programs-surveys/acs", @@ -387,6 +400,7 @@ def to_manifest(self) -> dict[str, object]: US_IMMIGRATION_STAGE_NAME, US_PUF_SUPPORT_STAGE_NAME, "puf_tax_detail", + "capital_gain_distributions", "scf_wealth", "sipp_tips", "org_wages", diff --git a/packages/populace-build/src/populace/build/us_runtime/capital_gain_distributions.py b/packages/populace-build/src/populace/build/us_runtime/capital_gain_distributions.py new file mode 100644 index 0000000..054115a --- /dev/null +++ b/packages/populace-build/src/populace/build/us_runtime/capital_gain_distributions.py @@ -0,0 +1,214 @@ +"""US capital gain distributions: split the Schedule D route out of LTCG. + +The IRS reports capital gain distributions (RIC and REIT capital gain +dividends) through two routes: directly on Form 1040 line 7 for filers with +no other Schedule D activity (PUF ``E01100`` — already a separate PUF-stage +output, ``non_sch_d_capital_gains``) and on Schedule D line 13 for everyone +else, where they fold into net long-term gains and are no longer separable +in any microdata. The SOI Sales of Capital Assets study (latest: TY2015) +publishes the only all-route national total; no by-AGI CGD-specific series +exists anywhere in published SOI data. See +``populace/build/us/soca_capital_gain_distribution_shares.json`` for the +anchor arithmetic, the by-AGI tables that do exist, and the caveats. + +The v1 split is therefore deterministic and proportional: eligible records +(positive long-term gains and no direct-route distributions — the two +routes are mutually exclusive on a real return) receive the national +Schedule-D share of long-term net gains. Participation margins and AGI-band +tilt are documented follow-ups in the resource file. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from importlib.resources import files +from typing import Any + +import numpy as np +import pandas as pd + +from populace.build.source_runtime import ( + SourceOperationSpec, + SourceRuntimeContext, + SourceRuntimeError, +) + +__all__ = [ + "CapitalGainDistributionShares", + "load_capital_gain_distribution_shares", + "split_us_component_by_share_from_manifest", +] + +_SPLIT_COMPONENT_PARAMETER_KEYS = frozenset( + { + "resource", + "share_field", + "source_column", + "output", + "exclusive_with", + } +) + +#: The only share resource the v1 handler knows how to read. +_SHARES_RESOURCE_NAME = "soca_capital_gain_distribution_shares" + +#: Fields of the resource the manifest may select as the split share. +_ALLOWED_SHARE_FIELDS = frozenset({"schedule_d_cgd_share_of_lt_net_gains"}) + +#: Relative tolerance for re-deriving the packaged anchor arithmetic. +_ANCHOR_RTOL = 1e-9 + + +@dataclass(frozen=True) +class CapitalGainDistributionShares: + """Validated split parameters from the packaged SOCA-derived resource.""" + + schedule_d_cgd_share_of_lt_net_gains: float + anchor: Mapping[str, Any] + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> CapitalGainDistributionShares: + anchor = raw.get("national_anchor_ty2015") + if not isinstance(anchor, Mapping): + raise ValueError( + "capital gain distribution shares resource requires a " + "'national_anchor_ty2015' mapping." + ) + required = ( + "soca_all_route_capital_gain_distributions_k", + "soca_long_term_total_net_gain_k", + "pub1304_direct_1040_route_k", + "schedule_d_route_residual_k", + "schedule_d_cgd_share_of_lt_net_gains", + ) + missing = [key for key in required if key not in anchor] + if missing: + raise ValueError(f"capital gain distribution anchor is missing {missing}.") + all_route = float(anchor["soca_all_route_capital_gain_distributions_k"]) + lt_total = float(anchor["soca_long_term_total_net_gain_k"]) + direct = float(anchor["pub1304_direct_1040_route_k"]) + residual = float(anchor["schedule_d_route_residual_k"]) + share = float(anchor["schedule_d_cgd_share_of_lt_net_gains"]) + derived_residual = all_route - direct + derived_share = derived_residual / (lt_total - direct) + if not np.isclose(residual, derived_residual, rtol=_ANCHOR_RTOL): + raise ValueError( + "capital gain distribution anchor residual " + f"{residual!r} does not equal all-route minus direct " + f"({derived_residual!r}); the packaged numbers drifted." + ) + if not np.isclose(share, derived_share, rtol=1e-6): + raise ValueError( + f"capital gain distribution anchor share {share!r} does not " + f"re-derive from its own inputs ({derived_share!r})." + ) + if not 0.0 < share < 1.0: + raise ValueError( + f"schedule_d_cgd_share_of_lt_net_gains must be in (0, 1), " + f"got {share!r}." + ) + return cls( + schedule_d_cgd_share_of_lt_net_gains=share, + anchor=dict(anchor), + ) + + +def load_capital_gain_distribution_shares() -> CapitalGainDistributionShares: + """Load and validate the packaged SOCA-derived share resource.""" + + path = files("populace.build.us") / f"{_SHARES_RESOURCE_NAME}.json" + return CapitalGainDistributionShares.from_dict( + json.loads(path.read_text(encoding="utf-8")) + ) + + +def split_us_component_by_share_from_manifest( + frame: pd.DataFrame | None, + operation: SourceOperationSpec, + context: SourceRuntimeContext, +) -> pd.DataFrame: + """Split a memo component out of a source column at a declared share. + + Eligibility is structural: the source column must be positive and every + ``exclusive_with`` column must be non-positive (the reporting routes are + mutually exclusive on a real return). Ineligible records get zero. The + split is deterministic — the declared share times the source column — + so the stage needs no seed. + """ + + if operation.kind != "split_component_by_share": + raise SourceRuntimeError( + f"component split received unexpected operation {operation.kind!r}." + ) + if frame is None: + raise SourceRuntimeError( + "split_component_by_share must follow read_table; there is no " + "frame to split." + ) + params = operation.parameters + unexpected = sorted(set(params) - _SPLIT_COMPONENT_PARAMETER_KEYS) + if unexpected: + raise SourceRuntimeError( + f"component split got unexpected parameter(s) {unexpected}; " + f"allowed keys are {sorted(_SPLIT_COMPONENT_PARAMETER_KEYS)}." + ) + resource = params.get("resource") + if resource != _SHARES_RESOURCE_NAME: + raise SourceRuntimeError( + f"component split knows only the {_SHARES_RESOURCE_NAME!r} " + f"resource, got {resource!r}." + ) + share_field = params.get("share_field") + if share_field not in _ALLOWED_SHARE_FIELDS: + raise SourceRuntimeError( + f"component split share_field must be one of " + f"{sorted(_ALLOWED_SHARE_FIELDS)}, got {share_field!r}." + ) + source_column = params.get("source_column") + output = params.get("output") + exclusive_with = params.get("exclusive_with", []) + if not isinstance(source_column, str) or not source_column: + raise SourceRuntimeError( + "component split requires a non-empty string 'source_column'." + ) + if not isinstance(output, str) or not output: + raise SourceRuntimeError( + "component split requires a non-empty string 'output'." + ) + if not isinstance(exclusive_with, (list, tuple)) or not all( + isinstance(name, str) and name for name in exclusive_with + ): + raise SourceRuntimeError( + "component split 'exclusive_with' must be a list of column names." + ) + if source_column not in frame.columns: + raise SourceRuntimeError( + f"component split source column {source_column!r} is not in the " + "frame; the stage must run after the stage that produces it." + ) + if output in frame.columns: + raise SourceRuntimeError( + f"component split output {output!r} already exists in the frame; " + "refusing to overwrite." + ) + missing_exclusive = [name for name in exclusive_with if name not in frame.columns] + if missing_exclusive: + raise SourceRuntimeError( + f"component split exclusive_with column(s) {missing_exclusive} " + "are not in the frame." + ) + + shares = load_capital_gain_distribution_shares() + share = getattr(shares, share_field) + + values = pd.to_numeric(frame[source_column], errors="raise").astype(float) + eligible = values > 0 + for name in exclusive_with: + other = pd.to_numeric(frame[name], errors="raise").astype(float) + eligible &= ~(other > 0) + + result = frame.copy() + result[output] = np.where(eligible, values * share, 0.0) + return result diff --git a/packages/populace-build/src/populace/build/us_runtime/source_runtime.py b/packages/populace-build/src/populace/build/us_runtime/source_runtime.py index 9848c05..94bb161 100644 --- a/packages/populace-build/src/populace/build/us_runtime/source_runtime.py +++ b/packages/populace-build/src/populace/build/us_runtime/source_runtime.py @@ -14,6 +14,9 @@ SourceRuntimeContext, SourceRuntimeError, ) +from populace.build.us_runtime.capital_gain_distributions import ( + split_us_component_by_share_from_manifest, +) from populace.build.us_runtime.immigration import ( derive_us_immigration_status_from_manifest, ) @@ -183,6 +186,7 @@ def us_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: "disaggregate_aggregate_records": ( disaggregate_us_puf_aggregate_records_from_manifest ), + "split_component_by_share": split_us_component_by_share_from_manifest, "support_clip": support_clip_us_source_output_from_manifest, } diff --git a/packages/populace-build/tests/test_us_capital_gain_distributions.py b/packages/populace-build/tests/test_us_capital_gain_distributions.py new file mode 100644 index 0000000..23801e4 --- /dev/null +++ b/packages/populace-build/tests/test_us_capital_gain_distributions.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from populace.build.source_manifest import SourceOperationSpec +from populace.build.source_runtime import ( + SourceRuntimeConfig, + SourceRuntimeContext, + SourceRuntimeError, + run_source_stage, +) +from populace.build.us_runtime import US_SOURCE_MANIFEST +from populace.build.us_runtime.capital_gain_distributions import ( + load_capital_gain_distribution_shares, + split_us_component_by_share_from_manifest, +) +from populace.build.us_runtime.source_runtime import us_source_operation_handlers + + +def _toy_tax_units() -> pd.DataFrame: + return pd.DataFrame( + { + "tax_unit_id": [1, 2, 3, 4, 5], + "tax_unit_weight": [100.0, 100.0, 100.0, 100.0, 100.0], + "long_term_capital_gains_before_response": [ + 10_000.0, + 0.0, + -4_000.0, + 50_000.0, + 8_000.0, + ], + "non_sch_d_capital_gains": [0.0, 0.0, 0.0, 0.0, 3_000.0], + } + ) + + +def _split_operation(**overrides) -> SourceOperationSpec: + raw = { + "kind": "split_component_by_share", + "resource": "soca_capital_gain_distribution_shares", + "share_field": "schedule_d_cgd_share_of_lt_net_gains", + "source_column": "long_term_capital_gains_before_response", + "output": "schedule_d_capital_gain_distributions", + "exclusive_with": ["non_sch_d_capital_gains"], + } + raw.update(overrides) + return SourceOperationSpec.from_mapping(raw) + + +def _context() -> SourceRuntimeContext: + return SourceRuntimeContext(config=SourceRuntimeConfig(), tables={}) + + +class TestSharesResource: + def test_packaged_resource_rederives_its_own_anchor(self) -> None: + shares = load_capital_gain_distribution_shares() + share = shares.schedule_d_cgd_share_of_lt_net_gains + anchor = shares.anchor + residual = ( + anchor["soca_all_route_capital_gain_distributions_k"] + - anchor["pub1304_direct_1040_route_k"] + ) + assert np.isclose(residual, anchor["schedule_d_route_residual_k"]) + assert np.isclose( + share, + residual + / ( + anchor["soca_long_term_total_net_gain_k"] + - anchor["pub1304_direct_1040_route_k"] + ), + rtol=1e-6, + ) + # TY2015: $68.1B Schedule-D route over $690.7B of Schedule-D LT + # net gains — the split is a bit under ten percent. + assert 0.09 < share < 0.11 + + +class TestCapitalGainDistributionsStage: + def test_manifest_stage_splits_the_schedule_d_route(self) -> None: + stage = US_SOURCE_MANIFEST.stage_map()["capital_gain_distributions"] + toy = _toy_tax_units() + + result = run_source_stage( + stage, + tables={"tax_unit": toy}, + operation_handlers=us_source_operation_handlers(), + config=SourceRuntimeConfig(seed=0, target_year=2024), + ) + + share = ( + load_capital_gain_distribution_shares() + .schedule_d_cgd_share_of_lt_net_gains + ) + out = result["schedule_d_capital_gain_distributions"] + assert np.isclose(out.iloc[0], 10_000.0 * share) + # Zero and negative long-term gains get nothing. + assert out.iloc[1] == 0.0 + assert out.iloc[2] == 0.0 + assert np.isclose(out.iloc[3], 50_000.0 * share) + # The two reporting routes are mutually exclusive on a real return. + assert out.iloc[4] == 0.0 + # The source column is untouched and the split never exceeds it. + pd.testing.assert_series_equal( + result["long_term_capital_gains_before_response"], + toy["long_term_capital_gains_before_response"], + ) + assert ( + out + <= result["long_term_capital_gains_before_response"].clip(lower=0.0) + + 1e-9 + ).all() + # National reconstruction: the split is exactly proportional. + eligible_lt = toy.loc[ + (toy["long_term_capital_gains_before_response"] > 0) + & ~(toy["non_sch_d_capital_gains"] > 0), + "long_term_capital_gains_before_response", + ].sum() + assert np.isclose(out.sum(), share * eligible_lt) + + def test_split_requires_a_frame(self) -> None: + with pytest.raises(SourceRuntimeError, match="must follow read_table"): + split_us_component_by_share_from_manifest( + None, _split_operation(), _context() + ) + + def test_split_rejects_unknown_parameters(self) -> None: + with pytest.raises(SourceRuntimeError, match="unexpected parameter"): + split_us_component_by_share_from_manifest( + _toy_tax_units(), _split_operation(bogus=1), _context() + ) + + def test_split_rejects_unknown_resource_and_share_field(self) -> None: + with pytest.raises(SourceRuntimeError, match="knows only"): + split_us_component_by_share_from_manifest( + _toy_tax_units(), + _split_operation(resource="somewhere_else"), + _context(), + ) + with pytest.raises(SourceRuntimeError, match="share_field"): + split_us_component_by_share_from_manifest( + _toy_tax_units(), + _split_operation(share_field="not_a_field"), + _context(), + ) + + def test_split_refuses_to_overwrite_an_existing_output(self) -> None: + frame = _toy_tax_units() + frame["schedule_d_capital_gain_distributions"] = 1.0 + with pytest.raises(SourceRuntimeError, match="already exists"): + split_us_component_by_share_from_manifest( + frame, _split_operation(), _context() + ) + + def test_split_requires_source_and_exclusive_columns(self) -> None: + missing_source = _toy_tax_units().drop( + columns=["long_term_capital_gains_before_response"] + ) + with pytest.raises(SourceRuntimeError, match="source column"): + split_us_component_by_share_from_manifest( + missing_source, _split_operation(), _context() + ) + missing_exclusive = _toy_tax_units().drop( + columns=["non_sch_d_capital_gains"] + ) + with pytest.raises(SourceRuntimeError, match="exclusive_with"): + split_us_component_by_share_from_manifest( + missing_exclusive, _split_operation(), _context() + ) + + def test_split_handler_is_registered(self) -> None: + handlers = us_source_operation_handlers() + assert ( + handlers["split_component_by_share"] + is split_us_component_by_share_from_manifest + )