diff --git a/.gitignore b/.gitignore index 70d8396d3..b0b3a6ab6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ !policyengine_uk_data/datasets/local_areas/**/*.csv **/_build !policyengine_uk_data/storage/*.csv +!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv **/version.json # Build output: household weights by local area, derived from FRS microdata policyengine_uk_data/storage/local_geography_weights.csv.gz @@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local build-job notes, caches and hand-off artefacts +.job/ diff --git a/changelog.d/593-age-topcode.fixed.md b/changelog.d/593-age-topcode.fixed.md new file mode 100644 index 000000000..49bfb27ce --- /dev/null +++ b/changelog.d/593-age-topcode.fixed.md @@ -0,0 +1 @@ +Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups. diff --git a/changelog.d/spi-tax-year-start.fixed.md b/changelog.d/spi-tax-year-start.fixed.md new file mode 100644 index 000000000..a2013007d --- /dev/null +++ b/changelog.d/spi-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571. diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md new file mode 100644 index 000000000..bcda429e8 --- /dev/null +++ b/changelog.d/target-coverage-gate.changed.md @@ -0,0 +1 @@ +Fail the calibration build when a national target is dropped (its name already taken, no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue, for every build or only for testing or release builds. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset. diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index aa9bda2d5..ad0e914ca 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -1,85 +1,99 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year -12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024 -15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024 -20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024 -30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024 -40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024 -50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024 -70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024 -100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024 -150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024 -200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024 -300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024 -500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 -1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 -12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17795688094,449188,4843188790,1153320,12201198978,1082502,4506163433,102180,699345737,1264606,286001092,116344,182664931,2025 -15000,20000.0,3136222,51175765762,658607,8085406650,2073953,22954798078,2165005,14088235102,234711,1764021335,2873184,1430005461,376347,892448664,2025 -20000,30000.0,7557283,181115878830,972229,15808329316,2347108,25746597844,2691081,31284169123,436036,3987314498,5311344,3267092769,852850,3475852690,2025 -30000,40000.0,5432746,179009880239,601952,12686939451,1173554,12718198935,1466943,25793901032,368253,3966438506,3733116,2640813005,674794,4905858152,2025 -40000,50000.0,3540896,146366902077,363195,8790236620,574637,6224679479,788102,17506703913,328797,3945562513,2397692,1951905265,634326,8767916697,2025 -50000,70000.0,3470078,183221877421,314634,8276717642,446153,4818439597,650513,18024653733,401639,5636517876,2256056,2442491080,650513,13047495084,2025 -70000,100000.0,1659163,124253916871,144671,5396983767,178056,1954259836,262026,9809969589,228641,3935124517,1082502,1690955363,339926,9853468288,2025 -100000,150000.0,800242,86661842021,85993,5014361783,74865,875797927,110274,5624935044,138601,2661688997,494714,1315187504,224594,7170903298,2025 -150000,200000.0,267085,41277572384,40467,3675184841,21245,274009977,29339,1874978348,54631,1200369548,147706,678469744,93075,3601108643,2025 -200000,300000.0,182103,38539774216,34397,4420290809,14164,185085985,20234,1626362435,41479,1095989587,112297,765105112,75876,3862058545,2025 -300000,500000.0,94087,30536979570,21245,4873395789,8093,99263992,10117,1160207597,23269,798506699,68795,702477135,49573,3580232651,2025 -500000,1000000.0,44514,24324283727,15175,6716022709,3035,44461996,4047,662975769,13152,494761014,38444,681601143,32374,3987314498,2025 -1000000,inf,21245,37381474991,11129,19634549151,2023,20679998,2023,383282867,7082,458228027,21245,1294311512,19222,11586175635,2025 -12570,inf,27581555,1141661836204,3712882,108221607319,8070207,88117472623,9282205,132346537984,2378470,30643868854,19801701,19146416186,4139813,74913497775,2025 -12570,15000.0,1385161,18400750621,452214,4991391904,1161091,12481831123,1089796,4690916055,102869,719766739,1273126,294352368,117128,187998775,2026 -15000,20000.0,3157352,52915768057,663044,8332822659,2087926,23482767029,2179591,14665852495,236292,1815531028,2892541,1471761839,378882,918508301,2026 -20000,30000.0,7608199,187273911644,978779,16292069209,2362921,26338779235,2709211,32566819510,438974,4103744689,5347128,3362492376,858596,3577348119,2026 -30000,40000.0,5469348,185096308020,606008,13075163824,1181461,13010722273,1476826,26851450524,370734,4082259115,3758267,2717925148,679340,5049109958,2026 -40000,50000.0,3564752,151343451852,365642,9059220649,578508,6367849438,793412,18224478468,331013,4060773541,2413846,2008901196,638600,9023941202,2026 -50000,70000.0,3493457,189451515267,316754,8529987828,449159,4929265511,654896,18763664221,404345,5801105058,2271256,2513812192,654896,13428483931,2026 -70000,100000.0,1670341,128478613802,145646,5562133182,179256,1999208544,263792,10212178171,230181,4050030754,1089796,1740331518,342216,10141191065,2026 -100000,150000.0,805634,89608389118,86573,5167802845,75369,895941607,111017,5855557283,139535,2739410722,498047,1353591180,226107,7380294769,2026 -150000,200000.0,268884,42681031026,40740,3787646663,21389,280312309,29537,1951852428,54999,1235420522,148701,698281164,93702,3706261565,2026 -200000,300000.0,183330,39850146315,34629,4555553110,14259,189343031,20370,1693043266,41759,1127992650,113054,787446298,76388,3974831244,2026 -300000,500000.0,94721,31575252545,21389,5022523247,8148,101547101,10185,1207776088,23426,821823217,69258,722989575,49907,3684775991,2026 -500000,1000000.0,44814,25151321854,15278,6921535135,3056,45484639,4074,690157764,13241,509208111,38703,701504001,32592,4103744689,2026 -1000000,inf,21389,38652464322,11204,20235372584,2037,21155646,2037,398997458,7130,471608356,21389,1332105606,19352,11924493731,2026 -12570,inf,27767379,1180478924442,3737897,111533222839,8124578,90144207486,9344742,137772743731,2394495,31538674501,19935110,19705494460,4167704,77100983340,2026 -12570,15000.0,1395755,18842365727,455673,5176573740,1169971,12731467935,1098130,4836334659,103655,743015304,1282863,303859990,118023,194071161,2027 -15000,20000.0,3181500,54185738125,668115,8641972377,2103895,23952422725,2196261,15120494566,238099,1874172930,2914664,1519299949,381780,948176246,2027 -20000,30000.0,7666388,191768455916,986265,16896508881,2380993,26865555219,2729932,33576392346,442331,4236296209,5388023,3471101344,865163,3692896957,2027 -30000,40000.0,5511178,189538590149,610643,13560255535,1190497,13270936915,1488121,27683846669,373570,4214116647,3787011,2805714505,684536,5212196906,2027 -40000,50000.0,3592016,154975670769,368438,9395319907,582933,6495206523,799480,18789438101,333544,4191937086,2432308,2073788982,643484,9315415747,2027 -50000,70000.0,3520175,193998321682,319176,8846452421,452594,5027850896,659905,19345338636,407437,5988481552,2288627,2595008672,659905,13862225814,2027 -70000,100000.0,1683116,131562080221,146759,5768489656,180627,2039192746,265809,10528756143,231942,4180847305,1098130,1796544465,344833,10468752935,2027 -100000,150000.0,811796,91758976290,87235,5359529569,75945,913860453,111866,6037079816,140602,2827894066,501856,1397312362,227836,7618679307,2027 -150000,200000.0,270941,43705369023,41052,3928169262,21552,285918560,29762,2012359939,55420,1275324775,149838,720835742,94419,3825974325,2027 -200000,300000.0,184732,40806543526,34894,4724565222,14368,193129895,20526,1745527682,42078,1164426968,113918,812880922,76972,4103218841,2027 -300000,500000.0,95445,32333053614,21552,5208860063,8210,103578044,10263,1245217200,23605,848368220,69788,746342238,50288,3803794763,2027 -500000,1000000.0,45157,25754949603,15394,7178325748,3079,46394332,4105,711552685,13342,525655603,38999,724162677,32841,4236296209,2027 -1000000,inf,21552,39580117355,11289,20986109757,2053,21578759,2053,411366396,7184,486841371,21552,1375132801,19500,12309656523,2027 -12570,inf,27979749,1208810232000,3766485,115671132138,8186717,91947093002,9416212,142043704838,2412808,32557378036,20087577,20341984648,4199579,79591355733,2027 -12570,15000.0,1401052,19238055016,457402,5364999173,1174411,12986098807,1102298,4981427321,104049,766048722,1287731,313279627,118471,200087353,2028 -15000,20000.0,3193574,55323637503,670650,8956537080,2111879,24431474027,2204596,15574117602,239003,1932272150,2925725,1566398134,383229,977569638,2028 -20000,30000.0,7695482,195795589515,990008,17511535761,2390029,27402869517,2740292,34583702323,444010,4367621074,5408471,3578705225,868446,3807376486,2028 -30000,40000.0,5532093,193518896614,612960,14053843987,1195015,13536357231,1493768,28514377081,374987,4344753948,3801383,2892691444,687133,5373774619,2028 -40000,50000.0,3605648,158230156643,369836,9737306191,585145,6625111425,802514,19353131432,334810,4321886822,2441538,2138076285,645926,9604192937,2028 -50000,70000.0,3533535,198072282416,320388,9168460125,454312,5128408512,662409,19925709286,408983,6174124031,2297313,2675453747,662409,14291953775,2028 -70000,100000.0,1689503,134324881179,147316,5978460617,181313,2079976843,266818,10844624536,232822,4310453259,1102298,1852237209,346142,10793283491,2028 -100000,150000.0,814876,93685912890,87566,5554614528,76234,932137770,112290,6218195484,141135,2915558570,503760,1440628941,228701,7854857795,2028 -150000,200000.0,271969,44623180866,41207,4071153218,21634,291636965,29875,2072731828,55630,1314859747,150407,743181596,94777,3944579242,2028 -200000,300000.0,185433,41663480094,35026,4896537706,14423,196992516,20604,1797894459,42238,1200524117,114351,838080169,77264,4230418317,2028 -300000,500000.0,95807,33012047069,21634,5398460706,8241,105649617,10302,1282574391,23694,874667571,70053,769478791,50479,3921712116,2028 -500000,1000000.0,45328,26295803010,15453,7439614237,3091,47322224,4121,732899652,13392,541950887,39147,746611665,32966,4367621074,2028 -1000000,inf,21634,40411298999,11332,21749996646,2060,22010337,2060,423707611,7211,501933417,21634,1417761814,19574,12691254952,2028 -12570,inf,28085934,1234195221815,3780779,119881519976,8217786,93786045792,9451947,146305093007,2421965,33566654314,20163811,20972584647,4215517,82058681794,2028 -12570,15000.0,1408997,19661286328,459996,5554383918,1181071,13245818324,1108549,5120904049,104639,788723905,1295034,322552761,119143,206009975,2029 -15000,20000.0,3211685,56540740564,674454,9272703298,2123856,24920098881,2217098,16010182775,240358,1989467760,2942317,1612763805,385402,1006505878,2029 -20000,30000.0,7739124,200103032447,995622,18129693871,2403583,27950921718,2755833,35552023516,446528,4496903457,5439143,3684635555,873371,3920075527,2029 -30000,40000.0,5563466,197776253000,616436,14549945400,1201792,13807081812,1502240,29312761111,377114,4473359460,3822941,2978315640,691030,5532839332,2029 -40000,50000.0,3626095,161711171571,371934,10081033598,588463,6757612399,807065,19895006537,336709,4449815463,2455385,2201363734,649589,9888478806,2029 -50000,70000.0,3553574,202429811895,322204,9492107237,456888,5230975711,666166,20483616198,411303,6356879232,2310341,2754647667,666166,14714998223,2029 -70000,100000.0,1699085,137279987377,148152,6189500583,182341,2121575986,268331,11148266977,234142,4438043464,1108549,1907063770,348105,11112766658,2029 -100000,150000.0,819498,95746974247,88062,5750692706,76666,950780349,112927,6392300917,141936,3001859637,506617,1483271821,229998,8087363023,2029 -150000,200000.0,273511,45604877162,41441,4214865136,21757,297469649,30045,2130766972,55945,1353779836,151260,765179908,95315,4061339509,2029 -200000,300000.0,186485,42580063881,35225,5069385738,14504,200932329,20721,1848234335,42477,1236059851,114999,862887496,77702,4355639474,2029 -300000,500000.0,96351,33738301982,21757,5589026646,8288,107762590,10360,1318485640,23829,900557891,70450,792255504,50765,4037795512,2029 -500000,1000000.0,45585,26874302614,15540,7702233001,3108,48268660,4144,753420366,13468,557992733,39369,768711507,33153,4496903457,2029 -1000000,inf,21757,41300335185,11396,22517772642,2072,22450540,2072,435571149,7252,516790738,21757,1459727824,19685,13066918422,2029 -12570,inf,28245211,1261347138253,3802220,124113343773,8264389,95661748947,9505550,150401540542,2435700,34560233426,20278162,21593376992,4239424,84487633796,2029 +12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023 +15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023 +20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023 +30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023 +40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023 +50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023 +70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023 +100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023 +150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023 +200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023 +300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023 +500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023 +1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023 +12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023 +12570,15000.0,1373373,17761903015,448366,4941310379,1151209,12094996607,1080521,4567498426,101993,694923847,1262291,284192738,116131,181509960,2024 +15000,20000.0,3130482,51078608670,657401,8249214625,2070157,22754993616,2161042,14279995081,234281,1752867615,2867925,1420963688,375658,886805805,2024 +20000,30000.0,7543451,180772030685,970449,16128601446,2342812,25522492840,2686155,31709989076,435238,3962103130,5301622,3246435287,851289,3453875241,2024 +30000,40000.0,5422802,178670030328,600851,12943973135,1171406,12607496463,1464258,26144990993,367579,3941359134,3726283,2624115424,673559,4874838929,2024 +40000,50000.0,3534415,146089024798,362530,8968324244,573585,6170498269,786660,17744993887,328196,3920615139,2393304,1939563574,633165,8712478086,2024 +50000,70000.0,3463727,182874031042,314058,8444401521,445336,4776498660,649323,18269993706,400904,5600878770,2251927,2427047467,649323,12964997152,2024 +70000,100000.0,1656126,124018021051,144406,5506325080,177731,1937249457,261547,9943496574,228222,3910243141,1080521,1680263631,339304,9791165849,2024 +100000,150000.0,798778,86497314682,85836,5115951287,74728,868174756,110072,5701498036,138347,2644859419,493808,1306871713,224183,7125562435,2024 +150000,200000.0,266596,41199206993,40393,3749643011,21206,271624924,29285,1900499345,54531,1192779738,147436,674179852,92905,3578339214,2024 +200000,300000.0,181770,38466606529,34334,4509844608,14138,183474949,20197,1648499432,41403,1089059761,112091,760267433,75737,3837639157,2024 +300000,500000.0,93914,30479005174,21206,4972129363,8079,98399972,10098,1175999595,23226,793457826,68669,698035447,49482,3557595219,2024 +500000,1000000.0,44433,24278104121,15147,6852087366,3029,44074988,4039,671999768,13128,491632692,38374,677291451,32315,3962103130,2024 +1000000,inf,21206,37310506333,11108,20032339376,2020,20499994,2020,388499866,7069,455330700,21206,1286127717,19187,11512917471,2024 +12570,inf,27531073,1139494393423,3706087,110414145442,8055437,87350475494,9265216,134147953786,2374117,30450110911,19765458,19025355421,4132236,74439827648,2024 +12570,15000.0,1389420,18703271362,453605,4975405206,1164661,12506225444,1093147,4731469975,103185,725361239,1277040,296640268,117488,189460025,2025 +15000,20000.0,3167060,53785738946,665083,8306133847,2094346,23528661429,2186293,14792641761,237019,1829642527,2901436,1483201339,380047,925647551,2025 +20000,30000.0,7631593,190352820961,981789,16239888094,2370187,26390255387,2717542,32848366262,440324,4135641689,5363569,3388627876,861236,3605153619,2025 +30000,40000.0,5486165,188139416066,607871,13033285986,1185093,13036150251,1481367,27083586753,371874,4113989114,3769823,2739050647,681429,5088354957,2025 +40000,50000.0,3575713,153831640195,366766,9030205290,580287,6380294676,795852,18382032776,332030,4092336540,2421268,2024515696,640563,9094081200,2025 +50000,70000.0,3504199,192566225856,317728,8502667524,450540,4938899201,656910,18925879900,405588,5846195057,2278240,2533351191,656910,13532858929,2025 +70000,100000.0,1675477,130590888799,146093,5544318483,179807,2003115770,264603,10300464520,230889,4081510253,1093147,1753858517,343268,10220015063,2025 +100000,150000.0,808111,91081611425,86839,5151251128,75601,897692623,111358,5906179762,139964,2760703221,499578,1364112180,226802,7437659267,2025 +150000,200000.0,269711,43382735940,40865,3775515385,21454,280860148,29627,1968726587,55168,1245023021,149158,703708664,93990,3735069064,2025 +200000,300000.0,183894,40505309577,34735,4540962340,14303,189713081,20433,1707679968,41887,1136760150,113401,793566848,76622,4005726243,2025 +300000,500000.0,95012,32094370976,21454,5006436839,8173,101745563,10216,1218217557,23498,828210966,69471,728609125,50060,3713416490,2025 +500000,1000000.0,44952,25564826536,15324,6899366470,3065,45573533,4087,696124318,13281,513166011,38822,706956550,32692,4135641689,2025 +1000000,inf,21454,39287936884,11238,20170561645,2043,21196992,2043,402446871,7151,475274006,21454,1342459606,19411,12017178729,2025 +12570,inf,27852760,1199886793524,3749390,111175998236,8149560,90320384099,9373476,138963817010,2401857,31783813794,19996408,19858658506,4180519,77700262824,2025 +12570,15000.0,1398781,19339192185,456661,5127654184,1172507,12793873312,1100511,4925460161,103880,746541897,1285644,305302209,118279,194992287,2026 +15000,20000.0,3188397,55614481669,669563,8560304178,2108456,24069829452,2201023,15399139815,238616,1883068368,2920983,1526511044,382608,952676600,2026 +20000,30000.0,7683009,196824914547,988403,16736833822,2386155,26997241142,2735851,34195148706,443290,4256403057,5399705,3487576327,867038,3710424654,2026 +30000,40000.0,5523127,194536252750,611967,13432108672,1193078,13335986588,1491347,28194013337,374380,4234118224,3795221,2819031344,686020,5236935698,2026 +40000,50000.0,3599803,159061994896,369237,9306532437,584197,6527043842,801213,19135695799,334267,4211833391,2437581,2083631863,644879,9359629759,2026 +50000,70000.0,3527807,199113576344,319868,8762851848,453575,5052495732,661335,19701840645,408321,6016904845,2293589,2607325433,661335,13928020474,2026 +70000,100000.0,1686765,135031046027,147078,5713976388,181019,2049188183,266385,10722783386,232444,4200690975,1100511,1805071453,345581,10518441062,2026 +100000,150000.0,813556,94178432949,87424,5308881047,76110,918339889,112108,6148333029,140907,2841316177,502944,1403944464,228330,7654840053,2026 +150000,200000.0,271528,44857771222,41141,3891047353,21599,287320036,29827,2049444343,55540,1281377884,150163,724257065,94623,3844133651,2026 +200000,300000.0,185133,41882510886,34970,4679917228,14399,194076553,20570,1777694817,42169,1169953720,114165,816739121,77139,4122694060,2026 +300000,500000.0,95652,33185596057,21599,5159635395,8228,104085749,10285,1268164455,23656,852394853,69939,749884622,50397,3821848818,2026 +500000,1000000.0,45255,26434043756,15428,7110489273,3086,46621742,4114,724665403,13371,528150536,39084,727599790,32912,4256403057,2026 +1000000,inf,21599,40623746898,11314,20787787231,2057,21684531,2057,418947186,7200,489152079,21599,1381659631,19542,12368082181,2026 +12570,inf,28040411,1240683560188,3774651,114578019055,8204466,92397786751,9436627,144661331081,2418039,32711906006,20131129,20438534364,4208684,79969122354,2026 +12570,15000.0,1409479,19803329740,460153,5317891383,1181475,13049750972,1108928,5078149643,104675,770655304,1295477,315163512,119184,201290564,2027 +15000,20000.0,3212783,56949220436,674684,8877893514,2124582,24551226406,2217856,15876513825,240441,1943891736,2943324,1575817561,385534,983448186,2027 +20000,30000.0,7741770,201548681379,995963,17357774368,2404405,27537186374,2756775,35255199818,446680,4393885462,5441003,3600225523,873670,3830271882,2027 +30000,40000.0,5565369,199205092061,616647,13930443123,1202203,13602706522,1502753,29068028989,377243,4370880827,3824248,2910086445,691266,5406089443,2027 +40000,50000.0,3627335,162879457626,372061,9651807021,588665,6657584818,807341,19728903209,336824,4347876191,2456224,2150933459,649811,9661947090,2027 +50000,70000.0,3554789,203892270698,322315,9087955752,457044,5153545723,666393,20312598570,411443,6211251701,2311131,2691542404,666393,14377897456,2027 +70000,100000.0,1699666,138271769783,148203,5925966281,182403,2090171978,268423,11055190142,234222,4336373873,1108928,1863375510,348224,10858188159,2027 +100000,150000.0,819778,96438700451,88092,5505841806,76692,936706701,112966,6338931623,141984,2933091081,506791,1449292064,230077,7902092442,2027 +150000,200000.0,273605,45934350640,41455,4035406143,21764,293066441,30055,2112977208,55965,1322766566,151312,747650668,95347,3968299698,2027 +200000,300000.0,186549,42887684526,35237,4853543279,14509,197958087,20728,1832803434,42492,1207743386,115038,843119907,77729,4255857647,2027 +300000,500000.0,96383,33982045116,21764,5351059105,8291,106167466,10364,1307477609,23837,879927324,70474,774105999,50783,3945295062,2027 +500000,1000000.0,45601,27068456627,15546,7374290130,3109,47554177,4146,747130062,13473,545209872,39382,751101363,33164,4393885462,2027 +1000000,inf,21764,41598710401,11400,21559019120,2073,22118222,2073,431934567,7255,504951759,21764,1426287428,19691,12767572941,2027 +12570,inf,28254869,1270459769485,3803520,118828891025,8267215,94245743887,9508800,149145838699,2436533,33768505081,20285095,21098701843,4240873,82552136032,2027 +12570,15000.0,1414828,20219199254,461900,5511460727,1185959,13310747543,1113137,5230496886,105072,794545560,1300393,324933557,119636,207530557,2028 +15000,20000.0,3224975,58145152885,677245,9201045663,2132645,25042253852,2226273,16352817849,241353,2004152234,2954494,1624667787,386997,1013935006,2028 +20000,30000.0,7771150,205781199511,999742,17989591147,2413530,28087933375,2767237,36312874929,448376,4530095582,5461652,3711832244,876985,3949010023,2028 +30000,40000.0,5586490,203388394865,618987,14437506271,1206765,13874762269,1508456,29940085621,378675,4506377804,3838761,3000298907,693890,5573677811,2028 +40000,50000.0,3641101,166299922860,373473,10003129345,590899,6790737306,810405,20320781004,338102,4482660027,2465546,2217612235,652277,9961466726,2028 +50000,70000.0,3568279,208174004156,323538,9418754091,458779,5256617250,668922,20921987542,413005,6403800038,2319902,2774980016,668922,14823611199,2028 +70000,100000.0,1706116,141175474083,148765,6141669334,183095,2131975666,269441,11386851840,235111,4470801138,1113137,1921140011,349546,11194791177,2028 +100000,150000.0,822889,98463911161,88427,5706252478,76983,955440947,113394,6529103009,142523,3024016685,508714,1494220009,230950,8147056715,2028 +150000,200000.0,274643,46898971051,41613,4182293483,21847,298927805,30169,2176367670,56177,1363772230,151886,770827782,95709,4091316691,2028 +200000,300000.0,187257,43788325012,35371,5030210518,14564,201917272,20806,1887788531,42653,1245183341,115475,869256561,78024,4387788915,2028 +300000,500000.0,96749,34695667359,21847,5545835742,8323,108290827,10403,1346702646,23927,907205005,70741,798103227,50975,4067598913,2028 +500000,1000000.0,45774,27636893655,15605,7642711653,3121,48505266,4161,769544369,13524,562111337,39532,774385449,33290,4530095582,2028 +1000000,inf,21847,42472282457,11443,22343759705,2081,22560589,2081,444892839,7282,520605225,21847,1470502231,19766,13163366745,2028 +12570,inf,28362098,1297139398310,3817955,123154220156,8298590,96130669967,9544887,153620294735,2445780,34815326206,20362079,21752760018,4256968,85111246060,2028 +12570,15000.0,1422852,20664015438,464519,5706015573,1192684,13576959973,1119449,5376947400,105668,818064254,1307768,334551650,120315,213673499,2029 +15000,20000.0,3243265,59424328419,681086,9525843046,2144739,25543094187,2238899,16810686123,242722,2063475507,2971249,1672758251,389192,1043947668,2029 +20000,30000.0,7815221,210308322801,1005412,18624624636,2427217,28649686723,2782930,37329611831,450918,4664187241,5492625,3821703158,881959,4065901443,2029 +30000,40000.0,5618171,207862877187,622498,14947150982,1213609,14152254887,1517011,30778388563,380822,4639767412,3860531,3089108304,697825,5738659694,2029 +40000,50000.0,3661750,169958470171,375591,10356240323,594250,6926550766,815001,20889749667,340020,4615347584,2479528,2283253963,655976,10256327964,2029 +50000,70000.0,3588515,212753768415,325373,9751236593,461381,5361748600,672716,21507789598,415347,6593353691,2333058,2857119933,672716,15262392804,2029 +70000,100000.0,1715792,144281291224,149609,6358470576,184134,2174614775,270970,11705676293,236444,4603137670,1119449,1978006107,351528,11526159046,2029 +100000,150000.0,827556,100630087015,88928,5907683483,77420,974549585,114037,6711913650,143331,3113528132,511599,1538449195,232260,8388211085,2029 +150000,200000.0,276201,47930734034,41849,4329928657,21971,304906304,30340,2237304550,56496,1404140138,152747,793644426,96252,4212420414,2029 +200000,300000.0,188319,44751654736,35571,5207777207,14647,205955579,20924,1940645383,42895,1282040996,116130,894986714,78466,4517668270,2029 +300000,500000.0,97298,35458961402,21971,5741604028,8370,110456624,10462,1384409445,24063,934058440,71143,821727229,51265,4188000585,2029 +500000,1000000.0,46033,28244896841,15693,7912499766,3139,49475363,4185,791091112,13601,578749935,39756,797307400,33479,4664187241,2029 +1000000,inf,21971,43406659648,11508,23132495567,2092,23011797,2092,457349549,7324,536015235,21971,1514029366,19878,13553004810,2029 +12570,inf,28522942,1325676067330,3839607,127501570438,8345652,98053265162,9599017,157921563165,2459650,35845866235,20477554,22396645696,4281109,87630554524,2029 diff --git a/policyengine_uk_data/storage/regional_age_80_plus.csv b/policyengine_uk_data/storage/regional_age_80_plus.csv new file mode 100644 index 000000000..4afa81cb2 --- /dev/null +++ b/policyengine_uk_data/storage/regional_age_80_plus.csv @@ -0,0 +1,13 @@ +name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url +east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland +scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland +south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales +west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 980223540..b9d19759c 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -16,7 +16,12 @@ import numpy as np import pandas as pd -from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.coverage import ( + TargetStatus, + check_coverage, + column_status, +) +from policyengine_uk_data.targets.registry import emitted_targets from policyengine_uk_data.targets.schema import ( GeographicLevel, Target, @@ -64,6 +69,7 @@ def create_target_matrix( dataset, time_period: str = None, reform=None, + require_coverage: bool = True, ) -> tuple[pd.DataFrame, pd.Series]: """Create (matrix, target_values) for calibration. @@ -71,12 +77,68 @@ def create_target_matrix( dataset: a UKSingleYearDataset instance. time_period: calendar year as string; defaults to dataset year. reform: optional PolicyEngine reform. + require_coverage: fail unless every target is in the matrix with + signal or listed in targets/known_gaps.yaml (targets/coverage.py). Returns: (df, targets) where df has one column per target and one row per household, and targets is a Series of scalar target values indexed by target name. """ + df, targets, statuses = build_target_matrix(dataset, time_period, reform) + if require_coverage: + check_coverage(statuses) + return df, targets + + +MATRIX_LEVELS = ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, +) + + +def select_matrix_targets(targets): + """The targets the matrix offers, and a ``duplicate`` status for each other. + + Targets are taken level by level (national, region, country), each level + in the order the sources emit them, and the first target of each name + wins. A later target of that name, at the same level or another, is + dropped and recorded as a duplicate (targets/coverage.py). + """ + kept, statuses = {}, [] + for level in MATRIX_LEVELS: + for target in targets: + if target.geographic_level != level: + continue + if target.name not in kept: + kept[target.name] = target + continue + first = kept[target.name] + statuses.append( + TargetStatus( + target.name, + "duplicate", + f"{target.source} {level.value} target for {_years(target)} " + f"dropped for the {first.source} " + f"{first.geographic_level.value} one for {_years(first)}", + ) + ) + return list(kept.values()), statuses + + +def _years(target: Target) -> str: + return ", ".join(str(year) for year in sorted(target.values)) + + +def build_target_matrix(dataset, time_period: str = None, reform=None): + """The target matrix, its values, and each offered target's status. + + A target that is dropped (a duplicate name, no value for the year, no + column, an error) or that enters with a non-finite value or column, or + with no household contributing, is recorded with that status + (targets/coverage.py). Each offered target gets exactly one status. + """ from policyengine_uk import Microsimulation if time_period is None: @@ -88,17 +150,7 @@ def create_target_matrix( ctx = _SimContext(sim, time_period, dataset, reform) - all_targets = [] - seen = set() - for level in ( - GeographicLevel.NATIONAL, - GeographicLevel.REGION, - GeographicLevel.COUNTRY, - ): - for t in get_all_targets(geographic_level=level): - if t.name not in seen: - seen.add(t.name) - all_targets.append(t) + all_targets, statuses = select_matrix_targets(emitted_targets()) df = pd.DataFrame() target_names = [] @@ -108,17 +160,34 @@ def create_target_matrix( try: val = _resolve_value(target, year) if val is None: + statuses.append( + TargetStatus( + target.name, "no_value_for_year", f"values for {_years(target)}" + ) + ) continue col = _compute_column(target, ctx, year) if col is None: + statuses.append(TargetStatus(target.name, "no_column")) continue df[target.name] = restrict_to_countries(col, ctx.country, target.countries) + # The status is of the column as it entered the matrix; a column + # whose status cannot be computed leaves the matrix again. + try: + status = column_status(val, df[target.name]) + except Exception: + del df[target.name] + raise + statuses.append(TargetStatus(target.name, status)) target_names.append(target.name) target_values.append(val) except Exception as e: logger.warning("Skipping target %s: %s", target.name, e) + statuses.append( + TargetStatus(target.name, "error", f"{type(e).__name__}: {e}") + ) - return df, pd.Series(target_values, index=target_names) + return df, pd.Series(target_values, index=target_names), statuses def restrict_to_countries(column, household_country, countries): diff --git a/policyengine_uk_data/targets/compute/demographics.py b/policyengine_uk_data/targets/compute/demographics.py index 4d0b80923..f405f24a5 100644 --- a/policyengine_uk_data/targets/compute/demographics.py +++ b/policyengine_uk_data/targets/compute/demographics.py @@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None: region_name = name[:idx] age_part = name[idx + 5 :] lower, upper = age_part.split("_") - lower, upper = int(lower), int(upper) + lower = int(lower) + upper = np.inf if upper == "plus" else int(upper) pe_region = _REGION_INV.get(region_name) if pe_region is None: @@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray: parts = name.split("_") sex = parts[0] lower = int(parts[1]) - upper = int(parts[2]) + upper = np.inf if parts[2] == "plus" else int(parts[2]) gender = ctx.sim.calculate("gender").values sex_match = gender == ("FEMALE" if sex == "female" else "MALE") diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py new file mode 100644 index 000000000..501470032 --- /dev/null +++ b/policyengine_uk_data/targets/coverage.py @@ -0,0 +1,271 @@ +"""Required coverage of the national calibration targets. + +Every target the registry offers must enter the national loss matrix with +a finite value and a column that some household contributes to, under a +name no other target has. The exceptions are the targets +``targets/known_gaps.yaml`` lists, each with the one status it may have, a +reason and an issue, for every build or only for testing or release builds. +``create_target_matrix`` checks this, so a build that would calibrate +without a target, or against one the weights cannot move, fails instead of +logging a warning. + + python -m policyengine_uk_data.targets.coverage [out.json] + +reports every target's status on a dataset: names, statuses and counts only, +never household values. Set TESTING=1 to check it as a testing build. +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +from collections import Counter, defaultdict +from dataclasses import dataclass +from fnmatch import fnmatchcase +from pathlib import Path +from typing import Iterable, Optional, Sequence + +import numpy as np +import yaml + +logger = logging.getLogger(__name__) + +KNOWN_GAPS_PATH = Path(__file__).with_name("known_gaps.yaml") +IN_MATRIX = "in_matrix" +# Statuses a known gap may excuse. An error or a non-finite value is never +# a known gap: it is a broken target. +EXCUSABLE = ("duplicate", "no_value_for_year", "no_column", "zero_signal") +NEVER_EXCUSED = ("error", "non_finite") +STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED +# Pull request builds set TESTING=1 (smaller, faster); release builds don't. +BUILDS = ("testing", "release") + + +class TargetCoverageError(Exception): + """The national target matrix is missing targets it must have.""" + + +@dataclass(frozen=True) +class TargetStatus: + name: str + status: str + detail: str = "" + + +@dataclass(frozen=True) +class KnownGap: + status: str + targets: frozenset + reason: str + issue: str + builds: frozenset = frozenset(BUILDS) + # fnmatch-style globs over target names; testing builds only. + patterns: tuple = () + + +def current_build() -> str: + """The kind of build running, read from TESTING as the build itself does.""" + return "testing" if os.environ.get("TESTING", "0") == "1" else "release" + + +def parse_known_gaps(raw) -> list: + if (raw or {}).get("schema_version") != 1: + raise ValueError("known gaps: schema_version must be 1") + gaps, seen = [], set() + for index, entry in enumerate(raw.get("gaps") or []): + where = f"known gaps[{index}]" + if entry.get("status") not in EXCUSABLE: + raise ValueError(f"{where}: status must be one of {', '.join(EXCUSABLE)}") + if not entry.get("reason") or not str(entry.get("issue", "")).startswith( + "https://github.com/" + ): + raise ValueError(f"{where}: needs a reason and an issue URL") + builds = entry.get("builds", list(BUILDS)) + if not isinstance(builds, list) or not builds or not set(builds) <= set(BUILDS): + raise ValueError(f"{where}: builds must list testing and/or release") + targets = list(entry.get("targets") or []) + patterns = list(entry.get("patterns") or []) + if not targets and not patterns: + raise ValueError(f"{where}: lists no targets") + if patterns and set(builds) != {"testing"}: + # A release gap names each target, so stale entries show up. + raise ValueError(f"{where}: patterns may only excuse testing builds") + if any(not set("*?[") & set(pattern) for pattern in patterns): + raise ValueError(f"{where}: a pattern needs a wildcard") + # A name may be listed under two statuses: a target without signal + # whose duplicate is dropped, say. Under one status, it is listed once. + names = [(name, entry["status"]) for name in targets + patterns] + repeated = seen.intersection(names) | { + t for t, n in Counter(names).items() if n > 1 + } + if repeated: + raise ValueError( + f"{where}: {sorted(name for name, _ in repeated)} listed twice" + ) + seen.update(names) + gaps.append( + KnownGap( + entry["status"], + frozenset(targets), + str(entry["reason"]), + str(entry["issue"]), + frozenset(builds), + tuple(patterns), + ) + ) + return gaps + + +def load_known_gaps(path=None) -> list: + with open(path or KNOWN_GAPS_PATH) as f: + return parse_known_gaps(yaml.safe_load(f)) + + +def column_status(value, column) -> str: + """A matrix column's status: non-finite, without signal, or fine. + + A column with no non-zero entry has no signal unless its target is + zero too: the weights cannot move it. + """ + column = np.asarray(column, dtype=float) + if not np.isfinite(value) or not np.isfinite(column).all(): + return "non_finite" + if value != 0 and not np.any(column != 0): + return "zero_signal" + return IN_MATRIX + + +def coverage_problems( + statuses: Sequence[TargetStatus], + gaps: Iterable[KnownGap], + build: Optional[str] = None, +): + """(problems, stale) in one kind of build, by default the one running. + + Problems are the targets out of the matrix that no gap for this build + excuses under their own status. Stale gaps are named targets that no + longer have that status, and patterns that match no offered target: + what a pattern excuses varies from one testing build to the next, so + matching nothing in one build is not stale. + """ + build = build or current_build() + gaps = [gap for gap in gaps if build in gap.builds] + named = {(name, gap.status) for gap in gaps for name in gap.targets} + patterns = [(pattern, gap.status) for gap in gaps for pattern in gap.patterns] + problems, matched, present = [], set(), defaultdict(list) + for row in statuses: + present[row.name].append(row.status) + if row.status == IN_MATRIX: + continue + if row.status not in NEVER_EXCUSED: + if (row.name, row.status) in named: + matched.add((row.name, row.status)) + continue + if any( + status == row.status and fnmatchcase(row.name, pattern) + for pattern, status in patterns + ): + continue + detail = f" ({row.detail})" if row.detail else "" + problems.append(f"{row.name}: {row.status}{detail}") + stale = [] + for gap in gaps: + for name in sorted(gap.targets): + if (name, gap.status) in matched: + continue + now = ", ".join(present.get(name, ["not offered by the registry"])) + stale.append(f"{name}: listed as {gap.status}, now {now}") + for pattern in gap.patterns: + if not any(fnmatchcase(name, pattern) for name in present): + stale.append(f"{pattern}: listed as {gap.status}, matches no target") + return problems, stale + + +def check_coverage( + statuses: Sequence[TargetStatus], + gaps: Optional[Iterable[KnownGap]] = None, + build: Optional[str] = None, +) -> list: + """Raise unless every target is in the matrix or a known gap; return stale gaps.""" + build = build or current_build() + problems, stale = coverage_problems( + statuses, load_known_gaps() if gaps is None else gaps, build + ) + counts = Counter(row.status for row in statuses) + logger.info( + "Calibration targets (%s build): %s", + build, + ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]), + ) + for line in stale: + logger.warning("Stale known calibration-target gap, remove it: %s", line) + write_step_summary(build, stale, problems) + if problems: + raise TargetCoverageError( + f"{len(problems)} calibration targets are missing or cannot be fitted, " + "and targets/known_gaps.yaml does not list them: " + "; ".join(problems) + ) + return stale + + +def write_step_summary(build: str, stale: Sequence[str], problems: Sequence[str]): + """Add stale gaps and problems to the GitHub Actions job summary, once each. + + A long build log buries a warning; the job summary shows it on the run's + page. Lines already in the summary (the local-authority calibration + repeats the constituency check) are not written again. + """ + path = os.environ.get("GITHUB_STEP_SUMMARY") + lines = [f"- Stale, remove from targets/known_gaps.yaml: `{s}`" for s in stale] + lines += [f"- Not in the matrix and not a known gap: `{p}`" for p in problems] + if not path or not lines: + return + summary = Path(path) + written = summary.read_text() if summary.exists() else "" + lines = [line for line in lines if line not in written] + if lines: + with summary.open("a") as f: + f.write(f"\n### Calibration target coverage ({build} build)\n\n") + f.write("\n".join(lines) + "\n") + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if len(argv) not in (2, 3): + print(__doc__) + return 2 + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.targets.build_loss_matrix import build_target_matrix + + _, _, statuses = build_target_matrix( + UKSingleYearDataset(file_path=argv[0]), argv[1] + ) + problems, stale = coverage_problems(statuses, load_known_gaps()) + report = { + "dataset": Path(argv[0]).name, + "year": int(argv[1]), + "build": current_build(), + "by_status": Counter(row.status for row in statuses), + "problems": problems, + "stale_known_gaps": stale, + "targets": [row.__dict__ for row in statuses], + } + if len(argv) == 3: + Path(argv[2]).write_text(json.dumps(report, indent=1)) + print( + json.dumps( + { + k: report[k] + for k in ("build", "by_status", "problems", "stale_known_gaps") + }, + indent=1, + ) + ) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml new file mode 100644 index 000000000..6728d0286 --- /dev/null +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -0,0 +1,113 @@ +# Calibration targets allowed to stay out of the national loss matrix, or to +# enter it with no signal. targets/coverage.py fails the build on any other +# target that is dropped (a name already taken, no value for the year, no +# column, an error), has a non-finite value or column, or has a column no +# household contributes to. +# +# Each entry names the exact targets, the one status they may have, why and +# the issue that tracks removing them. Errors and non-finite values are never +# allowed. An entry applies to every build unless `builds` limits it to +# testing builds (pull requests, TESTING=1) or release builds. A +# testing-only entry may name `patterns` instead, fnmatch globs over target +# names, for targets whose status varies from one testing build to the next. +# A named target that no longer has its status, or a pattern that matches +# no target at all, is reported as stale: remove it. +# +# Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with +# policyengine-uk 2.122.2 at the calibration year 2025, with #557 and #604, +# as a release build and as a testing build: the registry offers 658 names. +# 643 enter the matrix, 15 have no value for 2025 and 17 of those entering +# have no signal. 3 more targets reuse a name already taken and are dropped +# as duplicates (661 targets in all). +schema_version: 1 +gaps: + - status: no_value_for_year + reason: >- + Dated 2026 only. The calibration year is 2025, and _resolve_value does + not carry a later year back. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/456 + targets: + - dwp/uc/two_child_limit/3_children_households + - dwp/uc/two_child_limit/3_children_households_total_children + - dwp/uc/two_child_limit/4_children_households + - dwp/uc/two_child_limit/4_children_households_total_children + - dwp/uc/two_child_limit/5_children_households + - dwp/uc/two_child_limit/5_children_households_total_children + - dwp/uc/two_child_limit/6_children_households + - dwp/uc/two_child_limit/6_children_households_total_children + - dwp/uc/two_child_limit/adult_pip_children + - dwp/uc/two_child_limit/adult_pip_households + - dwp/uc/two_child_limit/children_affected + - dwp/uc/two_child_limit/children_in_affected_households + - dwp/uc/two_child_limit/disabled_child_element_children + - dwp/uc/two_child_limit/disabled_child_element_households + - dwp/uc/two_child_limit/households_affected + - status: duplicate + reason: >- + ons_demographics emits these names twice. The matrix keeps the national + ons/uk_population and the region Scottish targets. It drops a region + ons/uk_population read from demographics.csv 1,000 times too small, and + NRS country copies of the Scottish targets with identical values. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/605 + targets: + - ons/scotland_babies_under_1 + - ons/scotland_children_under_16 + - ons/uk_population + - status: zero_signal + reason: >- + Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - hmrc/dividend_income_count_income_band_1_000_000_to_inf + - hmrc/dividend_income_income_band_1_000_000_to_inf + - hmrc/employment_income_count_income_band_1_000_000_to_inf + - hmrc/employment_income_income_band_1_000_000_to_inf + - hmrc/private_pension_income_count_income_band_1_000_000_to_inf + - hmrc/private_pension_income_income_band_1_000_000_to_inf + - hmrc/property_income_count_income_band_1_000_000_to_inf + - hmrc/property_income_income_band_1_000_000_to_inf + - hmrc/state_pension_count_income_band_1_000_000_to_inf + - hmrc/state_pension_income_band_1_000_000_to_inf + - status: zero_signal + reason: >- + No band-H household in these regions. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - voa/council_tax/NORTH_EAST/H + - voa/council_tax/NORTH_WEST/H + - voa/council_tax/YORKSHIRE/H + - status: zero_signal + reason: >- + impute_student_loan_plan (datasets/imputations/student_loans.py) assigns + only NONE, PLAN_1, PLAN_2 and PLAN_5, never POSTGRADUATE, so no record + has a postgraduate loan in any build. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - slc/student_loan_repayment/england/postgraduate + - status: zero_signal + reason: >- + No couple without children receives UC payments above £27,600 a year. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf + - status: zero_signal + reason: >- + In pull request (TESTING) builds the UC payment bands with no signal + vary from one build to the next, and builds of the same code take one of + two pre-calibration states (#598). Those builds hold one output-area + clone of each household (PE_UK_DATA_OA_CLONES=1, as release builds do). + Of 21 pull request builds from 8 to 15 October 2026, the national + estimate in the final calibration log was exactly zero for + COUPLE_NO_CHILDREN 27,600 to 28,800 in 20, and 28,800 to 30,000 and + 30,000 and over in all 21; SINGLE 24,000 to 25,200 and 25,200 to 26,400 + in 20, and 27,600 to 28,800 in all 21; and COUPLE_WITH_CHILDREN 27,600 + to 28,800 in 3. The gate failed on the three SINGLE bands in run + 38726993359 and on the COUPLE_WITH_CHILDREN band in run 38732018053. In + release builds only the three COUPLE_NO_CHILDREN bands above may lack + signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598 + builds: [testing] + patterns: + - "dwp/uc_payment_dist/*" diff --git a/policyengine_uk_data/targets/registry.py b/policyengine_uk_data/targets/registry.py index b4c971084..40941ae30 100644 --- a/policyengine_uk_data/targets/registry.py +++ b/policyengine_uk_data/targets/registry.py @@ -31,6 +31,11 @@ def discover_source_modules() -> list: return modules +def emitted_targets() -> list[Target]: + """Every target the source modules emit, in order, duplicate names included.""" + return [target for mod in discover_source_modules() for target in mod.get_targets()] + + def get_all_targets( year: int | None = None, geographic_level: GeographicLevel | None = None, @@ -43,23 +48,20 @@ def get_all_targets( geographic_level: if provided, filter to this geographic level. Returns: - De-duplicated list of Target objects. + De-duplicated list of Target objects: of the targets that pass the + filters, the first of each name. ``emitted_targets`` keeps them all. """ all_targets: list[Target] = [] seen_names: set[str] = set() - for mod in discover_source_modules(): - for target in mod.get_targets(): - if target.name in seen_names: - continue - if year is not None and year not in target.values: - continue - if ( - geographic_level is not None - and target.geographic_level != geographic_level - ): - continue - seen_names.add(target.name) - all_targets.append(target) + for target in emitted_targets(): + if target.name in seen_names: + continue + if year is not None and year not in target.values: + continue + if geographic_level is not None and target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) return all_targets diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 17b375b8e..c76c60bc1 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -53,8 +53,11 @@ ] _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] -# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024 -_SPI_YEAR = 2024 +# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is +# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters +# reads each parameter at 30 April N), so the outturn belongs to 2023. +# Later years come from incomes_projection.csv. +_SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers @@ -229,8 +232,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame: def get_targets() -> list[Target]: """Build income-band targets from the live HMRC SPI ODS. - Also reads incomes_projection.csv if available, which contains - projected future year data generated by the microsimulation. + Also reads incomes_projection.csv if available, which projects the + same bands to later years with PolicyEngine's uprating indices. Each + band is returned as one target holding the base year and every + projected year: the registry keeps only the first target of a name, + and calibration reads targets without a year filter, so a band split + across two targets would lose its projected years and be calibrated + to the base-year outturn carried forward. """ config = load_config() ref = config["hmrc"]["spi_collated"] @@ -291,7 +299,18 @@ def get_targets() -> list[Target]: if proj_path.exists(): targets.extend(_read_projection_csv(proj_path, ref)) - return targets + return _merge_by_name(targets) + + +def _merge_by_name(targets: list[Target]) -> list[Target]: + """Combine targets that share a name into one, merging their years.""" + merged: dict[str, Target] = {} + for t in targets: + if t.name in merged: + merged[t.name].values.update(t.values) + else: + merged[t.name] = t + return list(merged.values()) def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: @@ -355,12 +374,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: ) ) - # Merge targets with the same name across years - merged: dict[str, Target] = {} - for t in targets: - if t.name in merged: - merged[t.name].values.update(t.values) - else: - merged[t.name] = t - - return list(merged.values()) + return _merge_by_name(targets) diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 0d04f6315..a40a0a4dd 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -3,10 +3,9 @@ Downloads the ONS 2024-based principal population projection for the UK to extract total population and gender × age band targets. -For regional age breakdowns (12 regions × 9 age bands), reads the -pre-existing demographics.csv which was extracted from ONS subnational -projections. The subnational projections don't have a stable machine- -readable download URL, so this is the pragmatic compromise. +Regional age breakdowns use demographics.csv below age 80 and a committed +ONS projection extract for 80+, including the terminal age group. See +tools/ons_age_targets for the sources and extraction method. Household type and tenure targets are from ONS families & households datasets (also lacking stable machine-readable URLs). @@ -52,7 +51,7 @@ _YEARS = list(range(2022, 2030)) -# Age band boundaries +# FRS ages are top-coded at 80, so the last band includes every older age. _AGE_BANDS = [ (0, 9), (10, 19), @@ -62,16 +61,18 @@ (50, 59), (60, 69), (70, 79), - (80, 89), + (80, None), ] # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only # the band totals, leaving the age composition *within* a band unconstrained # nationally: the calibration could satisfy 15-29 while starving 18-24, which # is what collapsed the modelled young-adult population to ~3.4M against ONS's -# ~5.4M. Regional targets already use finer bands, but they do not constrain -# the national weights. The ONS projection is published by single year of age, -# so this is a change of aggregation only - no new source data. +# ~5.4M. Regional ten-year bands also constrain the shared household weights, +# including summed local weights in dense local calibration, but provide less +# age resolution than these five-year sex bands. The ONS projection is published +# by single year of age, so this is a change of aggregation only - no new source +# data. _GENDER_BANDS = [ (0, 4), (5, 9), @@ -89,8 +90,7 @@ (65, 69), (70, 74), (75, 79), - (80, 84), - (85, 90), + (80, None), ] @@ -121,12 +121,19 @@ def _find_projection_member(names: list[str]) -> str: def _aggregate_ages( - df: pd.DataFrame, sex: str, low: int, high: int, years: list[int] + df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int] ) -> dict[int, float]: - """Sum population for a sex and age range across years.""" + """Sum an inclusive age band; None includes the projection's terminal age.""" sex_filter = "Females" if sex == "female" else "Males" - ages = pd.to_numeric(df["Age"], errors="coerce") - mask = (df["Sex"] == sex_filter) & ages.between(low, high) + # Use each group's lower age: the 2024-based workbook ends with + # "105 - 109" and "110 and over"; older vintages can use "90+". + ages = pd.to_numeric( + df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce" + ) + age_mask = ages >= low + if high is not None: + age_mask &= ages <= high + mask = (df["Sex"] == sex_filter) & age_mask subset = df[mask] result = {} for y in years: @@ -176,7 +183,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: if values: targets.append( Target( - name=f"ons/{sex}_{low}_{high}", + name=f"ons/{sex}_{low}_{'plus' if high is None else high}", variable="age", source="ons", unit=Unit.COUNT, @@ -190,17 +197,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: def _parse_regional_from_csv() -> list[Target]: - """Read regional age band targets from demographics.csv. - - This CSV was extracted from ONS subnational projections which - lack a stable machine-readable download URL. - """ + """Read regional bands, replacing the legacy bounded top band with 80+.""" csv_path = STORAGE / "demographics.csv" if not csv_path.exists(): logger.warning("demographics.csv not found, skipping regional") return [] demographics = pd.read_csv(csv_path) + # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums, + # including 90+, instead of relabelling totals with a different scope. + demographics = demographics[~demographics["name"].str.endswith("_age_80_89")] + top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv") + top_bands["unit"] = "person" + demographics = pd.concat([demographics, top_bands], ignore_index=True) targets = [] # Skip rows now handled by dedicated modules (ons_households.py, @@ -231,8 +240,8 @@ def _parse_regional_from_csv() -> list[Target]: for y in _YEARS: col = str(y) if col in row.index and pd.notna(row[col]): - # Values in CSV are in thousands - values[y] = float(row[col]) * 1e3 + scale = 1 if row["unit"] == "person" else 1e3 + values[y] = float(row[col]) * scale if values: targets.append( Target( @@ -243,7 +252,11 @@ def _parse_regional_from_csv() -> list[Target]: geographic_level=GeographicLevel.REGION, values=values, is_count=True, - reference_url=_REF_REGION, + reference_url=( + row["reference_url"] + if pd.notna(row.get("reference_url")) + else _REF_REGION + ), ) ) diff --git a/policyengine_uk_data/tests/conftest.py b/policyengine_uk_data/tests/conftest.py index 9de52820d..8c579975a 100644 --- a/policyengine_uk_data/tests/conftest.py +++ b/policyengine_uk_data/tests/conftest.py @@ -32,3 +32,9 @@ def baseline(enhanced_frs: UKSingleYearDataset): from policyengine_uk import Microsimulation return Microsimulation(dataset=enhanced_frs) + + +@pytest.fixture(autouse=True) +def no_job_summary(monkeypatch): + """Keep tests out of the GitHub Actions job summary the build writes to.""" + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv new file mode 100644 index 000000000..911aae4f5 --- /dev/null +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -0,0 +1,215 @@ +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 6370578cc..5ca8756c7 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -5,9 +5,12 @@ to have been generated and will be skipped otherwise. """ +import re + import pytest import pandas as pd from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources._common import load_config from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv" @@ -152,3 +155,74 @@ def test_projection_is_the_spi_table_uprated_with_the_committed_table( from policyengine_uk_data.utils.incomes_projection import project_income_table pd.testing.assert_frame_equal(project_income_table(base_targets), projections) + + +def test_spi_year_is_the_start_of_the_configured_tax_year(): + """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24). + + PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its + value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it + to the end year would treat it as 2024-25 and understate every later + year by a year's growth. A refreshed URL without a matching _SPI_YEAR + fails here. + """ + url = load_config()["hmrc"]["spi_collated"] + match = re.search(r"_(\d{2})(\d{2})\.ods$", url) + assert match, url + start, end = int(match[1]), int(match[2]) + assert end == (start + 1) % 100, url + assert _SPI_YEAR == 2000 + start + + +def test_calibration_reads_the_projected_spi_targets( + projections, base_targets, monkeypatch +): + """Calibration (build_loss_matrix) fetches targets without a year filter, + and the registry keeps only the first target of each name. Every SPI band + must therefore be one target carrying its outturn and every projected + year; if the projected years sit on a second target of the same name they + are dropped, and the outturn is carried forward unuprated instead. + + The official table is served from the committed incomes.csv, so the + outturn targets are always present and the test needs no download. + """ + from policyengine_uk_data.targets import get_all_targets + from policyengine_uk_data.targets import registry + from policyengine_uk_data.targets.build_loss_matrix import _resolve_value + from policyengine_uk_data.targets.sources import hmrc_spi + + def committed_table(include_aggregate: bool = True) -> pd.DataFrame: + if include_aggregate: + return base_targets + return _without_aggregate(base_targets).reset_index(drop=True) + + monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table) + monkeypatch.setattr(registry, "discover_source_modules", lambda: [hmrc_spi]) + + names = [t.name for t in hmrc_spi.get_targets()] + assert len(names) == len(set(names)) + + spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR]) + checked = 0 + for row in expected.itertuples(): + label = hmrc_spi._format_band_label( + row.total_income_lower_bound, row.total_income_upper_bound + ) + for variable in hmrc_spi.INCOME_VARIABLES: + amount = getattr(row, f"{variable}_amount") + if variable == "property_income": + amount *= hmrc_spi._PROPERTY_INCOME_SCALE + amount_target = spi[f"hmrc/{variable}_income_band_{label}"] + count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] + assert _resolve_value(amount_target, row.year) == pytest.approx(amount) + assert _resolve_value(count_target, row.year) == pytest.approx( + getattr(row, f"{variable}_count") + ) + checked += 2 + assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected) + + years = set(projections["year"]) + assert _SPI_YEAR in years + for target in spi.values(): + assert set(target.values) == years, target.name diff --git a/policyengine_uk_data/tests/test_national_age_bands.py b/policyengine_uk_data/tests/test_national_age_bands.py index f636255a6..354929a6f 100644 --- a/policyengine_uk_data/tests/test_national_age_bands.py +++ b/policyengine_uk_data/tests/test_national_age_bands.py @@ -4,8 +4,8 @@ 15-29, ...). A band that wide pins only its own total: the calibration could satisfy 15-29 while distributing those people any way it liked inside the band, and it settled on one that starved 18-24 down to ~3.4M against ONS's -~5.4M. The finer regional bands do not help, because they constrain regional -weights rather than the national ones. +~5.4M. Regional ten-year age bands constrain the same household weights, +but provide less age detail than national five-year sex bands. These tests pin the five-year granularity and the no-overlap property, so a future edit cannot quietly widen the bands or reintroduce the superseded CSV @@ -13,13 +13,17 @@ """ import re +from pathlib import Path +import pandas as pd + +from policyengine_uk_data.targets.sources import ons_demographics from policyengine_uk_data.targets.sources.ons_demographics import ( _GENDER_BANDS, get_targets, ) -_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+)$") +_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+|plus)$") def _gender_age_targets(): @@ -27,9 +31,11 @@ def _gender_age_targets(): def test_bands_are_at_most_five_years_wide(): - """A wider band leaves its internal age composition unconstrained.""" - for low, high in _GENDER_BANDS: - assert high - low <= 5, f"band {low}-{high} is too wide to pin composition" + """Only the terminal top-coded band may be wider than five years.""" + assert _GENDER_BANDS[-1] == (80, None) + for low, high in _GENDER_BANDS[:-1]: + assert high is not None, "only the terminal band may be open-ended" + assert 0 <= high - low < 5, f"band {low}-{high} is too wide to pin composition" def test_bands_are_contiguous_and_non_overlapping(): @@ -41,7 +47,7 @@ def test_bands_are_contiguous_and_non_overlapping(): def test_young_adult_ages_are_covered_by_narrow_bands(): """18-24 must not sit inside one wide band, which is the failure mode.""" - covering = [b for b in _GENDER_BANDS if b[0] <= 24 and b[1] >= 18] + covering = [b for b in _GENDER_BANDS if b[0] <= 24 and (b[1] is None or b[1] >= 18)] assert len(covering) >= 2, ( "18-24 is covered by a single band; its composition would be free" ) @@ -54,9 +60,13 @@ def test_national_gender_age_targets_are_emitted_once_per_band(): assert len({t.name for t in targets}) == len(targets) -def test_national_gender_age_targets_sum_to_uk_population(): - """Double-counted bands would show up here as a doubled total.""" +def test_national_gender_age_targets_sum_to_uk_population(monkeypatch): + """Every fixture age must be counted exactly once across both sexes.""" + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + monkeypatch.setattr(ons_demographics, "_download_uk_projection", lambda: projection) targets = _gender_age_targets() year = 2025 - total = sum(t.values.get(year, 0) for t in targets) - assert 65e6 < total < 73e6, f"national gender-age total {total / 1e6:.1f}M" + total = sum(t.values[year] for t in targets) + assert total == projection[str(year)].sum() diff --git a/policyengine_uk_data/tests/test_obr_universal_credit_target.py b/policyengine_uk_data/tests/test_obr_universal_credit_target.py index 8fcdb51e0..6ce352490 100644 --- a/policyengine_uk_data/tests/test_obr_universal_credit_target.py +++ b/policyengine_uk_data/tests/test_obr_universal_credit_target.py @@ -23,7 +23,7 @@ _compute_column, restrict_to_countries, ) -from policyengine_uk_data.targets.schema import GREAT_BRITAIN, GeographicLevel +from policyengine_uk_data.targets.schema import GREAT_BRITAIN from policyengine_uk_data.targets.sources import obr COUNTRIES = ("ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND") @@ -180,13 +180,7 @@ def map_result(self, values, source, target_entity): ) monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) - monkeypatch.setattr( - build_loss_matrix, - "get_all_targets", - lambda geographic_level=None: ( - [target] if geographic_level == GeographicLevel.NATIONAL else [] - ), - ) + monkeypatch.setattr(build_loss_matrix, "emitted_targets", lambda: [target]) matrix, values = build_loss_matrix.create_target_matrix( SimpleNamespace(time_period="2025"), time_period="2025" diff --git a/policyengine_uk_data/tests/test_ons_demographics_targets.py b/policyengine_uk_data/tests/test_ons_demographics_targets.py index 433bf3e25..1df25ff45 100644 --- a/policyengine_uk_data/tests/test_ons_demographics_targets.py +++ b/policyengine_uk_data/tests/test_ons_demographics_targets.py @@ -1,9 +1,21 @@ +from pathlib import Path +from types import SimpleNamespace + +import numpy as np import pandas as pd import pytest +from hypothesis import given +from hypothesis import strategies as st +from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources.ons_demographics import ( + _AGE_BANDS, + _GENDER_BANDS, _aggregate_ages, _find_projection_member, + _parse_regional_from_csv, + _parse_uk_totals, ) @@ -22,3 +34,124 @@ def test_aggregate_ages_accepts_string_age_values(): def test_find_projection_member_fails_loudly(): with pytest.raises(RuntimeError, match="uk_ppp_machine_readable"): _find_projection_member(["uk/readme.txt"]) + + +@pytest.mark.parametrize("bands", [_AGE_BANDS, _GENDER_BANDS]) +@given(age=st.integers(min_value=0, max_value=80)) +def test_age_bands_partition_frs_ages_and_respect_top_code(bands, age): + assert bands[-1] == (80, None) + for low, high in bands: + assert low <= 80, "No target may start above the FRS top code" + assert high is None or high < 80, "80+ must be open-ended" + assert sum(low <= age and (high is None or age <= high) for low, high in bands) == 1 + + +@pytest.mark.parametrize("sex,multiplier", [("female", 1), ("male", 10)]) +def test_80_plus_target_sums_every_projection_age(sex, multiplier): + projection = pd.DataFrame( + [ + {"Sex": label, "Age": str(age), 2025: (age + 1) * factor} + for label, factor in [("Females", 1), ("Males", 10)] + for age in range(101) + ] + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == sum( + (age + 1) * multiplier for age in range(80, 101) + ) + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +def test_80_plus_matches_published_ons_2025_projection(sex, label): + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + # The fixture retains ONS's single ages 0-104 and terminal 105-109/110+. + older_ages = {str(age) for age in range(80, 105)} | { + "105 - 109", + "110 and over", + } + expected = projection.loc[ + (projection.Sex == label) & projection.Age.isin(older_ages), "2025" + ].sum() + assert targets[f"ons/{sex}_80_plus"].values[2025] == expected + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +@pytest.mark.parametrize( + "terminal_ages", [["89", "90+"], ["105 - 109", "110 and over"]] +) +def test_80_plus_target_includes_terminal_age_groups(sex, label, terminal_ages): + projection = pd.DataFrame( + { + "Sex": [label] * 4, + "Age": ["79", "80", *terminal_ages], + 2025: [1, 2, 4, 8], + } + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == 14 + + +def test_emitted_age_target_names_respect_frs_top_code(): + projection = pd.DataFrame( + {"Sex": ["Females", "Males"], "Age": [80, 80], 2025: [1, 1]} + ) + targets = _parse_uk_totals(projection) + _parse_regional_from_csv() + top_bands = [] + for target in targets: + if "_age_" not in target.name and not target.name.startswith( + ("ons/female_", "ons/male_") + ): + continue + low, high = target.name.rsplit("_", 2)[-2:] + assert int(low) <= 80, target.name + assert high == "plus" or int(high) < 80, target.name + if int(low) == 80: + top_bands.append(target.name) + assert len(top_bands) == 14 # Both sexes and all twelve regions. + + +@pytest.mark.parametrize( + "name,expected", + [ + ("ons/female_80_plus", [1, 1, 1, 0]), + ("ons/male_80_plus", [0, 1, 1, 1]), + ("ons/london_age_80_plus", [1, 1, 1, 1]), + ], +) +def test_80_plus_dispatch_counts_matching_household_members(name, expected): + household = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + gender = pd.Series( + ["FEMALE", "FEMALE", "MALE", "FEMALE", "MALE", "FEMALE", "MALE", "MALE"] + ) + ctx = SimpleNamespace( + age=np.array([79, 80, 81, 100, 80, 80, 79, 85]), + region=pd.Series( + [ + "LONDON", + "LONDON", + "LONDON", + "WALES", + "WALES", + "LONDON", + "LONDON", + "LONDON", + ] + ), + sim=SimpleNamespace(calculate=lambda variable: gender), + household_from_person=lambda values: np.bincount(household, weights=values), + ) + target = Target( + name=name, + variable="age", + source="ons", + unit=Unit.COUNT, + values={2025: 1}, + is_count=True, + ) + + np.testing.assert_array_equal(_compute_column(target, ctx, 2025), expected) diff --git a/policyengine_uk_data/tests/test_policybench_transfer.py b/policyengine_uk_data/tests/test_policybench_transfer.py index a8579c0ec..4fa05852d 100644 --- a/policyengine_uk_data/tests/test_policybench_transfer.py +++ b/policyengine_uk_data/tests/test_policybench_transfer.py @@ -170,7 +170,11 @@ def test_policybench_transfer_calibration_improves_loss(tmp_path: Path): def test_policybench_transfer_calibration_uses_iterative_solver(monkeypatch): calls = {} - def fake_create_target_matrix(dataset, time_period, reform=None): + def fake_create_target_matrix( + dataset, time_period, reform=None, require_coverage=True + ): + # The transfer dataset is not the Enhanced FRS, so it is not gated. + assert require_coverage is False return pd.DataFrame( [[1.0, 0.0], [0.0, 1.0]], ), pd.Series([1.0, 2.0]) diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py index 5a2f6208c..b5a1e2eec 100644 --- a/policyengine_uk_data/tests/test_property_income_targets.py +++ b/policyengine_uk_data/tests/test_property_income_targets.py @@ -5,6 +5,7 @@ """ from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR def test_property_income_targets_scaled(): @@ -13,7 +14,7 @@ def test_property_income_targets_scaled(): Raw SPI 2023-24 total is scaled up to better match HMRC rental income statistics, which cover more landlords than SPI. """ - base_year = 2024 + base_year = _SPI_YEAR targets = get_all_targets(year=base_year) total = sum( t.values[base_year] diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py new file mode 100644 index 000000000..1f63ec006 --- /dev/null +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -0,0 +1,633 @@ +"""Required coverage of the national calibration targets (targets/coverage.py). + +create_target_matrix used to skip a target with a warning, or with nothing at +all when its year did not resolve (#456) or its name was already taken, and to +accept a column no household contributes to (#593). These tests pin the gate +that replaces that: every offered target is in the matrix with signal, or +listed in known_gaps.yaml with the status it may have, in the builds it may +have it. +""" + +from collections import Counter +from fnmatch import fnmatchcase +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import build_loss_matrix, registry +from policyengine_uk_data.targets.coverage import ( + BUILDS, + EXCUSABLE, + NEVER_EXCUSED, + STATUSES, + KnownGap, + TargetCoverageError, + TargetStatus, + check_coverage, + column_status, + coverage_problems, + current_build, + load_known_gaps, + parse_known_gaps, +) +from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit + +UC_BANDS = "dwp/uc_payment_dist/*" + + +def test_column_status(): + assert column_status(5.0, [0, 1, 0]) == "in_matrix" + assert column_status(5.0, [0, 0, 0]) == "zero_signal" + # A zero target met by a zero column needs no signal. + assert column_status(0.0, [0, 0, 0]) == "in_matrix" + assert column_status(float("nan"), [1, 1]) == "non_finite" + assert column_status(5.0, [1, np.inf]) == "non_finite" + + +def test_the_shipped_known_gaps_load_and_cite_issues(): + gaps = load_known_gaps() + for gap in gaps: + assert gap.status in EXCUSABLE + assert gap.issue.startswith("https://github.com/PolicyEngine/") + assert gap.reason + listed = [(n, gap.status) for gap in gaps for n in gap.targets] + assert len(listed) == len(set(listed)) + + def named(status): + return sum(len(g.targets) for g in gaps if g.status == status) + + # The 1.58.1 inventory at 2025, in every build, with #557 (SPI bands + # emitted once) and #604 (80-and-over ages in place of 85-90): 15 + # two-child-limit targets with no 2025 value, 17 without signal, and 3 ONS + # targets dropped under a name already taken (#605). + assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [ + 15, + 17, + 3, + ] + assert all(g.builds == set(BUILDS) for g in gaps if g.targets) + # Only in testing builds, any UC payment band may lack signal (#598). + [uc] = [g for g in gaps if g.patterns] + assert (uc.status, uc.patterns, uc.builds) == ( + "zero_signal", + (UC_BANDS,), + {"testing"}, + ) + + +def test_the_shipped_gaps_excuse_any_uc_band_without_signal_in_testing_builds_only(): + gaps = load_known_gaps() + flickers = TargetStatus( + "dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800", + "zero_signal", + ) + listed = TargetStatus( + "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf", + "zero_signal", + ) + broken = TargetStatus(flickers.name, "error", "ValueError: x") + assert coverage_problems([flickers, listed], gaps, "testing")[0] == [] + assert coverage_problems([flickers, listed], gaps, "release")[0] == [ + f"{flickers.name}: zero_signal" + ] + assert coverage_problems([broken], gaps, "testing")[0] == [ + f"{flickers.name}: error (ValueError: x)" + ] + + +def _entry(**kwargs): + return { + "status": "zero_signal", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t"], + **kwargs, + } + + +@pytest.mark.parametrize( + "entry, match", + [ + (_entry(status="error"), "status"), + (_entry(status="non_finite"), "status"), + (_entry(reason=""), "reason"), + (_entry(issue="#456"), "issue"), + (_entry(targets=[]), "no targets"), + (_entry(targets=["t", "t"]), "twice"), + (_entry(builds=["nightly"]), "builds"), + (_entry(builds=[]), "builds"), + (_entry(builds="testing"), "builds"), + # A pattern excuses testing builds only, and must be a pattern. + (_entry(targets=[], patterns=["a/*"]), "testing builds"), + (_entry(targets=[], patterns=["a/*"], builds=["release"]), "testing builds"), + (_entry(targets=[], patterns=["a/b"], builds=["testing"]), "wildcard"), + ( + _entry(status="error", targets=[], patterns=["a/*"], builds=["testing"]), + "status", + ), + ], +) +def test_malformed_known_gaps_are_refused(entry, match): + with pytest.raises(ValueError, match=match): + parse_known_gaps({"schema_version": 1, "gaps": [entry]}) + + +def _gap(status, *names, builds=BUILDS, patterns=()): + return KnownGap( + status, + frozenset(names), + "reason", + "https://github.com/x", + frozenset(builds), + tuple(patterns), + ) + + +def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported(): + statuses = [ + TargetStatus("a", "in_matrix"), + TargetStatus("b", "zero_signal"), + TargetStatus("c", "no_value_for_year"), + TargetStatus("d", "error", "KeyError: x"), + ] + problems, stale = coverage_problems( + statuses, + [_gap("zero_signal", "b", "c"), _gap("no_column", "gone")], + ) + assert problems == ["c: no_value_for_year", "d: error (KeyError: x)"] + assert stale == [ + "c: listed as zero_signal, now no_value_for_year", + "gone: listed as no_column, now not offered by the registry", + ] + + +def test_a_gap_for_one_kind_of_build_excuses_nothing_in_the_other(monkeypatch): + rows = [TargetStatus("dwp/uc_payment_dist/A", "zero_signal")] + for gap in ( + _gap("zero_signal", rows[0].name, builds=["testing"]), + _gap("zero_signal", builds=["testing"], patterns=[UC_BANDS]), + ): + assert check_coverage(rows, [gap], build="testing") == [] + with pytest.raises(TargetCoverageError, match="A: zero_signal"): + check_coverage(rows, [gap], build="release") + release_only = _gap("no_column", "b", builds=["release"]) + rows = [TargetStatus("b", "no_column")] + assert check_coverage(rows, [release_only], build="release") == [] + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [release_only], build="testing") + # Nor is it stale there: it does not apply. + assert coverage_problems([], [release_only], "testing") == ([], []) + # The build kind is read from TESTING, as create_datasets reads it. + monkeypatch.setenv("TESTING", "1") + assert current_build() == "testing" + monkeypatch.setenv("TESTING", "0") + assert current_build() == "release" + monkeypatch.delenv("TESTING") + assert current_build() == "release" + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [_gap("no_column", "b", builds=["testing"])]) + + +def test_a_pattern_excuses_only_its_status_and_is_stale_only_if_it_matches_nothing(): + [gap] = parse_known_gaps( + { + "schema_version": 1, + "gaps": [_entry(targets=[], patterns=[UC_BANDS], builds=["testing"])], + } + ) + rows = [ + TargetStatus("dwp/uc_payment_dist/A", "zero_signal"), + TargetStatus("dwp/uc_payment_dist/B", "in_matrix"), + TargetStatus("dwp/uc_payment_dist/C", "error", "KeyError: x"), + TargetStatus("dwp/uc_payment_dist/D", "non_finite"), + TargetStatus("dwp/uc_payment_dist/E", "no_column"), + TargetStatus("dwp/uc/F", "zero_signal"), + ] + assert coverage_problems(rows, [gap], "testing") == ( + [ + "dwp/uc_payment_dist/C: error (KeyError: x)", + "dwp/uc_payment_dist/D: non_finite", + "dwp/uc_payment_dist/E: no_column", + "dwp/uc/F: zero_signal", + ], + [], + ) + # Excusing nothing in one build is not stale; matching no target is. + assert coverage_problems(rows[1:2], [gap], "testing") == ([], []) + assert coverage_problems( + [TargetStatus("obr/x", "in_matrix")], [gap], "testing" + ) == ( + [], + [f"{UC_BANDS}: listed as zero_signal, matches no target"], + ) + + +NAMES = ["a/x", "a/y", "b/x", "c"] +PATTERNS = ["a/*", "*/x", "z*"] + + +@settings(max_examples=300, deadline=None) +@given( + rows=st.lists(st.tuples(st.sampled_from(NAMES), st.sampled_from(STATUSES))), + build=st.sampled_from(BUILDS), + data=st.data(), +) +def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix( + rows, build, data +): + # Hand-built gaps, so even statuses the file refuses: none may excuse an + # error or a non-finite value. + status = st.sampled_from(EXCUSABLE + NEVER_EXCUSED) + builds = st.sampled_from([BUILDS, ("testing",), ("release",)]) + # A name may be listed under several statuses, each once. + listed = data.draw( + st.dictionaries(st.tuples(st.sampled_from(NAMES + ["zz"]), status), builds) + ) + globbed = data.draw(st.dictionaries(st.sampled_from(PATTERNS), status)) + gaps = [_gap(s, n, builds=b) for (n, s), b in listed.items()] + gaps += [_gap(s, builds=["testing"], patterns=[p]) for p, s in globbed.items()] + statuses = [TargetStatus(n, s) for n, s in rows] + problems, stale = coverage_problems(statuses, gaps, build) + + # The model: only the gaps for this build apply. + named = {pair for pair, b in listed.items() if build in b} + patterns = globbed if build == "testing" else {} + + def excused(name, status): + return status not in NEVER_EXCUSED and ( + (name, status) in named + or any(s == status and fnmatchcase(name, p) for p, s in patterns.items()) + ) + + assert problems == [ + f"{n}: {s}" for n, s in rows if s != "in_matrix" and not excused(n, s) + ] + assert all(f"{n}: {s}" in problems for n, s in rows if s in NEVER_EXCUSED) + stale_pairs = { + (line.split(": listed as ")[0], line.split(": listed as ")[1].split(",")[0]) + for line in stale + } + assert stale_pairs == { + (n, s) for n, s in named if s in NEVER_EXCUSED or (n, s) not in rows + } | { + (p, s) + for p, s in patterns.items() + if not any(fnmatchcase(n, p) for n, _ in rows) + } + # Order of the statuses never changes the verdict. + shuffled = data.draw(st.permutations(statuses)) + assert sorted(coverage_problems(shuffled, gaps, build)[0]) == sorted(problems) + + +def _target(name, values={2025: 10.0}, **kwargs): + return Target( + **{ + "name": name, + "variable": "employment_income", + "source": "test", + "unit": Unit.GBP, + "values": values, + **kwargs, + } + ) + + +class FakeMicrosimulation: + """Two households, in England and Wales.""" + + def __init__(self, dataset=None, reform=None): + pass + + def calculate(self, variable, *args, **kwargs): + assert variable == "country" + return pd.Series(["ENGLAND", "WALES"]) + + +def _sources(*sources): + """Fake source modules, one per list of targets.""" + return lambda: [SimpleNamespace(get_targets=lambda s=s: list(s)) for s in sources] + + +@pytest.fixture +def fake_matrix(monkeypatch): + """The target matrix from chosen sources, with columns chosen by name.""" + import policyengine_uk + + monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + + def configure(columns, *sources): + monkeypatch.setattr(registry, "discover_source_modules", _sources(*sources)) + + def compute(target, ctx, year): + column = columns[target.name] + if isinstance(column, Exception): + raise column + return None if column is None else np.asarray(column) + + monkeypatch.setattr(build_loss_matrix, "_compute_column", compute) + return SimpleNamespace(time_period="2025") + + return configure + + +def _statuses_by_name(statuses): + by_name = {} + for row in statuses: + by_name.setdefault(row.name, []).append(row.status) + return by_name + + +def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")}, + [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")], + ) + with pytest.raises(TargetCoverageError) as raised: + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + message = str(raised.value) + assert "later: no_value_for_year (values for 2026)" in message + assert "broken: error (KeyError: 'no such variable')" in message + df, values = build_loss_matrix.create_target_matrix( + dataset, time_period="2025", require_coverage=False + ) + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("no_value_for_year", "later")], + ) + with pytest.raises(TargetCoverageError, match="broken"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_without_signal_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}, [_target("ok"), _target("flat")] + ) + with pytest.raises(TargetCoverageError, match="flat: zero_signal"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "flat")], + ) + df, values = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + # Known gaps still enter the matrix: the change is that they are declared. + assert list(df.columns) == ["ok", "flat"] + + +def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch): + dataset = fake_matrix({"nan": [np.nan, 1.0]}, [_target("nan")]) + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "nan")], + ) + with pytest.raises(TargetCoverageError, match="nan: non_finite"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_name_two_sources_emit_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + """Scenario A: two source modules emit the same national name.""" + dataset = fake_matrix( + {"obr/foo": [1.0, 2.0]}, + [_target("obr/foo")], + [_target("obr/foo", values={2025: 20.0})], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"obr/foo": 10.0} + assert statuses == [ + TargetStatus( + "obr/foo", + "duplicate", + "test national target for 2025 dropped for the test national one for 2025", + ), + TargetStatus("obr/foo", "in_matrix"), + ] + with pytest.raises(TargetCoverageError, match="obr/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("duplicate", "obr/foo")], + ) + df, _ = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + assert list(df.columns) == ["obr/foo"] + + +def test_a_regional_target_named_like_a_national_one_fails(fake_matrix): + """Scenario B: the national target wins even when emitted second.""" + dataset = fake_matrix( + {"ons/foo": [1.0, 2.0]}, + [ + _target( + "ons/foo", + values={2024: 5.0}, + geographic_level=GeographicLevel.REGION, + source="ons", + ), + _target("ons/foo"), + ], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"ons/foo": 10.0} + assert _statuses_by_name(statuses) == {"ons/foo": ["duplicate", "in_matrix"]} + assert statuses[0].detail == ( + "ons region target for 2024 dropped for the test national one for 2025" + ) + with pytest.raises(TargetCoverageError, match="ons/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_that_cannot_enter_the_matrix_has_one_error_status(fake_matrix): + """The status follows the column into the matrix, so a column that fails + to go in, or whose status fails, leaves only an error and no column.""" + dataset = fake_matrix( + { + "ok": [1.0, 2.0], + "long": [1.0, 2.0, 3.0], + "text": np.array(["a", "b"], dtype=object), + }, + [_target("ok"), _target("long"), _target("text")], + ) + df, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + assert _statuses_by_name(statuses) == { + "ok": ["in_matrix"], + "long": ["error"], + "text": ["error"], + } + + +def _main_create_target_matrix(dataset, time_period): + """create_target_matrix and get_all_targets as on main before the gate + (daad71cd), with module references made explicit: the differential + test's reference.""" + from policyengine_uk import Microsimulation + + def get_all_targets(geographic_level): + all_targets, seen_names = [], set() + for mod in registry.discover_source_modules(): + for target in mod.get_targets(): + if target.name in seen_names: + continue + if target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) + return all_targets + + year = int(time_period) + sim = Microsimulation(dataset=dataset) + sim.default_calculation_period = time_period + ctx = build_loss_matrix._SimContext(sim, time_period, dataset, None) + all_targets = [] + seen = set() + for level in ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + ): + for t in get_all_targets(geographic_level=level): + if t.name not in seen: + seen.add(t.name) + all_targets.append(t) + df = pd.DataFrame() + target_names = [] + target_values = [] + for target in all_targets: + try: + val = build_loss_matrix._resolve_value(target, year) + if val is None: + continue + col = build_loss_matrix._compute_column(target, ctx, year) + if col is None: + continue + df[target.name] = build_loss_matrix.restrict_to_countries( + col, ctx.country, target.countries + ) + target_names.append(target.name) + target_values.append(val) + except Exception: + pass + return df, pd.Series(target_values, index=target_names) + + +def _column(kind): + """A custom_compute giving a fixed column, no column, or an error.""" + if kind == "none": + return lambda ctx, target, year: None + if kind == "error": + + def raises(ctx, target, year): + raise KeyError("no such variable") + + return raises + return lambda ctx, target, year: np.asarray(kind, dtype=float) + + +emitted_target = st.builds( + lambda name, level, year, value, column, countries: _target( + name, + values={year: value}, + geographic_level=level, + countries=countries, + custom_compute=_column(column), + ), + name=st.sampled_from(["a", "b", "c"]), + level=st.sampled_from( + [ + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + GeographicLevel.CONSTITUENCY, + ] + ), + # 2023 carries forward to 2025; 2026 does not carry back. + year=st.sampled_from([2023, 2025, 2026]), + value=st.sampled_from([0.0, 10.0, float("nan")]), + column=st.one_of( + st.sampled_from(["none", "error"]), + st.lists( + st.sampled_from([0.0, 1.0, -2.5, np.nan, np.inf]), min_size=2, max_size=2 + ), + ), + countries=st.sampled_from([None, ("ENGLAND",)]), +) + + +@settings(max_examples=200, deadline=None) +@given(sources=st.lists(st.lists(emitted_target, max_size=5), min_size=1, max_size=3)) +def test_without_the_gate_the_matrix_is_mains(sources): + """require_coverage=False builds exactly the matrix main built, from the + same sources, duplicates included; and every target offered at a matrix + level gets exactly one status. + + The one intended difference, not generated here: a column whose status + cannot be computed (a non-numeric column) now leaves the matrix as an + error, where main kept it. + """ + import policyengine_uk + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + patch.setattr(registry, "discover_source_modules", _sources(*sources)) + dataset = SimpleNamespace(time_period="2025") + df, values = build_loss_matrix.create_target_matrix( + dataset, "2025", require_coverage=False + ) + main_df, main_values = _main_create_target_matrix(dataset, "2025") + _, _, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + pd.testing.assert_frame_equal(df, main_df) + pd.testing.assert_series_equal(values, main_values) + offered = Counter( + t.name + for source in sources + for t in source + if t.geographic_level in build_loss_matrix.MATRIX_LEVELS + ) + assert Counter(row.name for row in statuses) == offered + + +def test_stale_gaps_warn_without_failing(caplog): + caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage") + stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")]) + assert stale == ["a: listed as zero_signal, now in_matrix"] + assert "Stale known calibration-target gap" in caplog.text + + +def test_stale_gaps_and_problems_reach_the_job_summary_once(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + gaps = [_gap("zero_signal", "a")] + # The constituency and local authority calibrations both check. + for _ in range(2): + check_coverage([TargetStatus("a", "in_matrix")], gaps, build="release") + text = summary.read_text() + assert text.count("`a: listed as zero_signal, now in_matrix`") == 1 + assert text.count("### Calibration target coverage (release build)") == 1 + with pytest.raises(TargetCoverageError): + check_coverage([TargetStatus("b", "no_column")], gaps, build="testing") + assert ( + "Not in the matrix and not a known gap: `b: no_column`" in summary.read_text() + ) + monkeypatch.delenv("GITHUB_STEP_SUMMARY") + check_coverage([TargetStatus("c", "in_matrix")], [_gap("zero_signal", "c")]) + assert "`c:" not in summary.read_text() + + +def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs): + """On the built dataset the gate passes: the known gaps are complete.""" + from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE + + _, _, statuses = build_loss_matrix.build_target_matrix( + enhanced_frs, str(CURRENT_FRS_RELEASE.calibration_year) + ) + problems, _ = coverage_problems(statuses, load_known_gaps()) + assert problems == [] diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index c2a1c1c37..9c8e93cf5 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -7,8 +7,12 @@ 4. Target values match the current system's hardcoded values """ +from collections import Counter + from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.coverage import load_known_gaps +from policyengine_uk_data.targets.registry import emitted_targets def test_registry_loads(): @@ -18,11 +22,20 @@ def test_registry_loads(): def test_no_duplicate_names(): - """Target names should be unique across all sources.""" - targets = get_all_targets() - names = [t.name for t in targets] - duplicates = [n for n in names if names.count(n) > 1] - assert len(duplicates) == 0, f"Duplicate target names: {set(duplicates)}" + """Each name is emitted once, apart from the duplicates known_gaps.yaml lists. + + get_all_targets keeps only the first target of each name, so this counts + what the sources emit before that. + """ + counts = Counter(t.name for t in emitted_targets()) + listed = { + name + for gap in load_known_gaps() + if gap.status == "duplicate" + for name in gap.targets + } + duplicates = sorted(n for n, k in counts.items() if k > 1 and n not in listed) + assert not duplicates, f"Duplicate target names: {duplicates}" def test_obr_income_tax_exists(): diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index 27eb919fd..cae347e82 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -23,7 +23,11 @@ def get_loss_results(dataset, time_period, reform=None, household_weights=None): Returns: DataFrame with estimate vs target comparisons and error metrics. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # A diagnostic of any dataset: report every target, fitted or not, + # rather than enforce the Enhanced FRS build's coverage gate. + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) from policyengine_uk import Microsimulation if household_weights is None: diff --git a/policyengine_uk_data/utils/reweight.py b/policyengine_uk_data/utils/reweight.py index 0fc9da8bd..e94f9203b 100644 --- a/policyengine_uk_data/utils/reweight.py +++ b/policyengine_uk_data/utils/reweight.py @@ -38,7 +38,11 @@ def calibrate_household_weights( datasets such as the UK enhanced CPS. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # Only the CPS-based transfer dataset is reweighted here; the coverage + # gate is for the Enhanced FRS build (datasets/create_datasets.py). + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) household_count = len(dataset.household) if household_count == 0: raise ValueError("Dataset must contain at least one household") diff --git a/tools/ons_age_targets/README.md b/tools/ons_age_targets/README.md new file mode 100644 index 000000000..7d223513f --- /dev/null +++ b/tools/ons_age_targets/README.md @@ -0,0 +1,56 @@ +# ONS age target sources + +`policyengine_uk_data/storage/regional_age_80_plus.csv` contains public population +projection aggregates in persons. It contains no FRS data. Reproduce it from the +repository root with the locked environment: + +```sh +uv run python tools/ons_age_targets/build.py +``` + +Downloads are cached in `.job/`; SHA-256 checks pin the source files. The generator +uses pandas, requests and openpyxl, which are already locked dependencies. + +The English regions use the [ONS 2022-based single-year subnational +projection](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1), +five-year migration variant, published 24 June 2025. The [ONS release bulletin](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/subnationalpopulationprojectionsforengland/2022based) +identifies this variant as the one aligned with the 2022-based principal national +projection. For each of +the 309 English local authorities, the extract sums ages 80 through 89 and the +terminal `90 and over` group. The committed `storage/oa_crosswalk.csv.gz` maps all +309 authorities unambiguously to the nine English regions. Summing those regions +therefore reproduces the sum of all 309 authorities, without omitting or counting +an authority twice. Published counts have three decimal places; the output retains +that precision. + +Scotland, Wales and Northern Ireland use the ONS **2022-based principal national +projections**, so all 12 regional 80+ targets have the same projection vintage and +cover 2022–2029. Each country total sums both sexes at ages 80–104 and the terminal +groups `105 - 109` and `110 and over`. The regional CSV includes the reference URL +for each row. This replaces the legacy regional top-band values; it is a source +refresh as well as a correction to the open-ended band definition. The lower +regional age bands remain in `demographics.csv`. + +Pinned downloads, relative to +`https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/`: + +| Source | Download suffix | SHA-256 | +| --- | --- | --- | +| English local authorities | `localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip` | `171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b` | +| Scotland | `z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip` | `6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d` | +| Wales | `z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip` | `32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051` | +| Northern Ireland | `z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip` | `69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489` | + +The national targets continue to use the existing **2024-based UK principal +projection**. The test fixture `policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv` preserves +its complete `Sex`, `Age` and `2025` population columns, including `105 - 109` and +`110 and over`. Its source ZIP is +`z1zippedpopulationprojectionsdatafilesuk/2024based/uk.zip`, SHA-256 +`dea79703b9caddd560f1958b5a8ca12aa817b2396d244e6a2c775b1af7f48ab1`. + +The old regional rows first appeared in commit `c98c0ccf` (9 September 2024); +`63dfba48` (10 June 2025) reformatted them without changing their values. Their +reference `ons_age_sex_region` does not identify a projection vintage or extraction +method. Their finite `80_89` names alone do not prove that their published values +excluded 90+: the replacement uses directly reproducible sums rather than adding +an estimated 90+ population to those legacy values. diff --git a/tools/ons_age_targets/build.py b/tools/ons_age_targets/build.py new file mode 100644 index 000000000..b29be1a57 --- /dev/null +++ b/tools/ons_age_targets/build.py @@ -0,0 +1,138 @@ +"""Reproduce the public ONS regional 80+ target extract (no survey data).""" + +import argparse +import hashlib +import io +import zipfile +from pathlib import Path + +import pandas as pd +import requests + +ROOT = Path(__file__).resolve().parents[2] +YEARS = [str(year) for year in range(2022, 2030)] +ONS = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" +SOURCES = { + "regions": ( + "localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip", + "171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b", + ), + "scotland": ( + "z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip", + "6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d", + ), + "wales": ( + "z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip", + "32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051", + ), + "ni": ( + "z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip", + "69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489", + ), +} +REGIONS = { + "E12000001": "north_east", + "E12000002": "north_west", + "E12000003": "yorkshire_and_the_humber", + "E12000004": "east_midlands", + "E12000005": "west_midlands", + "E12000006": "east", + "E12000007": "london", + "E12000008": "south_east", + "E12000009": "south_west", +} + + +def aggregate_english_regions( + population: pd.DataFrame, crosswalk: pd.DataFrame +) -> pd.DataFrame: + """Sum ages 80 through the terminal 90+ row in every English LA.""" + lookup = crosswalk[["la_code", "region_code"]].drop_duplicates() + assert lookup.la_code.is_unique, "Local authorities have ambiguous regions" + ages = population.AGE_GROUP.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[pd.to_numeric(ages, errors="coerce") >= 80].copy() + assert set(older.AGE_GROUP) == {str(age) for age in range(80, 90)} | {"90 and over"} + older = older.merge( + lookup, + left_on="AREA_CODE", + right_on="la_code", + how="left", + validate="many_to_one", + ) + assert older.region_code.notna().all(), "Some local authorities lack a region" + assert set(older.region_code) == set(REGIONS) + assert older.AREA_CODE.nunique() == 309 + assert older.groupby("AREA_CODE").AGE_GROUP.nunique().eq(11).all() + result = older.groupby("region_code")[YEARS].sum().round(3) + assert result.sum().round(3).equals(older[YEARS].sum().round(3)) + result.index = result.index.map(REGIONS) + return result + + +def aggregate_country(population: pd.DataFrame) -> pd.Series: + """Include all single years and terminal age groups from 80 upwards.""" + ages = population.Age.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[ + (pd.to_numeric(ages, errors="coerce") >= 80) + & population.Sex.isin(["Females", "Males"]) + ] + assert {"105 - 109", "110 and over"}.issubset(set(older.Age.astype(str))) + older = older.rename(columns=lambda column: str(column)) + return older[YEARS].sum() + + +def build(source_dir: Path, output: Path) -> None: + """Download pinned public sources and save their exact published sums.""" + source_dir.mkdir(parents=True, exist_ok=True) + for name, (suffix, expected_sha256) in SOURCES.items(): + path = source_dir / f"ons-{name}-2022.zip" + if not path.exists(): + response = requests.get( + ONS + suffix, + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + }, + timeout=120, + ) + response.raise_for_status() + path.write_bytes(response.content) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256 + + with zipfile.ZipFile(source_dir / "ons-regions-2022.zip") as archive: + population = pd.read_csv(archive.open("2022 SNPP Population persons.csv")) + crosswalk = pd.read_csv(ROOT / "policyengine_uk_data/storage/oa_crosswalk.csv.gz") + totals = aggregate_english_regions(population, crosswalk) + for country, prefix in [("scotland", "sc"), ("wales", "wa"), ("ni", "ni")]: + with zipfile.ZipFile(source_dir / f"ons-{country}-2022.zip") as archive: + population = pd.read_excel( + io.BytesIO( + archive.read(f"{prefix}/{prefix}_ppp_machine_readable.xlsx") + ), + sheet_name="Population", + engine="openpyxl", + ) + totals.loc["northern_ireland" if country == "ni" else country] = ( + aggregate_country(population) + ) + totals.index = [f"{region}_age_80_plus" for region in totals.index] + totals.index.name = "name" + reference_base = "https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" + totals["reference_url"] = reference_base + "localauthoritiesinenglandz1" + for country in ["scotland", "wales", "ni"]: + region = "northern_ireland" if country == "ni" else country + totals.loc[f"{region}_age_80_plus", "reference_url"] = ( + reference_base + SOURCES[country][0].split("/")[0] + ) + totals.sort_index().to_csv(output, float_format="%.3f") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", type=Path, default=ROOT / ".job") + parser.add_argument( + "--output", + type=Path, + default=ROOT / "policyengine_uk_data/storage/regional_age_80_plus.csv", + ) + args = parser.parse_args() + build(args.source_dir, args.output)