From ca87af441bcf5d6830d2255ac84f9e230a422197 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 17:29:48 -0400 Subject: [PATCH 01/13] Map the 2023-24 SPI and CGT outturns to PolicyEngine year 2023 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PolicyEngine UK's year N is tax year N to N+1: convert_to_fiscal_year_parameters gives every parameter, the uprating indices included, its value at 30 April N. hmrc_spi mapped the 2023-24 SPI to 2024 and hmrc_cgt followed, so both treated 2023-24 outturns as 2024-25 values and every projected year carried one year too little uprating. The FRS release (2024-25 -> 2024), OBR targets, the SPI microdata (2022-23 -> 2022) and #533's salary sacrifice tables already use the start year; the end-year mapping arrived without stated reasoning in the February 2026 registry refactor (de061fc). - _SPI_YEAR and _CGT_BASE_YEAR move to 2023; incomes_projection.csv is regenerated from the same ODS (its 2023 rows equal main's 2024 rows). - The CGT base year is now gated at the £6,000 AEA the outturn was realised under. - Tests tie _SPI_YEAR to the tax year in the configured ODS name, check the CGT base-year AEA against policyengine-uk, and (as in #541) check that the committed projection is the SPI table uprated with the committed factors. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-cgt-tax-year-start.fixed.md | 1 + .../storage/incomes_projection.csv | 182 ++++++++++-------- .../targets/sources/hmrc_cgt.py | 30 +-- .../targets/sources/hmrc_spi.py | 6 +- .../tests/test_hmrc_cgt_targets.py | 16 +- .../tests/test_income_projection.py | 29 +++ .../tests/test_property_income_targets.py | 3 +- 7 files changed, 165 insertions(+), 102 deletions(-) create mode 100644 changelog.d/spi-cgt-tax-year-start.fixed.md diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md new file mode 100644 index 00000000..b49b12b5 --- /dev/null +++ b/changelog.d/spi-cgt-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating. diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index b8db818e..961476a1 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -1,85 +1,99 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employment_income_amount,self_employment_income_count,self_employment_income_amount,state_pension_count,state_pension_amount,private_pension_income_count,private_pension_income_amount,property_income_count,property_income_amount,savings_interest_income_count,savings_interest_income_amount,dividend_income_count,dividend_income_amount,year -12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2024 -15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2024 -20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2024 -30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2024 -40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2024 -50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2024 -70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2024 -100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2024 -150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2024 -200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2024 -300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2024 -500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 -1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 -12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17529896907,449188,5033426819,1153320,12181568310,1082502,4556072351,102180,688579832,1264606,281598319,116344,179852941,2025 -15000,20000.0,3136222,50411419508,658607,8402997371,2073953,22917865804,2165005,14244272179,234711,1736865546,2873184,1407991597,376347,878710084,2025 -20000,30000.0,7557283,178410785091,972229,16429272568,2347108,25705173808,2691081,31630663221,436036,3925932773,5311344,3216798319,852850,3422344538,2025 -30000,40000.0,5432746,176336241079,601952,13185276074,1173554,12697736459,1466943,26079586563,368253,3905378151,3733116,2600159664,674794,4830336134,2025 -40000,50000.0,3540896,144180808882,363195,9135512708,574637,6214664511,788102,17700602929,328797,3884823529,2397692,1921857143,634326,8632941176,2025 -50000,70000.0,3470078,180485329104,314634,8601822962,446153,4810687146,650513,18224289406,401639,5549747899,2256056,2404890756,650513,12846638655,2025 -70000,100000.0,1659163,122398096749,144671,5608974584,178056,1951115602,262026,9918621878,228641,3874546218,1082502,1664924370,339926,9701781513,2025 -100000,150000.0,800242,85367486122,85993,5211323401,74865,874388844,110274,5687235142,138601,2620714286,494714,1294941176,224594,7060512605,2025 -150000,200000.0,267085,40661062649,40467,3819544259,21245,273569119,29339,1895745047,54631,1181890756,147706,668025210,93075,3545672269,2025 -200000,300000.0,182103,37964155432,34397,4593917616,14164,184788197,20234,1644375538,41479,1079117647,112297,753326891,75876,3802605042,2025 -300000,500000.0,94087,30080888184,21245,5064820333,8093,99104285,10117,1173057709,23269,786214286,68795,691663025,49573,3525117647,2025 -500000,1000000.0,44514,23960983347,15175,6979824715,3035,44390461,4047,670318691,13152,487144538,38444,671108403,32374,3925932773,2025 -1000000,inf,21245,36823156225,11129,20405784400,2023,20646726,2023,387527993,7082,451173950,21245,1274386555,19222,11407815126,2025 -12570,inf,27581555,1124610309278,3712882,112472497809,8070207,87975699272,9282205,133812368648,2378470,30172129412,19801701,18851671429,4139813,73760260504,2025 -12570,15000.0,1385161,17918556701,452214,5189404032,1161091,12410509297,1089796,4724677003,102869,708285714,1273126,289657143,117128,185000000,2026 -15000,20000.0,3157352,51529103886,663044,8663391762,2087926,23348585287,2179591,14771403962,236292,1786571429,2892541,1448285714,378882,903857143,2026 -20000,30000.0,7608199,182366375892,978779,16938387379,2362921,26188278092,2709211,32801205857,438974,4038285714,5347128,3308857143,858596,3520285714,2026 -30000,40000.0,5469348,180245836638,606008,13593865031,1181461,12936378335,1476826,27044702842,370734,4017142857,3758267,2674571429,679340,4968571429,2026 -40000,50000.0,3564752,147377478192,365642,9418606486,578508,6331463217,793412,18355641688,331013,3996000000,2413846,1976857143,638600,8880000000,2026 -50000,70000.0,3493457,184486915147,316754,8868378615,449159,4901099434,654896,18898708010,404345,5708571429,2271256,2473714286,654896,13214285714,2026 -70000,100000.0,1670341,125111816019,145646,5782787029,179256,1987784964,263792,10285676141,230181,3985428571,1089796,1712571429,342216,9979428571,2026 -100000,150000.0,805634,87260190325,86573,5372813322,75369,890822150,111017,5897700258,139535,2695714286,498047,1332000000,226107,7262571429,2026 -150000,200000.0,268884,41562569389,40740,3937905346,21389,278710590,29537,1965900086,54999,1215714286,148701,687142857,93702,3647142857,2026 -200000,300000.0,183330,38805868358,34629,4736275197,14259,188261116,20370,1705228252,41759,1110000000,113054,774885714,76388,3911428571,2026 -300000,500000.0,94721,30747819191,21389,5221770377,8148,100966855,10185,1216468562,23426,808714286,69258,711457143,49907,3626000000,2026 -500000,1000000.0,44814,24492228390,15278,7196117441,3056,45224737,4074,695124892,13241,501085714,38703,690314286,32592,4038285714,2026 -1000000,inf,21389,37639571768,11204,21038124452,2037,21034762,2037,401869078,7130,464085714,21389,1310857143,19352,11734285714,2026 -12570,inf,27767379,1149544329897,3737897,115957826468,8124578,89629118836,9344742,138764306632,2394495,31035600000,19935110,19391171429,4167704,75871142857,2026 -12570,15000.0,1395755,18293814433,455673,5353812445,1169971,12658528698,1098130,4825839793,103655,730243697,1282863,298636975,118023,190735294,2027 -15000,20000.0,3181500,52608247423,668115,8937861525,2103895,23815198060,2196261,15087683032,238099,1841957983,2914664,1493184874,381780,931878151,2027 -20000,30000.0,7666388,186185567010,986265,17475021911,2380993,26711641067,2729932,33503531438,442331,4163478992,5388023,3411436975,865163,3629420168,2027 -30000,40000.0,5511178,184020618557,610643,14024539877,1190497,13194907033,1488121,27623772610,373570,4141680672,3787011,2757487395,684536,5122605042,2027 -40000,50000.0,3592016,150463917526,368438,9717002629,582933,6457995150,799480,18748664944,333544,4119882353,2432308,2038142857,643484,9155294118,2027 -50000,70000.0,3520175,188350515464,319176,9149342682,452594,4999046079,659905,19303359173,407437,5885546218,2288627,2550403361,659905,13623949580,2027 -70000,100000.0,1683116,127731958763,146759,5965994741,180627,2027510105,265809,10505908699,231942,4108983193,1098130,1765663866,344833,10288806723,2027 -100000,150000.0,811796,89087628866,87235,5543032428,75945,908624899,111866,6023979328,140602,2779285714,501856,1373294118,227836,7487722689,2027 -150000,200000.0,270941,42432989691,41052,4062664330,21552,284280517,29762,2007993109,55420,1253403361,149838,708445378,94419,3760210084,2027 -200000,300000.0,184732,39618556701,34894,4886327783,14368,192023444,20526,1741739879,42078,1144411765,113918,798908403,76972,4032689076,2027 -300000,500000.0,95445,31391752577,21552,5387204207,8210,102984640,10263,1242515073,23605,833785714,69788,733513445,50288,3738411765,2027 -500000,1000000.0,45157,25005154639,15394,7424101665,3079,46128537,4105,710008613,13342,516620168,38999,711715126,32841,4163478992,2027 -1000000,inf,21552,38427835052,11289,21704645048,2053,21455133,2053,410473730,7184,478473109,21552,1351495798,19500,12098067227,2027 -12570,inf,27979749,1173618556701,3766485,119631551271,8186717,91420323363,9416212,141735469423,2412808,31997752941,20087577,19992328571,4199579,78223268908,2027 -12570,15000.0,1401052,18709278351,457402,5543514461,1174411,12906548100,1102298,4927002584,104049,754453782,1287731,308537815,118471,197058824,2028 -15000,20000.0,3193574,53803013481,670650,9254557406,2111879,24281810833,2204596,15403962102,239003,1903025210,2925725,1542689076,383229,962773109,2028 -20000,30000.0,7695482,190413957177,990008,18094215600,2390029,27235004042,2740292,34205857020,444010,4301512605,5408471,3524537815,868446,3749747899,2028 -30000,40000.0,5532093,188199841396,612960,14521472393,1195015,13453435732,1493768,28202842377,374987,4278991597,3801383,2848907563,687133,5292436975,2028 -40000,50000.0,3605648,153881046788,369836,10061305872,585145,6584527082,802514,19141688200,334810,4256470588,2441538,2105714286,645926,9458823529,2028 -50000,70000.0,3533535,192628072958,320388,9473531989,454312,5096992724,662409,19708010336,408983,6080672269,2297313,2634957983,662409,14075630252,2028 -70000,100000.0,1689503,130632831086,147316,6177388256,181313,2067235247,266818,10726141258,232822,4245210084,1102298,1824201681,346142,10629915966,2028 -100000,150000.0,814876,91110864393,87566,5739439089,76234,926427648,112290,6150258398,141135,2871428571,503760,1418823529,228701,7735966387,2028 -150000,200000.0,271969,43396669310,41207,4206617003,21634,289850445,29875,2050086133,55630,1294957983,150407,731932773,94777,3884873950,2028 -200000,300000.0,185433,40518318795,35026,5059465381,14423,195785772,20604,1778251507,42238,1182352941,114351,825394958,77264,4166386555,2028 -300000,500000.0,95807,32104678826,21634,5578089395,8241,105002425,10302,1268561585,23694,861428571,70053,757831933,50479,3862352941,2028 -500000,1000000.0,45328,25573037272,15453,7687160386,3091,47032336,4121,724892334,13392,533747899,39147,735310924,32966,4301512605,2028 -1000000,inf,21634,39300555115,11332,22473707274,2060,21875505,2060,419078381,7211,494336134,21634,1396302521,19574,12499159664,2028 -12570,inf,28085934,1200272164948,3780779,123870464505,8217786,93211527890,9451947,144706632214,2421965,33058588235,20163811,20655142857,4215517,80816638655,2028 -12570,15000.0,1408997,19178350515,459996,5754294479,1181071,13164106710,1108549,5043152455,104639,779226891,1295034,318668908,119143,203529412,2029 -15000,20000.0,3211685,55151942902,674454,9606441718,2123856,24766370251,2217098,15767097330,240358,1965512605,2942317,1593344538,385402,994386555,2029 -20000,30000.0,7739124,195187946075,995622,18782208589,2403583,27778496362,2755833,35012230835,446528,4442756303,5439143,3640268908,873371,3872873950,2029 -30000,40000.0,5563466,192918318795,616436,15073619632,1201792,13721907842,1502240,28867700258,377114,4419495798,3822941,2942453782,691030,5466218487,2029 -40000,50000.0,3626095,157739095956,371934,10443865031,588463,6715925627,807065,19592937123,336709,4396235294,2455385,2174857143,649589,9769411765,2029 -50000,70000.0,3553574,197457573354,322204,9833742331,456888,5198706548,666166,20172609819,411303,6280336134,2310341,2721478992,666166,14537815126,2029 -70000,100000.0,1699085,133908009516,148152,6412269939,182341,2108488278,268331,10979000861,234142,4384605042,1108549,1884100840,348105,10978957983,2029 -100000,150000.0,819498,93395162569,88062,5957668712,76666,944915117,112927,6295245478,141936,2965714286,506617,1465411765,229998,7989983193,2029 -150000,200000.0,273511,44484694687,41441,4366564417,21757,295634600,30045,2098415159,55945,1337478992,151260,755966387,95315,4012436975,2029 -200000,300000.0,186485,41534179223,35225,5251840491,14504,199692805,20721,1820172265,42477,1221176471,114999,852497479,77702,4303193277,2029 -300000,500000.0,96351,32909595559,21757,5790184049,8288,107097817,10360,1298466839,23829,889714286,70450,782715966,50765,3989176471,2029 -500000,1000000.0,45585,26214195083,15540,7979447853,3108,47970897,4144,741981051,13468,551273950,39369,759455462,33153,4442756303,2029 -1000000,inf,21757,40285884219,11396,23328220859,2072,22312045,2072,428957795,7252,510568067,21757,1442151261,19685,12909579832,2029 -12570,inf,28245211,1230364948454,3802220,128580368098,8264389,95071624899,9505550,148117967270,2435700,34144094118,20278162,21333371429,4239424,83470319328,2029 +12570,15000.0,1360000,16900000000,444000,4810000000,1140000,11800000000,1070000,4350000000,101000,670000000,1250000,274000000,115000,175000000,2023 +15000,20000.0,3100000,48600000000,651000,8030000000,2050000,22200000000,2140000,13600000000,232000,1690000000,2840000,1370000000,372000,855000000,2023 +20000,30000.0,7470000,172000000000,961000,15700000000,2320000,24900000000,2660000,30200000000,431000,3820000000,5250000,3130000000,843000,3330000000,2023 +30000,40000.0,5370000,170000000000,595000,12600000000,1160000,12300000000,1450000,24900000000,364000,3800000000,3690000,2530000000,667000,4700000000,2023 +40000,50000.0,3500000,139000000000,359000,8730000000,568000,6020000000,779000,16900000000,325000,3780000000,2370000,1870000000,627000,8400000000,2023 +50000,70000.0,3430000,174000000000,311000,8220000000,441000,4660000000,643000,17400000000,397000,5400000000,2230000,2340000000,643000,12500000000,2023 +70000,100000.0,1640000,118000000000,143000,5360000000,176000,1890000000,259000,9470000000,226000,3770000000,1070000,1620000000,336000,9440000000,2023 +100000,150000.0,791000,82300000000,85000,4980000000,74000,847000000,109000,5430000000,137000,2550000000,489000,1260000000,222000,6870000000,2023 +150000,200000.0,264000,39200000000,40000,3650000000,21000,265000000,29000,1810000000,54000,1150000000,146000,650000000,92000,3450000000,2023 +200000,300000.0,180000,36600000000,34000,4390000000,14000,179000000,20000,1570000000,41000,1050000000,111000,733000000,75000,3700000000,2023 +300000,500000.0,93000,29000000000,21000,4840000000,8000,96000000,10000,1120000000,23000,765000000,68000,673000000,49000,3430000000,2023 +500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2023 +1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2023 +12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2023 +12570,15000.0,1373373,17685394191,448366,5039678604,1151209,12073283706,1080521,4566320072,101993,695117698,1262291,284272014,116131,181560593,2024 +15000,20000.0,3130482,50858589212,657401,8413434343,2070157,22714143921,2161042,14276311031,234281,1753356582,2867925,1421360070,375658,887053182,2024 +20000,30000.0,7543451,179993360996,970449,16449678604,2342812,25476674938,2686155,31701808318,435238,3963208370,5301622,3247340889,851289,3454838710,2024 +30000,40000.0,5422802,177900414938,600851,13201652893,1171406,12584863524,1464258,26138245931,367579,3942458588,3726283,2624847428,673559,4876198779,2024 +40000,50000.0,3534415,145459751037,362530,9146859504,573585,6159421009,786660,17740415913,328196,3921708806,2393304,1940104621,633165,8714908457,2024 +50000,70000.0,3463727,182086307054,314058,8612506887,445336,4767923904,649323,18265280289,400904,5602441151,2251927,2427724499,649323,12968613775,2024 +70000,100000.0,1656126,123483817427,144406,5615941230,177731,1933771712,261547,9940931284,228222,3911333915,1080521,1680732345,339304,9793897123,2024 +100000,150000.0,798778,86124730290,85836,5217796143,74728,866616212,110072,5700027125,138347,2645597210,493808,1307236269,224183,7127550131,2024 +150000,200000.0,266596,41021742739,40393,3824288338,21206,271137304,29285,1900009042,54531,1193112467,147436,674367916,92905,3579337402,2024 +200000,300000.0,181770,38300912863,34334,4599623508,14138,183145575,20197,1648074141,41403,1089363557,112091,760479512,75737,3838709677,2024 +300000,500000.0,93914,30347717842,21206,5071111111,8079,98223325,10098,1175696203,23226,793679163,68669,698230166,49482,3558587620,2024 +500000,1000000.0,44433,24173526971,15147,6988494031,3029,43995864,4039,671826401,13128,491769834,38374,677480384,32315,3963208370,2024 +1000000,inf,21206,37149792531,11108,20431129477,2020,20463193,2020,388399638,7069,455457716,21206,1286486486,19187,11516129032,2024 +12570,inf,27531073,1134586058091,3706087,112612194674,8055437,87193664185,9265216,134113345389,2374117,30458605057,19765458,19030662598,4132236,74460592851,2024 +12570,15000.0,1389420,18344564315,453605,5273774105,1164661,12463688999,1093147,4782640145,103185,714394071,1277040,292155187,117488,186595466,2025 +15000,20000.0,3167060,52754190871,665083,8804242424,2094346,23448635236,2186293,14952622061,237019,1801979076,2901436,1460775937,380047,911652136,2025 +20000,30000.0,7631593,186702074689,981789,17213774105,2370187,26300496278,2717542,33203616637,440324,4073112467,5363569,3337393200,861236,3550645161,2025 +30000,40000.0,5486165,184531120332,607871,13814876033,1185093,12991811414,1481367,27376491863,371874,4051787271,3769823,2697637315,681429,5011421099,2025 +40000,50000.0,3575713,150881327801,366766,9571735537,580287,6358593879,795852,18580831826,332030,4030462075,2421268,1993905841,640563,8956582389,2025 +50000,70000.0,3504199,188873029046,317728,9012561983,450540,4922100910,656910,19130560579,405588,5757802964,2278240,2495047951,656910,13328247602,2025 +70000,100000.0,1675477,128086307054,146093,5876804408,179807,1996302730,264603,10411862568,230889,4019799477,1093147,1727340889,343268,10065492589,2025 +100000,150000.0,808111,89334771784,86839,5460165289,75601,894639371,111358,5970054250,139964,2718962511,499578,1343487358,226802,7325204882,2025 +150000,200000.0,269711,42550705394,40865,4001928375,21454,279904880,29627,1990018083,55168,1226198779,149158,693068875,93990,3678596338,2025 +200000,300000.0,183894,39728464730,34735,4813278237,14303,189067825,20433,1726148282,41887,1119572799,113401,781568439,76622,3945161290,2025 +300000,500000.0,95012,31478838174,21454,5306666667,8173,101399504,10216,1231392405,23498,815688753,69471,717592851,50060,3657271142,2025 +500000,1000000.0,44952,25074522822,15324,7313112948,3065,45418528,4087,703652803,13281,505407149,38822,696267655,32692,4073112467,2025 +1000000,inf,21454,38534439834,11238,21380165289,2043,21124897,2043,406799277,7151,468088056,21454,1322162162,19411,11835483871,2025 +12570,inf,27852760,1176874356846,3749390,117843085399,8149560,90013184450,9373476,140466690778,2401857,31303255449,19996408,19558403662,4180519,76525466434,2025 +12570,15000.0,1398781,18751286307,456661,5437199265,1172507,12697932175,1100511,4959629295,103880,734838710,1285644,300516129,118279,191935484,2026 +15000,20000.0,3188397,53923817427,669563,9077070707,2108456,23889330025,2201023,15505967450,238616,1853548387,2920983,1502580645,382608,937741935,2026 +20000,30000.0,7683009,190841493776,988403,17747199265,2386155,26794789082,2735851,34432368897,443290,4189677419,5399705,3432903226,867038,3652258065,2026 +30000,40000.0,5523127,188622406639,611967,14242975207,1193078,13235980149,1491347,28389602170,374380,4167741935,3795221,2774838710,686020,5154838710,2026 +40000,50000.0,3599803,154226556017,369237,9868347107,584197,6478097601,801213,19268444846,334267,4145806452,2437581,2050967742,644879,9212903226,2026 +50000,70000.0,3527807,193060580913,319868,9291845730,453575,5014607113,661335,19838517179,408321,5922580645,2293589,2566451613,661335,13709677419,2026 +70000,100000.0,1686765,130926141079,147078,6058916437,181019,2033821340,266385,10797169982,232444,4134838710,1100511,1776774194,345581,10353548387,2026 +100000,150000.0,813556,91315435685,87424,5629366391,76110,911453267,112108,6190985533,140907,2796774194,502944,1381935484,228330,7534838710,2026 +150000,200000.0,271528,43494107884,41141,4125941230,21599,285165426,29827,2063661844,55540,1261290323,150163,712903226,94623,3783870968,2026 +200000,300000.0,185133,40609294606,34970,4962433425,14399,192621175,20570,1790027125,42169,1151612903,114165,803935484,77139,4058064516,2026 +300000,500000.0,95652,32176763485,21599,5471111111,8228,103305211,10285,1276962025,23656,839032258,69939,738129032,50397,3761935484,2026 +500000,1000000.0,45255,25630456432,15428,7539733701,3086,46272126,4114,729692586,13371,519870968,39084,716193548,32912,4189677419,2026 +1000000,inf,21599,39388796680,11314,22042699725,2057,21521919,2057,421853526,7200,481483871,21599,1360000000,19542,12174193548,2026 +12570,inf,28040411,1202967136929,3774651,121494839302,8204466,91704896609,9436627,145664882459,2418039,32199096774,20131129,20118129032,4208684,78715483871,2026 +12570,15000.0,1409479,19143983402,460153,5609458219,1181475,12951695616,1108928,5065822785,104675,757619878,1295477,309832607,119184,197885789,2027 +15000,20000.0,3212783,55053112033,674684,9364646465,2124582,24366749380,2217856,15837974684,240441,1911011334,2943324,1549163034,385534,966813426,2027 +20000,30000.0,7741770,194838174274,995963,18309458219,2404405,27330272953,2756775,35169620253,446680,4319564080,5441003,3539328684,873670,3765483871,2027 +30000,40000.0,5565369,192572614108,616647,14694214876,1202203,13500496278,1502753,28997468354,377243,4296948561,3824248,2860863121,691266,5314646905,2027 +40000,50000.0,3627335,157456431535,372061,10180991736,588665,6607559967,807341,19681012658,336824,4274333043,2456224,2114551003,649811,9498517873,2027 +50000,70000.0,3554789,197103734440,322315,9586225895,457044,5114822167,666393,20263291139,411443,6106190061,2311131,2646015693,666393,14134699215,2027 +70000,100000.0,1699666,133668049793,148203,6250872360,182403,2074466501,268423,11028354430,234222,4263025283,1108928,1831857018,348224,10674524847,2027 +100000,150000.0,819778,93227800830,88092,5807713499,76692,929668321,112966,6323544304,141984,2883478640,506791,1424777681,230077,7768430689,2027 +150000,200000.0,273605,44404979253,41455,4256657484,21764,290864351,30055,2107848101,55965,1300392328,151312,735004359,95347,3901176983,2027 +200000,300000.0,186549,41459751037,35237,5119651056,14509,196470637,20728,1828354430,42492,1187314734,115038,828858762,77729,4183870968,2027 +300000,500000.0,96383,32850622407,21764,5644444444,8291,105369727,10364,1304303797,23837,865043592,70474,761012206,50783,3878561465,2027 +500000,1000000.0,45601,26167219917,15546,7778604224,3109,47196857,4146,745316456,13473,535987794,39382,738396687,33164,4319564080,2027 +1000000,inf,21764,40213692946,11400,22741046832,2073,21952026,2073,430886076,7255,496410636,21764,1402162162,19691,12551612903,2027 +12570,inf,28254869,1228160165975,3803520,125343985308,8267215,93537584781,9508800,148783797468,2436533,33197319965,20285095,20741823017,4240873,81155789015,2027 +12570,15000.0,1414828,19578755187,461900,5808218549,1185959,13205459057,1113137,5172016275,105072,782737576,1300393,320104621,119636,204446382,2028 +15000,20000.0,3224975,56303402490,677245,9696464646,2132645,24844168734,2226273,16169981917,241353,1974367916,2954494,1600523104,386997,998866609,2028 +20000,30000.0,7771150,199263070539,999742,18958218549,2413530,27865756824,2767237,35906871609,448376,4462772450,5461652,3656669573,876985,3890322581,2028 +30000,40000.0,5586490,196946058091,618987,15214876033,1206765,13765012407,1508456,29605334539,378675,4439407149,3838761,2955710549,693890,5490845684,2028 +40000,50000.0,3641101,161032365145,373473,10541735537,590899,6737022333,810405,20093580470,338102,4416041848,2465546,2184655623,652277,9813426330,2028 +50000,70000.0,3568279,201580082988,323538,9925895317,458779,5215037221,668922,20688065099,413005,6308631212,2319902,2733740192,668922,14603312990,2028 +70000,100000.0,1706116,136703734440,148765,6472359963,183095,2115111663,269441,11259538879,235111,4404359198,1113137,1892589364,349546,11028421970,2028 +100000,150000.0,822889,95345062241,88427,6013498623,76983,947883375,113394,6456103074,142523,2979075850,508714,1472013949,230950,8025980820,2028 +150000,200000.0,274643,45413443983,41613,4407483930,21847,296563275,30169,2152034358,56177,1343504795,151886,759372276,95709,4030514385,2028 +200000,300000.0,187257,42401327801,35371,5301056015,14564,200320099,20806,1866681736,42653,1226678291,115475,856338274,78024,4322580645,2028 +300000,500000.0,96749,33596680498,21847,5844444444,8323,107434243,10403,1331645570,23927,893722755,70741,786242371,50975,4007149085,2028 +500000,1000000.0,45774,26761493776,15605,8054224059,3121,48121588,4161,760940325,13524,553757629,39532,762877071,33290,4462772450,2028 +1000000,inf,21847,41126970954,11443,23546831956,2081,22382134,2081,439918626,7282,512868352,21847,1448648649,19766,12967741935,2028 +12570,inf,28362098,1256052448133,3817955,129785307622,8298590,95370272953,9544887,151902712477,2445780,34297925022,20362079,21429485615,4256968,83846381866,2028 +12570,15000.0,1422852,20069626556,464519,6029063361,1192684,13468982630,1119449,5293942134,105668,808439407,1307768,330615519,120315,211159547,2029 +15000,20000.0,3243265,57715020747,681086,10065151515,2144739,25339950372,2238899,16551175407,242722,2039197908,2971249,1653077594,389192,1031665214,2029 +20000,30000.0,7815221,204258921162,1005412,19679063361,2427217,28421836228,2782930,36753345389,450918,4609311247,5492625,3776739320,881959,4018064516,2029 +30000,40000.0,5618171,201883817427,622498,15793388430,1213609,14039702233,1517011,30303254973,380822,4585178727,3860531,3052763731,697825,5671142110,2029 +40000,50000.0,3661750,165069709544,375591,10942561983,594250,6871464020,815001,20567269439,340020,4561046207,2479528,2256390584,655976,10135658239,2029 +50000,70000.0,3588515,206634024896,325373,10303305785,461381,5319106700,672716,21175768535,415347,6515780296,2333058,2823504795,672716,15082824760,2029 +70000,100000.0,1715792,140131120332,149609,6718457300,184134,2157320099,270970,11524972875,236444,4548979948,1119449,1954734089,351528,11390549259,2029 +100000,150000.0,827556,97735518672,88928,6242148760,77420,966799007,114037,6608300181,143331,3076896251,511599,1520348736,232260,8289520488,2029 +150000,200000.0,276201,46552033195,41849,4575068871,21971,302481390,30340,2202766727,56496,1387619878,152747,784306888,96252,4162859634,2029 +200000,300000.0,188319,43464398340,35571,5502617080,14647,204317618,20924,1910687161,42895,1266957280,116130,884456844,78466,4464516129,2029 +300000,500000.0,97298,34439004149,21971,6066666667,8370,109578164,10462,1363037975,24063,923068875,71143,812059285,51265,4138727114,2029 +500000,1000000.0,46033,27432448133,15693,8360468320,3139,49081886,4185,778878843,13601,571940715,39756,787926765,33479,4609311247,2029 +1000000,inf,21971,42158091286,11508,24442148760,2092,22828784,2092,450289331,7324,529708806,21971,1496216216,19878,13393548387,2029 +12570,inf,28522942,1287543734440,3839607,134720110193,8345652,97273449132,9599017,155483688969,2459650,35424125545,20477554,22133140366,4281109,86599546643,2029 diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py index f2936068..8b39a8e3 100644 --- a/policyengine_uk_data/targets/sources/hmrc_cgt.py +++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py @@ -29,17 +29,19 @@ for the simulation year, so the target's meaning tracks the policy in force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails. -Year handling: HMRC's figures are the 2023-24 outturn, mapped to -calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected -to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the -target constrains the projection years the calibration actually runs -for; the taxpayer count is held flat, as the repo has no administrative -basis for forecasting CGT taxpayer numbers and count uprating factors -would be a nominal-income index applied to a headcount. - -Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the -gate applied in later years uses the £3,000 AEA then in force. The -targets are therefore approximate for the projection years; the +Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine +UK's year N is tax year N to N+1, so the outturn is the 2023 value, the +year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR`` +follows the same convention). The amount is projected to 2029 with +PolicyEngine's ``capital_gains`` uprating factors so the target +constrains the projection years the calibration actually runs for; the +taxpayer count is held flat, as the repo has no administrative basis for +forecasting CGT taxpayer numbers and count uprating factors would be a +nominal-income index applied to a headcount. + +Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the +projected values carry the 2023-24 outturn realised under a £6,000 AEA. +The targets are therefore approximate for the projection years; the alternative (a fixed £6,000 gate) would be wrong in a different and less transparent way, because it would not describe any year's actual policy. @@ -55,9 +57,9 @@ logger = logging.getLogger(__name__) -# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024 -# following the hmrc_spi._SPI_YEAR convention. -_CGT_BASE_YEAR = 2024 +# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N +# is tax year N to N+1, so the outturn is the 2023 value. +_CGT_BASE_YEAR = 2023 _MAX_YEAR = 2029 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn). diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 17b375b8..2e1d1207 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -53,8 +53,10 @@ ] _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] -# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024 -_SPI_YEAR = 2024 +# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is +# tax year N to N+1 (parameters take their 6 April N value), so the +# outturn belongs to 2023. Later years come from incomes_projection.csv. +_SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py index 66a15b08..70bb85db 100644 --- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py @@ -89,6 +89,20 @@ def test_aea_tracks_the_year(): assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0 +def test_base_year_is_the_tax_year_the_outturn_was_realised_in(): + """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn + is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA, + read from the real parameter tree; mapping the outturn to 2024 would gate + it at 2024-25's £3,000.""" + from policyengine_uk.system import system + + from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount + + ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system)) + assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000 + assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000 + + def test_targets_load_with_expected_base_year_values(): targets = {t.name: t for t in get_targets()} # Aggregate targets plus the per-size-band targets (tested in @@ -296,7 +310,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years(): def test_registry_year_filter_excludes_years_before_the_outturn(): - """The targets describe a 2023-24 outturn mapped to calendar 2024, + """The targets describe the 2023-24 outturn, PolicyEngine year 2023, so they must not appear for earlier years.""" from policyengine_uk_data.targets import get_all_targets diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 686604e0..faa6159a 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -5,9 +5,12 @@ to have been generated and will be skipped otherwise. """ +import re + import pytest import pandas as pd from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.targets.sources._common import load_config from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv" @@ -143,3 +146,29 @@ def test_projection_keeps_top_open_ended_band(projections): ] assert len(top_band) == 1 assert top_band.iloc[0]["dividend_income_amount"] > 0 + + +def test_projection_is_the_spi_table_uprated_with_the_committed_table( + projections, base_targets +): + """Regenerate incomes_projection.csv whenever uprating_factors.csv changes.""" + from policyengine_uk_data.utils.incomes_projection import project_income_table + + pd.testing.assert_frame_equal(project_income_table(base_targets), projections) + + +def test_spi_year_is_the_start_of_the_configured_tax_year(): + """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24). + + PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its + value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it + to the end year would treat it as 2024-25 and understate every later + year by a year's growth. A refreshed URL without a matching _SPI_YEAR + fails here. + """ + url = load_config()["hmrc"]["spi_collated"] + match = re.search(r"_(\d{2})(\d{2})\.ods$", url) + assert match, url + start, end = int(match[1]), int(match[2]) + assert end == (start + 1) % 100, url + assert _SPI_YEAR == 2000 + start diff --git a/policyengine_uk_data/tests/test_property_income_targets.py b/policyengine_uk_data/tests/test_property_income_targets.py index 5a2f6208..b5a1e2ee 100644 --- a/policyengine_uk_data/tests/test_property_income_targets.py +++ b/policyengine_uk_data/tests/test_property_income_targets.py @@ -5,6 +5,7 @@ """ from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR def test_property_income_targets_scaled(): @@ -13,7 +14,7 @@ def test_property_income_targets_scaled(): Raw SPI 2023-24 total is scaled up to better match HMRC rental income statistics, which cover more landlords than SPI. """ - base_year = 2024 + base_year = _SPI_YEAR targets = get_all_targets(year=base_year) total = sum( t.values[base_year] From a608ddbab0c85f56d6779c2bdfd1f91bae20b3e7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 17:41:11 -0400 Subject: [PATCH 02/13] Calibrate SPI income bands to their projected values build_loss_matrix fetches targets without a year filter, and the registry keeps only the first target of each name. hmrc_spi emitted each band twice: the ODS outturn (base year only), then the projection (later years). The projected target was dropped, so _resolve_value carried the base-year outturn forward unuprated: the 2025 calibration has targeted raw 2023-24 SPI values since the February 2026 registry refactor. get_targets now merges the two into one target per band, and a test checks every projected band, variable and year through the calibration's own resolution path. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-cgt-tax-year-start.fixed.md | 2 +- .../targets/sources/hmrc_spi.py | 32 ++++++++------ .../tests/test_income_projection.py | 42 +++++++++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md index b49b12b5..514abf0d 100644 --- a/changelog.d/spi-cgt-tax-year-start.fixed.md +++ b/changelog.d/spi-cgt-tax-year-start.fixed.md @@ -1 +1 @@ -Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Income-band, local income and capital gains targets for 2024 onwards now carry one more year of uprating. +Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%). diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 2e1d1207..080320cd 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -231,8 +231,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame: def get_targets() -> list[Target]: """Build income-band targets from the live HMRC SPI ODS. - Also reads incomes_projection.csv if available, which contains - projected future year data generated by the microsimulation. + Also reads incomes_projection.csv if available, which projects the + same bands to later years with PolicyEngine's uprating indices. Each + band is returned as one target holding the base year and every + projected year: the registry keeps only the first target of a name, + and calibration reads targets without a year filter, so a band split + across two targets would lose its projected years and be calibrated + to the base-year outturn carried forward. """ config = load_config() ref = config["hmrc"]["spi_collated"] @@ -293,7 +298,18 @@ def get_targets() -> list[Target]: if proj_path.exists(): targets.extend(_read_projection_csv(proj_path, ref)) - return targets + return _merge_by_name(targets) + + +def _merge_by_name(targets: list[Target]) -> list[Target]: + """Combine targets that share a name into one, merging their years.""" + merged: dict[str, Target] = {} + for t in targets: + if t.name in merged: + merged[t.name].values.update(t.values) + else: + merged[t.name] = t + return list(merged.values()) def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: @@ -357,12 +373,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]: ) ) - # Merge targets with the same name across years - merged: dict[str, Target] = {} - for t in targets: - if t.name in merged: - merged[t.name].values.update(t.values) - else: - merged[t.name] = t - - return list(merged.values()) + return _merge_by_name(targets) diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index faa6159a..ad7c71ef 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -172,3 +172,45 @@ def test_spi_year_is_the_start_of_the_configured_tax_year(): start, end = int(match[1]), int(match[2]) assert end == (start + 1) % 100, url assert _SPI_YEAR == 2000 + start + + +def test_calibration_reads_the_projected_spi_targets(projections): + """Calibration (build_loss_matrix) fetches targets without a year filter, + and the registry keeps only the first target of each name. Every SPI band + must therefore be one target carrying all its years; if the projected + years sit on a second target of the same name they are dropped, and the + base-year outturn is carried forward unuprated instead.""" + from policyengine_uk_data.targets import get_all_targets + from policyengine_uk_data.targets.build_loss_matrix import _resolve_value + from policyengine_uk_data.targets.sources.hmrc_spi import ( + _PROPERTY_INCOME_SCALE, + INCOME_VARIABLES, + _format_band_label, + get_targets, + ) + + names = [t.name for t in get_targets()] + assert len(names) == len(set(names)) + + spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + checked = 0 + for row in _without_aggregate(projections).itertuples(): + if row.year <= _SPI_YEAR: + continue + label = _format_band_label( + row.total_income_lower_bound, row.total_income_upper_bound + ) + for variable in INCOME_VARIABLES: + amount = getattr(row, f"{variable}_amount") + if variable == "property_income": + amount *= _PROPERTY_INCOME_SCALE + amount_target = spi[f"hmrc/{variable}_income_band_{label}"] + count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] + assert _resolve_value(amount_target, row.year) == pytest.approx(amount) + assert _resolve_value(count_target, row.year) == pytest.approx( + getattr(row, f"{variable}_count") + ) + checked += 2 + assert checked == 2 * len(INCOME_VARIABLES) * len( + _without_aggregate(projections[projections["year"] > _SPI_YEAR]) + ) From 47fa84eeac7060a863c1ebd3a8a46f7b49ce5b30 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:32:02 -0400 Subject: [PATCH 03/13] Serve the SPI merge test the committed table, and fix the sampling date Review r2 (APPROVE_WITH_NITS): with the live ODS download failing, the merge test passed even without the merge, because only the projection targets were emitted. The test now serves get_income_band_table from incomes.csv, checks the outturn year as well as the projected years through the calibration resolver, and requires every SPI target to carry all of them. The comment now names 30 April N, the date convert_to_fiscal_year_parameters samples. Co-Authored-By: Claude Opus 5.5 --- .../targets/sources/hmrc_spi.py | 5 +- .../tests/test_income_projection.py | 50 +++++++++++-------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/policyengine_uk_data/targets/sources/hmrc_spi.py b/policyengine_uk_data/targets/sources/hmrc_spi.py index 080320cd..c76c60bc 100644 --- a/policyengine_uk_data/targets/sources/hmrc_spi.py +++ b/policyengine_uk_data/targets/sources/hmrc_spi.py @@ -54,8 +54,9 @@ _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")] # SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is -# tax year N to N+1 (parameters take their 6 April N value), so the -# outturn belongs to 2023. Later years come from incomes_projection.csv. +# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters +# reads each parameter at 30 April N), so the outturn belongs to 2023. +# Later years come from incomes_projection.csv. _SPI_YEAR = 2023 # HMRC Property Rental Income Statistics show ~1.9x more property income diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index ad7c71ef..944bab16 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -174,36 +174,43 @@ def test_spi_year_is_the_start_of_the_configured_tax_year(): assert _SPI_YEAR == 2000 + start -def test_calibration_reads_the_projected_spi_targets(projections): +def test_calibration_reads_the_projected_spi_targets( + projections, base_targets, monkeypatch +): """Calibration (build_loss_matrix) fetches targets without a year filter, and the registry keeps only the first target of each name. Every SPI band - must therefore be one target carrying all its years; if the projected - years sit on a second target of the same name they are dropped, and the - base-year outturn is carried forward unuprated instead.""" + must therefore be one target carrying its outturn and every projected + year; if the projected years sit on a second target of the same name they + are dropped, and the outturn is carried forward unuprated instead. + + The official table is served from the committed incomes.csv, so the + outturn targets are always present and the test needs no download. + """ from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.build_loss_matrix import _resolve_value - from policyengine_uk_data.targets.sources.hmrc_spi import ( - _PROPERTY_INCOME_SCALE, - INCOME_VARIABLES, - _format_band_label, - get_targets, - ) + from policyengine_uk_data.targets.sources import hmrc_spi - names = [t.name for t in get_targets()] + def committed_table(include_aggregate: bool = True) -> pd.DataFrame: + if include_aggregate: + return base_targets + return _without_aggregate(base_targets).reset_index(drop=True) + + monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table) + + names = [t.name for t in hmrc_spi.get_targets()] assert len(names) == len(set(names)) spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"} + expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR]) checked = 0 - for row in _without_aggregate(projections).itertuples(): - if row.year <= _SPI_YEAR: - continue - label = _format_band_label( + for row in expected.itertuples(): + label = hmrc_spi._format_band_label( row.total_income_lower_bound, row.total_income_upper_bound ) - for variable in INCOME_VARIABLES: + for variable in hmrc_spi.INCOME_VARIABLES: amount = getattr(row, f"{variable}_amount") if variable == "property_income": - amount *= _PROPERTY_INCOME_SCALE + amount *= hmrc_spi._PROPERTY_INCOME_SCALE amount_target = spi[f"hmrc/{variable}_income_band_{label}"] count_target = spi[f"hmrc/{variable}_count_income_band_{label}"] assert _resolve_value(amount_target, row.year) == pytest.approx(amount) @@ -211,6 +218,9 @@ def test_calibration_reads_the_projected_spi_targets(projections): getattr(row, f"{variable}_count") ) checked += 2 - assert checked == 2 * len(INCOME_VARIABLES) * len( - _without_aggregate(projections[projections["year"] > _SPI_YEAR]) - ) + assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected) + + years = set(projections["year"]) + assert _SPI_YEAR in years + for target in spi.values(): + assert set(target.values) == years, target.name From 39e652a81b4b101894ae0ab270730529d3425b7c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 07:23:24 -0400 Subject: [PATCH 04/13] Fail the calibration build on targets it drops or cannot fit create_target_matrix skipped a target whose year did not resolve with no log line (#456), skipped one whose column raised with a warning, and accepted columns no household contributes to (#593). It now records each offered target's status and fails unless every target is in the matrix with a finite value and signal, or is listed in targets/known_gaps.yaml under that status with a reason and an issue. Errors and non-finite values can never be listed; a listed gap that no longer applies is logged as stale. known_gaps.yaml lists the 34 gaps measured on uk-data 1.58.1 at the calibration year 2025: 15 two-child-limit targets dated 2026 only, and 19 targets with no signal. The matrix built is unchanged. The diagnostic get_loss_results and the CPS transfer reweighting opt out. `python -m policyengine_uk_data.targets.coverage` reports every target's status on a dataset (names, statuses and counts only). Co-Authored-By: Claude Opus 5.5 --- changelog.d/target-coverage-gate.changed.md | 1 + .../targets/build_loss_matrix.py | 39 ++- policyengine_uk_data/targets/coverage.py | 187 ++++++++++++ policyengine_uk_data/targets/known_gaps.yaml | 80 +++++ .../tests/test_policybench_transfer.py | 6 +- .../tests/test_target_coverage.py | 284 ++++++++++++++++++ policyengine_uk_data/utils/loss.py | 6 +- policyengine_uk_data/utils/reweight.py | 6 +- 8 files changed, 604 insertions(+), 5 deletions(-) create mode 100644 changelog.d/target-coverage-gate.changed.md create mode 100644 policyengine_uk_data/targets/coverage.py create mode 100644 policyengine_uk_data/targets/known_gaps.yaml create mode 100644 policyengine_uk_data/tests/test_target_coverage.py diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md new file mode 100644 index 00000000..769f3a14 --- /dev/null +++ b/changelog.d/target-coverage-gate.changed.md @@ -0,0 +1 @@ +Fail the calibration build when a national target is dropped (no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset. diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 98022354..49f22a4c 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -17,6 +17,11 @@ import pandas as pd from policyengine_uk_data.targets import get_all_targets +from policyengine_uk_data.targets.coverage import ( + TargetStatus, + check_coverage, + column_status, +) from policyengine_uk_data.targets.schema import ( GeographicLevel, Target, @@ -64,6 +69,7 @@ def create_target_matrix( dataset, time_period: str = None, reform=None, + require_coverage: bool = True, ) -> tuple[pd.DataFrame, pd.Series]: """Create (matrix, target_values) for calibration. @@ -71,12 +77,27 @@ def create_target_matrix( dataset: a UKSingleYearDataset instance. time_period: calendar year as string; defaults to dataset year. reform: optional PolicyEngine reform. + require_coverage: fail unless every target is in the matrix with + signal or listed in targets/known_gaps.yaml (targets/coverage.py). Returns: (df, targets) where df has one column per target and one row per household, and targets is a Series of scalar target values indexed by target name. """ + df, targets, statuses = build_target_matrix(dataset, time_period, reform) + if require_coverage: + check_coverage(statuses) + return df, targets + + +def build_target_matrix(dataset, time_period: str = None, reform=None): + """The target matrix, its values, and each offered target's status. + + A target that is dropped (no value for the year, no column, an error) + or that enters with a non-finite value or column, or with no household + contributing, is recorded with that status (targets/coverage.py). + """ from policyengine_uk import Microsimulation if time_period is None: @@ -103,22 +124,36 @@ def create_target_matrix( df = pd.DataFrame() target_names = [] target_values = [] + statuses = [] for target in all_targets: try: val = _resolve_value(target, year) if val is None: + statuses.append( + TargetStatus( + target.name, + "no_value_for_year", + f"values for {', '.join(str(y) for y in sorted(target.values))}", + ) + ) continue col = _compute_column(target, ctx, year) if col is None: + statuses.append(TargetStatus(target.name, "no_column")) continue - df[target.name] = restrict_to_countries(col, ctx.country, target.countries) + col = restrict_to_countries(col, ctx.country, target.countries) + statuses.append(TargetStatus(target.name, column_status(val, col))) + df[target.name] = col target_names.append(target.name) target_values.append(val) except Exception as e: logger.warning("Skipping target %s: %s", target.name, e) + statuses.append( + TargetStatus(target.name, "error", f"{type(e).__name__}: {e}") + ) - return df, pd.Series(target_values, index=target_names) + return df, pd.Series(target_values, index=target_names), statuses def restrict_to_countries(column, household_country, countries): diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py new file mode 100644 index 00000000..35bf5729 --- /dev/null +++ b/policyengine_uk_data/targets/coverage.py @@ -0,0 +1,187 @@ +"""Required coverage of the national calibration targets. + +Every target the registry offers must enter the national loss matrix with +a finite value and a column that some household contributes to. The +exceptions are the targets ``targets/known_gaps.yaml`` lists, each with the +one status it may have, a reason and an issue. ``create_target_matrix`` +checks this, so a build that would calibrate without a target, or against +one the weights cannot move, fails instead of logging a warning. + + python -m policyengine_uk_data.targets.coverage [out.json] + +reports every target's status on a dataset: names, statuses and counts only, +never household values. +""" + +from __future__ import annotations + +import json +import logging +import sys +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional, Sequence + +import numpy as np +import yaml + +logger = logging.getLogger(__name__) + +KNOWN_GAPS_PATH = Path(__file__).with_name("known_gaps.yaml") +IN_MATRIX = "in_matrix" +# Statuses a known gap may excuse. An error or a non-finite value is never +# a known gap: it is a broken target. +EXCUSABLE = ("no_value_for_year", "no_column", "zero_signal") +NEVER_EXCUSED = ("error", "non_finite") +STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED + + +class TargetCoverageError(Exception): + """The national target matrix is missing targets it must have.""" + + +@dataclass(frozen=True) +class TargetStatus: + name: str + status: str + detail: str = "" + + +@dataclass(frozen=True) +class KnownGap: + status: str + targets: frozenset + reason: str + issue: str + + +def parse_known_gaps(raw) -> list: + if (raw or {}).get("schema_version") != 1: + raise ValueError("known gaps: schema_version must be 1") + gaps, seen = [], set() + for index, entry in enumerate(raw.get("gaps") or []): + where = f"known gaps[{index}]" + if entry.get("status") not in EXCUSABLE: + raise ValueError(f"{where}: status must be one of {', '.join(EXCUSABLE)}") + if not entry.get("reason") or not str(entry.get("issue", "")).startswith( + "https://github.com/" + ): + raise ValueError(f"{where}: needs a reason and an issue URL") + targets = list(entry.get("targets") or []) + if not targets: + raise ValueError(f"{where}: lists no targets") + repeated = seen.intersection(targets) | { + t for t, n in Counter(targets).items() if n > 1 + } + if repeated: + raise ValueError(f"{where}: {sorted(repeated)} listed twice") + seen.update(targets) + gaps.append( + KnownGap( + entry["status"], + frozenset(targets), + str(entry["reason"]), + str(entry["issue"]), + ) + ) + return gaps + + +def load_known_gaps(path=None) -> list: + with open(path or KNOWN_GAPS_PATH) as f: + return parse_known_gaps(yaml.safe_load(f)) + + +def column_status(value, column) -> str: + """A matrix column's status: non-finite, without signal, or fine. + + A column with no non-zero entry has no signal unless its target is + zero too: the weights cannot move it. + """ + column = np.asarray(column, dtype=float) + if not np.isfinite(value) or not np.isfinite(column).all(): + return "non_finite" + if value != 0 and not np.any(column != 0): + return "zero_signal" + return IN_MATRIX + + +def coverage_problems(statuses: Sequence[TargetStatus], gaps: Iterable[KnownGap]): + """(problems, stale): targets out of place, and known gaps that no longer apply.""" + gaps = list(gaps) + excused = {name: gap.status for gap in gaps for name in gap.targets} + problems, matched = [], set() + for row in statuses: + if row.status == IN_MATRIX: + continue + if excused.get(row.name) == row.status: + matched.add(row.name) + continue + detail = f" ({row.detail})" if row.detail else "" + problems.append(f"{row.name}: {row.status}{detail}") + present = {row.name: row.status for row in statuses} + stale = [] + for gap in gaps: + for name in sorted(gap.targets - matched): + now = present.get(name, "not offered by the registry") + stale.append(f"{name}: listed as {gap.status}, now {now}") + return problems, stale + + +def check_coverage( + statuses: Sequence[TargetStatus], gaps: Optional[Iterable[KnownGap]] = None +) -> list: + """Raise unless every target is in the matrix or a known gap; return stale gaps.""" + problems, stale = coverage_problems( + statuses, load_known_gaps() if gaps is None else gaps + ) + counts = Counter(row.status for row in statuses) + logger.info( + "Calibration targets: %s", + ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]), + ) + for line in stale: + logger.warning("Stale known calibration-target gap, remove it: %s", line) + if problems: + raise TargetCoverageError( + f"{len(problems)} calibration targets are missing or cannot be fitted, " + "and targets/known_gaps.yaml does not list them: " + "; ".join(problems) + ) + return stale + + +def main(argv=None) -> int: + argv = sys.argv[1:] if argv is None else argv + if len(argv) not in (2, 3): + print(__doc__) + return 2 + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.targets.build_loss_matrix import build_target_matrix + + _, _, statuses = build_target_matrix( + UKSingleYearDataset(file_path=argv[0]), argv[1] + ) + problems, stale = coverage_problems(statuses, load_known_gaps()) + report = { + "dataset": Path(argv[0]).name, + "year": int(argv[1]), + "by_status": Counter(row.status for row in statuses), + "problems": problems, + "stale_known_gaps": stale, + "targets": [row.__dict__ for row in statuses], + } + if len(argv) == 3: + Path(argv[2]).write_text(json.dumps(report, indent=1)) + print( + json.dumps( + {k: report[k] for k in ("by_status", "problems", "stale_known_gaps")}, + indent=1, + ) + ) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml new file mode 100644 index 00000000..a1e58e17 --- /dev/null +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -0,0 +1,80 @@ +# Calibration targets allowed to stay out of the national loss matrix, or to +# enter it with no signal. targets/coverage.py fails the build on any other +# target that is dropped (no value for the year, no column, an error), has a +# non-finite value or column, or has a column no household contributes to. +# +# Each entry names the exact targets, the one status they may have, why and +# the issue that tracks removing them. Errors and non-finite values are never +# allowed. An entry that no longer matches is reported as stale: remove it. +# +# Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with +# policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645 +# enter the matrix, 15 have no value for 2025 and 19 of those entering have +# no signal. +schema_version: 1 +gaps: + - status: no_value_for_year + reason: >- + Dated 2026 only. The calibration year is 2025, and _resolve_value does + not carry a later year back. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/456 + targets: + - dwp/uc/two_child_limit/3_children_households + - dwp/uc/two_child_limit/3_children_households_total_children + - dwp/uc/two_child_limit/4_children_households + - dwp/uc/two_child_limit/4_children_households_total_children + - dwp/uc/two_child_limit/5_children_households + - dwp/uc/two_child_limit/5_children_households_total_children + - dwp/uc/two_child_limit/6_children_households + - dwp/uc/two_child_limit/6_children_households_total_children + - dwp/uc/two_child_limit/adult_pip_children + - dwp/uc/two_child_limit/adult_pip_households + - dwp/uc/two_child_limit/children_affected + - dwp/uc/two_child_limit/children_in_affected_households + - dwp/uc/two_child_limit/disabled_child_element_children + - dwp/uc/two_child_limit/disabled_child_element_households + - dwp/uc/two_child_limit/households_affected + - status: zero_signal + reason: >- + Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - hmrc/dividend_income_count_income_band_1_000_000_to_inf + - hmrc/dividend_income_income_band_1_000_000_to_inf + - hmrc/employment_income_count_income_band_1_000_000_to_inf + - hmrc/employment_income_income_band_1_000_000_to_inf + - hmrc/private_pension_income_count_income_band_1_000_000_to_inf + - hmrc/private_pension_income_income_band_1_000_000_to_inf + - hmrc/property_income_count_income_band_1_000_000_to_inf + - hmrc/property_income_income_band_1_000_000_to_inf + - hmrc/state_pension_count_income_band_1_000_000_to_inf + - hmrc/state_pension_income_band_1_000_000_to_inf + - status: zero_signal + reason: >- + No record is aged 85 or over; the oldest records fall in the 80-84 band. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - ons/female_85_90 + - ons/male_85_90 + - status: zero_signal + reason: >- + No band-H household in these regions. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - voa/council_tax/NORTH_EAST/H + - voa/council_tax/NORTH_WEST/H + - voa/council_tax/YORKSHIRE/H + - status: zero_signal + reason: >- + No record repays an English postgraduate loan. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - slc/student_loan_repayment/england/postgraduate + - status: zero_signal + reason: >- + No couple without children receives UC payments above £27,600 a year. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000 + - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf diff --git a/policyengine_uk_data/tests/test_policybench_transfer.py b/policyengine_uk_data/tests/test_policybench_transfer.py index a8579c0e..4fa05852 100644 --- a/policyengine_uk_data/tests/test_policybench_transfer.py +++ b/policyengine_uk_data/tests/test_policybench_transfer.py @@ -170,7 +170,11 @@ def test_policybench_transfer_calibration_improves_loss(tmp_path: Path): def test_policybench_transfer_calibration_uses_iterative_solver(monkeypatch): calls = {} - def fake_create_target_matrix(dataset, time_period, reform=None): + def fake_create_target_matrix( + dataset, time_period, reform=None, require_coverage=True + ): + # The transfer dataset is not the Enhanced FRS, so it is not gated. + assert require_coverage is False return pd.DataFrame( [[1.0, 0.0], [0.0, 1.0]], ), pd.Series([1.0, 2.0]) diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py new file mode 100644 index 00000000..ba48fe25 --- /dev/null +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -0,0 +1,284 @@ +"""Required coverage of the national calibration targets (targets/coverage.py). + +create_target_matrix used to skip a target with a warning, or with nothing at +all when its year did not resolve (#456), and to accept a column no household +contributes to (#593). These tests pin the gate that replaces that: every +offered target is in the matrix with signal, or listed in known_gaps.yaml with +the status it may have. +""" + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.targets import build_loss_matrix +from policyengine_uk_data.targets.coverage import ( + EXCUSABLE, + STATUSES, + KnownGap, + TargetCoverageError, + TargetStatus, + check_coverage, + column_status, + coverage_problems, + load_known_gaps, + parse_known_gaps, +) +from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit + + +def test_column_status(): + assert column_status(5.0, [0, 1, 0]) == "in_matrix" + assert column_status(5.0, [0, 0, 0]) == "zero_signal" + # A zero target met by a zero column needs no signal. + assert column_status(0.0, [0, 0, 0]) == "in_matrix" + assert column_status(float("nan"), [1, 1]) == "non_finite" + assert column_status(5.0, [1, np.inf]) == "non_finite" + + +def test_the_shipped_known_gaps_load_and_cite_issues(): + gaps = load_known_gaps() + assert gaps + for gap in gaps: + assert gap.status in EXCUSABLE + assert gap.issue.startswith("https://github.com/PolicyEngine/") + assert gap.reason + names = [n for gap in gaps for n in gap.targets] + assert len(names) == len(set(names)) + # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19 + # targets without signal. + assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15 + assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 19 + + +@pytest.mark.parametrize( + "entry, match", + [ + ( + { + "status": "error", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t"], + }, + "status", + ), + ( + { + "status": "non_finite", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t"], + }, + "status", + ), + ( + { + "status": "zero_signal", + "reason": "", + "issue": "https://github.com/a", + "targets": ["t"], + }, + "reason", + ), + ( + {"status": "zero_signal", "reason": "x", "issue": "#456", "targets": ["t"]}, + "issue", + ), + ( + { + "status": "zero_signal", + "reason": "x", + "issue": "https://github.com/a", + "targets": [], + }, + "no targets", + ), + ( + { + "status": "zero_signal", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t", "t"], + }, + "twice", + ), + ], +) +def test_malformed_known_gaps_are_refused(entry, match): + with pytest.raises(ValueError, match=match): + parse_known_gaps({"schema_version": 1, "gaps": [entry]}) + + +def _gap(status, *names): + return KnownGap(status, frozenset(names), "reason", "https://github.com/x") + + +def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported(): + statuses = [ + TargetStatus("a", "in_matrix"), + TargetStatus("b", "zero_signal"), + TargetStatus("c", "no_value_for_year"), + TargetStatus("d", "error", "KeyError: x"), + ] + problems, stale = coverage_problems( + statuses, + [_gap("zero_signal", "b", "c"), _gap("no_column", "gone")], + ) + assert problems == ["c: no_value_for_year", "d: error (KeyError: x)"] + assert stale == [ + "c: listed as zero_signal, now no_value_for_year", + "gone: listed as no_column, now not offered by the registry", + ] + + +statuses_strategy = st.dictionaries( + st.text("abcdef", min_size=1, max_size=3), st.sampled_from(STATUSES), max_size=12 +) + + +@settings(max_examples=300, deadline=None) +@given(statuses=statuses_strategy, data=st.data()) +def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix(statuses, data): + names = sorted(statuses) + excused = data.draw( + st.dictionaries( + st.sampled_from(names + ["zz"]) if names else st.just("zz"), + st.sampled_from(EXCUSABLE), + ) + ) + gaps = [_gap(status, name) for name, status in excused.items()] + rows = [TargetStatus(name, statuses[name]) for name in names] + problems, stale = coverage_problems(rows, gaps) + expected = [ + f"{n}: {statuses[n]}" + for n in names + if statuses[n] != "in_matrix" and excused.get(n) != statuses[n] + ] + assert problems == expected + assert {s.split(":")[0] for s in stale} == { + n for n, status in excused.items() if statuses.get(n) != status + } + # Order of the statuses never changes the verdict. + shuffled = data.draw(st.permutations(rows)) + assert sorted(coverage_problems(shuffled, gaps)[0]) == sorted(problems) + + +def _target(name, values={2025: 10.0}, **kwargs): + return Target( + name=name, + variable="employment_income", + source="test", + unit=Unit.GBP, + values=values, + **kwargs, + ) + + +@pytest.fixture +def fake_matrix(monkeypatch): + """create_target_matrix over two households, with chosen targets and columns.""" + import policyengine_uk + + class FakeMicrosimulation: + def __init__(self, dataset=None, reform=None): + pass + + def calculate(self, variable, *args, **kwargs): + assert variable == "country" + return pd.Series(["ENGLAND", "WALES"]) + + monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + + def configure(targets, columns): + monkeypatch.setattr( + build_loss_matrix, + "get_all_targets", + lambda geographic_level=None: ( + targets if geographic_level == GeographicLevel.NATIONAL else [] + ), + ) + + def compute(target, ctx, year): + column = columns[target.name] + if isinstance(column, Exception): + raise column + return None if column is None else np.asarray(column, dtype=float) + + monkeypatch.setattr(build_loss_matrix, "_compute_column", compute) + return SimpleNamespace(time_period="2025") + + return configure + + +def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")], + {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")}, + ) + with pytest.raises(TargetCoverageError) as raised: + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + message = str(raised.value) + assert "later: no_value_for_year (values for 2026)" in message + assert "broken: error (KeyError: 'no such variable')" in message + df, values = build_loss_matrix.create_target_matrix( + dataset, time_period="2025", require_coverage=False + ) + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("no_value_for_year", "later")], + ) + with pytest.raises(TargetCoverageError, match="broken"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_without_signal_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + dataset = fake_matrix( + [_target("ok"), _target("flat")], {"ok": [1.0, 2.0], "flat": [0.0, 0.0]} + ) + with pytest.raises(TargetCoverageError, match="flat: zero_signal"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "flat")], + ) + df, values = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + # Known gaps still enter the matrix: the change is that they are declared. + assert list(df.columns) == ["ok", "flat"] + + +def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch): + dataset = fake_matrix([_target("nan")], {"nan": [np.nan, 1.0]}) + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("zero_signal", "nan")], + ) + with pytest.raises(TargetCoverageError, match="nan: non_finite"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_stale_gaps_warn_without_failing(caplog): + caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage") + stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")]) + assert stale == ["a: listed as zero_signal, now in_matrix"] + assert "Stale known calibration-target gap" in caplog.text + + +def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs): + """On the built dataset the gate passes: the known gaps are complete.""" + from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE + + _, _, statuses = build_loss_matrix.build_target_matrix( + enhanced_frs, str(CURRENT_FRS_RELEASE.calibration_year) + ) + problems, _ = coverage_problems(statuses, load_known_gaps()) + assert problems == [] diff --git a/policyengine_uk_data/utils/loss.py b/policyengine_uk_data/utils/loss.py index 27eb919f..cae347e8 100644 --- a/policyengine_uk_data/utils/loss.py +++ b/policyengine_uk_data/utils/loss.py @@ -23,7 +23,11 @@ def get_loss_results(dataset, time_period, reform=None, household_weights=None): Returns: DataFrame with estimate vs target comparisons and error metrics. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # A diagnostic of any dataset: report every target, fitted or not, + # rather than enforce the Enhanced FRS build's coverage gate. + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) from policyengine_uk import Microsimulation if household_weights is None: diff --git a/policyengine_uk_data/utils/reweight.py b/policyengine_uk_data/utils/reweight.py index 0fc9da8b..e94f9203 100644 --- a/policyengine_uk_data/utils/reweight.py +++ b/policyengine_uk_data/utils/reweight.py @@ -38,7 +38,11 @@ def calibrate_household_weights( datasets such as the UK enhanced CPS. """ - matrix, targets = create_target_matrix(dataset, time_period, reform) + # Only the CPS-based transfer dataset is reweighted here; the coverage + # gate is for the Enhanced FRS build (datasets/create_datasets.py). + matrix, targets = create_target_matrix( + dataset, time_period, reform, require_coverage=False + ) household_count = len(dataset.household) if household_count == 0: raise ValueError("Dataset must contain at least one household") From eb247c646e51fa882d6365ef67beb226666aebd3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 08:05:12 -0400 Subject: [PATCH 05/13] List the three UC payment bands with no signal in pull request builds The first CI build of this branch failed the gate on three single-claimant UC payment bands (24,000 to 28,800 a year). Pull request (TESTING) builds hold fewer records than the release: across the final calibration logs of 12 recent ones, these three have no signal in 11 or all 12, while the released 1.58.1 build has signal in all three. A listed gap that applies to one kind of build only warns as stale in the other. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/targets/known_gaps.yaml | 16 +++++++++++++++- .../tests/test_target_coverage.py | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml index a1e58e17..de36f53a 100644 --- a/policyengine_uk_data/targets/known_gaps.yaml +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -10,7 +10,10 @@ # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with # policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645 # enter the matrix, 15 have no value for 2025 and 19 of those entering have -# no signal. +# no signal. Pull request (TESTING) builds hold fewer records: across the +# final calibration logs of 12 recent ones (targets with a national estimate +# of exactly zero), 22 targets have no signal in at least one, the 19 above +# and 3 single-claimant UC payment bands. schema_version: 1 gaps: - status: no_value_for_year @@ -78,3 +81,14 @@ gaps: - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_27_600_to_28_800 - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_28_800_to_30_000 - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf + - status: zero_signal + reason: >- + In pull request (TESTING) builds no single claimant's UC payments + fall in these bands: in all 12 recent builds for 27,600 to 28,800, + and in 11 of 12 for the other two. The released 1.58.1 build has + signal in all three. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + targets: + - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200 + - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400 + - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800 diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py index ba48fe25..c3de4205 100644 --- a/policyengine_uk_data/tests/test_target_coverage.py +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -50,9 +50,9 @@ def test_the_shipped_known_gaps_load_and_cite_issues(): names = [n for gap in gaps for n in gap.targets] assert len(names) == len(set(names)) # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19 - # targets without signal. + # targets without signal, and 3 more without signal in TESTING builds. assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15 - assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 19 + assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 22 @pytest.mark.parametrize( From 61f8f7ef00297a1344345b87e0f97e9cf5efe398 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 11:11:46 -0400 Subject: [PATCH 06/13] Match ONS age targets to the FRS top code of 80 --- .gitignore | 4 + changelog.d/593-age-topcode.fixed.md | 1 + .../storage/regional_age_80_plus.csv | 13 ++ .../targets/compute/demographics.py | 5 +- .../targets/sources/ons_demographics.py | 53 +++-- .../tests/fixtures/ons_uk_population_2025.csv | 215 ++++++++++++++++++ .../tests/test_ons_demographics_targets.py | 133 +++++++++++ tools/ons_age_targets/README.md | 56 +++++ tools/ons_age_targets/build.py | 138 +++++++++++ 9 files changed, 595 insertions(+), 23 deletions(-) create mode 100644 changelog.d/593-age-topcode.fixed.md create mode 100644 policyengine_uk_data/storage/regional_age_80_plus.csv create mode 100644 policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv create mode 100644 tools/ons_age_targets/README.md create mode 100644 tools/ons_age_targets/build.py diff --git a/.gitignore b/.gitignore index 70d8396d..b0b3a6ab 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ !policyengine_uk_data/datasets/local_areas/**/*.csv **/_build !policyengine_uk_data/storage/*.csv +!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv **/version.json # Build output: household weights by local area, derived from FRS microdata policyengine_uk_data/storage/local_geography_weights.csv.gz @@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local build-job notes, caches and hand-off artefacts +.job/ diff --git a/changelog.d/593-age-topcode.fixed.md b/changelog.d/593-age-topcode.fixed.md new file mode 100644 index 00000000..49bfb27c --- /dev/null +++ b/changelog.d/593-age-topcode.fixed.md @@ -0,0 +1 @@ +Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups. diff --git a/policyengine_uk_data/storage/regional_age_80_plus.csv b/policyengine_uk_data/storage/regional_age_80_plus.csv new file mode 100644 index 00000000..4afa81cb --- /dev/null +++ b/policyengine_uk_data/storage/regional_age_80_plus.csv @@ -0,0 +1,13 @@ +name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url +east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland +scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland +south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales +west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 diff --git a/policyengine_uk_data/targets/compute/demographics.py b/policyengine_uk_data/targets/compute/demographics.py index 4d0b8092..f405f24a 100644 --- a/policyengine_uk_data/targets/compute/demographics.py +++ b/policyengine_uk_data/targets/compute/demographics.py @@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None: region_name = name[:idx] age_part = name[idx + 5 :] lower, upper = age_part.split("_") - lower, upper = int(lower), int(upper) + lower = int(lower) + upper = np.inf if upper == "plus" else int(upper) pe_region = _REGION_INV.get(region_name) if pe_region is None: @@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray: parts = name.split("_") sex = parts[0] lower = int(parts[1]) - upper = int(parts[2]) + upper = np.inf if parts[2] == "plus" else int(parts[2]) gender = ctx.sim.calculate("gender").values sex_match = gender == ("FEMALE" if sex == "female" else "MALE") diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 0d04f631..04a1463f 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -3,10 +3,9 @@ Downloads the ONS 2024-based principal population projection for the UK to extract total population and gender × age band targets. -For regional age breakdowns (12 regions × 9 age bands), reads the -pre-existing demographics.csv which was extracted from ONS subnational -projections. The subnational projections don't have a stable machine- -readable download URL, so this is the pragmatic compromise. +Regional age breakdowns use demographics.csv below age 80 and a committed +ONS projection extract for 80+, including the terminal age group. See +tools/ons_age_targets for the sources and extraction method. Household type and tenure targets are from ONS families & households datasets (also lacking stable machine-readable URLs). @@ -52,7 +51,7 @@ _YEARS = list(range(2022, 2030)) -# Age band boundaries +# FRS ages are top-coded at 80, so the last band includes every older age. _AGE_BANDS = [ (0, 9), (10, 19), @@ -62,7 +61,7 @@ (50, 59), (60, 69), (70, 79), - (80, 89), + (80, None), ] # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only @@ -89,8 +88,7 @@ (65, 69), (70, 74), (75, 79), - (80, 84), - (85, 90), + (80, None), ] @@ -121,12 +119,19 @@ def _find_projection_member(names: list[str]) -> str: def _aggregate_ages( - df: pd.DataFrame, sex: str, low: int, high: int, years: list[int] + df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int] ) -> dict[int, float]: - """Sum population for a sex and age range across years.""" + """Sum an inclusive age band; None includes the projection's terminal age.""" sex_filter = "Females" if sex == "female" else "Males" - ages = pd.to_numeric(df["Age"], errors="coerce") - mask = (df["Sex"] == sex_filter) & ages.between(low, high) + # Use each group's lower age: the 2024-based workbook ends with + # "105 - 109" and "110 and over"; older vintages can use "90+". + ages = pd.to_numeric( + df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce" + ) + age_mask = ages >= low + if high is not None: + age_mask &= ages <= high + mask = (df["Sex"] == sex_filter) & age_mask subset = df[mask] result = {} for y in years: @@ -176,7 +181,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: if values: targets.append( Target( - name=f"ons/{sex}_{low}_{high}", + name=f"ons/{sex}_{low}_{'plus' if high is None else high}", variable="age", source="ons", unit=Unit.COUNT, @@ -190,17 +195,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: def _parse_regional_from_csv() -> list[Target]: - """Read regional age band targets from demographics.csv. - - This CSV was extracted from ONS subnational projections which - lack a stable machine-readable download URL. - """ + """Read regional bands, replacing the legacy bounded top band with 80+.""" csv_path = STORAGE / "demographics.csv" if not csv_path.exists(): logger.warning("demographics.csv not found, skipping regional") return [] demographics = pd.read_csv(csv_path) + # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums, + # including 90+, instead of relabelling totals with a different scope. + demographics = demographics[~demographics["name"].str.endswith("_age_80_89")] + top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv") + top_bands["unit"] = "person" + demographics = pd.concat([demographics, top_bands], ignore_index=True) targets = [] # Skip rows now handled by dedicated modules (ons_households.py, @@ -231,8 +238,8 @@ def _parse_regional_from_csv() -> list[Target]: for y in _YEARS: col = str(y) if col in row.index and pd.notna(row[col]): - # Values in CSV are in thousands - values[y] = float(row[col]) * 1e3 + scale = 1 if row["unit"] == "person" else 1e3 + values[y] = float(row[col]) * scale if values: targets.append( Target( @@ -243,7 +250,11 @@ def _parse_regional_from_csv() -> list[Target]: geographic_level=GeographicLevel.REGION, values=values, is_count=True, - reference_url=_REF_REGION, + reference_url=( + row["reference_url"] + if pd.notna(row.get("reference_url")) + else _REF_REGION + ), ) ) diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv new file mode 100644 index 00000000..585b1c84 --- /dev/null +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -0,0 +1,215 @@ +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_ons_demographics_targets.py b/policyengine_uk_data/tests/test_ons_demographics_targets.py index 433bf3e2..1df25ff4 100644 --- a/policyengine_uk_data/tests/test_ons_demographics_targets.py +++ b/policyengine_uk_data/tests/test_ons_demographics_targets.py @@ -1,9 +1,21 @@ +from pathlib import Path +from types import SimpleNamespace + +import numpy as np import pandas as pd import pytest +from hypothesis import given +from hypothesis import strategies as st +from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources.ons_demographics import ( + _AGE_BANDS, + _GENDER_BANDS, _aggregate_ages, _find_projection_member, + _parse_regional_from_csv, + _parse_uk_totals, ) @@ -22,3 +34,124 @@ def test_aggregate_ages_accepts_string_age_values(): def test_find_projection_member_fails_loudly(): with pytest.raises(RuntimeError, match="uk_ppp_machine_readable"): _find_projection_member(["uk/readme.txt"]) + + +@pytest.mark.parametrize("bands", [_AGE_BANDS, _GENDER_BANDS]) +@given(age=st.integers(min_value=0, max_value=80)) +def test_age_bands_partition_frs_ages_and_respect_top_code(bands, age): + assert bands[-1] == (80, None) + for low, high in bands: + assert low <= 80, "No target may start above the FRS top code" + assert high is None or high < 80, "80+ must be open-ended" + assert sum(low <= age and (high is None or age <= high) for low, high in bands) == 1 + + +@pytest.mark.parametrize("sex,multiplier", [("female", 1), ("male", 10)]) +def test_80_plus_target_sums_every_projection_age(sex, multiplier): + projection = pd.DataFrame( + [ + {"Sex": label, "Age": str(age), 2025: (age + 1) * factor} + for label, factor in [("Females", 1), ("Males", 10)] + for age in range(101) + ] + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == sum( + (age + 1) * multiplier for age in range(80, 101) + ) + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +def test_80_plus_matches_published_ons_2025_projection(sex, label): + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + # The fixture retains ONS's single ages 0-104 and terminal 105-109/110+. + older_ages = {str(age) for age in range(80, 105)} | { + "105 - 109", + "110 and over", + } + expected = projection.loc[ + (projection.Sex == label) & projection.Age.isin(older_ages), "2025" + ].sum() + assert targets[f"ons/{sex}_80_plus"].values[2025] == expected + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +@pytest.mark.parametrize( + "terminal_ages", [["89", "90+"], ["105 - 109", "110 and over"]] +) +def test_80_plus_target_includes_terminal_age_groups(sex, label, terminal_ages): + projection = pd.DataFrame( + { + "Sex": [label] * 4, + "Age": ["79", "80", *terminal_ages], + 2025: [1, 2, 4, 8], + } + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == 14 + + +def test_emitted_age_target_names_respect_frs_top_code(): + projection = pd.DataFrame( + {"Sex": ["Females", "Males"], "Age": [80, 80], 2025: [1, 1]} + ) + targets = _parse_uk_totals(projection) + _parse_regional_from_csv() + top_bands = [] + for target in targets: + if "_age_" not in target.name and not target.name.startswith( + ("ons/female_", "ons/male_") + ): + continue + low, high = target.name.rsplit("_", 2)[-2:] + assert int(low) <= 80, target.name + assert high == "plus" or int(high) < 80, target.name + if int(low) == 80: + top_bands.append(target.name) + assert len(top_bands) == 14 # Both sexes and all twelve regions. + + +@pytest.mark.parametrize( + "name,expected", + [ + ("ons/female_80_plus", [1, 1, 1, 0]), + ("ons/male_80_plus", [0, 1, 1, 1]), + ("ons/london_age_80_plus", [1, 1, 1, 1]), + ], +) +def test_80_plus_dispatch_counts_matching_household_members(name, expected): + household = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + gender = pd.Series( + ["FEMALE", "FEMALE", "MALE", "FEMALE", "MALE", "FEMALE", "MALE", "MALE"] + ) + ctx = SimpleNamespace( + age=np.array([79, 80, 81, 100, 80, 80, 79, 85]), + region=pd.Series( + [ + "LONDON", + "LONDON", + "LONDON", + "WALES", + "WALES", + "LONDON", + "LONDON", + "LONDON", + ] + ), + sim=SimpleNamespace(calculate=lambda variable: gender), + household_from_person=lambda values: np.bincount(household, weights=values), + ) + target = Target( + name=name, + variable="age", + source="ons", + unit=Unit.COUNT, + values={2025: 1}, + is_count=True, + ) + + np.testing.assert_array_equal(_compute_column(target, ctx, 2025), expected) diff --git a/tools/ons_age_targets/README.md b/tools/ons_age_targets/README.md new file mode 100644 index 00000000..7d223513 --- /dev/null +++ b/tools/ons_age_targets/README.md @@ -0,0 +1,56 @@ +# ONS age target sources + +`policyengine_uk_data/storage/regional_age_80_plus.csv` contains public population +projection aggregates in persons. It contains no FRS data. Reproduce it from the +repository root with the locked environment: + +```sh +uv run python tools/ons_age_targets/build.py +``` + +Downloads are cached in `.job/`; SHA-256 checks pin the source files. The generator +uses pandas, requests and openpyxl, which are already locked dependencies. + +The English regions use the [ONS 2022-based single-year subnational +projection](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1), +five-year migration variant, published 24 June 2025. The [ONS release bulletin](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/subnationalpopulationprojectionsforengland/2022based) +identifies this variant as the one aligned with the 2022-based principal national +projection. For each of +the 309 English local authorities, the extract sums ages 80 through 89 and the +terminal `90 and over` group. The committed `storage/oa_crosswalk.csv.gz` maps all +309 authorities unambiguously to the nine English regions. Summing those regions +therefore reproduces the sum of all 309 authorities, without omitting or counting +an authority twice. Published counts have three decimal places; the output retains +that precision. + +Scotland, Wales and Northern Ireland use the ONS **2022-based principal national +projections**, so all 12 regional 80+ targets have the same projection vintage and +cover 2022–2029. Each country total sums both sexes at ages 80–104 and the terminal +groups `105 - 109` and `110 and over`. The regional CSV includes the reference URL +for each row. This replaces the legacy regional top-band values; it is a source +refresh as well as a correction to the open-ended band definition. The lower +regional age bands remain in `demographics.csv`. + +Pinned downloads, relative to +`https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/`: + +| Source | Download suffix | SHA-256 | +| --- | --- | --- | +| English local authorities | `localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip` | `171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b` | +| Scotland | `z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip` | `6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d` | +| Wales | `z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip` | `32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051` | +| Northern Ireland | `z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip` | `69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489` | + +The national targets continue to use the existing **2024-based UK principal +projection**. The test fixture `policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv` preserves +its complete `Sex`, `Age` and `2025` population columns, including `105 - 109` and +`110 and over`. Its source ZIP is +`z1zippedpopulationprojectionsdatafilesuk/2024based/uk.zip`, SHA-256 +`dea79703b9caddd560f1958b5a8ca12aa817b2396d244e6a2c775b1af7f48ab1`. + +The old regional rows first appeared in commit `c98c0ccf` (9 September 2024); +`63dfba48` (10 June 2025) reformatted them without changing their values. Their +reference `ons_age_sex_region` does not identify a projection vintage or extraction +method. Their finite `80_89` names alone do not prove that their published values +excluded 90+: the replacement uses directly reproducible sums rather than adding +an estimated 90+ population to those legacy values. diff --git a/tools/ons_age_targets/build.py b/tools/ons_age_targets/build.py new file mode 100644 index 00000000..b29be1a5 --- /dev/null +++ b/tools/ons_age_targets/build.py @@ -0,0 +1,138 @@ +"""Reproduce the public ONS regional 80+ target extract (no survey data).""" + +import argparse +import hashlib +import io +import zipfile +from pathlib import Path + +import pandas as pd +import requests + +ROOT = Path(__file__).resolve().parents[2] +YEARS = [str(year) for year in range(2022, 2030)] +ONS = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" +SOURCES = { + "regions": ( + "localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip", + "171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b", + ), + "scotland": ( + "z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip", + "6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d", + ), + "wales": ( + "z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip", + "32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051", + ), + "ni": ( + "z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip", + "69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489", + ), +} +REGIONS = { + "E12000001": "north_east", + "E12000002": "north_west", + "E12000003": "yorkshire_and_the_humber", + "E12000004": "east_midlands", + "E12000005": "west_midlands", + "E12000006": "east", + "E12000007": "london", + "E12000008": "south_east", + "E12000009": "south_west", +} + + +def aggregate_english_regions( + population: pd.DataFrame, crosswalk: pd.DataFrame +) -> pd.DataFrame: + """Sum ages 80 through the terminal 90+ row in every English LA.""" + lookup = crosswalk[["la_code", "region_code"]].drop_duplicates() + assert lookup.la_code.is_unique, "Local authorities have ambiguous regions" + ages = population.AGE_GROUP.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[pd.to_numeric(ages, errors="coerce") >= 80].copy() + assert set(older.AGE_GROUP) == {str(age) for age in range(80, 90)} | {"90 and over"} + older = older.merge( + lookup, + left_on="AREA_CODE", + right_on="la_code", + how="left", + validate="many_to_one", + ) + assert older.region_code.notna().all(), "Some local authorities lack a region" + assert set(older.region_code) == set(REGIONS) + assert older.AREA_CODE.nunique() == 309 + assert older.groupby("AREA_CODE").AGE_GROUP.nunique().eq(11).all() + result = older.groupby("region_code")[YEARS].sum().round(3) + assert result.sum().round(3).equals(older[YEARS].sum().round(3)) + result.index = result.index.map(REGIONS) + return result + + +def aggregate_country(population: pd.DataFrame) -> pd.Series: + """Include all single years and terminal age groups from 80 upwards.""" + ages = population.Age.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[ + (pd.to_numeric(ages, errors="coerce") >= 80) + & population.Sex.isin(["Females", "Males"]) + ] + assert {"105 - 109", "110 and over"}.issubset(set(older.Age.astype(str))) + older = older.rename(columns=lambda column: str(column)) + return older[YEARS].sum() + + +def build(source_dir: Path, output: Path) -> None: + """Download pinned public sources and save their exact published sums.""" + source_dir.mkdir(parents=True, exist_ok=True) + for name, (suffix, expected_sha256) in SOURCES.items(): + path = source_dir / f"ons-{name}-2022.zip" + if not path.exists(): + response = requests.get( + ONS + suffix, + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + }, + timeout=120, + ) + response.raise_for_status() + path.write_bytes(response.content) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256 + + with zipfile.ZipFile(source_dir / "ons-regions-2022.zip") as archive: + population = pd.read_csv(archive.open("2022 SNPP Population persons.csv")) + crosswalk = pd.read_csv(ROOT / "policyengine_uk_data/storage/oa_crosswalk.csv.gz") + totals = aggregate_english_regions(population, crosswalk) + for country, prefix in [("scotland", "sc"), ("wales", "wa"), ("ni", "ni")]: + with zipfile.ZipFile(source_dir / f"ons-{country}-2022.zip") as archive: + population = pd.read_excel( + io.BytesIO( + archive.read(f"{prefix}/{prefix}_ppp_machine_readable.xlsx") + ), + sheet_name="Population", + engine="openpyxl", + ) + totals.loc["northern_ireland" if country == "ni" else country] = ( + aggregate_country(population) + ) + totals.index = [f"{region}_age_80_plus" for region in totals.index] + totals.index.name = "name" + reference_base = "https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" + totals["reference_url"] = reference_base + "localauthoritiesinenglandz1" + for country in ["scotland", "wales", "ni"]: + region = "northern_ireland" if country == "ni" else country + totals.loc[f"{region}_age_80_plus", "reference_url"] = ( + reference_base + SOURCES[country][0].split("/")[0] + ) + totals.sort_index().to_csv(output, float_format="%.3f") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", type=Path, default=ROOT / ".job") + parser.add_argument( + "--output", + type=Path, + default=ROOT / "policyengine_uk_data/storage/regional_age_80_plus.csv", + ) + args = parser.parse_args() + build(args.source_dir, args.output) From 29ca75420b885ba7cd0ee3819817a974841a5a99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 23:10:08 -0400 Subject: [PATCH 07/13] Fix national age-band tests for open-ended 80+ targets --- .../targets/sources/ons_demographics.py | 8 +- .../tests/fixtures/ons_uk_population_2025.csv | 430 +++++++++--------- .../tests/test_national_age_bands.py | 32 +- 3 files changed, 241 insertions(+), 229 deletions(-) diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 04a1463f..a40a0a4d 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -68,9 +68,11 @@ # the band totals, leaving the age composition *within* a band unconstrained # nationally: the calibration could satisfy 15-29 while starving 18-24, which # is what collapsed the modelled young-adult population to ~3.4M against ONS's -# ~5.4M. Regional targets already use finer bands, but they do not constrain -# the national weights. The ONS projection is published by single year of age, -# so this is a change of aggregation only - no new source data. +# ~5.4M. Regional ten-year bands also constrain the shared household weights, +# including summed local weights in dense local calibration, but provide less +# age resolution than these five-year sex bands. The ONS projection is published +# by single year of age, so this is a change of aggregation only - no new source +# data. _GENDER_BANDS = [ (0, 4), (5, 9), diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv index 585b1c84..911aae4f 100644 --- a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -1,215 +1,215 @@ -Sex,Age,2025 -Females,0,317686 -Females,1,327935 -Females,2,338279 -Females,3,358658 -Females,4,356258 -Females,5,369761 -Females,6,376378 -Females,7,381376 -Females,8,390185 -Females,9,400480 -Females,10,397930 -Females,11,399560 -Females,12,408413 -Females,13,418244 -Females,14,417732 -Females,15,413729 -Females,16,410318 -Females,17,414288 -Females,18,399447 -Females,19,398620 -Females,20,406477 -Females,21,404249 -Females,22,401940 -Females,23,408447 -Females,24,412365 -Females,25,422024 -Females,26,444655 -Females,27,458010 -Females,28,469985 -Females,29,463354 -Females,30,470982 -Females,31,484020 -Females,32,486083 -Females,33,498400 -Females,34,504074 -Females,35,503593 -Females,36,500871 -Females,37,507256 -Females,38,496049 -Females,39,489353 -Females,40,485837 -Females,41,469975 -Females,42,469221 -Females,43,463843 -Females,44,467338 -Females,45,465639 -Females,46,444476 -Females,47,412021 -Females,48,402609 -Females,49,410811 -Females,50,417510 -Females,51,422475 -Females,52,438299 -Females,53,454957 -Females,54,473232 -Females,55,460650 -Females,56,470719 -Females,57,469471 -Females,58,470280 -Females,59,469617 -Females,60,471064 -Females,61,464261 -Females,62,452458 -Females,63,439638 -Females,64,421487 -Females,65,403839 -Females,66,393032 -Females,67,380880 -Females,68,364266 -Females,69,348875 -Females,70,334931 -Females,71,333201 -Females,72,324950 -Females,73,313788 -Females,74,312458 -Females,75,314249 -Females,76,317096 -Females,77,330339 -Females,78,350379 -Females,79,263767 -Females,80,249905 -Females,81,242088 -Females,82,219106 -Females,83,189777 -Females,84,163167 -Females,85,161170 -Females,86,150872 -Females,87,137597 -Females,88,121030 -Females,89,105738 -Females,90,90358 -Females,91,73607 -Females,92,61708 -Females,93,51510 -Females,94,41269 -Females,95,32160 -Females,96,23665 -Females,97,16907 -Females,98,11918 -Females,99,8188 -Females,100,5349 -Females,101,3385 -Females,102,2064 -Females,103,1211 -Females,104,677 -Females,105 - 109,546 -Females,110 and over,6 -Males,0,333572 -Males,1,344421 -Males,2,356607 -Males,3,376143 -Males,4,373747 -Males,5,386967 -Males,6,395428 -Males,7,399866 -Males,8,409003 -Males,9,420639 -Males,10,417531 -Males,11,418996 -Males,12,428612 -Males,13,438674 -Males,14,439496 -Males,15,433546 -Males,16,431615 -Males,17,436434 -Males,18,425336 -Males,19,422999 -Males,20,430550 -Males,21,430470 -Males,22,426240 -Males,23,423436 -Males,24,426041 -Males,25,435235 -Males,26,447885 -Males,27,452338 -Males,28,459538 -Males,29,451896 -Males,30,454060 -Males,31,462487 -Males,32,460659 -Males,33,470586 -Males,34,476185 -Males,35,473059 -Males,36,467528 -Males,37,469878 -Males,38,458073 -Males,39,457277 -Males,40,454111 -Males,41,439326 -Males,42,438740 -Males,43,436382 -Males,44,439505 -Males,45,441201 -Males,46,423905 -Males,47,393520 -Males,48,386917 -Males,49,395093 -Males,50,402512 -Males,51,406800 -Males,52,422253 -Males,53,436908 -Males,54,450989 -Males,55,439963 -Males,56,449773 -Males,57,447985 -Males,58,451039 -Males,59,446161 -Males,60,447763 -Males,61,440594 -Males,62,429773 -Males,63,418061 -Males,64,402529 -Males,65,384785 -Males,66,372436 -Males,67,360028 -Males,68,341388 -Males,69,326411 -Males,70,309308 -Males,71,305067 -Males,72,296642 -Males,73,281653 -Males,74,279520 -Males,75,277095 -Males,76,279193 -Males,77,286999 -Males,78,302900 -Males,79,223985 -Males,80,208431 -Males,81,199761 -Males,82,175241 -Males,83,147423 -Males,84,122567 -Males,85,118037 -Males,86,107840 -Males,87,95378 -Males,88,80913 -Males,89,67392 -Males,90,55358 -Males,91,43439 -Males,92,34474 -Males,93,26990 -Males,94,20167 -Males,95,14236 -Males,96,9666 -Males,97,6400 -Males,98,4165 -Males,99,2573 -Males,100,1535 -Males,101,844 -Males,102,456 -Males,103,227 -Males,104,109 -Males,105 - 109,73 -Males,110 and over,0 +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_national_age_bands.py b/policyengine_uk_data/tests/test_national_age_bands.py index f636255a..354929a6 100644 --- a/policyengine_uk_data/tests/test_national_age_bands.py +++ b/policyengine_uk_data/tests/test_national_age_bands.py @@ -4,8 +4,8 @@ 15-29, ...). A band that wide pins only its own total: the calibration could satisfy 15-29 while distributing those people any way it liked inside the band, and it settled on one that starved 18-24 down to ~3.4M against ONS's -~5.4M. The finer regional bands do not help, because they constrain regional -weights rather than the national ones. +~5.4M. Regional ten-year age bands constrain the same household weights, +but provide less age detail than national five-year sex bands. These tests pin the five-year granularity and the no-overlap property, so a future edit cannot quietly widen the bands or reintroduce the superseded CSV @@ -13,13 +13,17 @@ """ import re +from pathlib import Path +import pandas as pd + +from policyengine_uk_data.targets.sources import ons_demographics from policyengine_uk_data.targets.sources.ons_demographics import ( _GENDER_BANDS, get_targets, ) -_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+)$") +_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+|plus)$") def _gender_age_targets(): @@ -27,9 +31,11 @@ def _gender_age_targets(): def test_bands_are_at_most_five_years_wide(): - """A wider band leaves its internal age composition unconstrained.""" - for low, high in _GENDER_BANDS: - assert high - low <= 5, f"band {low}-{high} is too wide to pin composition" + """Only the terminal top-coded band may be wider than five years.""" + assert _GENDER_BANDS[-1] == (80, None) + for low, high in _GENDER_BANDS[:-1]: + assert high is not None, "only the terminal band may be open-ended" + assert 0 <= high - low < 5, f"band {low}-{high} is too wide to pin composition" def test_bands_are_contiguous_and_non_overlapping(): @@ -41,7 +47,7 @@ def test_bands_are_contiguous_and_non_overlapping(): def test_young_adult_ages_are_covered_by_narrow_bands(): """18-24 must not sit inside one wide band, which is the failure mode.""" - covering = [b for b in _GENDER_BANDS if b[0] <= 24 and b[1] >= 18] + covering = [b for b in _GENDER_BANDS if b[0] <= 24 and (b[1] is None or b[1] >= 18)] assert len(covering) >= 2, ( "18-24 is covered by a single band; its composition would be free" ) @@ -54,9 +60,13 @@ def test_national_gender_age_targets_are_emitted_once_per_band(): assert len({t.name for t in targets}) == len(targets) -def test_national_gender_age_targets_sum_to_uk_population(): - """Double-counted bands would show up here as a doubled total.""" +def test_national_gender_age_targets_sum_to_uk_population(monkeypatch): + """Every fixture age must be counted exactly once across both sexes.""" + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + monkeypatch.setattr(ons_demographics, "_download_uk_projection", lambda: projection) targets = _gender_age_targets() year = 2025 - total = sum(t.values.get(year, 0) for t in targets) - assert 65e6 < total < 73e6, f"national gender-age total {total / 1e6:.1f}M" + total = sum(t.values[year] for t in targets) + assert total == projection[str(year)].sum() From a5905739cfb325b562c85a6f7b4693189fbf9197 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 08/13] Report duplicate target names, scope known gaps to a build kind, and test the matrix against main's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers Sol's review of #595 (F2, F3, F5, F6, F7). - Duplicates (F2). build_target_matrix now reads every target the sources emit (registry.emitted_targets) and select_matrix_targets keeps the first of each name, national then region then country, exactly as before. Each target it drops gets the status `duplicate`, which fails the build unless known_gaps.yaml lists it. test_no_duplicate_names counts the raw emission, so it can fail. On the current registry 159 targets are dropped this way: 156 HMRC SPI projections (#557) and 3 ONS copies (#605). They are listed; a name may be listed under two statuses, since the ten SPI £1m+ bands are both without signal and duplicated. - Build kinds (F3). A known gap may set `builds: [testing]` or `[release]`; the build kind comes from TESTING, as in create_datasets. A testing-only gap may name fnmatch `patterns`, and one now covers every UC payment band in testing builds (#598), replacing the three SINGLE bands; release builds still fail on any band without signal but the three COUPLE_NO_CHILDREN ones. A pattern is stale only if it matches no target. Errors and non-finite values are never excused, whatever a gap says. Stale gaps and problems are also written, once each, to $GITHUB_STEP_SUMMARY. - Status order (F5). The status is computed after the column enters the matrix, and a column whose status fails leaves it, so each offered target ends with exactly one status. - Reasons (F6). The 85-90 age gaps are permanent while ages are top-coded at 80 (#604); no record can hold a postgraduate loan. - A Hypothesis differential test checks that require_coverage=False builds main's matrix from the same sources, duplicates included. Co-Authored-By: Claude Opus 5.5 --- changelog.d/target-coverage-gate.changed.md | 2 +- .../targets/build_loss_matrix.py | 78 ++- policyengine_uk_data/targets/coverage.py | 140 ++++- policyengine_uk_data/targets/known_gaps.yaml | 236 +++++++- policyengine_uk_data/targets/registry.py | 30 +- policyengine_uk_data/tests/conftest.py | 6 + .../tests/test_obr_universal_credit_target.py | 10 +- .../tests/test_target_coverage.py | 568 ++++++++++++++---- .../tests/test_target_registry.py | 23 +- 9 files changed, 884 insertions(+), 209 deletions(-) diff --git a/changelog.d/target-coverage-gate.changed.md b/changelog.d/target-coverage-gate.changed.md index 769f3a14..bcda429e 100644 --- a/changelog.d/target-coverage-gate.changed.md +++ b/changelog.d/target-coverage-gate.changed.md @@ -1 +1 @@ -Fail the calibration build when a national target is dropped (no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset. +Fail the calibration build when a national target is dropped (its name already taken, no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue, for every build or only for testing or release builds. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset. diff --git a/policyengine_uk_data/targets/build_loss_matrix.py b/policyengine_uk_data/targets/build_loss_matrix.py index 49f22a4c..b9d19759 100644 --- a/policyengine_uk_data/targets/build_loss_matrix.py +++ b/policyengine_uk_data/targets/build_loss_matrix.py @@ -16,12 +16,12 @@ import numpy as np import pandas as pd -from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.coverage import ( TargetStatus, check_coverage, column_status, ) +from policyengine_uk_data.targets.registry import emitted_targets from policyengine_uk_data.targets.schema import ( GeographicLevel, Target, @@ -91,12 +91,53 @@ def create_target_matrix( return df, targets +MATRIX_LEVELS = ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, +) + + +def select_matrix_targets(targets): + """The targets the matrix offers, and a ``duplicate`` status for each other. + + Targets are taken level by level (national, region, country), each level + in the order the sources emit them, and the first target of each name + wins. A later target of that name, at the same level or another, is + dropped and recorded as a duplicate (targets/coverage.py). + """ + kept, statuses = {}, [] + for level in MATRIX_LEVELS: + for target in targets: + if target.geographic_level != level: + continue + if target.name not in kept: + kept[target.name] = target + continue + first = kept[target.name] + statuses.append( + TargetStatus( + target.name, + "duplicate", + f"{target.source} {level.value} target for {_years(target)} " + f"dropped for the {first.source} " + f"{first.geographic_level.value} one for {_years(first)}", + ) + ) + return list(kept.values()), statuses + + +def _years(target: Target) -> str: + return ", ".join(str(year) for year in sorted(target.values)) + + def build_target_matrix(dataset, time_period: str = None, reform=None): """The target matrix, its values, and each offered target's status. - A target that is dropped (no value for the year, no column, an error) - or that enters with a non-finite value or column, or with no household - contributing, is recorded with that status (targets/coverage.py). + A target that is dropped (a duplicate name, no value for the year, no + column, an error) or that enters with a non-finite value or column, or + with no household contributing, is recorded with that status + (targets/coverage.py). Each offered target gets exactly one status. """ from policyengine_uk import Microsimulation @@ -109,22 +150,11 @@ def build_target_matrix(dataset, time_period: str = None, reform=None): ctx = _SimContext(sim, time_period, dataset, reform) - all_targets = [] - seen = set() - for level in ( - GeographicLevel.NATIONAL, - GeographicLevel.REGION, - GeographicLevel.COUNTRY, - ): - for t in get_all_targets(geographic_level=level): - if t.name not in seen: - seen.add(t.name) - all_targets.append(t) + all_targets, statuses = select_matrix_targets(emitted_targets()) df = pd.DataFrame() target_names = [] target_values = [] - statuses = [] for target in all_targets: try: @@ -132,9 +162,7 @@ def build_target_matrix(dataset, time_period: str = None, reform=None): if val is None: statuses.append( TargetStatus( - target.name, - "no_value_for_year", - f"values for {', '.join(str(y) for y in sorted(target.values))}", + target.name, "no_value_for_year", f"values for {_years(target)}" ) ) continue @@ -142,9 +170,15 @@ def build_target_matrix(dataset, time_period: str = None, reform=None): if col is None: statuses.append(TargetStatus(target.name, "no_column")) continue - col = restrict_to_countries(col, ctx.country, target.countries) - statuses.append(TargetStatus(target.name, column_status(val, col))) - df[target.name] = col + df[target.name] = restrict_to_countries(col, ctx.country, target.countries) + # The status is of the column as it entered the matrix; a column + # whose status cannot be computed leaves the matrix again. + try: + status = column_status(val, df[target.name]) + except Exception: + del df[target.name] + raise + statuses.append(TargetStatus(target.name, status)) target_names.append(target.name) target_values.append(val) except Exception as e: diff --git a/policyengine_uk_data/targets/coverage.py b/policyengine_uk_data/targets/coverage.py index 35bf5729..50147003 100644 --- a/policyengine_uk_data/targets/coverage.py +++ b/policyengine_uk_data/targets/coverage.py @@ -1,25 +1,29 @@ """Required coverage of the national calibration targets. Every target the registry offers must enter the national loss matrix with -a finite value and a column that some household contributes to. The -exceptions are the targets ``targets/known_gaps.yaml`` lists, each with the -one status it may have, a reason and an issue. ``create_target_matrix`` -checks this, so a build that would calibrate without a target, or against -one the weights cannot move, fails instead of logging a warning. +a finite value and a column that some household contributes to, under a +name no other target has. The exceptions are the targets +``targets/known_gaps.yaml`` lists, each with the one status it may have, a +reason and an issue, for every build or only for testing or release builds. +``create_target_matrix`` checks this, so a build that would calibrate +without a target, or against one the weights cannot move, fails instead of +logging a warning. python -m policyengine_uk_data.targets.coverage [out.json] reports every target's status on a dataset: names, statuses and counts only, -never household values. +never household values. Set TESTING=1 to check it as a testing build. """ from __future__ import annotations import json import logging +import os import sys -from collections import Counter +from collections import Counter, defaultdict from dataclasses import dataclass +from fnmatch import fnmatchcase from pathlib import Path from typing import Iterable, Optional, Sequence @@ -32,9 +36,11 @@ IN_MATRIX = "in_matrix" # Statuses a known gap may excuse. An error or a non-finite value is never # a known gap: it is a broken target. -EXCUSABLE = ("no_value_for_year", "no_column", "zero_signal") +EXCUSABLE = ("duplicate", "no_value_for_year", "no_column", "zero_signal") NEVER_EXCUSED = ("error", "non_finite") STATUSES = (IN_MATRIX,) + EXCUSABLE + NEVER_EXCUSED +# Pull request builds set TESTING=1 (smaller, faster); release builds don't. +BUILDS = ("testing", "release") class TargetCoverageError(Exception): @@ -54,6 +60,14 @@ class KnownGap: targets: frozenset reason: str issue: str + builds: frozenset = frozenset(BUILDS) + # fnmatch-style globs over target names; testing builds only. + patterns: tuple = () + + +def current_build() -> str: + """The kind of build running, read from TESTING as the build itself does.""" + return "testing" if os.environ.get("TESTING", "0") == "1" else "release" def parse_known_gaps(raw) -> list: @@ -68,21 +82,37 @@ def parse_known_gaps(raw) -> list: "https://github.com/" ): raise ValueError(f"{where}: needs a reason and an issue URL") + builds = entry.get("builds", list(BUILDS)) + if not isinstance(builds, list) or not builds or not set(builds) <= set(BUILDS): + raise ValueError(f"{where}: builds must list testing and/or release") targets = list(entry.get("targets") or []) - if not targets: + patterns = list(entry.get("patterns") or []) + if not targets and not patterns: raise ValueError(f"{where}: lists no targets") - repeated = seen.intersection(targets) | { - t for t, n in Counter(targets).items() if n > 1 + if patterns and set(builds) != {"testing"}: + # A release gap names each target, so stale entries show up. + raise ValueError(f"{where}: patterns may only excuse testing builds") + if any(not set("*?[") & set(pattern) for pattern in patterns): + raise ValueError(f"{where}: a pattern needs a wildcard") + # A name may be listed under two statuses: a target without signal + # whose duplicate is dropped, say. Under one status, it is listed once. + names = [(name, entry["status"]) for name in targets + patterns] + repeated = seen.intersection(names) | { + t for t, n in Counter(names).items() if n > 1 } if repeated: - raise ValueError(f"{where}: {sorted(repeated)} listed twice") - seen.update(targets) + raise ValueError( + f"{where}: {sorted(name for name, _ in repeated)} listed twice" + ) + seen.update(names) gaps.append( KnownGap( entry["status"], frozenset(targets), str(entry["reason"]), str(entry["issue"]), + frozenset(builds), + tuple(patterns), ) ) return gaps @@ -107,42 +137,71 @@ def column_status(value, column) -> str: return IN_MATRIX -def coverage_problems(statuses: Sequence[TargetStatus], gaps: Iterable[KnownGap]): - """(problems, stale): targets out of place, and known gaps that no longer apply.""" - gaps = list(gaps) - excused = {name: gap.status for gap in gaps for name in gap.targets} - problems, matched = [], set() +def coverage_problems( + statuses: Sequence[TargetStatus], + gaps: Iterable[KnownGap], + build: Optional[str] = None, +): + """(problems, stale) in one kind of build, by default the one running. + + Problems are the targets out of the matrix that no gap for this build + excuses under their own status. Stale gaps are named targets that no + longer have that status, and patterns that match no offered target: + what a pattern excuses varies from one testing build to the next, so + matching nothing in one build is not stale. + """ + build = build or current_build() + gaps = [gap for gap in gaps if build in gap.builds] + named = {(name, gap.status) for gap in gaps for name in gap.targets} + patterns = [(pattern, gap.status) for gap in gaps for pattern in gap.patterns] + problems, matched, present = [], set(), defaultdict(list) for row in statuses: + present[row.name].append(row.status) if row.status == IN_MATRIX: continue - if excused.get(row.name) == row.status: - matched.add(row.name) - continue + if row.status not in NEVER_EXCUSED: + if (row.name, row.status) in named: + matched.add((row.name, row.status)) + continue + if any( + status == row.status and fnmatchcase(row.name, pattern) + for pattern, status in patterns + ): + continue detail = f" ({row.detail})" if row.detail else "" problems.append(f"{row.name}: {row.status}{detail}") - present = {row.name: row.status for row in statuses} stale = [] for gap in gaps: - for name in sorted(gap.targets - matched): - now = present.get(name, "not offered by the registry") + for name in sorted(gap.targets): + if (name, gap.status) in matched: + continue + now = ", ".join(present.get(name, ["not offered by the registry"])) stale.append(f"{name}: listed as {gap.status}, now {now}") + for pattern in gap.patterns: + if not any(fnmatchcase(name, pattern) for name in present): + stale.append(f"{pattern}: listed as {gap.status}, matches no target") return problems, stale def check_coverage( - statuses: Sequence[TargetStatus], gaps: Optional[Iterable[KnownGap]] = None + statuses: Sequence[TargetStatus], + gaps: Optional[Iterable[KnownGap]] = None, + build: Optional[str] = None, ) -> list: """Raise unless every target is in the matrix or a known gap; return stale gaps.""" + build = build or current_build() problems, stale = coverage_problems( - statuses, load_known_gaps() if gaps is None else gaps + statuses, load_known_gaps() if gaps is None else gaps, build ) counts = Counter(row.status for row in statuses) logger.info( - "Calibration targets: %s", + "Calibration targets (%s build): %s", + build, ", ".join(f"{counts[s]} {s}" for s in STATUSES if counts[s]), ) for line in stale: logger.warning("Stale known calibration-target gap, remove it: %s", line) + write_step_summary(build, stale, problems) if problems: raise TargetCoverageError( f"{len(problems)} calibration targets are missing or cannot be fitted, " @@ -151,6 +210,27 @@ def check_coverage( return stale +def write_step_summary(build: str, stale: Sequence[str], problems: Sequence[str]): + """Add stale gaps and problems to the GitHub Actions job summary, once each. + + A long build log buries a warning; the job summary shows it on the run's + page. Lines already in the summary (the local-authority calibration + repeats the constituency check) are not written again. + """ + path = os.environ.get("GITHUB_STEP_SUMMARY") + lines = [f"- Stale, remove from targets/known_gaps.yaml: `{s}`" for s in stale] + lines += [f"- Not in the matrix and not a known gap: `{p}`" for p in problems] + if not path or not lines: + return + summary = Path(path) + written = summary.read_text() if summary.exists() else "" + lines = [line for line in lines if line not in written] + if lines: + with summary.open("a") as f: + f.write(f"\n### Calibration target coverage ({build} build)\n\n") + f.write("\n".join(lines) + "\n") + + def main(argv=None) -> int: argv = sys.argv[1:] if argv is None else argv if len(argv) not in (2, 3): @@ -167,6 +247,7 @@ def main(argv=None) -> int: report = { "dataset": Path(argv[0]).name, "year": int(argv[1]), + "build": current_build(), "by_status": Counter(row.status for row in statuses), "problems": problems, "stale_known_gaps": stale, @@ -176,7 +257,10 @@ def main(argv=None) -> int: Path(argv[2]).write_text(json.dumps(report, indent=1)) print( json.dumps( - {k: report[k] for k in ("by_status", "problems", "stale_known_gaps")}, + { + k: report[k] + for k in ("build", "by_status", "problems", "stale_known_gaps") + }, indent=1, ) ) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml index de36f53a..7b28512a 100644 --- a/policyengine_uk_data/targets/known_gaps.yaml +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -1,19 +1,23 @@ # Calibration targets allowed to stay out of the national loss matrix, or to # enter it with no signal. targets/coverage.py fails the build on any other -# target that is dropped (no value for the year, no column, an error), has a -# non-finite value or column, or has a column no household contributes to. +# target that is dropped (a name already taken, no value for the year, no +# column, an error), has a non-finite value or column, or has a column no +# household contributes to. # # Each entry names the exact targets, the one status they may have, why and # the issue that tracks removing them. Errors and non-finite values are never -# allowed. An entry that no longer matches is reported as stale: remove it. +# allowed. An entry applies to every build unless `builds` limits it to +# testing builds (pull requests, TESTING=1) or release builds. A +# testing-only entry may name `patterns` instead, fnmatch globs over target +# names, for targets whose status varies from one testing build to the next. +# A named target that no longer has its status, or a pattern that matches +# no target at all, is reported as stale: remove it. # # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with -# policyengine-uk 2.122.2 at the calibration year 2025: of 660 targets, 645 -# enter the matrix, 15 have no value for 2025 and 19 of those entering have -# no signal. Pull request (TESTING) builds hold fewer records: across the -# final calibration logs of 12 recent ones (targets with a national estimate -# of exactly zero), 22 targets have no signal in at least one, the 19 above -# and 3 single-claimant UC payment bands. +# policyengine-uk 2.122.2 at the calibration year 2025, as a release build: +# the registry offers 660 names. 645 enter the matrix, 15 have no value for +# 2025 and 19 of those entering have no signal. 159 more targets reuse a +# name already taken and are dropped as duplicates. schema_version: 1 gaps: - status: no_value_for_year @@ -37,6 +41,182 @@ gaps: - dwp/uc/two_child_limit/disabled_child_element_children - dwp/uc/two_child_limit/disabled_child_element_households - dwp/uc/two_child_limit/households_affected + - status: duplicate + reason: >- + hmrc_spi emits each income-band target twice: the 2023-24 outturn, with + a value for 2024 only, then the incomes_projection.csv copy for 2025 to + 2029. The matrix keeps the first, so calibration at 2025 carries the + 2024 outturn forward and never sees the projection. #557 emits each band + once. + issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/557 + targets: + - hmrc/dividend_income_count_income_band_100_000_to_150_000 + - hmrc/dividend_income_count_income_band_12_570_to_15_000 + - hmrc/dividend_income_count_income_band_150_000_to_200_000 + - hmrc/dividend_income_count_income_band_15_000_to_20_000 + - hmrc/dividend_income_count_income_band_1_000_000_to_inf + - hmrc/dividend_income_count_income_band_200_000_to_300_000 + - hmrc/dividend_income_count_income_band_20_000_to_30_000 + - hmrc/dividend_income_count_income_band_300_000_to_500_000 + - hmrc/dividend_income_count_income_band_30_000_to_40_000 + - hmrc/dividend_income_count_income_band_40_000_to_50_000 + - hmrc/dividend_income_count_income_band_500_000_to_1_000_000 + - hmrc/dividend_income_count_income_band_50_000_to_70_000 + - hmrc/dividend_income_count_income_band_70_000_to_100_000 + - hmrc/dividend_income_income_band_100_000_to_150_000 + - hmrc/dividend_income_income_band_12_570_to_15_000 + - hmrc/dividend_income_income_band_150_000_to_200_000 + - hmrc/dividend_income_income_band_15_000_to_20_000 + - hmrc/dividend_income_income_band_1_000_000_to_inf + - hmrc/dividend_income_income_band_200_000_to_300_000 + - hmrc/dividend_income_income_band_20_000_to_30_000 + - hmrc/dividend_income_income_band_300_000_to_500_000 + - hmrc/dividend_income_income_band_30_000_to_40_000 + - hmrc/dividend_income_income_band_40_000_to_50_000 + - hmrc/dividend_income_income_band_500_000_to_1_000_000 + - hmrc/dividend_income_income_band_50_000_to_70_000 + - hmrc/dividend_income_income_band_70_000_to_100_000 + - hmrc/employment_income_count_income_band_100_000_to_150_000 + - hmrc/employment_income_count_income_band_12_570_to_15_000 + - hmrc/employment_income_count_income_band_150_000_to_200_000 + - hmrc/employment_income_count_income_band_15_000_to_20_000 + - hmrc/employment_income_count_income_band_1_000_000_to_inf + - hmrc/employment_income_count_income_band_200_000_to_300_000 + - hmrc/employment_income_count_income_band_20_000_to_30_000 + - hmrc/employment_income_count_income_band_300_000_to_500_000 + - hmrc/employment_income_count_income_band_30_000_to_40_000 + - hmrc/employment_income_count_income_band_40_000_to_50_000 + - hmrc/employment_income_count_income_band_500_000_to_1_000_000 + - hmrc/employment_income_count_income_band_50_000_to_70_000 + - hmrc/employment_income_count_income_band_70_000_to_100_000 + - hmrc/employment_income_income_band_100_000_to_150_000 + - hmrc/employment_income_income_band_12_570_to_15_000 + - hmrc/employment_income_income_band_150_000_to_200_000 + - hmrc/employment_income_income_band_15_000_to_20_000 + - hmrc/employment_income_income_band_1_000_000_to_inf + - hmrc/employment_income_income_band_200_000_to_300_000 + - hmrc/employment_income_income_band_20_000_to_30_000 + - hmrc/employment_income_income_band_300_000_to_500_000 + - hmrc/employment_income_income_band_30_000_to_40_000 + - hmrc/employment_income_income_band_40_000_to_50_000 + - hmrc/employment_income_income_band_500_000_to_1_000_000 + - hmrc/employment_income_income_band_50_000_to_70_000 + - hmrc/employment_income_income_band_70_000_to_100_000 + - hmrc/private_pension_income_count_income_band_100_000_to_150_000 + - hmrc/private_pension_income_count_income_band_12_570_to_15_000 + - hmrc/private_pension_income_count_income_band_150_000_to_200_000 + - hmrc/private_pension_income_count_income_band_15_000_to_20_000 + - hmrc/private_pension_income_count_income_band_1_000_000_to_inf + - hmrc/private_pension_income_count_income_band_200_000_to_300_000 + - hmrc/private_pension_income_count_income_band_20_000_to_30_000 + - hmrc/private_pension_income_count_income_band_300_000_to_500_000 + - hmrc/private_pension_income_count_income_band_30_000_to_40_000 + - hmrc/private_pension_income_count_income_band_40_000_to_50_000 + - hmrc/private_pension_income_count_income_band_500_000_to_1_000_000 + - hmrc/private_pension_income_count_income_band_50_000_to_70_000 + - hmrc/private_pension_income_count_income_band_70_000_to_100_000 + - hmrc/private_pension_income_income_band_100_000_to_150_000 + - hmrc/private_pension_income_income_band_12_570_to_15_000 + - hmrc/private_pension_income_income_band_150_000_to_200_000 + - hmrc/private_pension_income_income_band_15_000_to_20_000 + - hmrc/private_pension_income_income_band_1_000_000_to_inf + - hmrc/private_pension_income_income_band_200_000_to_300_000 + - hmrc/private_pension_income_income_band_20_000_to_30_000 + - hmrc/private_pension_income_income_band_300_000_to_500_000 + - hmrc/private_pension_income_income_band_30_000_to_40_000 + - hmrc/private_pension_income_income_band_40_000_to_50_000 + - hmrc/private_pension_income_income_band_500_000_to_1_000_000 + - hmrc/private_pension_income_income_band_50_000_to_70_000 + - hmrc/private_pension_income_income_band_70_000_to_100_000 + - hmrc/property_income_count_income_band_100_000_to_150_000 + - hmrc/property_income_count_income_band_12_570_to_15_000 + - hmrc/property_income_count_income_band_150_000_to_200_000 + - hmrc/property_income_count_income_band_15_000_to_20_000 + - hmrc/property_income_count_income_band_1_000_000_to_inf + - hmrc/property_income_count_income_band_200_000_to_300_000 + - hmrc/property_income_count_income_band_20_000_to_30_000 + - hmrc/property_income_count_income_band_300_000_to_500_000 + - hmrc/property_income_count_income_band_30_000_to_40_000 + - hmrc/property_income_count_income_band_40_000_to_50_000 + - hmrc/property_income_count_income_band_500_000_to_1_000_000 + - hmrc/property_income_count_income_band_50_000_to_70_000 + - hmrc/property_income_count_income_band_70_000_to_100_000 + - hmrc/property_income_income_band_100_000_to_150_000 + - hmrc/property_income_income_band_12_570_to_15_000 + - hmrc/property_income_income_band_150_000_to_200_000 + - hmrc/property_income_income_band_15_000_to_20_000 + - hmrc/property_income_income_band_1_000_000_to_inf + - hmrc/property_income_income_band_200_000_to_300_000 + - hmrc/property_income_income_band_20_000_to_30_000 + - hmrc/property_income_income_band_300_000_to_500_000 + - hmrc/property_income_income_band_30_000_to_40_000 + - hmrc/property_income_income_band_40_000_to_50_000 + - hmrc/property_income_income_band_500_000_to_1_000_000 + - hmrc/property_income_income_band_50_000_to_70_000 + - hmrc/property_income_income_band_70_000_to_100_000 + - hmrc/self_employment_income_count_income_band_100_000_to_150_000 + - hmrc/self_employment_income_count_income_band_12_570_to_15_000 + - hmrc/self_employment_income_count_income_band_150_000_to_200_000 + - hmrc/self_employment_income_count_income_band_15_000_to_20_000 + - hmrc/self_employment_income_count_income_band_1_000_000_to_inf + - hmrc/self_employment_income_count_income_band_200_000_to_300_000 + - hmrc/self_employment_income_count_income_band_20_000_to_30_000 + - hmrc/self_employment_income_count_income_band_300_000_to_500_000 + - hmrc/self_employment_income_count_income_band_30_000_to_40_000 + - hmrc/self_employment_income_count_income_band_40_000_to_50_000 + - hmrc/self_employment_income_count_income_band_500_000_to_1_000_000 + - hmrc/self_employment_income_count_income_band_50_000_to_70_000 + - hmrc/self_employment_income_count_income_band_70_000_to_100_000 + - hmrc/self_employment_income_income_band_100_000_to_150_000 + - hmrc/self_employment_income_income_band_12_570_to_15_000 + - hmrc/self_employment_income_income_band_150_000_to_200_000 + - hmrc/self_employment_income_income_band_15_000_to_20_000 + - hmrc/self_employment_income_income_band_1_000_000_to_inf + - hmrc/self_employment_income_income_band_200_000_to_300_000 + - hmrc/self_employment_income_income_band_20_000_to_30_000 + - hmrc/self_employment_income_income_band_300_000_to_500_000 + - hmrc/self_employment_income_income_band_30_000_to_40_000 + - hmrc/self_employment_income_income_band_40_000_to_50_000 + - hmrc/self_employment_income_income_band_500_000_to_1_000_000 + - hmrc/self_employment_income_income_band_50_000_to_70_000 + - hmrc/self_employment_income_income_band_70_000_to_100_000 + - hmrc/state_pension_count_income_band_100_000_to_150_000 + - hmrc/state_pension_count_income_band_12_570_to_15_000 + - hmrc/state_pension_count_income_band_150_000_to_200_000 + - hmrc/state_pension_count_income_band_15_000_to_20_000 + - hmrc/state_pension_count_income_band_1_000_000_to_inf + - hmrc/state_pension_count_income_band_200_000_to_300_000 + - hmrc/state_pension_count_income_band_20_000_to_30_000 + - hmrc/state_pension_count_income_band_300_000_to_500_000 + - hmrc/state_pension_count_income_band_30_000_to_40_000 + - hmrc/state_pension_count_income_band_40_000_to_50_000 + - hmrc/state_pension_count_income_band_500_000_to_1_000_000 + - hmrc/state_pension_count_income_band_50_000_to_70_000 + - hmrc/state_pension_count_income_band_70_000_to_100_000 + - hmrc/state_pension_income_band_100_000_to_150_000 + - hmrc/state_pension_income_band_12_570_to_15_000 + - hmrc/state_pension_income_band_150_000_to_200_000 + - hmrc/state_pension_income_band_15_000_to_20_000 + - hmrc/state_pension_income_band_1_000_000_to_inf + - hmrc/state_pension_income_band_200_000_to_300_000 + - hmrc/state_pension_income_band_20_000_to_30_000 + - hmrc/state_pension_income_band_300_000_to_500_000 + - hmrc/state_pension_income_band_30_000_to_40_000 + - hmrc/state_pension_income_band_40_000_to_50_000 + - hmrc/state_pension_income_band_500_000_to_1_000_000 + - hmrc/state_pension_income_band_50_000_to_70_000 + - hmrc/state_pension_income_band_70_000_to_100_000 + - status: duplicate + reason: >- + ons_demographics emits these names twice. The matrix keeps the national + ons/uk_population and the region Scottish targets. It drops a region + ons/uk_population read from demographics.csv 1,000 times too small, and + NRS country copies of the Scottish targets with identical values. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/605 + targets: + - ons/scotland_babies_under_1 + - ons/scotland_children_under_16 + - ons/uk_population - status: zero_signal reason: >- Fewer than 10 records have total income of £1m or more, and they have self-employment income only, so the band's targets for every other income type have no signal. @@ -54,8 +234,12 @@ gaps: - hmrc/state_pension_income_band_1_000_000_to_inf - status: zero_signal reason: >- - No record is aged 85 or over; the oldest records fall in the 80-84 band. - issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 + Ages come from the FRS AGE80 field, which is top-coded at 80 + (datasets/frs.py), and the SPI-donor rows are copies of FRS households, + so no record can be 85 or over in any build. More records will not fix + this: the targets must match the top code, which #604 does with + 80-and-over targets. + issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/604 targets: - ons/female_85_90 - ons/male_85_90 @@ -69,7 +253,9 @@ gaps: - voa/council_tax/YORKSHIRE/H - status: zero_signal reason: >- - No record repays an English postgraduate loan. + impute_student_loan_plan (datasets/imputations/student_loans.py) assigns + only NONE, PLAN_1, PLAN_2 and PLAN_5, never POSTGRADUATE, so no record + has a postgraduate loan in any build. issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 targets: - slc/student_loan_repayment/england/postgraduate @@ -83,12 +269,20 @@ gaps: - dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf - status: zero_signal reason: >- - In pull request (TESTING) builds no single claimant's UC payments - fall in these bands: in all 12 recent builds for 27,600 to 28,800, - and in 11 of 12 for the other two. The released 1.58.1 build has - signal in all three. - issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/593 - targets: - - dwp/uc_payment_dist/SINGLE_annual_payment_24_000_to_25_200 - - dwp/uc_payment_dist/SINGLE_annual_payment_25_200_to_26_400 - - dwp/uc_payment_dist/SINGLE_annual_payment_27_600_to_28_800 + In pull request (TESTING) builds the UC payment bands with no signal + vary from one build to the next, and builds of the same code take one of + two pre-calibration states (#598). Those builds hold one output-area + clone of each household (PE_UK_DATA_OA_CLONES=1, as release builds do). + Of 21 pull request builds from 8 to 15 October 2026, the national + estimate in the final calibration log was exactly zero for + COUPLE_NO_CHILDREN 27,600 to 28,800 in 20, and 28,800 to 30,000 and + 30,000 and over in all 21; SINGLE 24,000 to 25,200 and 25,200 to 26,400 + in 20, and 27,600 to 28,800 in all 21; and COUPLE_WITH_CHILDREN 27,600 + to 28,800 in 3. The gate failed on the three SINGLE bands in run + 38726993359 and on the COUPLE_WITH_CHILDREN band in run 38732018053. In + release builds only the three COUPLE_NO_CHILDREN bands above may lack + signal. + issue: https://github.com/PolicyEngine/policyengine-uk-data/issues/598 + builds: [testing] + patterns: + - "dwp/uc_payment_dist/*" diff --git a/policyengine_uk_data/targets/registry.py b/policyengine_uk_data/targets/registry.py index b4c97108..40941ae3 100644 --- a/policyengine_uk_data/targets/registry.py +++ b/policyengine_uk_data/targets/registry.py @@ -31,6 +31,11 @@ def discover_source_modules() -> list: return modules +def emitted_targets() -> list[Target]: + """Every target the source modules emit, in order, duplicate names included.""" + return [target for mod in discover_source_modules() for target in mod.get_targets()] + + def get_all_targets( year: int | None = None, geographic_level: GeographicLevel | None = None, @@ -43,23 +48,20 @@ def get_all_targets( geographic_level: if provided, filter to this geographic level. Returns: - De-duplicated list of Target objects. + De-duplicated list of Target objects: of the targets that pass the + filters, the first of each name. ``emitted_targets`` keeps them all. """ all_targets: list[Target] = [] seen_names: set[str] = set() - for mod in discover_source_modules(): - for target in mod.get_targets(): - if target.name in seen_names: - continue - if year is not None and year not in target.values: - continue - if ( - geographic_level is not None - and target.geographic_level != geographic_level - ): - continue - seen_names.add(target.name) - all_targets.append(target) + for target in emitted_targets(): + if target.name in seen_names: + continue + if year is not None and year not in target.values: + continue + if geographic_level is not None and target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) return all_targets diff --git a/policyengine_uk_data/tests/conftest.py b/policyengine_uk_data/tests/conftest.py index 9de52820..8c579975 100644 --- a/policyengine_uk_data/tests/conftest.py +++ b/policyengine_uk_data/tests/conftest.py @@ -32,3 +32,9 @@ def baseline(enhanced_frs: UKSingleYearDataset): from policyengine_uk import Microsimulation return Microsimulation(dataset=enhanced_frs) + + +@pytest.fixture(autouse=True) +def no_job_summary(monkeypatch): + """Keep tests out of the GitHub Actions job summary the build writes to.""" + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) diff --git a/policyengine_uk_data/tests/test_obr_universal_credit_target.py b/policyengine_uk_data/tests/test_obr_universal_credit_target.py index 8fcdb51e..6ce35249 100644 --- a/policyengine_uk_data/tests/test_obr_universal_credit_target.py +++ b/policyengine_uk_data/tests/test_obr_universal_credit_target.py @@ -23,7 +23,7 @@ _compute_column, restrict_to_countries, ) -from policyengine_uk_data.targets.schema import GREAT_BRITAIN, GeographicLevel +from policyengine_uk_data.targets.schema import GREAT_BRITAIN from policyengine_uk_data.targets.sources import obr COUNTRIES = ("ENGLAND", "SCOTLAND", "WALES", "NORTHERN_IRELAND") @@ -180,13 +180,7 @@ def map_result(self, values, source, target_entity): ) monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) - monkeypatch.setattr( - build_loss_matrix, - "get_all_targets", - lambda geographic_level=None: ( - [target] if geographic_level == GeographicLevel.NATIONAL else [] - ), - ) + monkeypatch.setattr(build_loss_matrix, "emitted_targets", lambda: [target]) matrix, values = build_loss_matrix.create_target_matrix( SimpleNamespace(time_period="2025"), time_period="2025" diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py index c3de4205..0be18e3b 100644 --- a/policyengine_uk_data/tests/test_target_coverage.py +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -1,12 +1,15 @@ """Required coverage of the national calibration targets (targets/coverage.py). create_target_matrix used to skip a target with a warning, or with nothing at -all when its year did not resolve (#456), and to accept a column no household -contributes to (#593). These tests pin the gate that replaces that: every -offered target is in the matrix with signal, or listed in known_gaps.yaml with -the status it may have. +all when its year did not resolve (#456) or its name was already taken, and to +accept a column no household contributes to (#593). These tests pin the gate +that replaces that: every offered target is in the matrix with signal, or +listed in known_gaps.yaml with the status it may have, in the builds it may +have it. """ +from collections import Counter +from fnmatch import fnmatchcase from types import SimpleNamespace import numpy as np @@ -15,9 +18,11 @@ from hypothesis import given, settings from hypothesis import strategies as st -from policyengine_uk_data.targets import build_loss_matrix +from policyengine_uk_data.targets import build_loss_matrix, registry from policyengine_uk_data.targets.coverage import ( + BUILDS, EXCUSABLE, + NEVER_EXCUSED, STATUSES, KnownGap, TargetCoverageError, @@ -25,11 +30,14 @@ check_coverage, column_status, coverage_problems, + current_build, load_known_gaps, parse_known_gaps, ) from policyengine_uk_data.targets.schema import GeographicLevel, Target, Unit +UC_BANDS = "dwp/uc_payment_dist/*" + def test_column_status(): assert column_status(5.0, [0, 1, 0]) == "in_matrix" @@ -42,71 +50,84 @@ def test_column_status(): def test_the_shipped_known_gaps_load_and_cite_issues(): gaps = load_known_gaps() - assert gaps for gap in gaps: assert gap.status in EXCUSABLE assert gap.issue.startswith("https://github.com/PolicyEngine/") assert gap.reason - names = [n for gap in gaps for n in gap.targets] - assert len(names) == len(set(names)) - # The 1.58.1 inventory: 15 two-child-limit targets with no 2025 value, 19 - # targets without signal, and 3 more without signal in TESTING builds. - assert sum(len(g.targets) for g in gaps if g.status == "no_value_for_year") == 15 - assert sum(len(g.targets) for g in gaps if g.status == "zero_signal") == 22 + listed = [(n, gap.status) for gap in gaps for n in gap.targets] + assert len(listed) == len(set(listed)) + + def named(status): + return sum(len(g.targets) for g in gaps if g.status == status) + + # The 1.58.1 inventory at 2025, in every build: 15 two-child-limit targets + # with no 2025 value, 19 without signal, and 156 SPI and 3 ONS targets + # dropped under a name already taken. + assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [ + 15, + 19, + 159, + ] + assert all(g.builds == set(BUILDS) for g in gaps if g.targets) + # Only in testing builds, any UC payment band may lack signal (#598). + [uc] = [g for g in gaps if g.patterns] + assert (uc.status, uc.patterns, uc.builds) == ( + "zero_signal", + (UC_BANDS,), + {"testing"}, + ) + + +def test_the_shipped_gaps_excuse_any_uc_band_without_signal_in_testing_builds_only(): + gaps = load_known_gaps() + flickers = TargetStatus( + "dwp/uc_payment_dist/COUPLE_WITH_CHILDREN_annual_payment_27_600_to_28_800", + "zero_signal", + ) + listed = TargetStatus( + "dwp/uc_payment_dist/COUPLE_NO_CHILDREN_annual_payment_30_000_to_inf", + "zero_signal", + ) + broken = TargetStatus(flickers.name, "error", "ValueError: x") + assert coverage_problems([flickers, listed], gaps, "testing")[0] == [] + assert coverage_problems([flickers, listed], gaps, "release")[0] == [ + f"{flickers.name}: zero_signal" + ] + assert coverage_problems([broken], gaps, "testing")[0] == [ + f"{flickers.name}: error (ValueError: x)" + ] + + +def _entry(**kwargs): + return { + "status": "zero_signal", + "reason": "x", + "issue": "https://github.com/a", + "targets": ["t"], + **kwargs, + } @pytest.mark.parametrize( "entry, match", [ + (_entry(status="error"), "status"), + (_entry(status="non_finite"), "status"), + (_entry(reason=""), "reason"), + (_entry(issue="#456"), "issue"), + (_entry(targets=[]), "no targets"), + (_entry(targets=["t", "t"]), "twice"), + (_entry(builds=["nightly"]), "builds"), + (_entry(builds=[]), "builds"), + (_entry(builds="testing"), "builds"), + # A pattern excuses testing builds only, and must be a pattern. + (_entry(targets=[], patterns=["a/*"]), "testing builds"), + (_entry(targets=[], patterns=["a/*"], builds=["release"]), "testing builds"), + (_entry(targets=[], patterns=["a/b"], builds=["testing"]), "wildcard"), ( - { - "status": "error", - "reason": "x", - "issue": "https://github.com/a", - "targets": ["t"], - }, - "status", - ), - ( - { - "status": "non_finite", - "reason": "x", - "issue": "https://github.com/a", - "targets": ["t"], - }, + _entry(status="error", targets=[], patterns=["a/*"], builds=["testing"]), "status", ), - ( - { - "status": "zero_signal", - "reason": "", - "issue": "https://github.com/a", - "targets": ["t"], - }, - "reason", - ), - ( - {"status": "zero_signal", "reason": "x", "issue": "#456", "targets": ["t"]}, - "issue", - ), - ( - { - "status": "zero_signal", - "reason": "x", - "issue": "https://github.com/a", - "targets": [], - }, - "no targets", - ), - ( - { - "status": "zero_signal", - "reason": "x", - "issue": "https://github.com/a", - "targets": ["t", "t"], - }, - "twice", - ), ], ) def test_malformed_known_gaps_are_refused(entry, match): @@ -114,8 +135,15 @@ def test_malformed_known_gaps_are_refused(entry, match): parse_known_gaps({"schema_version": 1, "gaps": [entry]}) -def _gap(status, *names): - return KnownGap(status, frozenset(names), "reason", "https://github.com/x") +def _gap(status, *names, builds=BUILDS, patterns=()): + return KnownGap( + status, + frozenset(names), + "reason", + "https://github.com/x", + frozenset(builds), + tuple(patterns), + ) def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported(): @@ -136,78 +164,168 @@ def test_a_gap_excuses_only_its_own_status_and_stale_gaps_are_reported(): ] -statuses_strategy = st.dictionaries( - st.text("abcdef", min_size=1, max_size=3), st.sampled_from(STATUSES), max_size=12 -) +def test_a_gap_for_one_kind_of_build_excuses_nothing_in_the_other(monkeypatch): + rows = [TargetStatus("dwp/uc_payment_dist/A", "zero_signal")] + for gap in ( + _gap("zero_signal", rows[0].name, builds=["testing"]), + _gap("zero_signal", builds=["testing"], patterns=[UC_BANDS]), + ): + assert check_coverage(rows, [gap], build="testing") == [] + with pytest.raises(TargetCoverageError, match="A: zero_signal"): + check_coverage(rows, [gap], build="release") + release_only = _gap("no_column", "b", builds=["release"]) + rows = [TargetStatus("b", "no_column")] + assert check_coverage(rows, [release_only], build="release") == [] + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [release_only], build="testing") + # Nor is it stale there: it does not apply. + assert coverage_problems([], [release_only], "testing") == ([], []) + # The build kind is read from TESTING, as create_datasets reads it. + monkeypatch.setenv("TESTING", "1") + assert current_build() == "testing" + monkeypatch.setenv("TESTING", "0") + assert current_build() == "release" + monkeypatch.delenv("TESTING") + assert current_build() == "release" + with pytest.raises(TargetCoverageError, match="b: no_column"): + check_coverage(rows, [_gap("no_column", "b", builds=["testing"])]) + + +def test_a_pattern_excuses_only_its_status_and_is_stale_only_if_it_matches_nothing(): + [gap] = parse_known_gaps( + { + "schema_version": 1, + "gaps": [_entry(targets=[], patterns=[UC_BANDS], builds=["testing"])], + } + ) + rows = [ + TargetStatus("dwp/uc_payment_dist/A", "zero_signal"), + TargetStatus("dwp/uc_payment_dist/B", "in_matrix"), + TargetStatus("dwp/uc_payment_dist/C", "error", "KeyError: x"), + TargetStatus("dwp/uc_payment_dist/D", "non_finite"), + TargetStatus("dwp/uc_payment_dist/E", "no_column"), + TargetStatus("dwp/uc/F", "zero_signal"), + ] + assert coverage_problems(rows, [gap], "testing") == ( + [ + "dwp/uc_payment_dist/C: error (KeyError: x)", + "dwp/uc_payment_dist/D: non_finite", + "dwp/uc_payment_dist/E: no_column", + "dwp/uc/F: zero_signal", + ], + [], + ) + # Excusing nothing in one build is not stale; matching no target is. + assert coverage_problems(rows[1:2], [gap], "testing") == ([], []) + assert coverage_problems( + [TargetStatus("obr/x", "in_matrix")], [gap], "testing" + ) == ( + [], + [f"{UC_BANDS}: listed as zero_signal, matches no target"], + ) + + +NAMES = ["a/x", "a/y", "b/x", "c"] +PATTERNS = ["a/*", "*/x", "z*"] @settings(max_examples=300, deadline=None) -@given(statuses=statuses_strategy, data=st.data()) -def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix(statuses, data): - names = sorted(statuses) - excused = data.draw( - st.dictionaries( - st.sampled_from(names + ["zz"]) if names else st.just("zz"), - st.sampled_from(EXCUSABLE), - ) +@given( + rows=st.lists(st.tuples(st.sampled_from(NAMES), st.sampled_from(STATUSES))), + build=st.sampled_from(BUILDS), + data=st.data(), +) +def test_problems_are_exactly_the_unexcused_targets_out_of_the_matrix( + rows, build, data +): + # Hand-built gaps, so even statuses the file refuses: none may excuse an + # error or a non-finite value. + status = st.sampled_from(EXCUSABLE + NEVER_EXCUSED) + builds = st.sampled_from([BUILDS, ("testing",), ("release",)]) + # A name may be listed under several statuses, each once. + listed = data.draw( + st.dictionaries(st.tuples(st.sampled_from(NAMES + ["zz"]), status), builds) ) - gaps = [_gap(status, name) for name, status in excused.items()] - rows = [TargetStatus(name, statuses[name]) for name in names] - problems, stale = coverage_problems(rows, gaps) - expected = [ - f"{n}: {statuses[n]}" - for n in names - if statuses[n] != "in_matrix" and excused.get(n) != statuses[n] + globbed = data.draw(st.dictionaries(st.sampled_from(PATTERNS), status)) + gaps = [_gap(s, n, builds=b) for (n, s), b in listed.items()] + gaps += [_gap(s, builds=["testing"], patterns=[p]) for p, s in globbed.items()] + statuses = [TargetStatus(n, s) for n, s in rows] + problems, stale = coverage_problems(statuses, gaps, build) + + # The model: only the gaps for this build apply. + named = {pair for pair, b in listed.items() if build in b} + patterns = globbed if build == "testing" else {} + + def excused(name, status): + return status not in NEVER_EXCUSED and ( + (name, status) in named + or any(s == status and fnmatchcase(name, p) for p, s in patterns.items()) + ) + + assert problems == [ + f"{n}: {s}" for n, s in rows if s != "in_matrix" and not excused(n, s) ] - assert problems == expected - assert {s.split(":")[0] for s in stale} == { - n for n, status in excused.items() if statuses.get(n) != status + assert all(f"{n}: {s}" in problems for n, s in rows if s in NEVER_EXCUSED) + stale_pairs = { + (line.split(": listed as ")[0], line.split(": listed as ")[1].split(",")[0]) + for line in stale + } + assert stale_pairs == { + (n, s) for n, s in named if s in NEVER_EXCUSED or (n, s) not in rows + } | { + (p, s) + for p, s in patterns.items() + if not any(fnmatchcase(n, p) for n, _ in rows) } # Order of the statuses never changes the verdict. - shuffled = data.draw(st.permutations(rows)) - assert sorted(coverage_problems(shuffled, gaps)[0]) == sorted(problems) + shuffled = data.draw(st.permutations(statuses)) + assert sorted(coverage_problems(shuffled, gaps, build)[0]) == sorted(problems) def _target(name, values={2025: 10.0}, **kwargs): return Target( - name=name, - variable="employment_income", - source="test", - unit=Unit.GBP, - values=values, - **kwargs, + **{ + "name": name, + "variable": "employment_income", + "source": "test", + "unit": Unit.GBP, + "values": values, + **kwargs, + } ) +class FakeMicrosimulation: + """Two households, in England and Wales.""" + + def __init__(self, dataset=None, reform=None): + pass + + def calculate(self, variable, *args, **kwargs): + assert variable == "country" + return pd.Series(["ENGLAND", "WALES"]) + + +def _sources(*sources): + """Fake source modules, one per list of targets.""" + return lambda: [SimpleNamespace(get_targets=lambda s=s: list(s)) for s in sources] + + @pytest.fixture def fake_matrix(monkeypatch): - """create_target_matrix over two households, with chosen targets and columns.""" + """The target matrix from chosen sources, with columns chosen by name.""" import policyengine_uk - class FakeMicrosimulation: - def __init__(self, dataset=None, reform=None): - pass - - def calculate(self, variable, *args, **kwargs): - assert variable == "country" - return pd.Series(["ENGLAND", "WALES"]) - monkeypatch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) - def configure(targets, columns): - monkeypatch.setattr( - build_loss_matrix, - "get_all_targets", - lambda geographic_level=None: ( - targets if geographic_level == GeographicLevel.NATIONAL else [] - ), - ) + def configure(columns, *sources): + monkeypatch.setattr(registry, "discover_source_modules", _sources(*sources)) def compute(target, ctx, year): column = columns[target.name] if isinstance(column, Exception): raise column - return None if column is None else np.asarray(column, dtype=float) + return None if column is None else np.asarray(column) monkeypatch.setattr(build_loss_matrix, "_compute_column", compute) return SimpleNamespace(time_period="2025") @@ -215,12 +333,19 @@ def compute(target, ctx, year): return configure +def _statuses_by_name(statuses): + by_name = {} + for row in statuses: + by_name.setdefault(row.name, []).append(row.status) + return by_name + + def test_a_dropped_target_fails_the_matrix_unless_it_is_a_known_gap( fake_matrix, monkeypatch ): dataset = fake_matrix( - [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")], {"ok": [1.0, 0.0], "later": [1.0, 1.0], "broken": KeyError("no such variable")}, + [_target("ok"), _target("later", values={2026: 1.0}), _target("broken")], ) with pytest.raises(TargetCoverageError) as raised: build_loss_matrix.create_target_matrix(dataset, time_period="2025") @@ -243,7 +368,7 @@ def test_a_column_without_signal_fails_unless_it_is_a_known_gap( fake_matrix, monkeypatch ): dataset = fake_matrix( - [_target("ok"), _target("flat")], {"ok": [1.0, 2.0], "flat": [0.0, 0.0]} + {"ok": [1.0, 2.0], "flat": [0.0, 0.0]}, [_target("ok"), _target("flat")] ) with pytest.raises(TargetCoverageError, match="flat: zero_signal"): build_loss_matrix.create_target_matrix(dataset, time_period="2025") @@ -257,7 +382,7 @@ def test_a_column_without_signal_fails_unless_it_is_a_known_gap( def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch): - dataset = fake_matrix([_target("nan")], {"nan": [np.nan, 1.0]}) + dataset = fake_matrix({"nan": [np.nan, 1.0]}, [_target("nan")]) monkeypatch.setattr( "policyengine_uk_data.targets.coverage.load_known_gaps", lambda: [_gap("zero_signal", "nan")], @@ -266,6 +391,209 @@ def test_a_non_finite_column_is_never_excused(fake_matrix, monkeypatch): build_loss_matrix.create_target_matrix(dataset, time_period="2025") +def test_a_name_two_sources_emit_fails_unless_it_is_a_known_gap( + fake_matrix, monkeypatch +): + """Scenario A: two source modules emit the same national name.""" + dataset = fake_matrix( + {"obr/foo": [1.0, 2.0]}, + [_target("obr/foo")], + [_target("obr/foo", values={2025: 20.0})], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"obr/foo": 10.0} + assert statuses == [ + TargetStatus( + "obr/foo", + "duplicate", + "test national target for 2025 dropped for the test national one for 2025", + ), + TargetStatus("obr/foo", "in_matrix"), + ] + with pytest.raises(TargetCoverageError, match="obr/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + monkeypatch.setattr( + "policyengine_uk_data.targets.coverage.load_known_gaps", + lambda: [_gap("duplicate", "obr/foo")], + ) + df, _ = build_loss_matrix.create_target_matrix(dataset, time_period="2025") + assert list(df.columns) == ["obr/foo"] + + +def test_a_regional_target_named_like_a_national_one_fails(fake_matrix): + """Scenario B: the national target wins even when emitted second.""" + dataset = fake_matrix( + {"ons/foo": [1.0, 2.0]}, + [ + _target( + "ons/foo", + values={2024: 5.0}, + geographic_level=GeographicLevel.REGION, + source="ons", + ), + _target("ons/foo"), + ], + ) + _, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert values.to_dict() == {"ons/foo": 10.0} + assert _statuses_by_name(statuses) == {"ons/foo": ["duplicate", "in_matrix"]} + assert statuses[0].detail == ( + "ons region target for 2024 dropped for the test national one for 2025" + ) + with pytest.raises(TargetCoverageError, match="ons/foo: duplicate"): + build_loss_matrix.create_target_matrix(dataset, time_period="2025") + + +def test_a_column_that_cannot_enter_the_matrix_has_one_error_status(fake_matrix): + """The status follows the column into the matrix, so a column that fails + to go in, or whose status fails, leaves only an error and no column.""" + dataset = fake_matrix( + { + "ok": [1.0, 2.0], + "long": [1.0, 2.0, 3.0], + "text": np.array(["a", "b"], dtype=object), + }, + [_target("ok"), _target("long"), _target("text")], + ) + df, values, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + assert list(df.columns) == ["ok"] and list(values.index) == ["ok"] + assert _statuses_by_name(statuses) == { + "ok": ["in_matrix"], + "long": ["error"], + "text": ["error"], + } + + +def _main_create_target_matrix(dataset, time_period): + """create_target_matrix and get_all_targets as on main before the gate + (daad71cd), with module references made explicit: the differential + test's reference.""" + from policyengine_uk import Microsimulation + + def get_all_targets(geographic_level): + all_targets, seen_names = [], set() + for mod in registry.discover_source_modules(): + for target in mod.get_targets(): + if target.name in seen_names: + continue + if target.geographic_level != geographic_level: + continue + seen_names.add(target.name) + all_targets.append(target) + return all_targets + + year = int(time_period) + sim = Microsimulation(dataset=dataset) + sim.default_calculation_period = time_period + ctx = build_loss_matrix._SimContext(sim, time_period, dataset, None) + all_targets = [] + seen = set() + for level in ( + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + ): + for t in get_all_targets(geographic_level=level): + if t.name not in seen: + seen.add(t.name) + all_targets.append(t) + df = pd.DataFrame() + target_names = [] + target_values = [] + for target in all_targets: + try: + val = build_loss_matrix._resolve_value(target, year) + if val is None: + continue + col = build_loss_matrix._compute_column(target, ctx, year) + if col is None: + continue + df[target.name] = build_loss_matrix.restrict_to_countries( + col, ctx.country, target.countries + ) + target_names.append(target.name) + target_values.append(val) + except Exception: + pass + return df, pd.Series(target_values, index=target_names) + + +def _column(kind): + """A custom_compute giving a fixed column, no column, or an error.""" + if kind == "none": + return lambda ctx, target, year: None + if kind == "error": + + def raises(ctx, target, year): + raise KeyError("no such variable") + + return raises + return lambda ctx, target, year: np.asarray(kind, dtype=float) + + +emitted_target = st.builds( + lambda name, level, year, value, column, countries: _target( + name, + values={year: value}, + geographic_level=level, + countries=countries, + custom_compute=_column(column), + ), + name=st.sampled_from(["a", "b", "c"]), + level=st.sampled_from( + [ + GeographicLevel.NATIONAL, + GeographicLevel.REGION, + GeographicLevel.COUNTRY, + GeographicLevel.CONSTITUENCY, + ] + ), + # 2023 carries forward to 2025; 2026 does not carry back. + year=st.sampled_from([2023, 2025, 2026]), + value=st.sampled_from([0.0, 10.0, float("nan")]), + column=st.one_of( + st.sampled_from(["none", "error"]), + st.lists( + st.sampled_from([0.0, 1.0, -2.5, np.nan, np.inf]), min_size=2, max_size=2 + ), + ), + countries=st.sampled_from([None, ("ENGLAND",)]), +) + + +@settings(max_examples=200, deadline=None) +@given(sources=st.lists(st.lists(emitted_target, max_size=5), min_size=1, max_size=3)) +def test_without_the_gate_the_matrix_is_mains(sources): + """require_coverage=False builds exactly the matrix main built, from the + same sources, duplicates included; and every target offered at a matrix + level gets exactly one status. + + The one intended difference, not generated here: a column whose status + cannot be computed (a non-numeric column) now leaves the matrix as an + error, where main kept it. + """ + import policyengine_uk + + with pytest.MonkeyPatch.context() as patch: + patch.setattr(policyengine_uk, "Microsimulation", FakeMicrosimulation) + patch.setattr(registry, "discover_source_modules", _sources(*sources)) + dataset = SimpleNamespace(time_period="2025") + df, values = build_loss_matrix.create_target_matrix( + dataset, "2025", require_coverage=False + ) + main_df, main_values = _main_create_target_matrix(dataset, "2025") + _, _, statuses = build_loss_matrix.build_target_matrix(dataset, "2025") + pd.testing.assert_frame_equal(df, main_df) + pd.testing.assert_series_equal(values, main_values) + offered = Counter( + t.name + for source in sources + for t in source + if t.geographic_level in build_loss_matrix.MATRIX_LEVELS + ) + assert Counter(row.name for row in statuses) == offered + + def test_stale_gaps_warn_without_failing(caplog): caplog.set_level("WARNING", logger="policyengine_uk_data.targets.coverage") stale = check_coverage([TargetStatus("a", "in_matrix")], [_gap("zero_signal", "a")]) @@ -273,6 +601,26 @@ def test_stale_gaps_warn_without_failing(caplog): assert "Stale known calibration-target gap" in caplog.text +def test_stale_gaps_and_problems_reach_the_job_summary_once(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + gaps = [_gap("zero_signal", "a")] + # The constituency and local authority calibrations both check. + for _ in range(2): + check_coverage([TargetStatus("a", "in_matrix")], gaps, build="release") + text = summary.read_text() + assert text.count("`a: listed as zero_signal, now in_matrix`") == 1 + assert text.count("### Calibration target coverage (release build)") == 1 + with pytest.raises(TargetCoverageError): + check_coverage([TargetStatus("b", "no_column")], gaps, build="testing") + assert ( + "Not in the matrix and not a known gap: `b: no_column`" in summary.read_text() + ) + monkeypatch.delenv("GITHUB_STEP_SUMMARY") + check_coverage([TargetStatus("c", "in_matrix")], [_gap("zero_signal", "c")]) + assert "`c:" not in summary.read_text() + + def test_built_data_has_no_target_outside_the_matrix_and_the_known_gaps(enhanced_frs): """On the built dataset the gate passes: the known gaps are complete.""" from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE diff --git a/policyengine_uk_data/tests/test_target_registry.py b/policyengine_uk_data/tests/test_target_registry.py index c2a1c1c3..9c8e93cf 100644 --- a/policyengine_uk_data/tests/test_target_registry.py +++ b/policyengine_uk_data/tests/test_target_registry.py @@ -7,8 +7,12 @@ 4. Target values match the current system's hardcoded values """ +from collections import Counter + from policyengine_uk_data.targets import get_all_targets from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.coverage import load_known_gaps +from policyengine_uk_data.targets.registry import emitted_targets def test_registry_loads(): @@ -18,11 +22,20 @@ def test_registry_loads(): def test_no_duplicate_names(): - """Target names should be unique across all sources.""" - targets = get_all_targets() - names = [t.name for t in targets] - duplicates = [n for n in names if names.count(n) > 1] - assert len(duplicates) == 0, f"Duplicate target names: {set(duplicates)}" + """Each name is emitted once, apart from the duplicates known_gaps.yaml lists. + + get_all_targets keeps only the first target of each name, so this counts + what the sources emit before that. + """ + counts = Counter(t.name for t in emitted_targets()) + listed = { + name + for gap in load_known_gaps() + if gap.status == "duplicate" + for name in gap.targets + } + duplicates = sorted(n for n, k in counts.items() if k > 1 and n not in listed) + assert not duplicates, f"Duplicate target names: {duplicates}" def test_obr_income_tax_exists(): From c9229ee68e728e525f7702067182c662e99960c1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 02:07:07 -0400 Subject: [PATCH 09/13] Refresh SPI changelog after integrating current uprating factors --- changelog.d/spi-cgt-tax-year-start.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md index 514abf0d..9df02167 100644 --- a/changelog.d/spi-cgt-tax-year-start.fixed.md +++ b/changelog.d/spi-cgt-tax-year-start.fixed.md @@ -1 +1 @@ -Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%). +Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. From 4840083cab23e8bb1ec74aa0064b668551bedd0f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 03:41:01 -0400 Subject: [PATCH 10/13] Defer CGT target year changes to the follow-up after #571 --- .../targets/sources/hmrc_cgt.py | 30 +++++++++---------- .../tests/test_hmrc_cgt_targets.py | 16 +--------- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/policyengine_uk_data/targets/sources/hmrc_cgt.py b/policyengine_uk_data/targets/sources/hmrc_cgt.py index 8b39a8e3..f2936068 100644 --- a/policyengine_uk_data/targets/sources/hmrc_cgt.py +++ b/policyengine_uk_data/targets/sources/hmrc_cgt.py @@ -29,19 +29,17 @@ for the simulation year, so the target's meaning tracks the policy in force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails. -Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine -UK's year N is tax year N to N+1, so the outturn is the 2023 value, the -year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR`` -follows the same convention). The amount is projected to 2029 with -PolicyEngine's ``capital_gains`` uprating factors so the target -constrains the projection years the calibration actually runs for; the -taxpayer count is held flat, as the repo has no administrative basis for -forecasting CGT taxpayer numbers and count uprating factors would be a -nominal-income index applied to a headcount. - -Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the -projected values carry the 2023-24 outturn realised under a £6,000 AEA. -The targets are therefore approximate for the projection years; the +Year handling: HMRC's figures are the 2023-24 outturn, mapped to +calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected +to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the +target constrains the projection years the calibration actually runs +for; the taxpayer count is held flat, as the repo has no administrative +basis for forecasting CGT taxpayer numbers and count uprating factors +would be a nominal-income index applied to a headcount. + +Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the +gate applied in later years uses the £3,000 AEA then in force. The +targets are therefore approximate for the projection years; the alternative (a fixed £6,000 gate) would be wrong in a different and less transparent way, because it would not describe any year's actual policy. @@ -57,9 +55,9 @@ logger = logging.getLogger(__name__) -# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N -# is tax year N to N+1, so the outturn is the 2023 value. -_CGT_BASE_YEAR = 2023 +# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024 +# following the hmrc_spi._SPI_YEAR convention. +_CGT_BASE_YEAR = 2024 _MAX_YEAR = 2029 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn). diff --git a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py index 70bb85db..66a15b08 100644 --- a/policyengine_uk_data/tests/test_hmrc_cgt_targets.py +++ b/policyengine_uk_data/tests/test_hmrc_cgt_targets.py @@ -89,20 +89,6 @@ def test_aea_tracks_the_year(): assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0 -def test_base_year_is_the_tax_year_the_outturn_was_realised_in(): - """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn - is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA, - read from the real parameter tree; mapping the outturn to 2024 would gate - it at 2024-25's £3,000.""" - from policyengine_uk.system import system - - from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount - - ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system)) - assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000 - assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000 - - def test_targets_load_with_expected_base_year_values(): targets = {t.name: t for t in get_targets()} # Aggregate targets plus the per-size-band targets (tested in @@ -310,7 +296,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years(): def test_registry_year_filter_excludes_years_before_the_outturn(): - """The targets describe the 2023-24 outturn, PolicyEngine year 2023, + """The targets describe a 2023-24 outturn mapped to calendar 2024, so they must not appear for earlier years.""" from policyengine_uk_data.targets import get_all_targets From 473adafc622b40f46572ccbca3186b74108c6e74 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 03:41:33 -0400 Subject: [PATCH 11/13] Describe the SPI-only calibration fix and defer CGT follow-up --- changelog.d/spi-cgt-tax-year-start.fixed.md | 1 - changelog.d/spi-tax-year-start.fixed.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/spi-cgt-tax-year-start.fixed.md create mode 100644 changelog.d/spi-tax-year-start.fixed.md diff --git a/changelog.d/spi-cgt-tax-year-start.fixed.md b/changelog.d/spi-cgt-tax-year-start.fixed.md deleted file mode 100644 index 9df02167..00000000 --- a/changelog.d/spi-cgt-tax-year-start.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. diff --git a/changelog.d/spi-tax-year-start.fixed.md b/changelog.d/spi-tax-year-start.fixed.md new file mode 100644 index 00000000..a2013007 --- /dev/null +++ b/changelog.d/spi-tax-year-start.fixed.md @@ -0,0 +1 @@ +Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571. From 6e03d4c166d7b66e16653d80b37b3c92a3d3622b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 05:58:37 -0400 Subject: [PATCH 12/13] Drop the gaps #557 and #604 close (integration with both) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With #557 each SPI income band is emitted once, so its 156 duplicate entries go; with #604 the ages are 80 and over, so the two ons/*_85_90 zero-signal entries go. The shipped-file counts move from 19 to 17 without signal and from 159 to 3 duplicates (the ONS three, #605). The ten SPI bands of £1m and over keep their zero-signal entries: #557 gives them no signal. On uk-data 1.58.1 at 2025 the coverage CLI reports 626 in the matrix, 3 duplicates, 15 with no 2025 value and 17 without signal, with no problems and no stale gaps, as a release build and as a testing build. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/targets/known_gaps.yaml | 176 ------------------ .../tests/test_target_coverage.py | 11 +- 2 files changed, 6 insertions(+), 181 deletions(-) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml index 7b28512a..98613034 100644 --- a/policyengine_uk_data/targets/known_gaps.yaml +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -41,171 +41,6 @@ gaps: - dwp/uc/two_child_limit/disabled_child_element_children - dwp/uc/two_child_limit/disabled_child_element_households - dwp/uc/two_child_limit/households_affected - - status: duplicate - reason: >- - hmrc_spi emits each income-band target twice: the 2023-24 outturn, with - a value for 2024 only, then the incomes_projection.csv copy for 2025 to - 2029. The matrix keeps the first, so calibration at 2025 carries the - 2024 outturn forward and never sees the projection. #557 emits each band - once. - issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/557 - targets: - - hmrc/dividend_income_count_income_band_100_000_to_150_000 - - hmrc/dividend_income_count_income_band_12_570_to_15_000 - - hmrc/dividend_income_count_income_band_150_000_to_200_000 - - hmrc/dividend_income_count_income_band_15_000_to_20_000 - - hmrc/dividend_income_count_income_band_1_000_000_to_inf - - hmrc/dividend_income_count_income_band_200_000_to_300_000 - - hmrc/dividend_income_count_income_band_20_000_to_30_000 - - hmrc/dividend_income_count_income_band_300_000_to_500_000 - - hmrc/dividend_income_count_income_band_30_000_to_40_000 - - hmrc/dividend_income_count_income_band_40_000_to_50_000 - - hmrc/dividend_income_count_income_band_500_000_to_1_000_000 - - hmrc/dividend_income_count_income_band_50_000_to_70_000 - - hmrc/dividend_income_count_income_band_70_000_to_100_000 - - hmrc/dividend_income_income_band_100_000_to_150_000 - - hmrc/dividend_income_income_band_12_570_to_15_000 - - hmrc/dividend_income_income_band_150_000_to_200_000 - - hmrc/dividend_income_income_band_15_000_to_20_000 - - hmrc/dividend_income_income_band_1_000_000_to_inf - - hmrc/dividend_income_income_band_200_000_to_300_000 - - hmrc/dividend_income_income_band_20_000_to_30_000 - - hmrc/dividend_income_income_band_300_000_to_500_000 - - hmrc/dividend_income_income_band_30_000_to_40_000 - - hmrc/dividend_income_income_band_40_000_to_50_000 - - hmrc/dividend_income_income_band_500_000_to_1_000_000 - - hmrc/dividend_income_income_band_50_000_to_70_000 - - hmrc/dividend_income_income_band_70_000_to_100_000 - - hmrc/employment_income_count_income_band_100_000_to_150_000 - - hmrc/employment_income_count_income_band_12_570_to_15_000 - - hmrc/employment_income_count_income_band_150_000_to_200_000 - - hmrc/employment_income_count_income_band_15_000_to_20_000 - - hmrc/employment_income_count_income_band_1_000_000_to_inf - - hmrc/employment_income_count_income_band_200_000_to_300_000 - - hmrc/employment_income_count_income_band_20_000_to_30_000 - - hmrc/employment_income_count_income_band_300_000_to_500_000 - - hmrc/employment_income_count_income_band_30_000_to_40_000 - - hmrc/employment_income_count_income_band_40_000_to_50_000 - - hmrc/employment_income_count_income_band_500_000_to_1_000_000 - - hmrc/employment_income_count_income_band_50_000_to_70_000 - - hmrc/employment_income_count_income_band_70_000_to_100_000 - - hmrc/employment_income_income_band_100_000_to_150_000 - - hmrc/employment_income_income_band_12_570_to_15_000 - - hmrc/employment_income_income_band_150_000_to_200_000 - - hmrc/employment_income_income_band_15_000_to_20_000 - - hmrc/employment_income_income_band_1_000_000_to_inf - - hmrc/employment_income_income_band_200_000_to_300_000 - - hmrc/employment_income_income_band_20_000_to_30_000 - - hmrc/employment_income_income_band_300_000_to_500_000 - - hmrc/employment_income_income_band_30_000_to_40_000 - - hmrc/employment_income_income_band_40_000_to_50_000 - - hmrc/employment_income_income_band_500_000_to_1_000_000 - - hmrc/employment_income_income_band_50_000_to_70_000 - - hmrc/employment_income_income_band_70_000_to_100_000 - - hmrc/private_pension_income_count_income_band_100_000_to_150_000 - - hmrc/private_pension_income_count_income_band_12_570_to_15_000 - - hmrc/private_pension_income_count_income_band_150_000_to_200_000 - - hmrc/private_pension_income_count_income_band_15_000_to_20_000 - - hmrc/private_pension_income_count_income_band_1_000_000_to_inf - - hmrc/private_pension_income_count_income_band_200_000_to_300_000 - - hmrc/private_pension_income_count_income_band_20_000_to_30_000 - - hmrc/private_pension_income_count_income_band_300_000_to_500_000 - - hmrc/private_pension_income_count_income_band_30_000_to_40_000 - - hmrc/private_pension_income_count_income_band_40_000_to_50_000 - - hmrc/private_pension_income_count_income_band_500_000_to_1_000_000 - - hmrc/private_pension_income_count_income_band_50_000_to_70_000 - - hmrc/private_pension_income_count_income_band_70_000_to_100_000 - - hmrc/private_pension_income_income_band_100_000_to_150_000 - - hmrc/private_pension_income_income_band_12_570_to_15_000 - - hmrc/private_pension_income_income_band_150_000_to_200_000 - - hmrc/private_pension_income_income_band_15_000_to_20_000 - - hmrc/private_pension_income_income_band_1_000_000_to_inf - - hmrc/private_pension_income_income_band_200_000_to_300_000 - - hmrc/private_pension_income_income_band_20_000_to_30_000 - - hmrc/private_pension_income_income_band_300_000_to_500_000 - - hmrc/private_pension_income_income_band_30_000_to_40_000 - - hmrc/private_pension_income_income_band_40_000_to_50_000 - - hmrc/private_pension_income_income_band_500_000_to_1_000_000 - - hmrc/private_pension_income_income_band_50_000_to_70_000 - - hmrc/private_pension_income_income_band_70_000_to_100_000 - - hmrc/property_income_count_income_band_100_000_to_150_000 - - hmrc/property_income_count_income_band_12_570_to_15_000 - - hmrc/property_income_count_income_band_150_000_to_200_000 - - hmrc/property_income_count_income_band_15_000_to_20_000 - - hmrc/property_income_count_income_band_1_000_000_to_inf - - hmrc/property_income_count_income_band_200_000_to_300_000 - - hmrc/property_income_count_income_band_20_000_to_30_000 - - hmrc/property_income_count_income_band_300_000_to_500_000 - - hmrc/property_income_count_income_band_30_000_to_40_000 - - hmrc/property_income_count_income_band_40_000_to_50_000 - - hmrc/property_income_count_income_band_500_000_to_1_000_000 - - hmrc/property_income_count_income_band_50_000_to_70_000 - - hmrc/property_income_count_income_band_70_000_to_100_000 - - hmrc/property_income_income_band_100_000_to_150_000 - - hmrc/property_income_income_band_12_570_to_15_000 - - hmrc/property_income_income_band_150_000_to_200_000 - - hmrc/property_income_income_band_15_000_to_20_000 - - hmrc/property_income_income_band_1_000_000_to_inf - - hmrc/property_income_income_band_200_000_to_300_000 - - hmrc/property_income_income_band_20_000_to_30_000 - - hmrc/property_income_income_band_300_000_to_500_000 - - hmrc/property_income_income_band_30_000_to_40_000 - - hmrc/property_income_income_band_40_000_to_50_000 - - hmrc/property_income_income_band_500_000_to_1_000_000 - - hmrc/property_income_income_band_50_000_to_70_000 - - hmrc/property_income_income_band_70_000_to_100_000 - - hmrc/self_employment_income_count_income_band_100_000_to_150_000 - - hmrc/self_employment_income_count_income_band_12_570_to_15_000 - - hmrc/self_employment_income_count_income_band_150_000_to_200_000 - - hmrc/self_employment_income_count_income_band_15_000_to_20_000 - - hmrc/self_employment_income_count_income_band_1_000_000_to_inf - - hmrc/self_employment_income_count_income_band_200_000_to_300_000 - - hmrc/self_employment_income_count_income_band_20_000_to_30_000 - - hmrc/self_employment_income_count_income_band_300_000_to_500_000 - - hmrc/self_employment_income_count_income_band_30_000_to_40_000 - - hmrc/self_employment_income_count_income_band_40_000_to_50_000 - - hmrc/self_employment_income_count_income_band_500_000_to_1_000_000 - - hmrc/self_employment_income_count_income_band_50_000_to_70_000 - - hmrc/self_employment_income_count_income_band_70_000_to_100_000 - - hmrc/self_employment_income_income_band_100_000_to_150_000 - - hmrc/self_employment_income_income_band_12_570_to_15_000 - - hmrc/self_employment_income_income_band_150_000_to_200_000 - - hmrc/self_employment_income_income_band_15_000_to_20_000 - - hmrc/self_employment_income_income_band_1_000_000_to_inf - - hmrc/self_employment_income_income_band_200_000_to_300_000 - - hmrc/self_employment_income_income_band_20_000_to_30_000 - - hmrc/self_employment_income_income_band_300_000_to_500_000 - - hmrc/self_employment_income_income_band_30_000_to_40_000 - - hmrc/self_employment_income_income_band_40_000_to_50_000 - - hmrc/self_employment_income_income_band_500_000_to_1_000_000 - - hmrc/self_employment_income_income_band_50_000_to_70_000 - - hmrc/self_employment_income_income_band_70_000_to_100_000 - - hmrc/state_pension_count_income_band_100_000_to_150_000 - - hmrc/state_pension_count_income_band_12_570_to_15_000 - - hmrc/state_pension_count_income_band_150_000_to_200_000 - - hmrc/state_pension_count_income_band_15_000_to_20_000 - - hmrc/state_pension_count_income_band_1_000_000_to_inf - - hmrc/state_pension_count_income_band_200_000_to_300_000 - - hmrc/state_pension_count_income_band_20_000_to_30_000 - - hmrc/state_pension_count_income_band_300_000_to_500_000 - - hmrc/state_pension_count_income_band_30_000_to_40_000 - - hmrc/state_pension_count_income_band_40_000_to_50_000 - - hmrc/state_pension_count_income_band_500_000_to_1_000_000 - - hmrc/state_pension_count_income_band_50_000_to_70_000 - - hmrc/state_pension_count_income_band_70_000_to_100_000 - - hmrc/state_pension_income_band_100_000_to_150_000 - - hmrc/state_pension_income_band_12_570_to_15_000 - - hmrc/state_pension_income_band_150_000_to_200_000 - - hmrc/state_pension_income_band_15_000_to_20_000 - - hmrc/state_pension_income_band_1_000_000_to_inf - - hmrc/state_pension_income_band_200_000_to_300_000 - - hmrc/state_pension_income_band_20_000_to_30_000 - - hmrc/state_pension_income_band_300_000_to_500_000 - - hmrc/state_pension_income_band_30_000_to_40_000 - - hmrc/state_pension_income_band_40_000_to_50_000 - - hmrc/state_pension_income_band_500_000_to_1_000_000 - - hmrc/state_pension_income_band_50_000_to_70_000 - - hmrc/state_pension_income_band_70_000_to_100_000 - status: duplicate reason: >- ons_demographics emits these names twice. The matrix keeps the national @@ -232,17 +67,6 @@ gaps: - hmrc/property_income_income_band_1_000_000_to_inf - hmrc/state_pension_count_income_band_1_000_000_to_inf - hmrc/state_pension_income_band_1_000_000_to_inf - - status: zero_signal - reason: >- - Ages come from the FRS AGE80 field, which is top-coded at 80 - (datasets/frs.py), and the SPI-donor rows are copies of FRS households, - so no record can be 85 or over in any build. More records will not fix - this: the targets must match the top code, which #604 does with - 80-and-over targets. - issue: https://github.com/PolicyEngine/policyengine-uk-data/pull/604 - targets: - - ons/female_85_90 - - ons/male_85_90 - status: zero_signal reason: >- No band-H household in these regions. diff --git a/policyengine_uk_data/tests/test_target_coverage.py b/policyengine_uk_data/tests/test_target_coverage.py index 0be18e3b..1f63ec00 100644 --- a/policyengine_uk_data/tests/test_target_coverage.py +++ b/policyengine_uk_data/tests/test_target_coverage.py @@ -60,13 +60,14 @@ def test_the_shipped_known_gaps_load_and_cite_issues(): def named(status): return sum(len(g.targets) for g in gaps if g.status == status) - # The 1.58.1 inventory at 2025, in every build: 15 two-child-limit targets - # with no 2025 value, 19 without signal, and 156 SPI and 3 ONS targets - # dropped under a name already taken. + # The 1.58.1 inventory at 2025, in every build, with #557 (SPI bands + # emitted once) and #604 (80-and-over ages in place of 85-90): 15 + # two-child-limit targets with no 2025 value, 17 without signal, and 3 ONS + # targets dropped under a name already taken (#605). assert [named(s) for s in ("no_value_for_year", "zero_signal", "duplicate")] == [ 15, - 19, - 159, + 17, + 3, ] assert all(g.builds == set(BUILDS) for g in gaps if g.targets) # Only in testing builds, any UC payment band may lack signal (#598). From 0785612c27a9728f0dd4dc84f57d9e0ff5eb52ab Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 16 Oct 2026 06:21:40 -0400 Subject: [PATCH 13/13] Bring known_gaps.yaml's header counts up to the integrated inventory (review nit) Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/targets/known_gaps.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/policyengine_uk_data/targets/known_gaps.yaml b/policyengine_uk_data/targets/known_gaps.yaml index 98613034..6728d028 100644 --- a/policyengine_uk_data/targets/known_gaps.yaml +++ b/policyengine_uk_data/targets/known_gaps.yaml @@ -14,10 +14,11 @@ # no target at all, is reported as stale: remove it. # # Measured on uk-data 1.58.1 (enhanced_frs_2024_25.h5, sha256 056e80fd) with -# policyengine-uk 2.122.2 at the calibration year 2025, as a release build: -# the registry offers 660 names. 645 enter the matrix, 15 have no value for -# 2025 and 19 of those entering have no signal. 159 more targets reuse a -# name already taken and are dropped as duplicates. +# policyengine-uk 2.122.2 at the calibration year 2025, with #557 and #604, +# as a release build and as a testing build: the registry offers 658 names. +# 643 enter the matrix, 15 have no value for 2025 and 17 of those entering +# have no signal. 3 more targets reuse a name already taken and are dropped +# as duplicates (661 targets in all). schema_version: 1 gaps: - status: no_value_for_year