Fail the calibration build on targets it drops or cannot fit

.gitignore

@@ -18,6 +18,7 @@
 !policyengine_uk_data/datasets/local_areas/**/*.csv
 **/_build
 !policyengine_uk_data/storage/*.csv
+!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
 **/version.json
 # Build output: household weights by local area, derived from FRS microdata
 policyengine_uk_data/storage/local_geography_weights.csv.gz
@@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz
 
 # Hypothesis example database (property-based tests)
 .hypothesis/
+
+# Local build-job notes, caches and hand-off artefacts
+.job/

changelog.d/593-age-topcode.fixed.md

@@ -0,0 +1 @@
+Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups.

changelog.d/spi-tax-year-start.fixed.md

@@ -0,0 +1 @@
+Map the HMRC Survey of Personal Incomes 2023-24 outturn to PolicyEngine year 2023, the tax year it describes, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Regenerate the SPI projections from the corrected base year with the current uprating factors. Defer the CGT year mapping and aggregate/band feasibility to a follow-up on top of #571.

changelog.d/target-coverage-gate.changed.md

@@ -0,0 +1 @@
+Fail the calibration build when a national target is dropped (its name already taken, no value for the year, no column, an error), has a non-finite value, or enters with no household contributing, unless `targets/known_gaps.yaml` lists it with a reason and an issue, for every build or only for testing or release builds. Add `python -m policyengine_uk_data.targets.coverage` to report every target's status on a dataset.

policyengine_uk_data/storage/regional_age_80_plus.csv

@@ -0,0 +1,13 @@
+name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url
+east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland
+scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland
+south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales
+west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1

policyengine_uk_data/targets/build_loss_matrix.py

@@ -16,7 +16,12 @@
 import numpy as np
 import pandas as pd
 
-from policyengine_uk_data.targets import get_all_targets
+from policyengine_uk_data.targets.coverage import (
+    TargetStatus,
+    check_coverage,
+    column_status,
+)
+from policyengine_uk_data.targets.registry import emitted_targets
 from policyengine_uk_data.targets.schema import (
     GeographicLevel,
     Target,
@@ -64,19 +69,76 @@ def create_target_matrix(
     dataset,
     time_period: str = None,
     reform=None,
+    require_coverage: bool = True,
 ) -> tuple[pd.DataFrame, pd.Series]:
     """Create (matrix, target_values) for calibration.
 
     Args:
         dataset: a UKSingleYearDataset instance.
         time_period: calendar year as string; defaults to dataset year.
         reform: optional PolicyEngine reform.
+        require_coverage: fail unless every target is in the matrix with
+            signal or listed in targets/known_gaps.yaml (targets/coverage.py).
 
     Returns:
         (df, targets) where df has one column per target and one row
         per household, and targets is a Series of scalar target values
         indexed by target name.
     """
+    df, targets, statuses = build_target_matrix(dataset, time_period, reform)
+    if require_coverage:
+        check_coverage(statuses)
+    return df, targets
+
+
+MATRIX_LEVELS = (
+    GeographicLevel.NATIONAL,
+    GeographicLevel.REGION,
+    GeographicLevel.COUNTRY,
+)
+
+
+def select_matrix_targets(targets):
+    """The targets the matrix offers, and a ``duplicate`` status for each other.
+
+    Targets are taken level by level (national, region, country), each level
+    in the order the sources emit them, and the first target of each name
+    wins. A later target of that name, at the same level or another, is
+    dropped and recorded as a duplicate (targets/coverage.py).
+    """
+    kept, statuses = {}, []
+    for level in MATRIX_LEVELS:
+        for target in targets:
+            if target.geographic_level != level:
+                continue
+            if target.name not in kept:
+                kept[target.name] = target
+                continue
+            first = kept[target.name]
+            statuses.append(
+                TargetStatus(
+                    target.name,
+                    "duplicate",
+                    f"{target.source} {level.value} target for {_years(target)} "
+                    f"dropped for the {first.source} "
+                    f"{first.geographic_level.value} one for {_years(first)}",
+                )
+            )
+    return list(kept.values()), statuses
+
+
+def _years(target: Target) -> str:
+    return ", ".join(str(year) for year in sorted(target.values))
+
+
+def build_target_matrix(dataset, time_period: str = None, reform=None):
+    """The target matrix, its values, and each offered target's status.
+
+    A target that is dropped (a duplicate name, no value for the year, no
+    column, an error) or that enters with a non-finite value or column, or
+    with no household contributing, is recorded with that status
+    (targets/coverage.py). Each offered target gets exactly one status.
+    """
     from policyengine_uk import Microsimulation
 
     if time_period is None:
@@ -88,17 +150,7 @@ def create_target_matrix(
 
     ctx = _SimContext(sim, time_period, dataset, reform)
 
-    all_targets = []
-    seen = set()
-    for level in (
-        GeographicLevel.NATIONAL,
-        GeographicLevel.REGION,
-        GeographicLevel.COUNTRY,
-    ):
-        for t in get_all_targets(geographic_level=level):
-            if t.name not in seen:
-                seen.add(t.name)
-                all_targets.append(t)
+    all_targets, statuses = select_matrix_targets(emitted_targets())
 
     df = pd.DataFrame()
     target_names = []
@@ -108,17 +160,34 @@ def create_target_matrix(
         try:
             val = _resolve_value(target, year)
             if val is None:
+                statuses.append(
+                    TargetStatus(
+                        target.name, "no_value_for_year", f"values for {_years(target)}"
+                    )
+                )
                 continue
             col = _compute_column(target, ctx, year)
             if col is None:
+                statuses.append(TargetStatus(target.name, "no_column"))
                 continue
             df[target.name] = restrict_to_countries(col, ctx.country, target.countries)
+            # The status is of the column as it entered the matrix; a column
+            # whose status cannot be computed leaves the matrix again.
+            try:
+                status = column_status(val, df[target.name])
+            except Exception:
+                del df[target.name]
+                raise
+            statuses.append(TargetStatus(target.name, status))
             target_names.append(target.name)
             target_values.append(val)
         except Exception as e:
             logger.warning("Skipping target %s: %s", target.name, e)
+            statuses.append(
+                TargetStatus(target.name, "error", f"{type(e).__name__}: {e}")
+            )
 
-    return df, pd.Series(target_values, index=target_names)
+    return df, pd.Series(target_values, index=target_names), statuses
 
 
 def restrict_to_countries(column, household_country, countries):

policyengine_uk_data/targets/compute/demographics.py

@@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None:
     region_name = name[:idx]
     age_part = name[idx + 5 :]
     lower, upper = age_part.split("_")
-    lower, upper = int(lower), int(upper)
+    lower = int(lower)
+    upper = np.inf if upper == "plus" else int(upper)
 
     pe_region = _REGION_INV.get(region_name)
     if pe_region is None:
@@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray:
     parts = name.split("_")
     sex = parts[0]
     lower = int(parts[1])
-    upper = int(parts[2])
+    upper = np.inf if parts[2] == "plus" else int(parts[2])
 
     gender = ctx.sim.calculate("gender").values
     sex_match = gender == ("FEMALE" if sex == "female" else "MALE")