Calibrate households' SDLT to the OBR total less corporate SDLT

changelog.d/sdlt-target-excludes-corporate.fixed.md

@@ -0,0 +1 @@
+Calibrate households' SDLT (`stamp_duty_land_tax`) to the OBR's SDLT forecast less policyengine-uk's corporate SDLT statistic for the same year, instead of the whole OBR line. policyengine-uk adds SDLT paid by companies and other non-natural persons separately, as `corporate_sdlt`, so calibration counted that share twice: on the 1.58.0 dataset with policyengine-uk 2.122.2, households' SDLT was £16.32bn in 2025-26 against the OBR's £15.41bn for all buyers, and `corporate_sdlt` added £3.13bn on top. With policyengine-uk 2.152.0 or later, which projects the corporate statistic with the OBR forecast (PolicyEngine/policyengine-uk#2322), the 2025-26 household target is £10.60bn. Full builds now need such a version and stop at the start with an earlier one, which holds the statistic at £3.13bn in every year.

policyengine_uk_data/datasets/create_datasets.py

@@ -57,8 +57,17 @@ def _materialize_base_year_dataset(dataset, frs_release, uprate_dataset):
 
 def main():
     """Create enhanced FRS dataset with rich progress tracking."""
+    # Imported before the try, whose handler reports any failure with it,
+    # the preflight's included.
+    from policyengine_uk_data.utils.progress import (
+        ProcessingProgress,
+        display_success_panel,
+        display_error_panel,
+    )
+
     try:
-        assert_local_build_environment()
+        # TESTING builds (PR CI) skip the full-build checks.
+        assert_local_build_environment(full_build=os.environ.get("TESTING", "0") != "1")
 
         from policyengine_uk.data import UKSingleYearDataset
         from policyengine_uk_data.datasets.disability_benefits import (
@@ -67,11 +76,6 @@ def main():
         from policyengine_uk_data.datasets.frs import create_frs
         from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE
         from policyengine_uk_data.storage import STORAGE_FOLDER
-        from policyengine_uk_data.utils.progress import (
-            ProcessingProgress,
-            display_success_panel,
-            display_error_panel,
-        )
         from policyengine_uk_data.utils.subsample import subsample_dataset
         from policyengine_uk_data.utils.uprating import uprate_dataset
 

policyengine_uk_data/targets/sources/obr.py

@@ -14,6 +14,7 @@
 
 from policyengine_uk_data.targets.obr_sources import (
     ObrManifest,
+    SourceError,
     active_manifest,
     check_coverage,
     load_workbook,
@@ -71,6 +72,77 @@ def _find_row(ws, label: str, col: str = "B", max_row: int = 80) -> int:
     raise ValueError(f"Row '{label}' not found in sheet")
 
 
+def corporate_sdlt_receipts(years=None) -> dict[int, float]:
+    """policyengine-uk's SDLT receipts from non-natural persons, by year.
+
+    These are the statistics `corporate_sdlt` allocates to households: each
+    household's `shareholding` divides its corporate-sector wealth by the
+    weighted total, so while that total is not zero the weighted sum of
+    `corporate_sdlt` is this statistic whatever the weights. Read from the
+    installed policyengine-uk, which projects it from
+    PolicyEngine/policyengine-uk#2322 on (see `unprojected_corporate_sdlt`).
+    Years are PolicyEngine years (2025 is 2025-26); the default is the active
+    OBR manifest's years.
+    """
+    from policyengine_uk.system import system
+
+    statistics = system.parameters.gov.hmrc.stamp_duty.statistics
+    return {
+        year: float(
+            statistics.residential.corporate.revenue(f"{year}-01-01")
+            + statistics.non_residential.corporate.revenue(f"{year}-01-01")
+        )
+        for year in (active_manifest().years if years is None else years)
+    }
+
+
+def unprojected_corporate_sdlt() -> list[str]:
+    """The corporate SDLT revenue parameters policyengine-uk does not project.
+
+    PolicyEngine/policyengine-uk#2322 (first released in 2.152.0) gives each
+    one an uprating index, the OBR's SDLT receipts for its property type, so
+    that load-time uprating projects it past HMRC's last outturn. Earlier
+    versions have no index and carry the 2020-21 figures forward unchanged,
+    £3.13bn in all, so the check is on the index, not on the values.
+    """
+    from policyengine_uk.system import system
+
+    statistics = system.parameters.gov.hmrc.stamp_duty.statistics
+    return [
+        f"gov.hmrc.stamp_duty.statistics.{kind}.corporate.revenue"
+        for kind in ("residential", "non_residential")
+        if not getattr(statistics, kind).corporate.revenue.metadata.get("uprating")
+    ]
+
+
+def household_sdlt(obr_sdlt: dict[int, float]) -> dict[int, float]:
+    """The OBR's SDLT receipts less policyengine-uk's corporate SDLT, by year.
+
+    The OBR's SDLT line covers every buyer. policyengine-uk counts receipts
+    from non-natural persons (companies, partnerships, collective investment
+    schemes) separately, as `corporate_sdlt`, so targeting the household
+    variable `stamp_duty_land_tax` at the whole line counted them twice (#564).
+    Raises SourceError, which the receipts parser does not catch, if the
+    statistic exceeds the OBR's line in any year.
+    """
+    unprojected = unprojected_corporate_sdlt()
+    if unprojected:
+        logger.warning(
+            "OBR receipts: policyengine-uk does not project %s, so it predates "
+            "PolicyEngine/policyengine-uk#2322 and the household SDLT target "
+            "overstates households' share. Full builds refuse it.",
+            " or ".join(unprojected),
+        )
+    corporate = corporate_sdlt_receipts(obr_sdlt)
+    household = {year: total - corporate[year] for year, total in obr_sdlt.items()}
+    negative = sorted(year for year, value in household.items() if value < 0)
+    if negative:
+        raise SourceError(
+            f"policyengine-uk's corporate SDLT exceeds the OBR's SDLT in {negative}"
+        )
+    return household
+
+
 def _parse_receipts(
     wb: openpyxl.Workbook, manifest: ObrManifest | None = None
 ) -> list[Target]:
@@ -136,6 +208,8 @@ def _parse_receipts(
         try:
             row_num = _find_row(ws39, label, col="B", max_row=80)
             values = read_row(ws39, row_num, manifest)
+            if values and name == "sdlt":
+                values = household_sdlt(values)
             if values:
                 recovered.append(label)
                 targets.append(

policyengine_uk_data/tests/test_build_environment.py

@@ -1,3 +1,5 @@
+import pytest
+
 from policyengine_uk_data.utils.build_environment import get_local_build_issues
 
 
@@ -29,3 +31,41 @@ def test_get_local_build_issues_accepts_supported_environment():
         )
         == []
     )
+
+
+def test_get_local_build_issues_requires_projected_corporate_sdlt():
+    """policyengine-uk before #2322 held corporate SDLT at £3.13bn."""
+    issues = get_local_build_issues(
+        python_version=(3, 13, 7),
+        variable_names={"num_vehicles"},
+        unprojected_corporate_sdlt=[
+            "gov.hmrc.stamp_duty.statistics.residential.corporate.revenue"
+        ],
+    )
+
+    assert len(issues) == 1
+    assert "PolicyEngine/policyengine-uk#2322" in issues[0]
+    assert "statistics.residential.corporate.revenue" in issues[0]
+
+
+def test_get_local_build_issues_accepts_projected_corporate_sdlt():
+    assert (
+        get_local_build_issues(
+            python_version=(3, 13, 7),
+            variable_names={"num_vehicles"},
+            unprojected_corporate_sdlt=[],
+        )
+        == []
+    )
+
+
+def test_a_refused_build_reports_the_preflight_error(monkeypatch):
+    """The build's error handler can report a failure of the preflight itself."""
+    from policyengine_uk_data.datasets import create_datasets
+
+    def refuse(full_build=False):
+        raise RuntimeError("preflight refused")
+
+    monkeypatch.setattr(create_datasets, "assert_local_build_environment", refuse)
+    with pytest.raises(RuntimeError, match="preflight refused"):
+        create_datasets.main()

policyengine_uk_data/tests/test_obr_receipts_sheet.py

@@ -105,8 +105,10 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook:
         row = 10 + offset
         ws[f"B{row}"] = label
         if populate:
+            # £100bn a year, above the corporate SDLT statistic the SDLT
+            # target subtracts.
             for col in ("D", "E", "F", "G", "H", "I", "J"):
-                ws[f"{col}{row}"] = 1.0
+                ws[f"{col}{row}"] = 100.0
     return wb