diff --git a/changelog.d/property-wealth-from-components.fixed.md b/changelog.d/property-wealth-from-components.fixed.md new file mode 100644 index 00000000..a671843a --- /dev/null +++ b/changelog.d/property-wealth-from-components.fixed.md @@ -0,0 +1 @@ +Derive worldwide gross property wealth from six uprated inputs covering all seven WAS terms; second homes and buy-to-let use the combined residential value and secured debt from #501. Keep the WAS total only for conditioning, remove inherited saved totals, and retain overseas property and other real estate separately. Requires a companion model stacked on policyengine-uk#1974 and #1969 for complete wealth, UK land and benefit-capital routing; domestic purchase proxies include buy-to-let through the combined residential input. Wealth, land, decile and reform outputs can change; calibration controls and objective terms are unchanged, but losses and fitted weights can change. Regional HPI still adjusts only main residence. Existing published datasets are not migrated. diff --git a/docs/imputations.md b/docs/imputations.md index 6af802ec..0668fdcf 100644 --- a/docs/imputations.md +++ b/docs/imputations.md @@ -42,7 +42,7 @@ Imputes household wealth components using demographic and income predictors. The | Variable | WAS round 8 source | Description | |----------|--------------------|-------------| | `owned_land` | `DVLUKValR8_sum` | Value of land in the UK | -| `property_wealth` | `DVPropertyR8` | Gross value of all property, main residence included | +| `property_wealth` | `DVPropertyR8` | Gross value of all property; sequential conditioning draw only, never saved | | `private_pension_wealth` | `totalpenr8_aggr` − `dvvaldbt_scaper8_aggr` | Private pension wealth other than current-employment defined benefit rights: defined contribution pots, retained rights, pensions in payment | | `directly_held_shares` | `DVFShUKVR8_aggr` + `DVFESHARESR8_aggr` | UK shares and employee shares and options, held outside ISAs and pooled funds | | `unit_and_investment_trusts` | `DVFCollVR8_aggr` | Unit trusts and investment trusts (one survey question) | @@ -53,6 +53,8 @@ Imputes household wealth components using demographic and income predictors. The | `main_residence_value` | `DVHValueR8` | Value of main home | | `other_residential_property_value` | `DVHseValR8_sum` + `DVBltValR8_sum` | Second homes and buy-to-let property | | `non_residential_property_value` | `DVBlDValR8_sum` | Other buildings, such as shops and garages | +| `overseas_property_value` | `DVLOSValR8_sum` | Overseas land and property | +| `other_property_value` | `DVOprValR8_sum` | Other real estate, unspecified geography/type | | `savings` | `DVSaValR8_aggr` | Savings account balances (cash ISAs excluded) | | `num_vehicles` | `vcarnr8` | Number of vehicles owned | | `student_loan_balance` | `Tot_LosR8_aggr` − `Tot_los_exc_SLCR8_aggr` | Student Loans Company balance, allocated to people | @@ -61,7 +63,7 @@ Imputes household wealth components using demographic and income predictors. The | `non_residential_property_secured_debt` | `DVBldDebtR8_sum` | Mortgages and loans secured on other buildings | | `owned_land_secured_debt` | `DVLUKDebtR8_sum` | Loans secured on UK land | -`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. Land and property overseas (`DVLOSValR8_sum`), other real estate (`DVOPrValR8_sum`) and overseas shares (`DVFShOSVR8_aggr`) are not exported separately; `property_wealth` and the financial wealth totals include them. +`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. The seven gross WAS property terms are retained in six saved inputs: second homes and buy-to-let share one residential input. `property_wealth` remains a conditioning draw only; the companion model calculates the total from those six uprated inputs. Overseas shares (`DVFShOSVR8_aggr`) remain included only in the financial wealth totals. Each target draws its quantile with its own seed. microimpute otherwise uses one seed for every target, so a household drawn high for one asset is drawn high for every later target conditioned on it, which makes sparse targets such as the debt secured on land near-certain for the households that hold the asset. diff --git a/docs/property-wealth-model-changes.md b/docs/property-wealth-model-changes.md new file mode 100644 index 00000000..ea974f2e --- /dev/null +++ b/docs/property-wealth-model-changes.md @@ -0,0 +1,395 @@ +# Property wealth: required companion model changes + +This document specifies the `policyengine-uk` changes required before building +a data release with all seven WAS property terms mapped to six saved household +components. [policyengine-uk-data#543](https://github.com/PolicyEngine/policyengine-uk-data/pull/543) +is stacked on [#501](https://github.com/PolicyEngine/policyengine-uk-data/pull/501) +and must merge after it. Its base tracks the `was-wealth-split` branch and must +be restacked whenever that branch moves, without rewriting #501's history. +The data PR remains a draft for a batched release. No model PR is opened by +this work. + +The property companion must stack on #501's two open model companions: +[policyengine-uk#1974 (`wealth-share-components`)](https://github.com/PolicyEngine/policyengine-uk/pull/1974) +and [#1969 (`uc-capital-sale-expenses`)](https://github.com/PolicyEngine/policyengine-uk/pull/1969). +Retain their share-component, secured-debt and capital-valuation treatment. +Historical file:line citations below describe the audited `policyengine-uk +2.93.0` wheel; they do not describe the current companion branches. Data +citations marked "original PR" refer to head +`af64d3757c2648e820219faee96186d3d7cd9a63`. Current companion behavior below +was checked against the live branches on 13 October 2026. + +## Source definition and routing + +The [round-8 derived-variable specification](https://doc.ukdataservice.ac.uk/doc/7215/mrdoc/excel/7215_property_dv_spec_r8.xlsx), +worksheet `DV_Spec_Property_R8`, row 49, defines: + +```text +DVPropertyR8 = DVhValueR8 + DVHseValR8_sum + DVBltValR8_sum + + DVBlDValR8_sum + DVLUKValR8_sum + DVLOSValR8_sum + + DVOprValR8_sum +``` + +This is gross property value. Row 45 defines the separate net measure +`HpropWR8 = DVPropertyR8 - HmortGR8`. The [user guide, p. 39](https://doc.ukdataservice.ac.uk/doc/7215/mrdoc/pdf/7215_was_userguide_round_8.pdf) +describes mortgage and equity-release deductions from net property wealth. + +| WAS field | Stored household component | Coverage | +|---|---|---| +| `DVhValueR8` | `main_residence_value` | Main residence | +| `DVHseValR8_sum` | `other_residential_property_value` | Second UK houses, including time-shares and holiday homes; combined with buy-to-let by #501 | +| `DVBltValR8_sum` | `other_residential_property_value` | UK residential property let for profit; combined with second homes by #501 | +| `DVBlDValR8_sum` | `non_residential_property_value` | Other UK buildings | +| `DVLUKValR8_sum` | `owned_land` | UK land-only holdings | +| `DVLOSValR8_sum` | `overseas_property_value` | Overseas land and property, including time-shares | +| `DVOprValR8_sum` | `other_property_value` | Other real estate; geography and property type are not separately identified | + +#501 renames the buy-to-let source to the internal `buy_to_let_value` and +combines it with `second_homes_value` in `other_residential_property_value`. +It also combines their debt in `other_residential_property_secured_debt`. +There is no separately exported buy-to-let model input: adding one to an +aggregate or capital list would count the same holding twice. + +Rows 54–59 identify the six other-property categories. The [round-8 dictionaries](https://doc.ukdataservice.ac.uk/doc/7215/mrdoc/ukda_data_dictionaries.zip), +person dictionary `UProp2R8` (position 3,709), identify category 5 as overseas +land **or property**, including time-shares. The [questionnaire](https://doc.ukdataservice.ac.uk/doc/7215/mrdoc/pdf/7215_was_questionnaire_round_8_covid19_version.pdf), +pp. 197–199 and 205–206, confirms the category routing and gross valuations. +The abbreviated `DVLOSValR8_sum` label "overseas land" must not be interpreted +as excluding overseas dwellings. + +At the original PR baseline, `DVLUKValR8_sum` already mapped directly to +`owned_land` (`datasets/imputations/wealth.py:73`), and the three building +components mapped at `:102–104`. Overseas assets were renamed `non_uk_land` +at `:97` but omitted from `IMPUTE_VARIABLES` at `:48–60`; buy-to-let and other +real estate had no mapping. The QRF trained on those output columns at +`:318–320`, predicted at `:370`, and wrote each non-conditioning household +output at `:288–297`. Thus `owned_land` means the WAS UK land component, +not estimated land under dwellings. + +The restacked mapping retains all seven source terms in six saved components. +The source total remains a sequential conditioning output and is not saved; +an inherited saved `property_wealth` column must also be removed. The QRF +wrapper passes configured targets to MicroImpute (`utils/qrf.py:52–56` at +the original baseline) and delegates prediction (`:68`). Sequential draws +do not impose an accounting constraint tying component draws to the earlier +conditioning-total draw. Source reconciliation is performed before prediction; +the model subsequently derives property wealth from the saved components. + +Before training, handle documented `-9` (not applicable), `-8` (don't know or +refusal), and missing property values consistently; report those counts in +source reconciliation. Household dictionary positions 828–829 explicitly +document these codes for overseas and other property. Complete-case and +operational zero-filled reconciliation have different interpretations and +must be reported separately. Do not infer a source residual from independently +imputed release columns or apply the regional HPI adjustment before measuring +the source identity. + +## Consumer audit + +### Historical 2.93.0 audit + +Paths in this table are relative to `policyengine_uk/` in the model repository. +These line numbers and behaviors belong to the historical audited wheel. + +| Consumer | File:line in 2.93.0 | Historical behavior | +|---|---|---| +| Residential property | `variables/household/wealth/residential_property_value.py:13` | Adds main residence and the other-residential component, which then contained second homes only | +| Property wealth | `variables/household/wealth/property_wealth.py:12` | Adds residential and non-residential property | +| Direct land | `variables/input/owned_land.py:7,12` | Land-only plots; GDP-per-capita uprating | +| Household land | `variables/household/wealth/household_land_value.py:17–18,56` | Multiplies property wealth by regional intensity, then adds direct land | +| Total wealth | `variables/household/wealth/total_wealth.py:12–17` | Includes property wealth but does not separately add `owned_land` | +| Wealth deciles | `variables/household/wealth/household_wealth_decile.py:12–16` | Reads total wealth with person-weighted household weights | +| Wealth tax | `variables/contrib/ubi_center/wealth_tax.py:13–15` | Reads total wealth | +| Non-primary-residence wealth tax | `variables/contrib/cec/non_primary_residence_wealth_tax.py:13–16` | Reads total wealth minus main residence | +| LVT | `variables/contrib/ubi_center/land_value_tax.py:13–16` | Reads household, corporate and combined land estimates | +| Main dwelling purchases | `variables/household/consumption/main_residential_property_purchased.py:14–15` | Reads main residence directly | +| Additional dwelling purchases | `variables/household/consumption/additional_residential_property_purchased.py:14–17` | Reads the other-residential component directly | +| Non-residential purchases | `variables/household/consumption/non_residential_property_purchased.py:12` | Reads the non-residential component directly | +| High-value council tax | `variables/gov/hmrc/high_value_council_tax_surcharge.py:23` | Reads main residence directly | +| UC capital | `variables/gov/dwp/universal_credit/uc_assessable_capital.py:19–21` | Sums configured capital sources | +| UC property-income gate | `variables/gov/dwp/universal_credit/income/uc_unearned_income.py:18–20,26–27` | Tested other-residential and non-residential capital; this gate has since been removed | +| Housing Benefit capital | `variables/gov/dwp/housing_benefit/housing_benefit_assessable_capital.py:20–21` | Sums configured capital sources | +| Pension Credit capital | `variables/gov/dwp/pension_credit/pension_credit_assessable_capital.py:21–22` | Sums configured capital sources | + +### Verified current companion behavior + +#1974 changes all six countable-capital lists to use `share_like_capital` +instead of `corporate_wealth`, retaining the share-component aggregate +fallback and avoiding duplicate holdings. #1969 uses +[`utils/capital_valuation.py`](https://github.com/PolicyEngine/policyengine-uk/blob/uc-capital-sale-expenses/policyengine_uk/utils/capital_valuation.py) +to apply each programme's source-specific sale-expense deduction and then +secured debt, flooring secured-property categories at zero. Its debt mapping +links `other_residential_property_value` to +`other_residential_property_secured_debt`; that single link covers second +homes and buy-to-let together after #501. Preserve the combined stack's share +routing when updating property sources; do not restore historical capital +lists or overwrite share sale-expense decisions. + +The current +[`uc_unearned_income.py`](https://github.com/PolicyEngine/policyengine-uk/blob/uc-capital-sale-expenses/policyengine_uk/variables/gov/dwp/universal_credit/income/uc_unearned_income.py) +sums configured claimant unearned-income sources. Actual interest, dividends +and rent are excluded at every capital level; capital enters through tariff +income. There is no property-income gate to extend for the new inputs. + +The current +[`additional_residential_property_purchased.py`](https://github.com/PolicyEngine/policyengine-uk/blob/uc-capital-sale-expenses/policyengine_uk/variables/household/consumption/additional_residential_property_purchased.py) +multiplies `other_residential_property_value` by `property_purchased`. #501 +therefore includes domestic buy-to-let automatically once in the existing +additional-dwelling purchase/SDLT stock proxy. No separate buy-to-let term is +needed. Overseas and unclassified other real estate remain outside that proxy. + +Income Support, income-based JSA and income-related ESA likewise use the +configured capital-source lists specified below. Benefit capital, +property-purchase proxies and high-value council tax do not read +`property_wealth` directly. Removing the stored aggregate alone has no direct +effect on these calculations at fixed weights; completing the component +mapping and companion routing can change them. + +In the original uk-data audit, `targets/build_loss_matrix.py:305–315` +dispatches existing national and regional land targets. No control values, +target weights or objective terms change. Calculated land columns, evaluated +losses and fitted constituency/local-authority weights can change. +Recalibration can change weighted outputs even where formulas have no direct +aggregate dependency. + +## Exact model implementation + +### New stock inputs and uprating + +Create only `policyengine_uk/variables/input/overseas_property_value.py` and +`other_property_value.py`. Each follows this template, substituting its class +name, label and category documentation: + +```python +from policyengine_uk.model_api import * + + +class overseas_property_value(Variable): + label = "Overseas property value" + documentation = "Gross value of overseas land and property, including time-shares" + entity = Household + definition_period = YEAR + value_type = float + unit = GBP + quantity_type = STOCK + default_value = 0 + uprating = "gov.economic_assumptions.indices.obr.per_capita.gdp" +``` + +Describe other property as gross other real estate with unspecified +geography/type. This GDP index matches existing main residence, +other-residential, non-residential and land input attributes. Add both new +names under `gov.economic_assumptions.yoy_growth.obr.per_capita.gdp` in +`policyengine_uk/data/uprating_indices.yaml`, preserving #501's model +companions' registrations. Preserve stock handling for monthly calculations. + +Both registrations matter. In the historical loading audit, single-year +dataset loading copied columns into each year (`simulation.py:468–470` and +`data/economic_assumptions.py:43–51`) and grew only names in the YAML list +(`data/economic_assumptions.py:89–98`). Those explicit annual inputs override +the generic core fallback using a variable's `uprating` attribute when only +an earlier-period input exists. A companion test must cover the dataset-load +route as well as situation-based calculations. Historical dataset `validate()` +checked missing values only (`data/dataset_schema.py:145–150`); the simulation +loader skipped unknown columns (`simulation.py:499–505`). The uk-data +compatibility guard must prevent release generation with a model that cannot +carry the new components. + +### Aggregates and land + +Keep `variables/household/wealth/residential_property_value.py` with its two +existing terms: + +```python +adds = [ + "main_residence_value", + "other_residential_property_value", +] +``` + +Document domestic residential property: the second term now includes both +second homes and buy-to-let through #501. Overseas and other real estate +cannot be classified as residential from their aggregate WAS values. + +In `variables/household/wealth/property_wealth.py`: + +```python +adds = [ + "residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", +] +``` + +This gives: + +```text +property_wealth = residential_property_value + non_residential_property_value + + owned_land + overseas_property_value + other_property_value +``` + +Document worldwide gross property value, including direct UK land holdings. +`total_wealth` receives these assets through `property_wealth`; do not also +add land, overseas property or other property separately to `total_wealth`. +Gross asset wealth remains separate from #1969's net countable-capital +valuation, which deducts sale expenses and identified secured debt. + +In `variables/household/wealth/household_land_value.py`, keep the existing +region selection and intensity parameters, add the overseas input and replace +the return expression: + +```python +overseas_property_value = household("overseas_property_value", period) +return ( + property_wealth - owned_land - overseas_property_value +) * property_wealth_intensity + owned_land +``` + +Thus `household_land_value = (property_wealth - owned_land - +overseas_property_value) * regional_intensity + owned_land`. Direct UK land +enters once at full value; explicitly identified overseas holdings enter zero +times. The remaining other-real-estate value uses the existing regional +intensity approximation, subject to the geography question below. + +### Benefit capital and domestic purchase routing + +Add `overseas_property_value` and `other_property_value` exactly once to each +countable-capital `sources.yaml` and its `sale_expenses/sources.yaml` below. +Preserve other sources, effective dates and the combined companions' share +treatment. Update descriptions to identify the new categories and retain +their broad-asset, beneficial-ownership and disregard limitations. + +| Programme | Capital directory under `policyengine_uk/parameters/` | Lists to update | +|---|---|---| +| Universal Credit | `gov/dwp/universal_credit/means_test/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | +| Housing Benefit | `gov/dwp/housing_benefit/means_test/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | +| Income Support | `gov/dwp/income_support/means_test/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | +| Income-based JSA | `gov/dwp/JSA/income/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | +| Income-related ESA | `gov/dwp/ESA/income/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | +| Pension Credit | `gov/dwp/pension_credit/income/capital/` | `sources.yaml`, `sale_expenses/sources.yaml` | + +Both types of list matter: #1969 deducts the programme's sale-expense rate +only when a counted source is also in its sale-expense list. Listing a source +for sale expenses alone does not make it countable capital. Overseas and other +real estate use the same real-property sale-expense treatment as the existing +property categories. Preserve #1969's helper and secured-debt mapping; do not +add a separate buy-to-let source or debt. No additional secured-debt input is +exported for overseas or other real estate, so debt on those holdings cannot +be identified or deducted by this mapping. + +The new categories enter non-main-home capital within the model's current +proxy, reported-capital and disregard semantics. Source aggregates cannot +identify additional statutory disregards or individual beneficial ownership. +This is not a claim that every holding is legally countable in every +household. Keep main residence and private pension rights excluded. + +UC currently omits `owned_land` from its countable-capital list, while the +other five lists include it. Preserve that existing UC treatment; correcting +it requires the explicit model decision recorded below. Its presence in a +sale-expense list does not override that omission. + +Retain the current additional residential purchase formula: + +```python +other_residential_property_value = household("other_residential_property_value", period) +return other_residential_property_value * property_purchased +``` + +The #501 component already includes second homes and domestic buy-to-let +once. Keep overseas assets and unclassified other real estate outside the UK +dwelling-purchase/SDLT proxy. This retains the existing stock-based purchase +approximation; it does not establish observed transaction values. + +## Required model tests and invariants + +Add `policyengine_uk/tests/test_property_wealth_components.py` with synthetic +single-household tests and Hypothesis tests of non-negative finite component +values. No licensed data is required. The invariants are: + +- The seven WAS source terms reconcile before imputation, with second homes + and buy-to-let summed into one saved other-residential component. +- Gross property wealth equals the sum of the six saved property inputs. +- Buy-to-let contributes exactly once to residential property, gross property + and total wealth, capital and the domestic additional-dwelling purchase proxy. +- With other wealth inputs zero, total wealth equals that property sum. +- UK land-only value contributes once to property and once to household land. +- Overseas value contributes to global property/total wealth and contributes + zero to household land and household LVT. +- Uprating each saved input preserves the aggregate identity at every tested + year; monthly stock values are not prorated. + +Cover each of the seven source categories alone and a mixed household. In +particular, verify second-home-only, buy-to-let-only and combined values use +the same other-residential input. In London, £100,000 domestic buildings, +£20,000 direct UK land and £30,000 overseas assets give £150,000 property +wealth and £105,000 household land with intensity 0.85. Repeat at 2024 and +2026 using the GDP index; check each input and aggregate growth against the +actual model index. Test monthly stock values to detect annual-value prorating. + +Add/extend `tests/policy/baseline/household/wealth/household_land_value.yaml` +with land-only, overseas-only and mixed cases. Preserve existing regional +cases; for consistent component inputs their expected land values remain +valid. Test household LVT with only its household rate enabled. + +Add separate overseas-only and other-property-only cases to each affected +benefit-capital test file, including sale-expense valuation, reported-capital +overrides and existing household-to-benunit allocation behavior. Assert each +new source occurs exactly once in both relevant lists, and main residence +remains excluded. A combined second-home/buy-to-let case must use #501's +single other-residential value and combined secured debt, with #1969's +sale-expense deduction before debt and zero floor. Preserve current UC +unearned-income behavior; no property-income gate test is required. + +Test that buy-to-let enters the existing additional domestic dwelling-purchase +proxy once, while overseas and unclassified other holdings do not. Keep the +uk-data stored-total override regression. Demonstrate separate mutations for +omitting a source category, double-counting buy-to-let, restoring saved-total +persistence, dropping the direct-land subtraction and dropping the overseas +subtraction; each must fail its relevant test before restoring source files +byte for byte. + +Until the property companion is released, a test-only Reform can exercise +the specified contract. Passing it proves the contract under that Reform, +not implementation by the installed model or production release. For +dataset-based contract tests, arrange for the new definitions to exist before +data loading and mock the YAML registration in the fixture; adding only +`Variable.uprating` to a Reform does not exercise the required dataset +uprating route. Keep test-only support out of production data files. + +## Reviewer decisions and release boundary + +The proposed treatment keeps overseas and direct UK land in global gross +`property_wealth`, matching WAS and the model's "all owned properties" +semantics, then corrects the domestic land base. Alternatives to consider +are keeping overseas assets only in `total_wealth`, or keeping UK land +outside `property_wealth` and adding it separately to `total_wealth`. Either +requires changing aggregate definitions and consumer expectations. + +The other-real-estate category has no explicit geography/type in its public +label. The selected treatment retains it as separate global wealth and uses +the current regional property-to-land approximation for land estimation. +Reviewers should decide whether additional source detail permits geographic +classification, or whether to exclude unclassified holdings from UK land +until that classification exists. The lack of identified secured debt for +overseas and other real estate also limits their benefit-capital valuation. + +Reviewers should separately decide whether to add `owned_land` to UC capital +sources; its present omission is inconsistent with the other listed benefits +but is not changed implicitly through the gross property aggregate. + +Merge #501 first. Implement and release this property companion on top of +the open #1974 and #1969 model companions, update the data dependency/lock to +a release carrying all required changes, and run its targeted tests before +the batched release. The data guard must reject a model that cannot carry the +new components. Licensed WAS reconciliation remains unrun in this workspace; +the seven-term reconciliation tool must be run on the licensed household +source before release. + +Production rebuilding, recalibration and LVT-impact measurement remain +separate release work. Existing data releases are not migrated; their saved +totals still override formulas. Only main-residence imputation receives the +regional HPI adjustment; the remaining source categories retain their mapped +values and the model's chosen uprating index, with second homes and buy-to-let +already combined by #501. diff --git a/policyengine_uk_data/datasets/imputations/regional_property_uprating.py b/policyengine_uk_data/datasets/imputations/regional_property_uprating.py index 68aa7e37..5921ab4b 100644 --- a/policyengine_uk_data/datasets/imputations/regional_property_uprating.py +++ b/policyengine_uk_data/datasets/imputations/regional_property_uprating.py @@ -3,13 +3,11 @@ The WAS QRF imputation compresses regional property values — London is undervalued by ~41% and the North East overvalued by ~39% relative to -UK HPI Dec 2025. This module rescales main_residence_value (and -property_wealth proportionally) so that regional means match official -average house prices, following the same pattern as -``_calibrate_energy_to_need`` in ``consumption.py``. +UK HPI Dec 2025. This module rescales main_residence_value so that +regional means match official average house prices, following the same +pattern as ``_calibrate_energy_to_need`` in ``consumption.py``. """ -import numpy as np import pandas as pd from policyengine_uk.data import UKSingleYearDataset @@ -32,8 +30,9 @@ def _calibrate_property_to_hpi(household: pd.DataFrame) -> pd.DataFrame: applies it multiplicatively. This preserves within-region distributional shape while anchoring the level to admin data. - property_wealth is adjusted by the same factor so that the land-to- - property ratio stays consistent. + property_wealth is not stored: policyengine-uk sums it from + main_residence_value and the other property components, so it picks up + this rescaling without being scaled here. """ hpi_prices = _load_regional_house_prices() region = household["region"].values @@ -56,7 +55,6 @@ def _calibrate_property_to_hpi(household: pd.DataFrame) -> pd.DataFrame: factor = hpi_price / imputed_mean household.loc[owners_mask, "main_residence_value"] *= factor - household.loc[owners_mask, "property_wealth"] *= factor return household @@ -77,7 +75,6 @@ def uprate_property_by_region( household = _calibrate_property_to_hpi(household) dataset.household["main_residence_value"] = household["main_residence_value"].values - dataset.household["property_wealth"] = household["property_wealth"].values dataset.validate() return dataset diff --git a/policyengine_uk_data/datasets/imputations/wealth.py b/policyengine_uk_data/datasets/imputations/wealth.py index 07e1d986..aeab0aa9 100644 --- a/policyengine_uk_data/datasets/imputations/wealth.py +++ b/policyengine_uk_data/datasets/imputations/wealth.py @@ -68,6 +68,9 @@ "other_residential_property_secured_debt", "non_residential_property_secured_debt", "owned_land_secured_debt", + # Append new outputs: retain #501's chain order and quantile seeds. + "overseas_property_value", + "other_property_value", ] # corporate_wealth is not imputed: it is the sum of its imputed components, @@ -108,6 +111,22 @@ "student_loan_balance": ("Tot_LosR8_aggr", "-Tot_los_exc_SLCR8_aggr"), } +# Seven WAS source terms become six model inputs: #501 combines second +# homes and buy-to-let. Keep direct UK land and overseas property distinct. +PROPERTY_COMPONENTS = ( + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", +) + +# Keep the source total as a sequential predictor, but let the companion +# model sum the six uprated inputs, covering all seven WAS source terms. +# Independent predictions need not reconcile to the conditioning draw. +CONDITIONING_ONLY_VARIABLES = ("property_wealth",) + WAS_RENAMES = { "R8xshhwgt": "household_weight", # Components for estimating land holdings. @@ -143,7 +162,8 @@ "DVPriRntR8": "is_renter", # {1, 2} TODO: Get codebook values. "CTAmtR8": "council_tax", # Other columns for reference. - "DVLOSValR8_sum": "non_uk_land", + # Overseas land AND property (including dwellings/time-shares), not land only. + "DVLOSValR8_sum": "overseas_property_value", "HFINWNTR8_Sum": "net_financial_wealth", "HFINWR8_SUM": "gross_financial_wealth", "TotalWlthR8": "wealth", @@ -156,6 +176,7 @@ "DVBLtDebtR8_sum": "buy_to_let_debt", "DVBlDValR8_sum": "non_residential_property_value", "DVBldDebtR8_sum": "non_residential_property_secured_debt", + "DVOprValR8_sum": "other_property_value", "DVTotinc_bhcR8": "household_net_income", "DVSaValR8_aggr": "savings", "vcarnr8": "num_vehicles", @@ -202,6 +223,29 @@ def generate_was_table(was: pd.DataFrame): was = was.rename(columns=RENAMES).fillna(0)[list(RENAMES.values())] + # WAS gross property fields use -9 (not applicable) and -8 (unknown/refused). + # Treat these as missing, like NaN. Gross asset values cannot be negative; + # reject undocumented codes without exposing any household records. + # Validate the seven raw terms before combining residential values: a + # missing/negative buy-to-let value must not reduce or hide in second homes. + gross_columns = [ + "property_wealth", + "main_residence_value", + "second_homes_value", + "buy_to_let_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", + ] + was[gross_columns] = was[gross_columns].replace({-9: 0, -8: 0}) + negative = was[gross_columns].lt(0).sum() + if negative.any(): + raise ValueError( + "Unexpected negative WAS gross property values (household counts): " + + ", ".join(f"{name}={count}" for name, count in negative.items() if count) + ) + was["is_renting"] = was["is_renter"] == 1 # Private pension wealth other than current-employment defined benefit @@ -228,6 +272,7 @@ def generate_was_table(was: pd.DataFrame): "impute_variables": tuple(IMPUTE_VARIABLES), "derived_targets": tuple(DERIVED_TARGETS.items()), "quantile_draw_seeds": tuple(QUANTILE_DRAW_SEEDS.items()), + "property_mapping_version": 2, } @@ -361,6 +406,135 @@ def use_independent_quantile_draws(model: QRF) -> QRF: return model +def store_wealth_predictions( + dataset: UKSingleYearDataset, predictions: pd.DataFrame +) -> UKSingleYearDataset: + """Write imputed wealth to the dataset, skipping conditioning-only outputs. + + `student_loan_balance` is allocated to people; every other saved output is + a household column. + """ + # Also remove an inherited saved total when augmenting an existing dataset. + dataset.household.drop( + columns=list(CONDITIONING_ONLY_VARIABLES), errors="ignore", inplace=True + ) + for column in predictions.columns: + if column in CONDITIONING_ONLY_VARIABLES: + continue + if column == "student_loan_balance": + dataset.person[column] = _allocate_student_loan_balance_to_people( + household_balances=predictions[column].clip(lower=0), + person=dataset.person, + ) + continue + dataset.household[column] = predictions[column].values + return dataset + + +def require_property_component_model(): + """Reject a model that would silently omit the newly retained categories. + + The companion model changes, including the UK-only land formula, are + specified in docs/property-wealth-model-changes.md. They must land before + building a release with this mapping. + """ + from policyengine_uk.system import system + + new_components = { + "overseas_property_value", + "other_property_value", + } + missing = new_components - system.variables.keys() + if missing: + raise ValueError( + "Property imputation requires the companion policyengine-uk change " + "in docs/property-wealth-model-changes.md; missing variables: " + + ", ".join(sorted(missing)) + ) + property_adds = sorted(system.variables["property_wealth"].adds or ()) + residential_adds = sorted(system.variables["residential_property_value"].adds or ()) + if property_adds != sorted( + [ + "residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", + ] + ) or residential_adds != sorted( + [ + "main_residence_value", + "other_residential_property_value", + ] + ): + raise ValueError( + "policyengine-uk property aggregates must include all seven WAS " + "categories; apply docs/property-wealth-model-changes.md" + ) + required_index = "gov.economic_assumptions.indices.obr.per_capita.gdp" + if any( + system.variables[name].uprating != required_index + for name in PROPERTY_COMPONENTS + ): + raise ValueError( + "Every WAS property component requires the GDP-per-capita model uprating index" + ) + + # The UK dataset loader materialises yearly inputs using this registry; + # variable attributes alone do not uprate those explicitly supplied arrays. + from pathlib import Path + import yaml + from policyengine_uk.data import economic_assumptions + + with open( + Path(economic_assumptions.__file__).parent / "uprating_indices.yaml" + ) as registry_file: + registry = yaml.safe_load(registry_file) + registered = [name for names in registry.values() for name in names] + gdp_registered = registry.get( + "gov.economic_assumptions.yoy_growth.obr.per_capita.gdp", () + ) + if any( + registered.count(name) != 1 or name not in gdp_registered + for name in PROPERTY_COMPONENTS + ): + raise ValueError( + "Every WAS property component requires UK dataset uprating registration" + ) + + # Exercise the installed formula directly: adding the inputs and aggregate + # without changing land estimation would still double-count UK land and + # tax overseas assets. These are synthetic formula probes, not survey data. + from policyengine_core.periods import period + from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE + + probe_period = period(CURRENT_FRS_RELEASE.base_year) + regions = system.variables["region"].possible_values + inputs = { + "region": regions.encode(np.array(["LONDON", "LONDON", "LONDON"])), + "property_wealth": np.array([100.0, 100.0, 150.0]), + "owned_land": np.array([100.0, 0.0, 20.0]), + "overseas_property_value": np.array([0.0, 100.0, 30.0]), + **{ + name: np.zeros(3) + for name in PROPERTY_COMPONENTS + if name not in ("owned_land", "overseas_property_value") + }, + "main_residence_value": np.array([0.0, 0.0, 100.0]), + } + formula = system.variables["household_land_value"].get_formula(probe_period) + land = formula(lambda name, _: inputs[name], probe_period, system.parameters) + regional = system.parameters( + probe_period + ).household.wealth.land.intensity.property_wealth_by_region + expected_land = [100.0, 0.0, 100.0 * regional.LONDON + 20.0] + if not np.allclose(land, expected_land): + raise ValueError( + "policyengine-uk land formula must count UK land once and exclude " + "overseas property; apply docs/property-wealth-model-changes.md" + ) + + def save_imputation_models(): """ Train and save wealth imputation model. @@ -422,6 +596,7 @@ def impute_wealth(dataset: UKSingleYearDataset) -> UKSingleYearDataset: """ dataset = dataset.copy() + require_property_component_model() model = create_wealth_model() sim = Microsimulation(dataset=dataset) predictors = model.input_columns @@ -432,16 +607,7 @@ def impute_wealth(dataset: UKSingleYearDataset) -> UKSingleYearDataset: "NORTHERN_IRELAND", "WALES" ) # WAS doesn't sample NI -> put NI households in Wales (closest aggregate) output_df = derive_wealth_outputs(model.predict(input_df)) - - for column in output_df.columns: - if column == "student_loan_balance": - dataset.person[column] = _allocate_student_loan_balance_to_people( - household_balances=output_df[column].clip(lower=0), - person=dataset.person, - ) - continue - dataset.household[column] = output_df[column].values - + dataset = store_wealth_predictions(dataset, output_df) dataset.validate() return dataset diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 5b5bf690..284b820a 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -56,7 +56,9 @@ mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.45 net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +other_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +overseas_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 pension_credit_reported_capital,1.0,1.0,1.102,1.161,1.204,1.256,1.293,1.335,1.376,1.417,1.462,1.508,1.555,1.604,1.655 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index b7037441..f7d63260 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -56,7 +56,9 @@ mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.04 net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +other_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +overseas_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 pension_credit_reported_capital,0,0.0,0.102,0.054,0.037,0.043,0.029,0.032,0.031,0.03,0.032,0.031,0.031,0.032,0.032 diff --git a/policyengine_uk_data/tests/test_property_wealth_components.py b/policyengine_uk_data/tests/test_property_wealth_components.py new file mode 100644 index 00000000..1aaf93c4 --- /dev/null +++ b/policyengine_uk_data/tests/test_property_wealth_components.py @@ -0,0 +1,869 @@ +"""Preserve the seven WAS gross-property categories without a saved total. + +The local Reform is the companion policyengine-uk contract specified in +docs/property-wealth-model-changes.md. It is used explicitly: passing these +synthetic tests does not mean the installed production model has that change. + +The synthetic tests need no private data. The built-dataset tests run after +`make data` and are skipped when the enhanced FRS is absent. Failure messages +report counts and maxima only, never individual records. +""" + +import builtins +import importlib +import io +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pandas as pd +import pytest +import yaml +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from policyengine_uk import Microsimulation +from policyengine_uk.data import UKSingleYearDataset +from policyengine_uk.model_api import GBP, STOCK, YEAR, Household, Reform, Variable +from policyengine_uk.utils.scenario import Scenario + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.datasets.imputations.regional_property_uprating import ( + _load_regional_house_prices, + uprate_property_by_region, +) +from policyengine_uk_data.datasets.imputations.wealth import ( + CONDITIONING_ONLY_VARIABLES, + DERIVED_TARGETS, + IMPUTE_VARIABLES, + PROPERTY_COMPONENTS, + QUANTILE_DRAW_SEEDS, + WAS_RENAMES, + derive_wealth_outputs, + generate_was_table, + require_property_component_model, + store_wealth_predictions, +) +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.utils.uprating import uprate_dataset + +COMPONENTS = ( + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", +) +# Explicit specification, independent of the implementation's rename table. +SOURCE_COMPONENTS = ( + "DVhvalueR8", + "DVHseValR8_sum", + "DVBltValR8_sum", + "DVBlDValR8_sum", + "DVLUKValR8_sum", + "DVLOSValR8_sum", + "DVOprValR8_sum", +) +# Seven independent source terms become six model inputs: #501 combines the +# two kinds of residential property other than the main residence. +SOURCE_INPUTS = ( + "main_residence_value", + "other_residential_property_value", + "other_residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", +) +NEW_COMPONENTS = ( + "overseas_property_value", + "other_property_value", +) +BASE_YEAR = CURRENT_FRS_RELEASE.base_year +# Two years past the base year: 2026 for the 2024-25 FRS. +PROJECTION_YEAR = BASE_YEAR + 2 +YEARS = sorted({BASE_YEAR, CURRENT_FRS_RELEASE.calibration_year, PROJECTION_YEAR}) +RTOL = 1e-5 + + +class PropertyComponentsContract(Reform): + """Executable contract for the model change required before release.""" + + def apply(self): + for name in NEW_COMPONENTS: + variable = type( + name, + (Variable,), + { + "label": name.replace("_", " ").capitalize(), + "value_type": float, + "entity": Household, + "definition_period": YEAR, + "unit": GBP, + "quantity_type": STOCK, + "uprating": "gov.economic_assumptions.indices.obr.per_capita.gdp", + }, + ) + if name in self.variables: + self.update_variable(variable) + else: + self.add_variable(variable) + + class residential_property_value(Variable): + value_type = float + entity = Household + definition_period = YEAR + unit = GBP + quantity_type = STOCK + adds = [ + "main_residence_value", + "other_residential_property_value", + ] + + class property_wealth(Variable): + value_type = float + entity = Household + definition_period = YEAR + unit = GBP + quantity_type = STOCK + adds = [ + "residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", + ] + + class household_land_value(Variable): + value_type = float + entity = Household + definition_period = YEAR + unit = GBP + quantity_type = STOCK + + def formula(household, period, parameters): + region = household("region", period) + regional = parameters( + period + ).household.wealth.land.intensity.property_wealth_by_region + regions = tuple(_load_regional_house_prices()) + ("NORTHERN_IRELAND",) + intensity = np.select( + [ + region == getattr(region.possible_values, name) + for name in regions + ], + [getattr(regional, name) for name in regions], + default=regional.UNKNOWN, + ) + land = household("owned_land", period) + overseas = household("overseas_property_value", period) + domestic_property = ( + household("property_wealth", period) - land - overseas + ) + return domestic_property * intensity + land + + self.update_variable(residential_property_value) + self.update_variable(property_wealth) + self.update_variable(household_land_value) + + +@contextmanager +def _contract_uprating_spec(omit=()): + # The model extends datasets before setting inputs. That extension reads + # this YAML, so Variable.uprating alone does not uprate new dataset columns. + # Exercise the companion YAML entries in memory, without changing the + # installed model or treating the production dependency as completed. + economic_assumptions = importlib.import_module( + "policyengine_uk.data.economic_assumptions" + ) + config_path = Path(economic_assumptions.__file__).with_name("uprating_indices.yaml") + indices = yaml.safe_load(config_path.read_text()) + for names in indices.values(): + names[:] = [name for name in names if name not in omit] + gdp_variables = indices["gov.economic_assumptions.yoy_growth.obr.per_capita.gdp"] + gdp_variables.extend( + name + for name in NEW_COMPONENTS + if name not in gdp_variables and name not in omit + ) + contents = yaml.safe_dump(indices) + + def contract_yaml(filename, *args, **kwargs): + if Path(filename).resolve() == config_path.resolve(): + return io.StringIO(contents) + return builtins.open(filename, *args, **kwargs) + + wealth = importlib.import_module("policyengine_uk_data.datasets.imputations.wealth") + with ( + patch.object(economic_assumptions, "open", contract_yaml, create=True), + patch.object(wealth, "open", contract_yaml, create=True), + ): + yield + + +def _contract_simulation(dataset): + # Apply the variable definitions before dataset loading can skip them. + scenario = Scenario( + applied_before_data_load=True, + simulation_modifier=lambda sim: setattr( + sim, + "tax_benefit_system", + PropertyComponentsContract(sim.tax_benefit_system), + ), + ) + simulation_module = importlib.import_module("policyengine_uk.simulation") + model = importlib.import_module("policyengine_uk.system").system + # Clone the real, already loaded model rather than reading its thousands + # of source/parameter files again for each Hypothesis example. + with ( + _contract_uprating_spec(), + patch.object(simulation_module, "CountryTaxBenefitSystem", model.clone), + ): + return Microsimulation(dataset=dataset, scenario=scenario) + + +def _source_table(component_values): + raw = pd.DataFrame( + 0.0, + index=range(len(component_values)), + columns=list(dict.fromkeys((*WAS_RENAMES, *SOURCE_COMPONENTS, "DVPropertyR8"))), + ) + raw["R8xshhwgt"] = 1.0 + raw["GORR8"] = 8 + raw.loc[:, list(SOURCE_COMPONENTS)] = component_values + raw["DVPropertyR8"] = np.asarray(component_values).sum(axis=1) + return raw + + +def test_source_mapping_preserves_each_of_seven_categories(): + """Each row owns only one category, including each formerly omitted one.""" + amounts = np.arange(1, 8) * 10_000.0 + mapped = generate_was_table(_source_table(np.diag(amounts))) + expected = np.zeros((7, 6)) + expected[np.arange(7), [COMPONENTS.index(name) for name in SOURCE_INPUTS]] = amounts + assert PROPERTY_COMPONENTS == COMPONENTS + np.testing.assert_array_equal(mapped[list(COMPONENTS)].to_numpy(), expected) + np.testing.assert_array_equal( + mapped["property_wealth"], mapped[list(COMPONENTS)].sum(axis=1) + ) + + +@settings(max_examples=30, deadline=None) +@given( + components=st.lists( + st.integers(min_value=0, max_value=5_000_000), min_size=7, max_size=7 + ) +) +def test_source_total_reconciles_before_independent_imputation(components): + mapped = generate_was_table(_source_table([components])) + assert mapped["property_wealth"].iloc[0] == sum(components) + assert mapped[list(COMPONENTS)].sum(axis=1).iloc[0] == sum(components) + + +@pytest.mark.parametrize("source", SOURCE_COMPONENTS) +def test_source_mapping_requires_every_property_category(source): + raw = _source_table([[0.0] * 7]).drop(columns=source) + with pytest.raises(ValueError, match="Could not find column"): + generate_was_table(raw) + + +@pytest.mark.parametrize("missing", [np.nan, -8.0, -9.0]) +def test_source_mapping_normalises_property_missing_codes(missing): + raw = _source_table([[0.0] * 7]) + raw.loc[0, list(SOURCE_COMPONENTS) + ["DVPropertyR8"]] = missing + mapped = generate_was_table(raw) + assert (mapped[list(COMPONENTS) + ["property_wealth"]] == 0).all().all() + + +@pytest.mark.parametrize("source", ["DVHseValR8_sum", "DVBltValR8_sum"]) +@pytest.mark.parametrize("missing", [np.nan, -8.0, -9.0]) +def test_combined_residential_sources_normalise_missing_codes(source, missing): + """A missing source does not reduce the positive value of its sibling.""" + raw = _source_table([[0.0, 100.0, 200.0, 0.0, 0.0, 0.0, 0.0]]) + raw.loc[0, source] = missing + expected = 200.0 if source == "DVHseValR8_sum" else 100.0 + raw.loc[0, "DVPropertyR8"] = expected + mapped = generate_was_table(raw) + assert mapped["other_residential_property_value"].iloc[0] == expected + assert mapped[list(COMPONENTS)].sum(axis=1).iloc[0] == expected + + +@pytest.mark.parametrize("source", (*SOURCE_COMPONENTS, "DVPropertyR8")) +def test_source_mapping_rejects_unrecognised_negative_gross_value(source): + raw = _source_table([[0.0] * 7]) + raw.loc[0, source] = -1.0 + with pytest.raises(ValueError, match="negative"): + generate_was_table(raw) + + +@pytest.mark.parametrize("source", ["DVHseValR8_sum", "DVBltValR8_sum"]) +def test_combined_residential_source_cannot_mask_a_negative_gross_value(source): + """Validate raw sources before a positive sibling can conceal a negative.""" + raw = _source_table([[0.0, 100.0, 200.0, 0.0, 0.0, 0.0, 0.0]]) + raw.loc[0, source] = -1.0 + with pytest.raises(ValueError, match="negative"): + generate_was_table(raw) + + +def test_build_requires_companion_model_change(monkeypatch): + from policyengine_uk.system import system + + monkeypatch.delitem(system.variables, "overseas_property_value", raising=False) + with pytest.raises(ValueError, match="companion policyengine-uk change"): + require_property_component_model() + + +def test_companion_model_contract_satisfies_build_guard(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + monkeypatch.setattr(system_module, "system", contract) + with _contract_uprating_spec(): + require_property_component_model() + + +def test_build_guard_rejects_obsolete_buy_to_let_addition(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + monkeypatch.setattr( + contract.variables["residential_property_value"], + "adds", + [ + "main_residence_value", + "other_residential_property_value", + "buy_to_let_property_value", + ], + ) + monkeypatch.setattr(system_module, "system", contract) + with _contract_uprating_spec(): + with pytest.raises(ValueError, match="property aggregates"): + require_property_component_model() + + +def test_build_guard_rejects_old_land_formula(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + + def old_london_formula(household, period, parameters): + regional = parameters( + period + ).household.wealth.land.intensity.property_wealth_by_region + return household("property_wealth", period) * regional.LONDON + household( + "owned_land", period + ) + + monkeypatch.setattr( + contract.variables["household_land_value"], + "get_formula", + lambda period: old_london_formula, + ) + monkeypatch.setattr(system_module, "system", contract) + with _contract_uprating_spec(): + with pytest.raises(ValueError, match="land"): + require_property_component_model() + + +def test_build_guard_rejects_missing_dataset_uprating_registration(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + monkeypatch.setattr(system_module, "system", contract) + with _contract_uprating_spec(omit=("overseas_property_value",)): + with pytest.raises(ValueError, match="uprating registration"): + require_property_component_model() + + +def test_build_guard_rejects_a_different_non_null_uprating_index(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + monkeypatch.setattr(system_module, "system", contract) + monkeypatch.setattr( + contract.variables["overseas_property_value"], + "uprating", + "gov.economic_assumptions.indices.ons.population", + ) + with _contract_uprating_spec(): + with pytest.raises(ValueError, match="model uprating index"): + require_property_component_model() + + +def test_build_guard_rejects_formula_discarding_domestic_property(monkeypatch): + system_module = importlib.import_module("policyengine_uk.system") + contract = PropertyComponentsContract(system_module.system) + monkeypatch.setattr(system_module, "system", contract) + monkeypatch.setattr( + contract.variables["household_land_value"], + "get_formula", + lambda period: ( + lambda household, period, parameters: household("owned_land", period) + ), + ) + with _contract_uprating_spec(): + with pytest.raises(ValueError, match="land formula"): + require_property_component_model() + + +@pytest.mark.parametrize( + "filename", ["uprating_factors.csv", "uprating_growth_factors.csv"] +) +def test_new_components_have_materialisation_uprating_indices(filename): + factors = pd.read_csv(STORAGE_FOLDER / filename, index_col="Variable") + for component in NEW_COMPONENTS: + np.testing.assert_array_equal( + factors.loc[component].to_numpy(), + factors.loc["non_residential_property_value"].to_numpy(), + ) + + +def _calculate(sim, variable, year) -> np.ndarray: + return np.asarray(sim.calculate(variable, year).values, dtype=float) + + +def _property_arrays(sim, years) -> dict: + out = {} + for year in years: + main = _calculate(sim, "main_residence_value", year) + components = sum(_calculate(sim, name, year) for name in COMPONENTS) + out[year] = { + "property_wealth": _calculate(sim, "property_wealth", year), + "main_residence_value": main, + "components": components, + } + return out + + +def _n_not_close(actual, expected) -> tuple[int, float]: + gap = np.abs(actual - expected) + bad = gap > RTOL * np.abs(expected) + 1.0 + return int(bad.sum()), float(gap[bad].sum()) + + +def _synthetic_dataset(household: pd.DataFrame) -> UKSingleYearDataset: + n = len(household) + ids = np.arange(n) + household = household.copy() + for name, default in { + "household_weight": 1.0, + "region": "LONDON", + "council_tax": 0.0, + "rent": 0.0, + "tenure_type": "OWNED_OUTRIGHT", + }.items(): + if name not in household: + household[name] = default + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": np.full(n, 45), + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=household.assign(household_id=ids), + fiscal_year=BASE_YEAR, + ) + + +@pytest.mark.parametrize("component", COMPONENTS) +def test_household_owning_only_one_category_keeps_its_value_and_growth(component): + household = pd.DataFrame( + { + "region": ["LONDON"], + **{name: [100_000.0 if name == component else 0.0] for name in COMPONENTS}, + } + ) + simulation = _contract_simulation(_synthetic_dataset(household)) + np.testing.assert_allclose( + _calculate(simulation, "property_wealth", BASE_YEAR), [100_000.0] + ) + projected = _calculate(simulation, component, PROJECTION_YEAR) + assert projected[0] > 100_000.0 + np.testing.assert_allclose( + _calculate(simulation, "property_wealth", PROJECTION_YEAR), projected + ) + + +def test_property_wealth_is_imputed_before_its_components(): + """Non-land components follow the total; the existing land draw stays first.""" + assert set(CONDITIONING_ONLY_VARIABLES) <= set(IMPUTE_VARIABLES) + position = IMPUTE_VARIABLES.index("property_wealth") + assert "property_wealth" in CONDITIONING_ONLY_VARIABLES + assert IMPUTE_VARIABLES.index("owned_land") < position + for component in set(COMPONENTS) - {"owned_land"}: + assert IMPUTE_VARIABLES.index(component) > position + + +def test_wealth_split_chain_and_definitions_are_preserved(): + """Appending #543 targets preserves #501's chain, draws and definitions.""" + wealth_split_targets = ( + "owned_land", + "property_wealth", + "private_pension_wealth", + "directly_held_shares", + "unit_and_investment_trusts", + "stocks_and_shares_isa", + "gross_financial_wealth", + "net_financial_wealth", + "main_residence_value", + "other_residential_property_value", + "non_residential_property_value", + "savings", + "num_vehicles", + "student_loan_balance", + "cash_isa", + "other_residential_property_secured_debt", + "non_residential_property_secured_debt", + "owned_land_secured_debt", + ) + assert tuple(IMPUTE_VARIABLES) == (*wealth_split_targets, *NEW_COMPONENTS) + assert {name: QUANTILE_DRAW_SEEDS[name] for name in wealth_split_targets} == { + name: 1_000 + index for index, name in enumerate(wealth_split_targets) + } + assert DERIVED_TARGETS["other_residential_property_value"] == ( + "DVHseValR8_sum", + "DVBltValR8_sum", + ) + assert DERIVED_TARGETS["other_residential_property_secured_debt"] == ( + "DVHseDebtR8_sum", + "DVBLtDebtR8_sum", + ) + + +def test_impute_wealth_does_not_save_property_wealth(monkeypatch): + """The imputation step drops the WAS total and keeps the components.""" + from policyengine_uk_data.datasets.imputations import wealth + + predicted = { + **{name: [0.0, 0.0] for name in IMPUTE_VARIABLES}, + "property_wealth": [250_000.0, 0.0], + **{name: [10_000.0 * (i + 1), 0.0] for i, name in enumerate(COMPONENTS)}, + } + + class DummyModel: + input_columns = ["region"] + + @staticmethod + def predict(input_df): + return pd.DataFrame(predicted, index=input_df.index) + + class DummyMicrosimulation: + def __init__(self, dataset): + self.regions = dataset.household["region"] + + def calculate_dataframe(self, predictors, map_to): + return pd.DataFrame({"region": self.regions.values}) + + monkeypatch.setattr(wealth, "create_wealth_model", lambda: DummyModel()) + monkeypatch.setattr(wealth, "Microsimulation", DummyMicrosimulation) + monkeypatch.setattr(wealth, "require_property_component_model", lambda: None) + household = pd.DataFrame( + {"household_weight": [1.0, 1.0], "region": ["LONDON", "WALES"]} + ) + imputed = wealth.impute_wealth(_synthetic_dataset(household)) + + assert "property_wealth" not in imputed.household.columns + for component in COMPONENTS: + assert imputed.household[component].tolist() == predicted[component] + + +def test_storage_removes_an_inherited_property_total(): + dataset = _synthetic_dataset(pd.DataFrame({"property_wealth": [999_999.0]})) + predictions = pd.DataFrame( + {**{name: [100.0] for name in COMPONENTS}, "property_wealth": [7.0]} + ) + stored = store_wealth_predictions(dataset, predictions) + assert "property_wealth" not in stored.household + assert set(COMPONENTS) <= set(stored.household) + + +@settings(max_examples=20, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given( + second_homes=st.integers(min_value=0, max_value=5_000_000), + buy_to_let=st.integers(min_value=1, max_value=5_000_000), + second_homes_debt=st.integers(min_value=0, max_value=5_000_000), + buy_to_let_debt=st.integers(min_value=0, max_value=5_000_000), +) +def test_buy_to_let_is_counted_exactly_once_with_secured_debt_separate( + second_homes, buy_to_let, second_homes_debt, buy_to_let_debt +): + """For every draw, gross values merge once and debt remains a separate input. + + Neither residential source is exported separately. Stored and projected + residential_property_value and property_wealth contain the merged gross + value exactly once, including where debt exceeds the value of the asset. + """ + raw = _source_table([[0, second_homes, buy_to_let, 0, 0, 0, 0]]) + raw.loc[0, "DVHseDebtR8_sum"] = second_homes_debt + raw.loc[0, "DVBLtDebtR8_sum"] = buy_to_let_debt + mapped = generate_was_table(raw) + gross = second_homes + buy_to_let + debt = second_homes_debt + buy_to_let_debt + assert mapped["other_residential_property_value"].iloc[0] == gross + assert mapped["other_residential_property_secured_debt"].iloc[0] == debt + assert mapped[list(COMPONENTS)].sum(axis=1).iloc[0] == gross + + intermediates = { + "second_homes_value", + "buy_to_let_value", + "buy_to_let_property_value", + } + assert not intermediates.intersection(IMPUTE_VARIABLES) + assert not intermediates.intersection(PROPERTY_COMPONENTS) + predictions = derive_wealth_outputs(mapped[IMPUTE_VARIABLES]) + stored = store_wealth_predictions( + _synthetic_dataset(pd.DataFrame({"region": ["LONDON"]})), predictions + ) + assert not intermediates.intersection(stored.household.columns) + assert "property_wealth" not in stored.household.columns + assert stored.household["other_residential_property_value"].iloc[0] == gross + assert stored.household["other_residential_property_secured_debt"].iloc[0] == debt + + simulation = _contract_simulation(stored) + np.testing.assert_allclose( + _calculate(simulation, "property_wealth", BASE_YEAR), + [gross], + rtol=RTOL, + atol=1.0, + ) + for year in YEARS: + residential = _calculate(simulation, "other_residential_property_value", year) + np.testing.assert_allclose( + _calculate(simulation, "residential_property_value", year), + residential, + rtol=RTOL, + atol=1.0, + ) + np.testing.assert_allclose( + _calculate(simulation, "property_wealth", year), + residential, + rtol=RTOL, + atol=1.0, + ) + np.testing.assert_allclose( + _calculate(simulation, "total_wealth", year), + residential, + rtol=RTOL, + atol=1.0, + ) + assert ( + _calculate(simulation, "other_residential_property_value", PROJECTION_YEAR)[0] + > gross + ) + + +value = st.one_of( + st.just(0.0), + st.floats(min_value=1_000, max_value=5_000_000, allow_nan=False), +) +household_rows = st.lists( + st.tuples( + st.sampled_from(sorted(_load_regional_house_prices())), + st.tuples(*([value] * len(COMPONENTS))), + value, # WAS property total, imputed independently of the components + ), + min_size=1, + max_size=8, +) + + +@settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +@given(rows=household_rows) +def test_saved_property_wealth_follows_its_components(rows): + """Invariants for any non-negative imputed components and any WAS total: + + - no `property_wealth` column is saved; + - property_wealth equals the sum of its components in every year, + including after uk-data materialises the calibration year; + - property_wealth >= main_residence_value (P4); + - property_wealth grows exactly as its components do. + """ + regions, components, was_total = map(list, zip(*rows)) + n = len(rows) + household = pd.DataFrame( + { + "household_weight": np.ones(n), + "region": regions, + "council_tax": np.zeros(n), + "rent": np.zeros(n), + "tenure_type": ["OWNED_OUTRIGHT"] * n, + } + ) + predictions = pd.DataFrame( + { + "property_wealth": was_total, + **dict(zip(COMPONENTS, zip(*components))), + } + ) + dataset = store_wealth_predictions(_synthetic_dataset(household), predictions) + dataset = uprate_property_by_region(dataset) + assert "property_wealth" not in dataset.household.columns + + simulation = _contract_simulation(dataset) + arrays = _property_arrays(simulation, YEARS) + calibration_year = CURRENT_FRS_RELEASE.calibration_year + materialised = uprate_dataset(dataset, calibration_year) + # uprate_dataset stores an int year; policyengine-uk expects a string. + materialised.time_period = str(materialised.time_period) + arrays_materialised = _property_arrays( + _contract_simulation(materialised), [calibration_year] + ) + + for a in [*arrays.values(), *arrays_materialised.values()]: + np.testing.assert_allclose( + a["property_wealth"], a["components"], rtol=RTOL, atol=1.0 + ) + assert np.all(a["property_wealth"] >= a["main_residence_value"]) + + base, later = arrays[BASE_YEAR], arrays[PROJECTION_YEAR] + owns = base["components"] > 0 + np.testing.assert_allclose( + later["property_wealth"][owns] / base["property_wealth"][owns], + later["components"][owns] / base["components"][owns], + rtol=RTOL, + ) + for component in COMPONENTS: + base_component = _calculate(simulation, component, BASE_YEAR) + projected_component = _calculate(simulation, component, PROJECTION_YEAR) + owners = base_component > 0 + assert np.all(projected_component[owners] > base_component[owners]) + + +@settings(max_examples=15, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given(amount=st.floats(min_value=1_000, max_value=5_000_000, allow_nan=False)) +def test_uk_land_counted_once_and_overseas_excluded_from_uk_land(amount): + household = pd.DataFrame( + { + "region": ["LONDON", "LONDON"], + "owned_land": [amount, 0.0], + "overseas_property_value": [0.0, amount], + **{ + name: [0.0, 0.0] + for name in set(COMPONENTS) - {"owned_land", "overseas_property_value"} + }, + } + ) + sim = _contract_simulation(_synthetic_dataset(household)) + for year in YEARS: + land = _calculate(sim, "owned_land", year) + overseas = _calculate(sim, "overseas_property_value", year) + np.testing.assert_allclose( + _calculate(sim, "property_wealth", year), land + overseas + ) + np.testing.assert_allclose( + _calculate(sim, "total_wealth", year), land + overseas + ) + np.testing.assert_allclose( + _calculate(sim, "household_land_value", year), land, rtol=RTOL, atol=1.0 + ) + + +def test_mixed_property_preserves_worldwide_wealth_and_domestic_land_accounting(): + source_values = [ + 100_000.0, + 20_000.0, + 60_000.0, + 40_000.0, + 10_000.0, + 30_000.0, + 70_000.0, + ] + mapped = generate_was_table(_source_table([source_values])) + dataset = store_wealth_predictions( + _synthetic_dataset(pd.DataFrame({"region": ["LONDON"]})), + mapped[IMPUTE_VARIABLES], + ) + sim = _contract_simulation(dataset) + np.testing.assert_allclose( + _calculate(sim, "property_wealth", BASE_YEAR), mapped["property_wealth"] + ) + for year in YEARS: + values = {name: _calculate(sim, name, year) for name in COMPONENTS} + domestic = sum( + values[name] + for name in COMPONENTS + if name not in {"owned_land", "overseas_property_value"} + ) + worldwide = sum(values.values()) + intensity = sim.tax_benefit_system.parameters( + str(year) + ).household.wealth.land.intensity.property_wealth_by_region.LONDON + np.testing.assert_allclose( + _calculate(sim, "property_wealth", year), worldwide, rtol=RTOL + ) + np.testing.assert_allclose( + _calculate(sim, "total_wealth", year), worldwide, rtol=RTOL + ) + np.testing.assert_allclose( + _calculate(sim, "household_land_value", year), + domestic * intensity + values["owned_land"], + rtol=RTOL, + ) + + +@pytest.fixture(scope="module") +def built_property_arrays(): + try: + dataset = UKSingleYearDataset( + STORAGE_FOLDER / CURRENT_FRS_RELEASE.enhanced_dataset_file + ) + except FileNotFoundError: + pytest.skip("Enhanced FRS dataset not available") + stored = "property_wealth" in dataset.household.columns + return stored, _property_arrays(Microsimulation(dataset=dataset), YEARS) + + +def test_built_dataset_does_not_save_property_wealth(built_property_arrays): + stored, _ = built_property_arrays + assert not stored, "the enhanced FRS saves a property_wealth column" + + +@pytest.mark.parametrize("year", YEARS, ids=map(str, YEARS)) +def test_built_property_wealth_is_the_sum_of_its_components( + built_property_arrays, year +): + _, arrays = built_property_arrays + a = arrays[year] + n_bad, total_gap = _n_not_close(a["property_wealth"], a["components"]) + assert n_bad == 0, ( + f"{year}: property_wealth differs from the sum of its components for " + f"{n_bad} of {len(a['components'])} households (unweighted absolute " + f"gap £{total_gap / 1e9:,.1f}bn)" + ) + + +@pytest.mark.parametrize("year", YEARS, ids=map(str, YEARS)) +def test_built_property_wealth_at_least_main_residence(built_property_arrays, year): + """P4: property_wealth >= main_residence_value for every household.""" + _, arrays = built_property_arrays + a = arrays[year] + below = a["property_wealth"] < a["main_residence_value"] + owners = a["main_residence_value"] > 0 + assert not below.any(), ( + f"{year}: property_wealth < main_residence_value for {int(below.sum())} " + f"households ({int((below & owners).sum())} of {int(owners.sum())} owners)" + ) + + +def test_built_property_wealth_grows_with_its_components(built_property_arrays): + """Per household, property_wealth(projection) / property_wealth(base) + equals the same ratio for the sum of its components.""" + _, arrays = built_property_arrays + base, later = arrays[BASE_YEAR], arrays[PROJECTION_YEAR] + owns = base["components"] > 0 + assert owns.any() + with np.errstate(divide="ignore", invalid="ignore"): + growth = later["property_wealth"][owns] / base["property_wealth"][owns] + component_growth = later["components"][owns] / base["components"][owns] + bad = ~(np.abs(growth - component_growth) <= RTOL * component_growth) + assert not bad.any(), ( + f"property_wealth growth {BASE_YEAR}-{PROJECTION_YEAR} differs from " + f"component growth for {int(bad.sum())} of {int(owns.sum())} households " + f"with property (mean property_wealth growth " + f"{np.nanmean(growth):.4f}, mean component growth " + f"{np.mean(component_growth):.4f})" + ) diff --git a/policyengine_uk_data/tests/test_regional_property_uprating.py b/policyengine_uk_data/tests/test_regional_property_uprating.py index dd49d9e9..9a3f840f 100644 --- a/policyengine_uk_data/tests/test_regional_property_uprating.py +++ b/policyengine_uk_data/tests/test_regional_property_uprating.py @@ -1,13 +1,25 @@ """Tests for regional property value calibration.""" +import importlib.util +from pathlib import Path + import numpy as np import pandas as pd -import pytest -from policyengine_uk_data.datasets.imputations.regional_property_uprating import ( - _load_regional_house_prices, - _calibrate_property_to_hpi, +# Load the current source, including deliberate byte-restored mutations. +_REGIONAL_PATH = ( + Path(__file__).resolve().parents[1] + / "datasets" + / "imputations" + / "regional_property_uprating.py" +) +_REGIONAL_SPEC = importlib.util.spec_from_file_location( + "regional_property_module", _REGIONAL_PATH ) +regional = importlib.util.module_from_spec(_REGIONAL_SPEC) +_REGIONAL_SPEC.loader.exec_module(regional) +_load_regional_house_prices = regional._load_regional_house_prices +_calibrate_property_to_hpi = regional._calibrate_property_to_hpi def test_csv_loads_all_regions(): @@ -42,7 +54,6 @@ def test_calibration_rescales_means(): { "region": region, "main_residence_value": v, - "property_wealth": v * 1.1, "household_weight": 1.0, } for v in values @@ -53,7 +64,6 @@ def test_calibration_rescales_means(): { "region": region, "main_residence_value": 0.0, - "property_wealth": 0.0, "household_weight": 1.0, } for _ in range(10) @@ -74,12 +84,10 @@ def test_calibration_rescales_means(): def test_renters_unchanged(): """Households with zero main_residence_value should not be scaled.""" - prices = _load_regional_house_prices() df = pd.DataFrame( { "region": ["LONDON", "LONDON", "LONDON"], "main_residence_value": [500_000.0, 0.0, 0.0], - "property_wealth": [500_000.0, 0.0, 0.0], "household_weight": [1.0, 1.0, 1.0], } ) @@ -88,20 +96,32 @@ def test_renters_unchanged(): assert calibrated.iloc[2]["main_residence_value"] == 0.0 -def test_property_wealth_scales_proportionally(): - """property_wealth should scale by the same factor as main_residence_value.""" - prices = _load_regional_house_prices() +def test_only_main_residence_value_is_rescaled(): + """Other property columns are untouched and no property_wealth is created. + + policyengine-uk sums property_wealth from main_residence_value and the + other property components, so the rescaling reaches it through them. + """ df = pd.DataFrame( { "region": ["LONDON", "LONDON"], "main_residence_value": [400_000.0, 300_000.0], - "property_wealth": [500_000.0, 400_000.0], + "other_residential_property_value": [150_000.0, 0.0], + "non_residential_property_value": [0.0, 80_000.0], + "owned_land": [20_000.0, 10_000.0], + "overseas_property_value": [30_000.0, 40_000.0], + "other_property_value": [5_000.0, 15_000.0], "household_weight": [1.0, 1.0], } ) - original_ratio = df["property_wealth"].values / df["main_residence_value"].values + original = df.copy(deep=True) calibrated = _calibrate_property_to_hpi(df) - new_ratio = ( - calibrated["property_wealth"].values / calibrated["main_residence_value"].values - ) - np.testing.assert_allclose(original_ratio, new_ratio, rtol=1e-10) + assert "property_wealth" not in calibrated.columns + for column in ( + "other_residential_property_value", + "non_residential_property_value", + "owned_land", + "overseas_property_value", + "other_property_value", + ): + np.testing.assert_array_equal(calibrated[column], original[column]) diff --git a/policyengine_uk_data/tests/test_student_loan_balance.py b/policyengine_uk_data/tests/test_student_loan_balance.py index 16000777..80135dac 100644 --- a/policyengine_uk_data/tests/test_student_loan_balance.py +++ b/policyengine_uk_data/tests/test_student_loan_balance.py @@ -194,6 +194,7 @@ def calculate_dataframe(self, predictors, map_to): monkeypatch.setattr(wealth, "create_wealth_model", lambda: DummyModel()) monkeypatch.setattr(wealth, "Microsimulation", DummyMicrosimulation) + monkeypatch.setattr(wealth, "require_property_component_model", lambda: None) imputed = wealth.impute_wealth(DummyDataset()) diff --git a/policyengine_uk_data/tests/test_was_property_reconciliation.py b/policyengine_uk_data/tests/test_was_property_reconciliation.py new file mode 100644 index 00000000..8f42a54b --- /dev/null +++ b/policyengine_uk_data/tests/test_was_property_reconciliation.py @@ -0,0 +1,249 @@ +"""Aggregate-only source checks do not need licensed WAS records.""" + +import importlib.util +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings, strategies as st + +SCRIPT = Path(__file__).resolve().parents[2] / "tools/reconcile_was_property.py" +SPEC = importlib.util.spec_from_file_location("reconcile_was_property", SCRIPT) +reconciliation = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(reconciliation) + +# Deliberately independent of both the source mapping and measurement script. +SEVEN_SOURCE_FIELDS = ( + "DVhValueR8", + "DVHseValR8_sum", + "DVBltValR8_sum", + "DVBlDValR8_sum", + "DVLUKValR8_sum", + "DVLOSValR8_sum", + "DVOprValR8_sum", +) +EXPECTED_SOURCE_TO_MODEL_INPUT = { + "DVhValueR8": "main_residence_value", + "DVHseValR8_sum": "other_residential_property_value", + "DVBltValR8_sum": "other_residential_property_value", + "DVBlDValR8_sum": "non_residential_property_value", + "DVLUKValR8_sum": "owned_land", + "DVLOSValR8_sum": "overseas_property_value", + "DVOprValR8_sum": "other_property_value", +} + + +def source_frame(values, weights=None): + frame = pd.DataFrame(values, columns=SEVEN_SOURCE_FIELDS) + frame["DVPropertyR8"] = frame.sum(axis=1) + frame["R8xshhwgt"] = 1.0 if weights is None else weights + return frame + + +def test_all_categories_and_households_owning_only_omitted_categories(): + frame = source_frame(np.eye(7) * 100, weights=np.arange(1, 8)) + result = reconciliation.reconcile_property(frame) + complete = result["complete_cases"] + assert complete["source_total"] == { + "unweighted_sum": 700.0, + "weighted_sum": 2800.0, + } + assert complete["source_total_minus_original_three"] == { + "unweighted_sum": 400.0, + "weighted_sum": 2100.0, + } + assert complete["source_total_minus_seven_components"]["unweighted_sum"] == 0 + assert complete["households_owning_only_omitted_categories"] == { + "unweighted_count": 4, + "weighted_count": 21.0, + } + assert complete["stack_501_four_component_sum"] == { + "unweighted_sum": 400.0, + "weighted_sum": 1000.0, + } + assert complete["source_total_minus_stack_501_four"] == { + "unweighted_sum": 300.0, + "weighted_sum": 1800.0, + } + assert ( + complete["source_total_minus_stack_501_four"] + == complete["three_remaining_components"] + ) + assert complete["households_owning_only_remaining_categories"] == { + "unweighted_count": 3, + "weighted_count": 18.0, + } + assert complete["model_input_sum"] == complete["source_total"] + assert complete["model_inputs"]["other_residential_property_value"] == { + "source_fields": ["DVHseValR8_sum", "DVBltValR8_sum"], + "unweighted_sum": 200.0, + "weighted_sum": 500.0, + "owning_households": {"unweighted_count": 2, "weighted_count": 5.0}, + "households_owning_only_this_input": { + "unweighted_count": 2, + "weighted_count": 5.0, + }, + } + for category in complete["components"].values(): + assert category["unweighted_sum"] == 100 + assert category["households_owning_only_this_category"]["unweighted_count"] == 1 + + +def test_mapping_metadata_distinguishes_historical_and_stack_501_baselines(): + report = reconciliation.reconcile_property(source_frame(np.zeros((1, 7)))) + assert report["source_to_model_input"] == EXPECTED_SOURCE_TO_MODEL_INPUT + assert len(set(report["source_to_model_input"].values())) == 6 + historical = report["baselines"]["historical_pre_501"] + assert "Historical" in historical["description"] + assert historical["source_fields"] == [ + "DVhValueR8", + "DVHseValR8_sum", + "DVBlDValR8_sum", + ] + assert report["baselines"]["stack_501"]["source_fields"] == [ + "DVhValueR8", + "DVHseValR8_sum", + "DVBltValR8_sum", + "DVBlDValR8_sum", + ] + + +@pytest.mark.parametrize("source_field", SEVEN_SOURCE_FIELDS) +def test_each_source_singleton_reaches_exactly_its_expected_model_input(source_field): + frame = source_frame(np.zeros((1, 7)), weights=[2]) + frame[source_field] = 125 + frame["DVPropertyR8"] = 125 + complete = reconciliation.reconcile_property(frame)["complete_cases"] + assert set(complete["model_inputs"]) == set(EXPECTED_SOURCE_TO_MODEL_INPUT.values()) + for model_input, summary in complete["model_inputs"].items(): + expected = ( + 125 if model_input == EXPECTED_SOURCE_TO_MODEL_INPUT[source_field] else 0 + ) + assert summary["unweighted_sum"] == expected + assert summary["weighted_sum"] == expected * 2 + assert summary["owning_households"]["unweighted_count"] == (expected > 0) + assert complete["model_input_sum"] == complete["source_total"] + assert complete["source_total_minus_model_inputs"]["unweighted_sum"] == 0 + + +def test_buy_to_let_and_second_homes_are_grouped_and_counted_exactly_once(): + frame = source_frame([[0, 130, 70, 0, 0, 0, 0]], weights=[3]) + complete = reconciliation.reconcile_property(frame)["complete_cases"] + other_residential = complete["model_inputs"]["other_residential_property_value"] + assert other_residential["source_fields"] == ["DVHseValR8_sum", "DVBltValR8_sum"] + assert other_residential["unweighted_sum"] == 200 + assert other_residential["weighted_sum"] == 600 + assert other_residential["owning_households"]["unweighted_count"] == 1 + assert complete["model_input_sum"] == complete["source_total"] + assert complete["source_total_minus_model_inputs"]["unweighted_sum"] == 0 + assert complete["source_total_minus_stack_501_four"]["unweighted_sum"] == 0 + + +@given( + st.lists( + st.lists(st.integers(0, 1_000_000), min_size=7, max_size=7), + min_size=1, + max_size=20, + ) +) +@settings(deadline=None) +def test_complete_source_identity_reconciles_before_any_transformation(values): + result = reconciliation.reconcile_property(source_frame(values)) + complete = result["complete_cases"] + assert complete["households"] == len(values) + assert complete["source_total_minus_seven_components"]["unweighted_sum"] == 0 + assert complete["absolute_seven_component_residual"]["weighted_sum"] == 0 + assert ( + complete["source_total_minus_original_three"] + == complete["four_previously_omitted_components"] + ) + assert ( + complete["source_total_minus_stack_501_four"] + == complete["three_remaining_components"] + ) + assert complete["model_input_sum"] == complete["source_total"] + assert complete["source_total_minus_model_inputs"]["unweighted_sum"] == 0 + for model_input, summary in complete["model_inputs"].items(): + expected_source_fields = [ + source_field + for source_field, expected_input in EXPECTED_SOURCE_TO_MODEL_INPUT.items() + if expected_input == model_input + ] + expected = sum( + row[SEVEN_SOURCE_FIELDS.index(source_field)] + for row in values + for source_field in expected_source_fields + ) + assert summary["source_fields"] == expected_source_fields + assert summary["unweighted_sum"] == expected + assert summary["weighted_sum"] == expected + + +def test_missing_and_negative_codes_reported_separately_from_complete_cases(): + frame = source_frame(np.full((6, 7), 10.0), weights=[1, 2, 3, -1, np.nan, 0]) + frame.loc[1, "DVLOSValR8_sum"] = np.nan + frame.loc[2, "DVLUKValR8_sum"] = -9 + frame["DVOprValR8_sum"] = frame["DVOprValR8_sum"].astype(object) + frame.loc[3, "DVOprValR8_sum"] = "unknown" + frame.loc[4, "DVBlDValR8_sum"] = np.inf + result = reconciliation.reconcile_property(frame) + assert result["gross_value_invalid_counts"]["DVLOSValR8_sum"]["missing"] == 1 + assert result["gross_value_invalid_counts"]["DVLUKValR8_sum"]["negative"] == 1 + assert ( + result["gross_value_invalid_counts"]["DVLUKValR8_sum"]["sentinel_minus_9"] == 1 + ) + assert result["gross_value_invalid_counts"]["DVOprValR8_sum"]["non_numeric"] == 1 + assert result["gross_value_invalid_counts"]["DVBlDValR8_sum"]["non_finite"] == 1 + assert result["households_excluded_from_complete_cases"] == 4 + assert result["complete_cases"]["households"] == 2 + assert result["weight_invalid_counts"]["invalid"] == 3 + assert result["cleaned"]["source_total_minus_seven_components"] == { + "unweighted_sum": 40.0, + "weighted_sum": 50.0, + } + + +def test_known_codes_and_other_negatives_are_distinguished(): + frame = source_frame(np.ones((3, 7))) + frame["DVLOSValR8_sum"] = [-9, -8, -1] + report = reconciliation.reconcile_property(frame) + counts = report["gross_value_invalid_counts"]["DVLOSValR8_sum"] + assert counts["sentinel_minus_9"] == 1 + assert counts["sentinel_minus_8"] == 1 + assert counts["other_negative"] == 1 + assert report["complete_cases"]["households"] == 0 + assert ( + report["cleaned"]["source_total_minus_seven_components"]["unweighted_sum"] == 3 + ) + + +def test_offsets_cannot_hide_household_discrepancies(): + frame = source_frame(np.ones((2, 7))) + frame["DVPropertyR8"] += [5, -5] + result = reconciliation.reconcile_property(frame)["complete_cases"] + assert result["source_total_minus_seven_components"]["unweighted_sum"] == 0 + assert result["absolute_seven_component_residual"]["unweighted_sum"] == 10 + assert result["households_with_residual_above_tolerance"]["unweighted_count"] == 2 + + +def test_cli_outputs_aggregates_without_household_identifiers(tmp_path, capsys): + frame = source_frame(np.ones((2, 7))) + frame["household_identifier"] = ["private-synthetic-id-1", "private-synthetic-id-2"] + path = tmp_path / "synthetic.tab" + frame.rename(columns=str.lower).to_csv(path, sep="\t", index=False) + reconciliation.main(["--household-tab", str(path)]) + output = capsys.readouterr().out + report = json.loads(output) + assert report["complete_cases"]["households"] == 2 + assert "private-synthetic-id" not in output + assert "household_identifier" not in output + + +@pytest.mark.parametrize("tolerance", [-1, np.inf, np.nan]) +def test_invalid_tolerance_rejected(tolerance): + with pytest.raises(ValueError, match="Tolerance"): + reconciliation.reconcile_property( + source_frame(np.ones((1, 7))), tolerance=tolerance + ) diff --git a/tools/reconcile_was_property.py b/tools/reconcile_was_property.py new file mode 100644 index 00000000..8b5d75b5 --- /dev/null +++ b/tools/reconcile_was_property.py @@ -0,0 +1,262 @@ +"""Report aggregate Round 8 WAS property reconciliation before imputation. + +Run with a licensed household TAB file; stdout contains aggregate JSON only:: + + python tools/reconcile_was_property.py --household-tab /licensed/was_r8.tab + +No survey records, household identifiers or row examples are emitted. Compare +complete cases first. The separate cleaned report replaces invalid gross +values with zero, so its residuals can reflect missing data as well as a source +identity discrepancy. Neither report applies imputation or uprating. + +The historical pre-#501 baseline has three source terms; the stacked #501 +baseline has four because it already includes buy-to-let. Model summaries group +the seven raw terms into six inputs, combining second homes and buy-to-let once. +""" + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd + +# Independent transcription of DV_Spec_Property_R8, rows 49–59. Do not derive +# these from the imputation mapping: reconciliation must detect omissions there. +SOURCE_TOTAL = "DVPropertyR8" +SOURCE_COMPONENTS = { + "main_residence": "DVhValueR8", + "second_uk_houses": "DVHseValR8_sum", + "uk_buy_to_let": "DVBltValR8_sum", + "other_uk_buildings": "DVBlDValR8_sum", + "uk_land": "DVLUKValR8_sum", + "overseas_land_and_property": "DVLOSValR8_sum", + "other_property": "DVOprValR8_sum", +} +# Historical pre-#501 baseline, retained to compare the earlier omissions. +ORIGINAL_COMPONENTS = ( + "main_residence", + "second_uk_houses", + "other_uk_buildings", +) +STACK_501_COMPONENTS = ( + "main_residence", + "second_uk_houses", + "uk_buy_to_let", + "other_uk_buildings", +) +# Independent source-to-model transcription. Seven source terms produce six +# inputs because #501 combines second homes and buy-to-let exactly once. +SOURCE_TO_MODEL_INPUT = { + "DVhValueR8": "main_residence_value", + "DVHseValR8_sum": "other_residential_property_value", + "DVBltValR8_sum": "other_residential_property_value", + "DVBlDValR8_sum": "non_residential_property_value", + "DVLUKValR8_sum": "owned_land", + "DVLOSValR8_sum": "overseas_property_value", + "DVOprValR8_sum": "other_property_value", +} +DEFAULT_WEIGHT_COLUMN = "R8xshhwgt" + + +def _column(frame: pd.DataFrame, name: str) -> pd.Series: + matches = [column for column in frame if str(column).lower() == name.lower()] + if len(matches) != 1: + raise ValueError(f"Expected exactly one source column named {name}") + return frame[matches[0]] + + +def _numeric_values(series: pd.Series, *, weight=False): + numeric = pd.to_numeric(series, errors="coerce").astype(float) + finite = pd.Series(np.isfinite(numeric), index=series.index) + invalid = ~finite | (numeric <= 0 if weight else numeric < 0) + counts = { + "missing": int(series.isna().sum()), + "non_numeric": int((series.notna() & numeric.isna()).sum()), + "non_finite": int((numeric.notna() & ~finite).sum()), + "negative": int((numeric < 0).sum()), + "zero": int((numeric == 0).sum()), + "invalid": int(invalid.sum()), + } + if not weight: + counts.update( + sentinel_minus_9=int((numeric == -9).sum()), + sentinel_minus_8=int((numeric == -8).sum()), + other_negative=int(((numeric < 0) & ~numeric.isin([-9, -8])).sum()), + ) + return numeric.mask(invalid, 0), ~invalid, counts + + +def _summary(total, components, weights, valid_weights, tolerance): + seven_sum = components.sum(axis=1) + original_sum = components[list(ORIGINAL_COMPONENTS)].sum(axis=1) + omitted = [name for name in components if name not in ORIGINAL_COMPONENTS] + omitted_sum = components[omitted].sum(axis=1) + stack_501_sum = components[list(STACK_501_COMPONENTS)].sum(axis=1) + remaining = [name for name in components if name not in STACK_501_COMPONENTS] + remaining_sum = components[remaining].sum(axis=1) + model_inputs = pd.DataFrame( + { + model_input: components[ + [ + name + for name, source_field in SOURCE_COMPONENTS.items() + if SOURCE_TO_MODEL_INPUT[source_field] == model_input + ] + ].sum(axis=1) + for model_input in dict.fromkeys(SOURCE_TO_MODEL_INPUT.values()) + } + ) + model_input_sum = model_inputs.sum(axis=1) + residual = total - seven_sum + + def money(values): + return { + "unweighted_sum": float(values.sum()), + "weighted_sum": float( + (values[valid_weights] * weights[valid_weights]).sum() + ), + } + + def households(mask): + return { + "unweighted_count": int(mask.sum()), + "weighted_count": float(weights[mask & valid_weights].sum()), + } + + result = { + "households": len(total), + "households_with_valid_weight": int(valid_weights.sum()), + "source_total": money(total), + "original_three_component_sum": money(original_sum), + "seven_component_sum": money(seven_sum), + "source_total_minus_original_three": money(total - original_sum), + "four_previously_omitted_components": money(omitted_sum), + "stack_501_four_component_sum": money(stack_501_sum), + "source_total_minus_stack_501_four": money(total - stack_501_sum), + "three_remaining_components": money(remaining_sum), + "model_input_sum": money(model_input_sum), + "source_total_minus_model_inputs": money(total - model_input_sum), + "source_total_minus_seven_components": money(residual), + "absolute_seven_component_residual": money(residual.abs()), + "households_with_residual_above_tolerance": households( + residual.abs() > tolerance + ), + "households_owning_only_omitted_categories": households( + (original_sum == 0) & (omitted_sum > 0) + ), + "households_owning_only_remaining_categories": households( + (stack_501_sum == 0) & (remaining_sum > 0) + ), + "components": {}, + "model_inputs": {}, + } + for name, values in components.items(): + result["components"][name] = { + "source_field": SOURCE_COMPONENTS[name], + **money(values), + "owning_households": households(values > 0), + "households_owning_only_this_category": households( + (values > 0) & (seven_sum == values) + ), + } + for name, values in model_inputs.items(): + result["model_inputs"][name] = { + "source_fields": [ + source_field + for source_field, model_input in SOURCE_TO_MODEL_INPUT.items() + if model_input == name + ], + **money(values), + "owning_households": households(values > 0), + "households_owning_only_this_input": households( + (values > 0) & (model_input_sum == values) + ), + } + return result + + +def reconcile_property( + frame: pd.DataFrame, + *, + weight_column: str = DEFAULT_WEIGHT_COLUMN, + tolerance: float = 0.01, +) -> dict: + """Summarise both complete-case and zero-filled raw-source identities.""" + if not np.isfinite(tolerance) or tolerance < 0: + raise ValueError("Tolerance must be finite and non-negative") + source_fields = {"source_total": SOURCE_TOTAL, **SOURCE_COMPONENTS} + cleaned, valid, invalid_counts = {}, {}, {} + for name, source_field in source_fields.items(): + cleaned[name], valid[name], invalid_counts[source_field] = _numeric_values( + _column(frame, source_field) + ) + weights, valid_weights, weight_counts = _numeric_values( + _column(frame, weight_column), weight=True + ) + complete_cases = pd.DataFrame(valid).all(axis=1) + components = pd.DataFrame({name: cleaned[name] for name in SOURCE_COMPONENTS}) + total = cleaned["source_total"] + return { + "round": 8, + "stage": "raw source before imputation or regional HPI adjustment", + "tolerance_pounds": tolerance, + "baselines": { + "historical_pre_501": { + "description": ( + "Historical three-source baseline before #501; " + "four omitted categories include buy-to-let" + ), + "source_fields": [ + SOURCE_COMPONENTS[name] for name in ORIGINAL_COMPONENTS + ], + }, + "stack_501": { + "description": ( + "#501 residential/building aggregate covers four source terms; " + "UK land is already exported separately. #543 includes land, " + "overseas property and other real estate in the gross aggregate" + ), + "source_fields": [ + SOURCE_COMPONENTS[name] for name in STACK_501_COMPONENTS + ], + }, + }, + "source_to_model_input": SOURCE_TO_MODEL_INPUT, + "handling": { + "complete_cases": "All eight gross value fields finite and non-negative", + "cleaned": "Diagnostic only: missing, non-numeric, non-finite and all negative gross values set to zero", + "known_negative_codes": "-9 not asked/applicable and -8 don't know/refusal (documented for overseas and other property); counted separately", + "other_negative_values": "Excluded from complete cases and zeroed for the diagnostic; production rejects undocumented negative gross values", + "weighted_statistics": "Only finite, strictly positive household weights included", + }, + "gross_value_invalid_counts": invalid_counts, + "weight_column": weight_column, + "weight_invalid_counts": weight_counts, + "households_excluded_from_complete_cases": int((~complete_cases).sum()), + "complete_cases": _summary( + total[complete_cases], + components[complete_cases], + weights[complete_cases], + valid_weights[complete_cases], + tolerance, + ), + "cleaned": _summary(total, components, weights, valid_weights, tolerance), + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--household-tab", type=Path, required=True) + parser.add_argument("--weight-column", default=DEFAULT_WEIGHT_COLUMN) + parser.add_argument("--tolerance", type=float, default=0.01) + args = parser.parse_args(argv) + frame = pd.read_csv(args.household_tab, sep="\t", low_memory=False) + result = reconcile_property( + frame, weight_column=args.weight_column, tolerance=args.tolerance + ) + print(json.dumps(result, indent=2, allow_nan=False)) + + +if __name__ == "__main__": + main()