diff --git a/changelog.d/2526.fixed.md b/changelog.d/2526.fixed.md new file mode 100644 index 0000000000..6311e348d1 --- /dev/null +++ b/changelog.d/2526.fixed.md @@ -0,0 +1 @@ +Added explicit energy price-period declarations for dataset projection using the existing annual CPI assumption, while preserving flat energy amounts in unlabelled inputs. diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index 6b54e39151..579827e429 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -52,6 +52,63 @@ configured environment variable. Keep default dataset behavior explicit so calculator-style simulations, microsimulations, and API workers do not accidentally diverge. +## Energy price declarations + +Unlabelled dataset energy spending stays flat through the projection horizon. +Do not infer its price year from the dataset year, filename, URL or file format. +Legacy builds can hold FY2026-27 energy prices in a dataset labelled 2024. + +An explicit declaration opts the three stored energy amounts into their +existing annual OBR consumer-price-index assumption. For an input whose +producer receipt declares calendar-2024 prices, a caller can label a local +UK single-year H5 before constructing a simulation: + +```python +from policyengine_uk import Microsimulation +from policyengine_uk.data import UKSingleYearDataset + +dataset = UKSingleYearDataset( + file_path="verified-input.h5", + energy_price_basis={"period_type": "calendar_year", "period_value": 2024}, +) +simulation = Microsimulation(dataset=dataset) +``` + +Use `fiscal_year` with the starting year when the receipt describes a UK +financial year. Both keys are required. The year must be an integer from 1 +through 9999; malformed or conflicting caller/file declarations raise a clear +error. A caller's `None` leaves an existing file declaration intact. + +The model retains the period kind as provenance but uses its existing annual +index-year convention: CY2024 and FY2024-25 both use `I[2024]`. Projection +multiplies the supplied amount by `I[target year] / I[price year]`. This is +the existing generic CPI assumption, with no new energy-specific forecast or +precise fiscal/calendar price adjustment. A 2026 basis therefore stays unchanged +at target 2026 and can grow at target 2027. The original observation remains +unchanged; a projected child's declaration records its effective fiscal-year +price basis so later extension does not apply the original ratio twice. + +Single-year H5 files store a JSON declaration in the root attribute +`energy_price_basis`. Multi-year files store it on each `/time_period/` +group. Copy, save/load and filtering preserve the declaration. A Core Dataset +can declare the same named field or H5 attribute before UK conversion. HF/GCS +materializers retain the existing local-file routing. External multi-year +amounts and DataFrame inputs remain authoritative when loaded. + +Disabling economic assumptions resets energy amounts and their declaration as +one group. A base year that supplies every stored later energy column replaces +those amounts and their declaration. With no overlapping base energy, later +amounts and metadata stay intact. A partial overlap resets shared columns only +when both declarations match exactly; otherwise the entire later energy group +and its declaration remain authoritative. Missing columns are never created. + +Existing metadata-free private Microcosm releases stay flat under this +compatibility rule. They require an explicit caller migration based on a +verified receipt, or a producer release that exports the canonical declaration +matching that receipt. The producer must carry both period kind and year into +the H5 artifact; this consumer change does not retrofit that metadata or verify +the released private file. + ## Tests For dataset source routing, prefer stubbed tests that prove dispatch behavior diff --git a/policyengine_uk/data/dataset_schema.py b/policyengine_uk/data/dataset_schema.py index ff86478278..b1f35ce85b 100644 --- a/policyengine_uk/data/dataset_schema.py +++ b/policyengine_uk/data/dataset_schema.py @@ -8,6 +8,13 @@ from pathlib import Path import h5py +from policyengine_uk.data.energy_price_basis import ( + EnergyPriceBasis, + merge_energy_price_basis, + parse_energy_price_basis, + read_energy_price_basis, + write_energy_price_basis, +) class UKSingleYearDataset: @@ -57,7 +64,9 @@ def __init__( benunit: pd.DataFrame = None, household: pd.DataFrame = None, fiscal_year: int = None, + energy_price_basis: dict[str, str | int] | EnergyPriceBasis | None = None, ): + basis = parse_energy_price_basis(energy_price_basis) file_path = str(file_path) if file_path else None if file_path is not None: if fiscal_year is not None: @@ -76,6 +85,7 @@ def __init__( self._benunit = f["benunit"] self._household = f["household"] self.time_period = str(f["time_period"].iloc[0]) + basis = merge_energy_price_basis(basis, read_energy_price_basis(file_path)) else: if person is None or benunit is None or household is None: raise ValueError( @@ -97,6 +107,7 @@ def __init__( self._household = household self.time_period = str(fiscal_year) + self._energy_price_basis = basis self.data_format = "arrays" # tables tuple points at the raw (int16) DataFrames used by the # simulation engine internals - do not decode these. @@ -106,6 +117,15 @@ def __init__( # dataset is loaded and enum columns have been converted to int16. self._enum_columns: dict = {} + @property + def energy_price_basis(self) -> dict[str, str | int] | None: + """A defensive copy of the stored amounts' declared price period.""" + return self._energy_price_basis.as_dict() if self._energy_price_basis else None + + @energy_price_basis.setter + def energy_price_basis(self, value: object) -> None: + self._energy_price_basis = parse_energy_price_basis(value) + def _decode_enum_df(self, df: pd.DataFrame) -> pd.DataFrame: """Return a copy of *df* with int16 enum columns decoded to strings.""" if not self._enum_columns: @@ -156,6 +176,7 @@ def save(self, file_path: str): f.put("benunit", self.benunit, format="table", data_columns=True) f.put("household", self.household, format="table", data_columns=True) f.put("time_period", pd.Series([self.time_period]), format="table") + write_energy_price_basis(file_path, self._energy_price_basis) def load(self): data = {} @@ -171,6 +192,7 @@ def copy(self): benunit=self.benunit.copy(), household=self.household.copy(), fiscal_year=self.time_period, + energy_price_basis=self._energy_price_basis, ) def validate(self): @@ -181,7 +203,11 @@ def validate(self): raise ValueError(f"Column '{col}' contains NaN values.") @staticmethod - def from_simulation(simulation: "Microsimulation", fiscal_year: int = 2025): + def from_simulation( + simulation: "Microsimulation", + fiscal_year: int = 2025, + energy_price_basis: dict[str, str | int] | EnergyPriceBasis | None = None, + ): entity_dfs = {} for entity in ["person", "benunit", "household"]: @@ -203,6 +229,7 @@ def from_simulation(simulation: "Microsimulation", fiscal_year: int = 2025): benunit=entity_dfs["benunit"], household=entity_dfs["household"], fiscal_year=fiscal_year, + energy_price_basis=energy_price_basis, ) @@ -212,6 +239,7 @@ def __init__( file_path: str = None, datasets: list[UKSingleYearDataset] | None = None, ): + file_path = str(file_path) if file_path else None if datasets is not None: self.datasets = {} for dataset in datasets: @@ -238,6 +266,10 @@ def __init__( household=household_df, fiscal_year=fiscal_year, ) + for year, dataset in self.datasets.items(): + dataset.energy_price_basis = read_energy_price_basis( + file_path, f"/time_period/{year}" + ) self.data_format = "time_period_arrays" self.time_period = list(sorted(self.datasets.keys()))[0] @@ -283,6 +315,10 @@ def save(self, file_path: str): format="table", data_columns=True, ) + for year, dataset in self.datasets.items(): + write_energy_price_basis( + file_path, dataset.energy_price_basis, f"/time_period/{year}" + ) def copy(self): new_datasets = {year: dataset.copy() for year, dataset in self.datasets.items()} diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index a59ebd9920..d85822a051 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -102,8 +102,8 @@ # Before this horizon was extended, Core uprated unmapped inputs from 2030. # Preserve that behavior in the newly materialized years without changing # projections for earlier years. Correcting historical mapping omissions is -# a separate change. Energy values already have FY26/27 prices (#1868), as -# documented in uprating_indices.yaml, and must remain flat in the dataset. +# a separate change. Energy has its own declaration-aware projection below, +# so it must not also grow through the generic variable-index fallback. LEGACY_PROJECTION_END_YEAR = 2030 DATASET_UPRATING_EXCLUSIONS = frozenset( { @@ -137,6 +137,7 @@ def _unmapped_uprating_indices(uprating): SOCIAL_RENT_GROWTH = "gov.economic_assumptions.yoy_growth.obr.social_rent" PRIVATE_RENT_GROWTH = "gov.economic_assumptions.yoy_growth.ons.private_rental_prices" RESIDENTIAL_PROPERTY_INDEX = "gov.economic_assumptions.indices.obr.house_prices" +ENERGY_PRICE_INDEX = "gov.economic_assumptions.indices.obr.consumer_price_index" RESIDENTIAL_PROPERTY_INPUTS = ( "main_residence_value", "other_residential_property_value", @@ -154,7 +155,7 @@ def uprating_input_paths() -> list: ``apply_single_year_uprating`` and the functions it calls read these, and no other parameters: the growth rates in ``uprating_indices.yaml``, the council tax and rent series, the cumulative residential property index, - the indices of the input variables that + the conditional energy CPI index, the indices of the input variables that file leaves out (after ``LEGACY_PROJECTION_END_YEAR``) and the capital gains projection's metadata (``CGT_INDEX_PATH``). A path may name a node (the regional private rent series), which covers its parameters. @@ -171,6 +172,7 @@ def uprating_input_paths() -> list: PRIVATE_RENT_GROWTH, CGT_INDEX_PATH, RESIDENTIAL_PROPERTY_INDEX, + ENERGY_PRICE_INDEX, } | set(_unmapped_uprating_indices(uprating).values()) ) @@ -270,6 +272,9 @@ def apply_uprating( uprate_residential_property( current_year, dataset.datasets[first_year], tax_benefit_system_parameters ) + uprate_energy( + current_year, dataset.datasets[first_year], tax_benefit_system_parameters + ) return dataset @@ -320,6 +325,8 @@ def apply_single_year_uprating( # Next, apply custom uprating. + current_year = uprate_energy(current_year, previous_year, parameters) + # apply_uprating replaces these chained stocks from the original observation. current_year = uprate_residential_property(current_year, previous_year, parameters) @@ -370,6 +377,42 @@ def apply_single_year_uprating( return current_year +def uprate_energy( + current_year: UKSingleYearDataset, + observed_year: UKSingleYearDataset, + parameters: ParameterNode, +) -> UKSingleYearDataset: + """Project declared energy prices with the existing annual CPI assumption.""" + basis = observed_year.energy_price_basis + variables = [ + name + for name in DATASET_UPRATING_EXCLUSIONS + if name in observed_year.household.columns + and name in current_year.household.columns + ] + ratio = 1 + projected = False + if basis is not None and variables: + index = parameters.get_child(ENERGY_PRICE_INDEX) + source_index = _uprating_index_value( + index, f"{basis['period_value']:04d}-01-01" + ) + target_index = _uprating_index_value(index, f"{current_year.time_period}-01-01") + projected = source_index not in (None, 0) and target_index is not None + if projected: + ratio = target_index / source_index + for variable in variables: + current_year.household[variable] = ( + observed_year.household[variable].to_numpy() * ratio + ) + current_year.energy_price_basis = ( + {"period_type": "fiscal_year", "period_value": int(current_year.time_period)} + if projected + else basis + ) + return current_year + + def uprate_residential_property( current_year: UKSingleYearDataset, previous_year: UKSingleYearDataset, @@ -666,19 +709,44 @@ def reset_growthfactor_uprating( growthfactor_uprated_variables.update({"council_tax", "rent"}) growthfactor_uprated_variables.update(COUNTRY_UPRATED_INPUTS) growthfactor_uprated_variables.update(_unmapped_uprating_indices(uprating)) + growthfactor_uprated_variables.difference_update(DATASET_UPRATING_EXCLUSIONS) first_year = min(dataset.datasets.keys()) base_year_dataset = dataset.datasets[first_year] + base_energy = DATASET_UPRATING_EXCLUSIONS.intersection( + base_year_dataset.household.columns + ) for year in dataset.datasets: if year == first_year: continue current_year_dataset = dataset.datasets[year] + current_energy = DATASET_UPRATING_EXCLUSIONS.intersection( + current_year_dataset.household.columns + ) + shared_energy = current_energy.intersection(base_energy) + replace_energy_group = bool(current_energy) and current_energy <= base_energy + reset_energy = replace_energy_group or ( + current_year_dataset.energy_price_basis + == base_year_dataset.energy_price_basis + ) + + # One declaration describes every stored energy amount. Keep the + # later group intact if a partial reset would mix different bases. for table_name in current_year_dataset.table_names: base_table = getattr(base_year_dataset, table_name) current_table = getattr(current_year_dataset, table_name) for variable in growthfactor_uprated_variables: if variable in current_table.columns and variable in base_table.columns: current_table[variable] = base_table[variable].values + if reset_energy: + for variable in shared_energy: + current_year_dataset.household[variable] = base_year_dataset.household[ + variable + ].values + if replace_energy_group: + current_year_dataset.energy_price_basis = ( + base_year_dataset.energy_price_basis + ) return dataset diff --git a/policyengine_uk/data/energy_price_basis.py b/policyengine_uk/data/energy_price_basis.py new file mode 100644 index 0000000000..bd9d241948 --- /dev/null +++ b/policyengine_uk/data/energy_price_basis.py @@ -0,0 +1,110 @@ +"""Declared price periods for stored annual energy spending.""" + +from collections.abc import Mapping +from dataclasses import dataclass +import inspect +import json +from pathlib import Path + +import h5py + + +ENERGY_PRICE_BASIS_ATTRIBUTE = "energy_price_basis" + + +@dataclass(frozen=True) +class EnergyPriceBasis: + """The declared annual price period of supplied energy amounts.""" + + period_type: str + period_value: int + + def __post_init__(self) -> None: + if not isinstance(self.period_type, str) or self.period_type not in ( + "calendar_year", + "fiscal_year", + ): + raise ValueError( + "energy_price_basis period_type must be calendar_year or fiscal_year" + ) + if type(self.period_value) is not int or not 1 <= self.period_value <= 9999: + raise ValueError( + "energy_price_basis period_value must be an integer year from 1 to 9999" + ) + + def as_dict(self) -> dict[str, str | int]: + return {"period_type": self.period_type, "period_value": self.period_value} + + +def parse_energy_price_basis(value: object) -> EnergyPriceBasis | None: + """Validate an explicit declaration without inferring a missing period.""" + if value is None or isinstance(value, EnergyPriceBasis): + return value + if not isinstance(value, Mapping) or set(value) != {"period_type", "period_value"}: + raise ValueError( + "energy_price_basis must contain only period_type and period_value" + ) + return EnergyPriceBasis(value["period_type"], value["period_value"]) + + +def merge_energy_price_basis( + declared: object, stored: object +) -> EnergyPriceBasis | None: + """Accept matching caller/file declarations and reject a conflict.""" + declared = parse_energy_price_basis(declared) + stored = parse_energy_price_basis(stored) + if declared is not None and stored is not None and declared != stored: + raise ValueError("Conflicting energy_price_basis declarations") + return declared if declared is not None else stored + + +def read_energy_price_basis( + file_path: str | Path, group: str = "/" +) -> EnergyPriceBasis | None: + """Read a declaration without modifying the source H5 file.""" + with h5py.File(file_path, "r") as file: + if group not in file: + return None + value = file[group].attrs.get(ENERGY_PRICE_BASIS_ATTRIBUTE) + if value is None: + return None + try: + if isinstance(value, bytes): + value = value.decode("utf-8") + if not isinstance(value, str): + raise ValueError("expected a JSON object") + value = json.loads(value) + except (UnicodeError, TypeError, ValueError) as error: + raise ValueError("Invalid energy_price_basis JSON declaration") from error + return parse_energy_price_basis(value) + + +def write_energy_price_basis( + file_path: str | Path, value: object, group: str = "/" +) -> None: + """Write only the energy declaration, preserving unrelated H5 attributes.""" + basis = parse_energy_price_basis(value) + with h5py.File(file_path, "r+") as file: + attributes = file[group].attrs + if basis is None: + if ENERGY_PRICE_BASIS_ATTRIBUTE in attributes: + del attributes[ENERGY_PRICE_BASIS_ATTRIBUTE] + else: + attributes[ENERGY_PRICE_BASIS_ATTRIBUTE] = json.dumps( + basis.as_dict(), sort_keys=True + ) + + +def core_dataset_energy_price_basis(dataset: object) -> EnergyPriceBasis | None: + """Read explicit Core object/file provenance before schema conversion.""" + # Core's __getattr__ reads HDF keys, so inspect only declared attributes. + declared = inspect.getattr_static(dataset, ENERGY_PRICE_BASIS_ATTRIBUTE, None) + file_path = inspect.getattr_static(dataset, "file_path", None) + stored = None + if ( + isinstance(file_path, (str, Path)) + and Path(file_path).suffix == ".h5" + and Path(file_path).is_file() + ): + stored = read_energy_price_basis(file_path) + return merge_energy_price_basis(declared, stored) diff --git a/policyengine_uk/data/filter_dataset.py b/policyengine_uk/data/filter_dataset.py index 3b94884842..8515c03688 100644 --- a/policyengine_uk/data/filter_dataset.py +++ b/policyengine_uk/data/filter_dataset.py @@ -69,4 +69,5 @@ def filter_dataset( household=new_dataset.household, benunit=new_dataset.benunit, fiscal_year=year, + energy_price_basis=new_dataset.energy_price_basis, ) diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index e82333b1a6..6bc7e8ea33 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -1,26 +1,12 @@ -# Energy variables are deliberately ABSENT from every index below. +# Energy variables use the declaration-aware projector in economic_assumptions.py, +# rather than an unconditional mapping here. Unlabelled dataset inputs remain +# flat: legacy builds can already carry FY2026-27 prices despite a 2024 data year. +# An explicit energy_price_basis opts into the variables' existing annual OBR-CPI +# assumption from the declared price year. This does not supply an energy-specific +# forecast or automatically label existing private Microcosm artifacts. # -# SCOPE: this file governs DATASET PROJECTION only — the uprating applied to -# a dataset's stored values when it is projected to later years. It does not -# govern the class-level `uprating` attribute on the Variable, which -# policyengine-core still uses to carry a direct situation input forward from -# an earlier period. That path is unchanged and out of scope here: a 2024 -# value a user supplies directly may be their own contemporaneous bill rather -# than this data build's artifact, so it has no claim on the build's price -# basis (#1868 review A1). -# -# policyengine-uk-data calibrates domestic_energy_consumption, -# electricity_consumption and gas_consumption to NEED mean kWh by income band -# and converts to pounds at Ofgem Q2 2026 unit rates, so the stored values -# already carry FY26/27 price levels. Its imputations/consumption.py says so -# outright: "the stored £/yr values represent FY26/27 price levels. Energy -# variables are not CPI-uprated, so this avoids the need for price-level -# adjustment at simulation time." -# -# Projecting them from the dataset's DATA year (2024) would therefore -# re-apply 2024-2026 price changes the build has already priced in (#1867). -# Other consumption variables come from the LCFS at data-year prices and do -# need uprating, which is why only these three are excluded. +# This map governs dataset projection. Core's class-level variable uprating still +# carries direct situation inputs forward independently of dataset declarations. # # Northern Ireland domestic rates grow with the combined regional and district # poundage (#1990). Households outside Northern Ireland have none, so the index diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 9cbaae3c9f..4aa8b39b12 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -111,9 +111,26 @@ projection through 2030 retains its previous handling; correcting those earlier mapping omissions is a separate change. `domestic_energy_consumption`, `electricity_consumption` and `gas_consumption` -are deliberately excluded from this fallback as well as the annual growth -mapping. Their stored amounts already use FY2026-27 unit prices, so dataset -projection holds them flat, as documented in `uprating_indices.yaml`. +use a separate declaration-aware projector. Unlabelled datasets retain their +stored amounts through the projection horizon, preserving legacy inputs that +already use FY2026-27 prices. A dataset's explicit `energy_price_basis` opts +into the variables' existing cumulative OBR-CPI index from its declared price +year. They remain outside both the unconditional growth mapping and fallback, +so no second index grows them after 2030. + +The declaration retains `calendar_year` or `fiscal_year` and its integer year. +The model's annual convention uses the same index level for CY2024 and +FY2024-25: `I[2024]`. It does not provide a precise fiscal/calendar price +alignment or an energy-specific forecast. The projector applies +`I[target year] / I[declared price year]` to the supplied observation. +Consequently a 2026 basis gives a ratio of one in 2026 and can grow in 2027; +there is no special 2026 rule. Each projected child records its effective +fiscal-year basis, while the source observation keeps its original declaration. + +See [dataset sources](../../../../docs/engineering/skills/dataset-sources.md) +for the declaration and migration contract. Existing metadata-free private +Microcosm files stay flat until a producer or caller supplies a verified +declaration; this consumer change does not label released artifacts. The additional indices have their own source limits: diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 3b1236e053..dc0a2b273a 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -37,6 +37,7 @@ uprating_input_fingerprint, ) from policyengine_uk.data.dataset_sources import materialize_gcs_dataset_url +from policyengine_uk.data.energy_price_basis import core_dataset_energy_price_basis from policyengine_uk.data.input_dataframe import ( dataset_from_input_dataframe, with_identifiers_in_each_year, @@ -930,9 +931,9 @@ def build_from_dataset_source( cache_key = self._dataset_cache_key(dataset_source) if cache_key is not None and cache_key in _url_dataset_cache: multi_year_dataset = _url_dataset_cache[cache_key] + self.dataset = multi_year_dataset self.build_from_multi_year_dataset(multi_year_dataset) self._restore_cgt_extension_provenance(multi_year_dataset) - self.dataset = multi_year_dataset return dataset_file = materialize_gcs_dataset_url(dataset_source) self.build_from_file(dataset_file, cache_key=dataset_source) @@ -971,9 +972,9 @@ def build_from_file( if key is not None: _url_dataset_cache[key] = multi_year_dataset + self.dataset = multi_year_dataset self.build_from_multi_year_dataset(multi_year_dataset) self._restore_cgt_extension_provenance(multi_year_dataset) - self.dataset = multi_year_dataset def build_from_url(self, url: str) -> None: """Build simulation from a HuggingFace dataset URL. @@ -992,9 +993,9 @@ def build_from_url(self, url: str) -> None: cache_key = self._dataset_cache_key(url) if cache_key is not None and cache_key in _url_dataset_cache: multi_year_dataset = _url_dataset_cache[cache_key] + self.dataset = multi_year_dataset self.build_from_multi_year_dataset(multi_year_dataset) self._restore_cgt_extension_provenance(multi_year_dataset) - self.dataset = multi_year_dataset return # Parse HuggingFace URL components @@ -1077,6 +1078,7 @@ def build_from_dataset(self, dataset: Dataset) -> None: Args: dataset: PolicyEngine Dataset object containing simulation data """ + energy_price_basis = core_dataset_energy_price_basis(dataset) data: Dict[str, Dict[str, Union[float, int, str]]] = dataset.load_dataset() first_variable = data[list(data.keys())[0]] @@ -1131,13 +1133,13 @@ def get_first_array(variable_name: str) -> np.ndarray: self.dataset = dataset dataset = UKSingleYearDataset.from_simulation( - self, fiscal_year=first_time_period + self, fiscal_year=first_time_period, energy_price_basis=energy_price_basis ) multi_year_dataset = self._extend_dataset_with_cgt_provenance(dataset) + self.dataset = multi_year_dataset self.build_from_multi_year_dataset(multi_year_dataset) self._restore_cgt_extension_provenance(multi_year_dataset) - self.dataset = multi_year_dataset def build_from_single_year_dataset(self, dataset: UKSingleYearDataset) -> None: """Build simulation from a single-year UK dataset. @@ -1148,9 +1150,9 @@ def build_from_single_year_dataset(self, dataset: UKSingleYearDataset) -> None: dataset = self._extend_dataset_with_cgt_provenance(dataset) _pre_encode_enum_columns(dataset, self.tax_benefit_system) + self.dataset = dataset self.build_from_multi_year_dataset(dataset) self._restore_cgt_extension_provenance(dataset) - self.dataset = dataset def build_from_multi_year_dataset(self, dataset: UKMultiYearDataset) -> None: """Build simulation from a multi-year UK dataset. @@ -1195,6 +1197,8 @@ def build_from_multi_year_dataset(self, dataset: UKMultiYearDataset) -> None: continue self.set_input(variable, year, table[variable]) + # Retain the dataset actually loaded, including the copied base amounts + # and price declaration when economic assumptions were disabled. self.dataset = dataset if self.disable_economic_assumptions: # The loader changed these future amounts even for an external diff --git a/policyengine_uk/tests/code_health/test_energy_price_level.py b/policyengine_uk/tests/code_health/test_energy_price_level.py index 3ff5a714b1..0c9dae1331 100644 --- a/policyengine_uk/tests/code_health/test_energy_price_level.py +++ b/policyengine_uk/tests/code_health/test_energy_price_level.py @@ -1,25 +1,13 @@ -"""Energy variables must not be uprated by DATASET PROJECTION. - -policyengine-uk-data calibrates the energy variables to NEED mean kWh and -converts to pounds at Ofgem Q2 2026 unit rates, so the stored values already -carry FY26/27 price levels. Its imputations/consumption.py states that they -"are not CPI-uprated, so this avoids the need for price-level adjustment at -simulation time". - -Projecting them from the dataset's data year re-applies price changes the -build has already applied. #1859 reported the resulting inconsistency and -#1860 resolved it the wrong way round — by uprating electricity and gas to -match domestic_energy_consumption, rather than by removing all three (#1867). - -SCOPE. This guard covers uprating_indices.yaml, which governs dataset -projection. It deliberately does NOT cover the class-level `uprating` -attribute, which policyengine-core still uses to carry a direct situation -input forward from an earlier period. That path is unchanged: a value a user -supplies directly for 2024 may be their own contemporaneous bill rather than -this build's artifact, so it has no claim on the build's price basis. Worth -noting because #1860 and #1862 described that attribute as dead metadata — -it is inert for dataset projection, but core does read it for direct inputs -(#1868 review A1). +"""Energy inputs must stay outside unconditional dataset uprating. + +Unlabelled files retain flat energy spending: their data year does not +establish their price year. An explicit energy_price_basis instead opts into +the conditional annual CPI projector (#2526). A YAML entry would grow the +unlabelled files and could grow a declared observation twice. + +This guard covers the unconditional YAML map. Core still uses the variables' +class-level uprating for direct situation inputs; that separate behavior is +preserved and tested in test_energy_dataset_projection.py. """ from pathlib import Path @@ -28,7 +16,7 @@ UPRATING_INDICES = Path(__file__).parents[2] / "data" / "uprating_indices.yaml" -# Calibrated to NEED at Ofgem Q2 2026 rates, so already at FY26/27 prices. +# Projection is conditional on the observation's explicit price declaration. ENERGY_VARIABLES_AT_BUILD_PRICE_LEVEL = { "domestic_energy_consumption", "electricity_consumption", @@ -45,7 +33,6 @@ def test_energy_variables_are_not_projected_by_any_index(): if variable in ENERGY_VARIABLES_AT_BUILD_PRICE_LEVEL ) assert not offenders, ( - "these variables are stored at the data build's own FY26/27 price " - f"level and must not be projected by an index: {offenders}. " - "See #1867." + "these variables must use the conditional energy_price_basis " + f"projector, not an unconditional YAML index: {offenders}. See #2526." ) diff --git a/policyengine_uk/tests/code_health/test_uprating_declarations.py b/policyengine_uk/tests/code_health/test_uprating_declarations.py index d5ba4fb1f0..0317076069 100644 --- a/policyengine_uk/tests/code_health/test_uprating_declarations.py +++ b/policyengine_uk/tests/code_health/test_uprating_declarations.py @@ -1,11 +1,11 @@ """Every variable declaring `uprating` should appear in uprating_indices.yaml. -Through 2030, `uprating_indices.yaml` is the only mechanism that applies -uprating, via `apply_single_year_uprating` and `reset_growthfactor_uprating` -in `data/economic_assumptions.py`. The class-level `uprating = "..."` -attribute on a Variable is read only by the fallback in -`_unmapped_uprating_indices`, which grows unlisted columns after -LEGACY_PROJECTION_END_YEAR (2030). +Through 2030, ordinary dataset uprating uses `uprating_indices.yaml` via +`apply_single_year_uprating` in `data/economic_assumptions.py`. Explicit +exceptions below use custom projectors, including declaration-aware energy +projection. The class-level `uprating = "..."` attribute also supports the +fallback in `_unmapped_uprating_indices`, which grows other unlisted columns +after LEGACY_PROJECTION_END_YEAR (2030). A variable can therefore declare an index and silently stay flat from the dataset year to 2030. That is what #1859 found for electricity and gas @@ -32,9 +32,9 @@ # Reported counterpart of employee_pension_contributions, which IS # uprated; the reported variable feeds imputation rather than results. "employee_pension_contributions_reported", - # policyengine-uk-data stores energy spend already priced at the dataset's - # target price level (Ofgem Q2 2026 unit rates), so CPI-uprating from the - # data year re-applies price changes the build has priced in (#1868). + # The custom energy projector uses an explicit energy_price_basis, while + # unlabelled observations stay flat (#2526). Unconditional YAML growth + # would break that compatibility and could project declared values twice. "domestic_energy_consumption", "electricity_consumption", "gas_consumption", diff --git a/policyengine_uk/tests/test_energy_dataset_projection.py b/policyengine_uk/tests/test_energy_dataset_projection.py new file mode 100644 index 0000000000..141af6ca72 --- /dev/null +++ b/policyengine_uk/tests/test_energy_dataset_projection.py @@ -0,0 +1,465 @@ +"""Dataset energy prices move only when their price basis is declared. + +The controlled CPI levels are test data for the agreed annual projection +contract, not forecasts or an energy-specific price index. +""" + +import json + +import h5py +import numpy as np +import pandas as pd +import pytest +from policyengine_core.data import Dataset + +import policyengine_uk.simulation as simulation_module +from policyengine_uk import CountryTaxBenefitSystem, Simulation +from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset +from policyengine_uk.data.economic_assumptions import ( + apply_uprating, + extend_single_year_dataset, + reset_growthfactor_uprating, + uprating_input_fingerprint, + uprating_input_paths, +) +from policyengine_uk.scenarios import no_economic_assumptions +from policyengine_uk.utils.scenario import Scenario + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +ENERGY = ( + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", +) +CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" +LEVELS = dict(enumerate([100, 110, 125, 150, 160, 170, 180, 190, 200], 2024)) +DECLARATION = {"period_type": "calendar_year", "period_value": 2024} + + +def energy_dataset(basis=None, year=2024): + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + "age": [40, 50], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1, 2], + "region": ["LONDON", "WALES"], + "tenure_type": ["OWNED_OUTRIGHT", "OWNED_OUTRIGHT"], + "council_tax": [0.0, 0.0], + "rent": [0.0, 0.0], + "household_weight": [1.0, 2.0], + **dict.fromkeys(ENERGY, [100.0, 0.0]), + } + ), + fiscal_year=year, + energy_price_basis=basis, + ) + + +@pytest.fixture +def parameters(): + parameters = CountryTaxBenefitSystem().parameters.clone() + index = parameters.get_child(CPI) + # In particular, 2026 / 2024 = 125 / 100 and 2027 / 2026 = 150 / 125. + for year, value in LEVELS.items(): + index.update(period=str(year), value=value) + return parameters + + +def energy_file(tmp_path, declaration=None): + path = tmp_path / "renamed-input.h5" + energy_dataset().save(path) + if declaration is not None: + with h5py.File(path, "a") as file: + file.attrs["energy_price_basis"] = json.dumps(declaration) + return path + + +@pytest.mark.parametrize("period_type", ["calendar_year", "fiscal_year"]) +def test_declared_2024_prices_reach_2026_without_changing_the_source( + tmp_path, parameters, period_type +): + declaration = {"period_type": period_type, "period_value": 2024} + source = UKSingleYearDataset(file_path=energy_file(tmp_path, declaration)) + projected = extend_single_year_dataset(source, parameters, end_year=2026) + for variable in ENERGY: + np.testing.assert_allclose(projected[2026].household[variable], [125.0, 0.0]) + np.testing.assert_array_equal(source.household[variable], [100.0, 0.0]) + assert source.energy_price_basis == declaration + + +def test_unlabelled_energy_stays_flat_through_the_dataset_horizon(parameters): + source = energy_dataset() + projected = extend_single_year_dataset(source, parameters, end_year=2032) + for year in [2026, 2027, 2031, 2032]: + for variable in ENERGY: + np.testing.assert_array_equal(projected[year].household[variable], [100, 0]) + + +def test_2026_price_basis_is_identity_in_2026_and_grows_afterwards( + tmp_path, parameters +): + source = UKSingleYearDataset( + file_path=energy_file( + tmp_path, {"period_type": "fiscal_year", "period_value": 2026} + ) + ) + projected = extend_single_year_dataset(source, parameters, end_year=2027) + for variable in ENERGY: + np.testing.assert_array_equal(projected[2026].household[variable], [100, 0]) + np.testing.assert_allclose(projected[2027].household[variable], [120, 0]) + + +@pytest.mark.parametrize("missing", [ENERGY[:1], ENERGY[1:], ENERGY]) +def test_partial_or_absent_energy_columns_do_not_create_inputs(parameters, missing): + source = energy_dataset(DECLARATION) + source.household = source.household.drop(columns=list(missing)) + unlabelled = source.copy() + unlabelled.energy_price_basis = None + control = extend_single_year_dataset(unlabelled, parameters, end_year=2032) + projected = extend_single_year_dataset(source, parameters, end_year=2032) + for year in (2026, 2032): + assert not set(missing) & set(projected[year].household.columns) + for variable in set(ENERGY) - set(missing): + np.testing.assert_allclose( + projected[year].household[variable], [LEVELS[year], 0] + ) + np.testing.assert_array_equal( + projected[year].household.household_weight, + control[year].household.household_weight, + ) + + +def test_energy_declaration_does_not_change_other_consumption_uprating(parameters): + food = "food_and_non_alcoholic_beverages_consumption" + growth = parameters.get_child( + "gov.economic_assumptions.yoy_growth.obr.consumer_price_index" + ) + growth.update(period="2025", value=0.1) + growth.update(period="2026", value=0.2) + unlabelled = energy_dataset() + unlabelled.household[food] = [50.0, 0.0] + labelled = unlabelled.copy() + labelled.energy_price_basis = DECLARATION + plain = extend_single_year_dataset(unlabelled, parameters, end_year=2026) + declared = extend_single_year_dataset(labelled, parameters, end_year=2026) + for projected in (plain, declared): + np.testing.assert_allclose(projected[2026].household[food], [66.0, 0.0]) + for variable in ENERGY: + np.testing.assert_allclose(plain[2026].household[variable], [100.0, 0.0]) + np.testing.assert_allclose(declared[2026].household[variable], [125.0, 0.0]) + + +def test_reapplying_uprating_recomputes_from_the_original_observation(parameters): + source = energy_dataset(DECLARATION) + once = extend_single_year_dataset(source, parameters, end_year=2032) + twice = apply_uprating(once, parameters) + for year in once.years: + for variable in ENERGY: + np.testing.assert_array_equal( + twice[year].household[variable], once[year].household[variable] + ) + assert twice[year].energy_price_basis == once[year].energy_price_basis + assert source.energy_price_basis == DECLARATION + assert once[2026].energy_price_basis == { + "period_type": "fiscal_year", + "period_value": 2026, + } + + +def test_serialized_projected_child_extends_without_double_projection( + tmp_path, parameters +): + source = energy_dataset(DECLARATION) + one_shot = extend_single_year_dataset(source, parameters, end_year=2027) + child = one_shot[2026].copy() + path = tmp_path / "child-with-an-unrelated-filename.h5" + child.save(path) + loaded = UKSingleYearDataset(file_path=path) + assert loaded.energy_price_basis == { + "period_type": "fiscal_year", + "period_value": 2026, + } + extended = extend_single_year_dataset(loaded, parameters, end_year=2027) + for variable in ENERGY: + np.testing.assert_allclose(extended[2027].household[variable], [150.0, 0.0]) + np.testing.assert_allclose( + extended[2027].household[variable], one_shot[2027].household[variable] + ) + np.testing.assert_array_equal(source.household[variable], [100.0, 0.0]) + assert source.energy_price_basis == DECLARATION + + +def test_reset_restores_original_amounts_and_original_kind(parameters): + projected = extend_single_year_dataset( + energy_dataset(DECLARATION), parameters, end_year=2027 + ) + frozen = reset_growthfactor_uprating(projected.copy()) + for year in frozen.years: + assert frozen[year].energy_price_basis == DECLARATION + for variable in ENERGY: + np.testing.assert_array_equal(frozen[year].household[variable], [100, 0]) + for variable in ENERGY: + np.testing.assert_allclose(projected[2027].household[variable], [150, 0]) + + +def controlled_changes(target=125): + return { + CPI: { + f"year:{year}-01-01:1": target if year == 2026 else value + for year, value in LEVELS.items() + } + } + + +def before_load(target=125): + return Scenario( + parameter_changes=controlled_changes(target), applied_before_data_load=True + ) + + +def assert_energy(simulation, year, expected): + for variable in ENERGY: + np.testing.assert_allclose(simulation.calculate(variable, year), [expected, 0]) + + +def core_arrays(source): + system = CountryTaxBenefitSystem() + arrays = {} + for table in source.tables: + for variable in table.columns: + values = table[variable].to_numpy() + possible_values = getattr( + system.variables[variable], "possible_values", None + ) + if possible_values is not None: + values = np.asarray(possible_values.encode(values)) + arrays[variable] = {source.time_period: values} + return arrays + + +class InMemoryCore(Dataset): + """Synthetic Core input with the public load_dataset interface, no file.""" + + def __init__(self, arrays): + self.arrays = arrays + + def load_dataset(self): + return self.arrays + + +def core_file(source, path, basis=None): + with h5py.File(path, "w") as file: + for variable, periods in core_arrays(source).items(): + for period, values in periods.items(): + file.create_dataset(f"{variable}/{period}", data=values) + if basis is not None: + file.attrs["energy_price_basis"] = json.dumps(basis) + return path + + +@pytest.mark.parametrize( + "source_kind", ["single", "uk_h5", "core_h5", "core_instance", "core_class"] +) +def test_declared_inputs_reach_the_same_projection(source_kind, tmp_path, monkeypatch): + original = energy_dataset(DECLARATION) + source = original + if source_kind == "uk_h5": + source = tmp_path / "producer-2026-does-not-define-price-year.h5" + original.save(source) + elif source_kind == "core_h5": + source = core_file(original, tmp_path / "renamed-core.h5", DECLARATION) + elif source_kind in ("core_instance", "core_class"): + source = InMemoryCore(core_arrays(original)) + if source_kind == "core_instance": + source.energy_price_basis = dict(DECLARATION) + else: + monkeypatch.setattr( + InMemoryCore, "energy_price_basis", dict(DECLARATION), raising=False + ) + simulation = Simulation(dataset=source, scenario=before_load()) + assert_energy(simulation, 2026, 125) + assert original.energy_price_basis == DECLARATION + for variable in ENERGY: + np.testing.assert_array_equal(original.household[variable], [100, 0]) + + +def test_unlabelled_in_memory_core_input_stays_flat(): + source = InMemoryCore(core_arrays(energy_dataset())) + simulation = Simulation(dataset=source, scenario=before_load()) + for year in (2026, 2027, 2032): + assert_energy(simulation, year, 100) + + +def test_core_file_and_explicit_field_must_agree(tmp_path): + path = core_file(energy_dataset(), tmp_path / "core.h5", DECLARATION) + source = Dataset.from_file(path) + source.energy_price_basis = {"period_type": "fiscal_year", "period_value": 2024} + with pytest.raises(ValueError, match="energy_price_basis"): + Simulation(dataset=source) + + +@pytest.fixture +def remote_energy(monkeypatch, tmp_path): + path = tmp_path / "arbitrary-price-basis.h5" + energy_dataset(DECLARATION).save(path) + monkeypatch.setattr(simulation_module, "_url_dataset_cache", {}) + monkeypatch.setattr( + simulation_module, "download_huggingface_dataset", lambda **_: str(path) + ) + monkeypatch.setattr( + simulation_module, "materialize_gcs_dataset_url", lambda url: str(path) + ) + return path + + +@pytest.mark.parametrize( + "url", ["hf://policyengine/mock/prices.h5", "gs://mock/prices.h5"] +) +@pytest.mark.parametrize("reformed_first", [False, True]) +def test_remote_projection_and_baseline_are_isolated_in_both_orders( + remote_energy, url, reformed_first +): + if reformed_first: + reformed = Simulation(dataset=url, reform=controlled_changes(160)) + current = Simulation(dataset=url) + else: + current = Simulation(dataset=url) + reformed = Simulation(dataset=url, reform=controlled_changes(160)) + assert_energy(reformed, 2026, 160) + for variable in ENERGY: + np.testing.assert_array_equal( + current.calculate(variable, 2026), + reformed.baseline.calculate(variable, 2026), + ) + repeated = Simulation(dataset=url) + for variable in ENERGY: + np.testing.assert_array_equal( + repeated.calculate(variable, 2026), current.calculate(variable, 2026) + ) + + +def test_index_only_reform_dict_matches_before_load_scenario(): + source = energy_dataset(DECLARATION) + by_dict = Simulation(dataset=source, reform=controlled_changes(160)) + by_scenario = Simulation(dataset=source, scenario=before_load(160)) + assert_energy(by_dict, 2026, 160) + for year in (2024, 2025, 2026, 2027, 2032): + for variable in ENERGY: + np.testing.assert_array_equal( + by_dict.calculate(variable, year), by_scenario.calculate(variable, year) + ) + + +def test_cpi_index_changes_the_projection_fingerprint(parameters): + assert CPI in uprating_input_paths() + changed = parameters.clone() + changed.get_child(CPI).update(period="2026", value=160) + assert uprating_input_fingerprint(parameters) != uprating_input_fingerprint(changed) + assert uprating_input_fingerprint(parameters) == uprating_input_fingerprint( + parameters.clone() + ) + + +@pytest.mark.parametrize("frozen_first", [False, True]) +def test_no_economic_assumptions_does_not_contaminate_cached_energy( + remote_energy, frozen_first +): + url = "hf://policyengine/mock/prices.h5" + if frozen_first: + frozen = Simulation(dataset=url, scenario=no_economic_assumptions) + current = Simulation(dataset=url) + else: + current = Simulation(dataset=url) + frozen = Simulation(dataset=url, scenario=no_economic_assumptions) + for year in (2026, 2027, 2032): + assert_energy(frozen, year, 100) + assert frozen.dataset[year].energy_price_basis == DECLARATION + repeated = Simulation(dataset=url) + for variable in ENERGY: + np.testing.assert_array_equal( + repeated.calculate(variable, 2026), current.calculate(variable, 2026) + ) + + +@pytest.mark.parametrize("source_kind", ["single", "local", "core", "gcs"]) +def test_no_economic_reset_keeps_loaded_amounts_and_metadata_coherent( + remote_energy, source_kind +): + original = energy_dataset(DECLARATION) + if source_kind == "single": + source = original + elif source_kind == "local": + source = remote_energy + elif source_kind == "core": + source = InMemoryCore(core_arrays(original)) + source.energy_price_basis = dict(DECLARATION) + else: + source = "gs://mock/prices.h5" + frozen = Simulation(dataset=source, scenario=no_economic_assumptions) + for year in (2024, 2026, 2032): + assert_energy(frozen, year, 100) + assert frozen.dataset[year].energy_price_basis == DECLARATION + for variable in ENERGY: + np.testing.assert_array_equal( + frozen.dataset[year].household[variable], [100.0, 0.0] + ) + assert original.energy_price_basis == DECLARATION + + +def test_external_multi_year_amounts_are_authoritative(): + first = energy_dataset(DECLARATION) + later = energy_dataset( + {"period_type": "fiscal_year", "period_value": 2026}, year=2026 + ) + for variable in ENERGY: + later.household[variable] = [777.0, 0.0] + source = UKMultiYearDataset(datasets=[first, later]) + simulation = Simulation(dataset=source, scenario=before_load()) + assert_energy(simulation, 2024, 100) + assert_energy(simulation, 2026, 777) + assert simulation.dataset[2026].energy_price_basis == later.energy_price_basis + + +def test_external_dataframe_amounts_are_authoritative(): + frame = pd.DataFrame( + { + "person_id__2024": [1, 2], + "person_benunit_id__2024": [1, 2], + "person_household_id__2024": [1, 2], + "age__2024": [40, 50], + **{f"{variable}__2024": [100.0, 0.0] for variable in ENERGY}, + **{f"{variable}__2026": [777.0, 0.0] for variable in ENERGY}, + } + ) + before = frame.copy(deep=True) + simulation = Simulation(dataset=frame, scenario=before_load()) + assert_energy(simulation, 2024, 100) + assert_energy(simulation, 2026, 777) + pd.testing.assert_frame_equal(frame, before) + + +def test_direct_situation_keeps_existing_cpi_uprating(): + simulation = Simulation( + situation={ + "people": {"person": {"age": {2024: 40}}}, + "benunits": {"benunit": {"members": ["person"]}}, + "households": { + "household": { + "members": ["person"], + **{variable: {2024: 100.0} for variable in ENERGY}, + } + }, + }, + scenario=before_load(), + ) + for variable in ENERGY: + np.testing.assert_allclose(simulation.calculate(variable, 2026), [125.0]) diff --git a/policyengine_uk/tests/test_energy_price_basis.py b/policyengine_uk/tests/test_energy_price_basis.py new file mode 100644 index 0000000000..8cf5842c95 --- /dev/null +++ b/policyengine_uk/tests/test_energy_price_basis.py @@ -0,0 +1,184 @@ +"""Explicit energy price provenance; unlabelled datasets retain flat amounts.""" + +import hashlib +import json +from types import SimpleNamespace + +import h5py +import pandas as pd +import pytest + +from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset +from policyengine_uk.data.filter_dataset import filter_dataset + + +def single(basis=None, year=2024): + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + "months_since_last_birthday": [3, 8], + "attends_private_school": [False, False], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + {"household_id": [1, 2], "domestic_energy_consumption": [100.0, 200.0]} + ), + fiscal_year=year, + energy_price_basis=basis, + ) + + +@pytest.mark.parametrize("period_type", ["calendar_year", "fiscal_year"]) +def test_constructor_preserves_the_declared_price_period(period_type): + declaration = {"period_type": period_type, "period_value": 2024} + dataset = UKSingleYearDataset( + person=pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [1]} + ), + benunit=pd.DataFrame({"benunit_id": [1]}), + household=pd.DataFrame({"household_id": [1]}), + fiscal_year=2024, + energy_price_basis=declaration, + ) + assert dataset.energy_price_basis == declaration + declaration["period_value"] = 2026 + assert dataset.energy_price_basis == { + "period_type": period_type, + "period_value": 2024, + } + exposed = dataset.energy_price_basis + exposed["period_value"] = 2030 + assert dataset.energy_price_basis["period_value"] == 2024 + + +@pytest.mark.parametrize("year", [1, 9999]) +def test_supported_integer_year_boundaries(year): + declaration = {"period_type": "calendar_year", "period_value": year} + assert single(declaration).energy_price_basis == declaration + + +@pytest.mark.parametrize( + "declaration", + [ + {}, + {"period_type": "calendar_year"}, + {"period_value": 2024}, + {"period_type": "month", "period_value": 2024}, + {"period_type": "calendar_year", "period_value": True}, + {"period_type": "calendar_year", "period_value": False}, + {"period_type": "calendar_year", "period_value": "2024"}, + {"period_type": "calendar_year", "period_value": 2024.0}, + {"period_type": "calendar_year", "period_value": 2024.5}, + {"period_type": "calendar_year", "period_value": 0}, + {"period_type": "calendar_year", "period_value": 10000}, + {"period_type": "calendar_year", "period_value": 2024, "legacy": True}, + "calendar_year:2024", + ["calendar_year", 2024], + 2024, + ], +) +def test_malformed_explicit_declarations_fail_clearly(declaration): + with pytest.raises(ValueError, match="energy_price_basis"): + single(declaration) + + +@pytest.mark.parametrize("period_type", ["calendar_year", "fiscal_year"]) +def test_single_copy_and_h5_roundtrip_preserve_kind_and_values(tmp_path, period_type): + declaration = {"period_type": period_type, "period_value": 2024} + source = single(declaration) + copied = source.copy() + path = tmp_path / "arbitrary-name.h5" + copied.save(path) + with h5py.File(path, "r") as file: + assert json.loads(file.attrs["energy_price_basis"]) == declaration + loaded = UKSingleYearDataset(file_path=path) + for item in (source, copied, loaded): + assert item.energy_price_basis == declaration + pd.testing.assert_frame_equal(item.household, source.household) + copied.energy_price_basis = None + assert source.energy_price_basis == declaration + assert loaded.energy_price_basis == declaration + + +def test_unlabelled_file_remains_readable_and_can_be_labelled_explicitly(tmp_path): + path = tmp_path / "fiscal-2099-producer-name-is-not-a-declaration.h5" + source = single() + source.save(path) + before = hashlib.sha256(path.read_bytes()).hexdigest() + with h5py.File(path, "r") as file: + assert "energy_price_basis" not in file.attrs + assert UKSingleYearDataset(file_path=path).energy_price_basis is None + declaration = {"period_type": "calendar_year", "period_value": 2024} + labelled = UKSingleYearDataset(file_path=path, energy_price_basis=declaration) + assert labelled.energy_price_basis == declaration + assert source.energy_price_basis is None + assert hashlib.sha256(path.read_bytes()).hexdigest() == before + + +def test_stored_declaration_agrees_with_caller_or_fails_on_conflict(tmp_path): + declaration = {"period_type": "calendar_year", "period_value": 2024} + path = tmp_path / "labelled.h5" + single(declaration).save(path) + for caller in (None, declaration): + assert ( + UKSingleYearDataset( + file_path=path, energy_price_basis=caller + ).energy_price_basis + == declaration + ) + for conflicting in ( + {"period_type": "fiscal_year", "period_value": 2024}, + {"period_type": "calendar_year", "period_value": 2025}, + ): + with pytest.raises(ValueError, match="energy_price_basis"): + UKSingleYearDataset(file_path=path, energy_price_basis=conflicting) + + +@pytest.mark.parametrize( + "stored", + ["not JSON", "[]", "2024", '{"period_type":"fiscal_year"}', 2024], +) +def test_malformed_persisted_declaration_fails_clearly(tmp_path, stored): + path = tmp_path / "invalid.h5" + single().save(path) + with h5py.File(path, "a") as file: + file.attrs["energy_price_basis"] = stored + with pytest.raises(ValueError, match="energy_price_basis"): + UKSingleYearDataset(file_path=path) + + +def test_multi_copy_and_h5_keep_each_year_declaration(tmp_path): + declarations = { + 2024: {"period_type": "calendar_year", "period_value": 2024}, + 2025: {"period_type": "fiscal_year", "period_value": 2025}, + 2026: None, + } + source = UKMultiYearDataset( + datasets=[single(basis, year) for year, basis in declarations.items()] + ) + copied = source.copy() + path = tmp_path / "several-years.h5" + copied.save(path) + loaded = UKMultiYearDataset(file_path=path) + for year, declaration in declarations.items(): + for item in (source, copied, loaded): + assert item[year].energy_price_basis == declaration + pd.testing.assert_frame_equal(item[year].household, source[year].household) + copied[2024].energy_price_basis = None + assert source[2024].energy_price_basis == declarations[2024] + assert loaded[2024].energy_price_basis == declarations[2024] + + +def test_filtered_household_preserves_the_observations_price_basis(): + declaration = {"period_type": "calendar_year", "period_value": 2024} + source = single(declaration) + simulation = SimpleNamespace(dataset=UKMultiYearDataset(datasets=[source])) + filtered = filter_dataset(simulation, household_id=2, year=2024) + assert filtered.energy_price_basis == declaration + assert filtered.household["domestic_energy_consumption"].tolist() == [200.0] + assert filtered.person.person_id.tolist() == [2] + assert source.household.household_id.tolist() == [1, 2] diff --git a/policyengine_uk/tests/test_energy_reset_coherence.py b/policyengine_uk/tests/test_energy_reset_coherence.py new file mode 100644 index 0000000000..83d2b05403 --- /dev/null +++ b/policyengine_uk/tests/test_energy_reset_coherence.py @@ -0,0 +1,415 @@ +"""External energy observations retain a coherent amount group and price basis. + +CPI levels 100/125/150 are controlled test inputs, not an economic forecast. +The later FY2026 observation must extend to 2027 by 150 / 125 after reset +and serialization; the original external observations must remain untouched. +""" + +from copy import deepcopy +import json + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk import CountryTaxBenefitSystem, Simulation +from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset +from policyengine_uk.data.economic_assumptions import extend_single_year_dataset +from policyengine_uk.scenarios import no_economic_assumptions + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +ENERGY = ( + "domestic_energy_consumption", + "electricity_consumption", + "gas_consumption", +) +LATER_BASIS = {"period_type": "fiscal_year", "period_value": 2026} + + +def _observation(year, amounts, basis): + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1], + "person_benunit_id": [1], + "person_household_id": [1], + "age": [40], + } + ), + benunit=pd.DataFrame({"benunit_id": [1]}), + household=pd.DataFrame( + { + "household_id": [1], + "region": ["LONDON"], + "tenure_type": ["OWNED_OUTRIGHT"], + "council_tax": [0.0], + "rent": [0.0], + "household_weight": [1.0], + **{name: [value] for name, value in amounts.items()}, + } + ), + fiscal_year=year, + energy_price_basis=basis, + ) + + +def _state(observation): + return { + "basis": deepcopy(observation.energy_price_basis), + "amounts": { + name: observation.household[name].tolist() + for name in ENERGY + if name in observation.household + }, + } + + +@pytest.fixture +def controlled_parameters(): + parameters = CountryTaxBenefitSystem().parameters.clone() + index = parameters.get_child( + "gov.economic_assumptions.indices.obr.consumer_price_index" + ) + for year, level in {2024: 100.0, 2026: 125.0, 2027: 150.0}.items(): + index.update(period=str(year), value=level) + return parameters + + +@pytest.mark.parametrize( + "base_amounts,base_basis,later_amounts", + [ + ({}, None, dict.fromkeys(ENERGY, 777.0)), + ( + {ENERGY[0]: 111.0}, + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + ), + ], + ids=["no-base-energy", "partial-base-different-basis"], +) +def test_external_energy_reset_survives_loading_and_extension( + tmp_path, controlled_parameters, base_amounts, base_basis, later_amounts +): + base = _observation(2024, base_amounts, base_basis) + later = _observation(2026, later_amounts, LATER_BASIS) + source = UKMultiYearDataset(datasets=[base, later]) + original = { + year: { + "state": _state(source[year]), + "tables": [ + table.copy(deep=True) + for table in ( + source[year].person, + source[year].benunit, + source[year].household, + ) + ], + } + for year in source.years + } + + simulation = Simulation(dataset=source, scenario=no_economic_assumptions) + child = simulation.dataset[2026] + loaded_inputs = {name: simulation.calculate(name, 2026).tolist() for name in ENERGY} + path = tmp_path / "external-child.h5" + child.save(path) + reloaded = UKSingleYearDataset(file_path=path) + extended = extend_single_year_dataset( + reloaded, controlled_parameters, end_year=2027 + )[2027] + + observations = { + "caller_before": {year: item["state"] for year, item in original.items()}, + "caller_after": {year: _state(source[year]) for year in source.years}, + "reset_child": _state(child), + "loaded_inputs_2026": loaded_inputs, + "saved_reloaded_child": _state(reloaded), + "extended_2027": _state(extended), + "expected_2027": { + name: [amount * 150.0 / 125.0] for name, amount in later_amounts.items() + }, + } + (tmp_path / "energy-reset-observations.json").write_text( + json.dumps(observations, indent=2) + "\n" + ) + + # Check caller ownership even when the returned reset child is defective. + for year in source.years: + assert _state(source[year]) == original[year]["state"] + for before, after in zip( + original[year]["tables"], + (source[year].person, source[year].benunit, source[year].household), + ): + # Public values stay exact; the loader may encode enum buffers. + pd.testing.assert_frame_equal(before, after, check_dtype=False) + + for name, amount in later_amounts.items(): + np.testing.assert_array_equal(child.household[name], [amount]) + np.testing.assert_array_equal(loaded_inputs[name], [amount]) + np.testing.assert_array_equal(reloaded.household[name], [amount]) + # Hand oracle: 777 * 150 / 125 = 932.4, likewise 888 and 999. + np.testing.assert_allclose(extended.household[name], [amount * 150.0 / 125.0]) + assert child.energy_price_basis == LATER_BASIS + assert reloaded.energy_price_basis == LATER_BASIS + assert extended.energy_price_basis == { + "period_type": "fiscal_year", + "period_value": 2027, + } + + +# Each row states the retained observations explicitly. These expected amounts +# are hand oracles, not a second implementation of the reset decision. +@pytest.mark.parametrize( + "base_amounts,base_basis,later_amounts,later_basis," + "expected_reset,expected_basis,expected_2027", + [ + pytest.param( + {ENERGY[1]: 222.0, ENERGY[2]: 333.0}, + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="two-base-columns-no-domestic-different-basis", + ), + pytest.param( + {ENERGY[0]: 111.0, ENERGY[2]: 333.0}, + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="two-base-columns-no-electricity-different-basis", + ), + pytest.param( + {ENERGY[0]: 111.0, ENERGY[1]: 222.0}, + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="two-base-columns-no-gas-different-basis", + ), + pytest.param( + {ENERGY[0]: 111.0}, + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [111.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [133.2, 1065.6, 1198.8])), + id="same-exact-basis-domestic-overlap-only", + ), + pytest.param( + {ENERGY[1]: 111.0}, + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 111.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 133.2, 1198.8])), + id="same-exact-basis-electricity-overlap-only", + ), + pytest.param( + {ENERGY[2]: 111.0}, + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 111.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 133.2])), + id="same-exact-basis-gas-overlap-only", + ), + pytest.param( + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [166.5, 333.0, 499.5])), + id="full-energy-group-resets-values-and-basis", + ), + pytest.param( + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[0]: 777.0}, + LATER_BASIS, + {ENERGY[0]: 111.0}, + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[0]: 166.5}, + id="one-present-domestic-no-invented-columns", + ), + pytest.param( + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[1]: 888.0}, + LATER_BASIS, + {ENERGY[1]: 222.0}, + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[1]: 333.0}, + id="one-present-electricity-no-invented-columns", + ), + pytest.param( + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[2]: 999.0}, + LATER_BASIS, + {ENERGY[2]: 333.0}, + {"period_type": "calendar_year", "period_value": 2024}, + {ENERGY[2]: 499.5}, + id="one-present-gas-no-invented-columns", + ), + pytest.param( + {ENERGY[0]: 111.0}, + {"period_type": "calendar_year", "period_value": 2026}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="same-year-calendar-and-fiscal-bases-remain-distinct", + ), + pytest.param( + {ENERGY[0]: 111.0}, + {"period_type": "calendar_year", "period_value": 2024}, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + None, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + None, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + id="labelled-base-unlabelled-later-remains-flat", + ), + pytest.param( + {ENERGY[0]: 111.0}, + None, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="unlabelled-base-labelled-later-retains-own-basis", + ), + pytest.param( + {ENERGY[0]: 111.0}, + None, + dict(zip(ENERGY, [777.0, 888.0, 999.0])), + None, + dict(zip(ENERGY, [111.0, 888.0, 999.0])), + None, + dict(zip(ENERGY, [111.0, 888.0, 999.0])), + id="both-unlabelled-overlap-reset-remains-flat", + ), + pytest.param( + dict(zip(ENERGY, [111.0, 222.0, 333.0])), + {"period_type": "calendar_year", "period_value": 2024}, + {}, + LATER_BASIS, + {}, + LATER_BASIS, + None, + id="no-later-energy-preserves-declaration-and-absence", + ), + pytest.param( + {ENERGY[0]: 111.0, "food_and_non_alcoholic_beverages_consumption": 10.0}, + {"period_type": "calendar_year", "period_value": 2024}, + { + **dict(zip(ENERGY, [777.0, 888.0, 999.0])), + "food_and_non_alcoholic_beverages_consumption": 20.0, + }, + LATER_BASIS, + { + **dict(zip(ENERGY, [777.0, 888.0, 999.0])), + "food_and_non_alcoholic_beverages_consumption": 10.0, + }, + LATER_BASIS, + dict(zip(ENERGY, [932.4, 1065.6, 1198.8])), + id="ordinary-consumption-still-resets-while-energy-is-retained", + ), + ], +) +def test_energy_reset_controls_preserve_coherent_public_observations( + tmp_path, + controlled_parameters, + base_amounts, + base_basis, + later_amounts, + later_basis, + expected_reset, + expected_basis, + expected_2027, +): + source = UKMultiYearDataset( + datasets=[ + _observation(2024, base_amounts, base_basis), + _observation(2026, later_amounts, later_basis), + ] + ) + original = { + year: { + "state": _state(source[year]), + "tables": [ + table.copy(deep=True) + for table in ( + source[year].person, + source[year].benunit, + source[year].household, + ) + ], + } + for year in source.years + } + + simulation = Simulation(dataset=source, scenario=no_economic_assumptions) + child = simulation.dataset[2026] + assert child is not source[2026] + path = tmp_path / "controlled-energy-child.h5" + child.save(path) + reloaded = UKSingleYearDataset(file_path=path) + + # Check the public observations rather than internal encoded enum buffers. + for year in source.years: + assert _state(source[year]) == original[year]["state"] + for before, after in zip( + original[year]["tables"], + (source[year].person, source[year].benunit, source[year].household), + ): + pd.testing.assert_frame_equal(before, after, check_dtype=False) + + present_energy = {name for name in expected_reset if name in ENERGY} + for observation in (child, reloaded): + assert { + name for name in ENERGY if name in observation.household + } == present_energy + assert observation.energy_price_basis == expected_basis + for name, amount in expected_reset.items(): + np.testing.assert_array_equal(observation.household[name], [amount]) + for name, amount in expected_reset.items(): + np.testing.assert_array_equal(simulation.calculate(name, 2026), [amount]) + + # Empty energy groups have no asserted projection-metadata contract here. + if expected_2027 is not None: + extended = extend_single_year_dataset( + reloaded, controlled_parameters, end_year=2027 + )[2027] + assert {name for name in ENERGY if name in extended.household} == present_energy + for name, amount in expected_2027.items(): + np.testing.assert_allclose(extended.household[name], [amount]) + assert extended.energy_price_basis == ( + {"period_type": "fiscal_year", "period_value": 2027} + if expected_basis is not None + else None + ) + + # Calculation and re-extension must also leave the caller's tables intact. + for year in source.years: + assert _state(source[year]) == original[year]["state"] + for before, after in zip( + original[year]["tables"], + (source[year].person, source[year].benunit, source[year].household), + ): + pd.testing.assert_frame_equal(before, after, check_dtype=False)