diff --git a/changelog.d/2526.fixed.md b/changelog.d/2526.fixed.md new file mode 100644 index 0000000000..83af32a0ec --- /dev/null +++ b/changelog.d/2526.fixed.md @@ -0,0 +1 @@ +- Kept declared energy projection and reset limited to the three energy inputs, rejected projections before the selected index's supported history, and documented generic CPI as an interim price proxy. diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index eb089a73d9..b30a440cc3 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -58,10 +58,12 @@ Unlabelled dataset energy spending stays flat through the projection horizon. Do not infer its price year from the dataset year, filename, URL or file format. Legacy builds can hold FY2026-27 energy prices in a dataset labelled 2024. -An explicit declaration opts the three stored energy amounts into their -existing annual OBR consumer-price-index assumption. For an input whose -producer receipt declares calendar-2024 prices, a caller can label a local -UK single-year H5 before constructing a simulation: +An explicit declaration opts only `domestic_energy_consumption`, +`electricity_consumption` and `gas_consumption` into the existing generic annual +OBR consumer-price-index assumption as an interim price proxy. Other generic +uprating exclusions do not join this declaration or its reset group. For an +input whose producer receipt declares calendar-2024 prices, a caller can label +a local UK single-year H5 before constructing a simulation: ```python from policyengine_uk import Microsimulation @@ -79,14 +81,24 @@ financial year. Both keys are required. The year must be an integer from 1 through 9999; malformed or conflicting caller/file declarations raise a clear error. A caller's `None` leaves an existing file declaration intact. -The model retains the period kind as provenance but uses its existing annual -index-year convention: CY2024 and FY2024-25 both use `I[2024]`. Projection -multiplies the supplied amount by `I[target year] / I[price year]`. This is -the existing generic CPI assumption, with no new energy-specific forecast or -precise fiscal/calendar price adjustment. A 2026 basis therefore stays unchanged -at target 2026 and can grow at target 2027. The original observation remains -unchanged; a projected child's declaration records its effective fiscal-year -price basis so later extension does not apply the original ratio twice. +The model retains the period kind as provenance but uses the January 1 index +level of the starting year for both calendar and fiscal price periods. +Projection multiplies the supplied amount by `I[target year] / I[price year]`. +This shared start-year convention approximates calendar/fiscal alignment; +generic CPI does not establish constant-consumption energy-price outturns or +an energy-specific forecast. Open [issue #2526](https://github.com/PolicyEngine/policyengine-uk/issues/2526) +tracks energy-specific index alignment and a sourced future assumption. The +original observation remains unchanged; a projected child's declaration records +its effective fiscal-year price basis so later extension does not apply the +original ratio twice. + +When a declaration projects stored energy inputs, both January 1 dates must +fall on or after the selected index's earliest non-null value. An earlier +basis or target, including January 1 before a midyear start, raises a clear +error rather than using Core's backward first-value fallback. An index with +no usable history also raises an error for that projection. Unlabelled +observations and declarations with no relevant stored inputs retain their +existing behaviour without requiring index history. Single-year H5 files store a JSON declaration in the root attribute `energy_price_basis`. Multi-year files store it on each `/time_period/` diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index d85822a051..01737ce720 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -105,13 +105,14 @@ # a separate change. Energy has its own declaration-aware projection below, # so it must not also grow through the generic variable-index fallback. LEGACY_PROJECTION_END_YEAR = 2030 -DATASET_UPRATING_EXCLUSIONS = frozenset( +DECLARED_ENERGY_INPUTS = frozenset( { "domestic_energy_consumption", "electricity_consumption", "gas_consumption", } ) +DATASET_UPRATING_EXCLUSIONS = DECLARED_ENERGY_INPUTS # Inputs that describe only the year they are stored for, so are not carried # into projected years. Unused annual allowance comes from the three tax years # before this one (Finance Act 2004 s. 228A), a window that moves each year; a @@ -386,7 +387,7 @@ def uprate_energy( basis = observed_year.energy_price_basis variables = [ name - for name in DATASET_UPRATING_EXCLUSIONS + for name in DECLARED_ENERGY_INPUTS if name in observed_year.household.columns and name in current_year.household.columns ] @@ -394,10 +395,27 @@ def uprate_energy( projected = False if basis is not None and variables: index = parameters.get_child(ENERGY_PRICE_INDEX) - source_index = _uprating_index_value( - index, f"{basis['period_value']:04d}-01-01" - ) - target_index = _uprating_index_value(index, f"{current_year.time_period}-01-01") + source_instant = f"{basis['period_value']:04d}-01-01" + target_instant = f"{int(current_year.time_period):04d}-01-01" + supported_instants = [ + value.instant_str + for value in getattr(index, "values_list", []) + if value.value is not None + ] + if not supported_instants: + raise ValueError( + f"Energy index {ENERGY_PRICE_INDEX!r} has no non-null history " + f"for declared basis {source_instant} and target {target_instant}." + ) + first_supported = min(supported_instants) + if source_instant < first_supported or target_instant < first_supported: + raise ValueError( + f"Energy index {ENERGY_PRICE_INDEX!r} first supports " + f"{first_supported}; declared basis {source_instant} and target " + f"{target_instant} must be on or after that instant." + ) + source_index = _uprating_index_value(index, source_instant) + target_index = _uprating_index_value(index, target_instant) projected = source_index not in (None, 0) and target_index is not None if projected: ratio = target_index / source_index @@ -713,7 +731,7 @@ def reset_growthfactor_uprating( first_year = min(dataset.datasets.keys()) base_year_dataset = dataset.datasets[first_year] - base_energy = DATASET_UPRATING_EXCLUSIONS.intersection( + base_energy = DECLARED_ENERGY_INPUTS.intersection( base_year_dataset.household.columns ) @@ -721,7 +739,7 @@ def reset_growthfactor_uprating( if year == first_year: continue current_year_dataset = dataset.datasets[year] - current_energy = DATASET_UPRATING_EXCLUSIONS.intersection( + current_energy = DECLARED_ENERGY_INPUTS.intersection( current_year_dataset.household.columns ) shared_energy = current_energy.intersection(base_energy) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 2b7468b887..09092747e2 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -114,18 +114,27 @@ earlier mapping omissions is a separate change. use a separate declaration-aware projector. Unlabelled datasets retain their stored amounts through the projection horizon, preserving legacy inputs that already use FY2026-27 prices. A dataset's explicit `energy_price_basis` opts -into the variables' existing cumulative OBR-CPI index from its declared price -year. They remain outside both the unconditional growth mapping and fallback, -so no second index grows them after 2030. +into the variables' existing cumulative annual OBR-CPI index as an interim +price proxy from its declared price year. The declaration covers only these +three inputs, independently of any other generic uprating exclusions. They +remain outside both the unconditional growth mapping and fallback, so no +second index grows them after 2030. The declaration retains `calendar_year` or `fiscal_year` and its integer year. -The model's annual convention uses the same index level for CY2024 and -FY2024-25: `I[2024]`. It does not provide a precise fiscal/calendar price -alignment or an energy-specific forecast. The projector applies -`I[target year] / I[declared price year]` to the supplied observation. -Consequently a 2026 basis gives a ratio of one in 2026 and can grow in 2027; -there is no special 2026 rule. Each projected child records its effective -fiscal-year basis, while the source observation keeps its original declaration. +The model uses the January 1 index level for the declared starting year for +both calendar and fiscal price periods. This shared start-year convention is +an approximation, rather than precise fiscal/calendar price alignment. The +projector applies `I[target year] / I[declared price year]` to the supplied +observation. A requested declared projection raises an error if either +January 1 precedes the selected index's earliest non-null value, including a +midyear start, or if the index has no usable history. Unlabelled inputs and +declarations with no stored energy inputs do not require index history. Each +projected child records its effective fiscal-year basis, while the source +observation keeps its original declaration. + +Generic CPI does not establish constant-consumption energy-price outturns or +an energy-specific forecast. Open [issue #2526](https://github.com/PolicyEngine/policyengine-uk/issues/2526) +tracks energy-specific index alignment and a sourced future assumption. See [dataset sources](../../../../docs/engineering/skills/dataset-sources.md) for the declaration and migration contract. Existing metadata-free private diff --git a/policyengine_uk/tests/test_energy_dataset_projection.py b/policyengine_uk/tests/test_energy_dataset_projection.py index 141af6ca72..31a2cc705d 100644 --- a/policyengine_uk/tests/test_energy_dataset_projection.py +++ b/policyengine_uk/tests/test_energy_dataset_projection.py @@ -463,3 +463,210 @@ def test_direct_situation_keeps_existing_cpi_uprating(): ) for variable in ENERGY: np.testing.assert_allclose(simulation.calculate(variable, 2026), [125.0]) + + +def _controlled_energy_history(monkeypatch, history): + """A real, clean index leaf with only the explicitly supplied history.""" + from policyengine_core.parameters import Parameter, ParameterNode + from policyengine_uk.data import economic_assumptions + + name = "controlled_energy_index" + parameters = ParameterNode("", data={}) + parameters.add_child(name, Parameter(name, data=history)) + monkeypatch.setattr(economic_assumptions, "ENERGY_PRICE_INDEX", name) + return parameters + + +def test_declared_energy_projection_ignores_widened_generic_exclusions( + monkeypatch, parameters +): + from policyengine_uk.data import economic_assumptions + + unrelated = "food_and_non_alcoholic_beverages_consumption" + monkeypatch.setattr( + economic_assumptions, + "DATASET_UPRATING_EXCLUSIONS", + economic_assumptions.DATASET_UPRATING_EXCLUSIONS | {unrelated}, + ) + observed = energy_dataset(DECLARATION) + observed.household[unrelated] = [20.0, 30.0] + current = observed.copy() + current.time_period = "2026" + current.household[unrelated] = [777.0, 888.0] + before = observed.household.copy(deep=True) + + economic_assumptions.uprate_energy(current, observed, parameters) + + for variable in ENERGY: + # Controlled annual CPI oracle: 100 * 125 / 100 = 125. + np.testing.assert_array_equal(current.household[variable], [125.0, 0.0]) + np.testing.assert_array_equal(current.household[unrelated], [777.0, 888.0]) + assert current.energy_price_basis == { + "period_type": "fiscal_year", + "period_value": 2026, + } + pd.testing.assert_frame_equal(observed.household, before) + assert observed.energy_price_basis == DECLARATION + + +@pytest.mark.parametrize( + "first_supported,basis_year,target_year,unsupported_year", + [ + pytest.param("2031-01-01", 2030, 2033, 2030, id="basis-before-first-value"), + pytest.param("2031-01-01", 2033, 2030, 2030, id="target-before-first-value"), + pytest.param("2031-07-01", 2031, 2033, 2031, id="basis-before-midyear-start"), + pytest.param("2031-07-01", 2033, 2031, 2031, id="target-before-midyear-start"), + ], +) +def test_energy_projection_rejects_pre_support_basis_and_target( + monkeypatch, first_supported, basis_year, target_year, unsupported_year +): + from policyengine_uk.data import economic_assumptions + + parameters = _controlled_energy_history( + monkeypatch, + {"2028-01-01": None, first_supported: 100, "2033-01-01": 125}, + ) + basis = {"period_type": "calendar_year", "period_value": basis_year} + observed = energy_dataset(basis, year=2030) + current = observed.copy() + current.time_period = str(target_year) + + with pytest.raises(ValueError, match="controlled_energy_index") as error: + economic_assumptions.uprate_energy(current, observed, parameters) + assert f"{unsupported_year}-01-01" in str(error.value) + assert first_supported in str(error.value) + for variable in ENERGY: + np.testing.assert_array_equal(observed.household[variable], [100.0, 0.0]) + assert observed.energy_price_basis == basis + + +@pytest.mark.parametrize( + "first_supported,basis_year,target_year,expected", + [ + pytest.param("2031-01-01", 2031, 2031, 100, id="first-supported-boundary"), + pytest.param("2031-01-01", 2031, 2033, 125, id="valid-ratio"), + pytest.param("2031-07-01", 2032, 2033, 125, id="after-midyear-start"), + pytest.param( + "2031-01-01", 2031, 2035, 125, id="existing-terminal-carry-forward" + ), + ], +) +def test_energy_projection_accepts_supported_boundary_and_future_ratio( + monkeypatch, first_supported, basis_year, target_year, expected +): + from policyengine_uk.data import economic_assumptions + + parameters = _controlled_energy_history( + monkeypatch, + { + "2028-01-01": None, + first_supported: 100, + "2033-01-01": 125, + "2034-01-01": None, + }, + ) + basis = {"period_type": "calendar_year", "period_value": basis_year} + observed = energy_dataset(basis, year=2030) + current = observed.copy() + current.time_period = str(target_year) + + economic_assumptions.uprate_energy(current, observed, parameters) + + for variable in ENERGY: + np.testing.assert_array_equal(current.household[variable], [expected, 0.0]) + np.testing.assert_array_equal(observed.household[variable], [100.0, 0.0]) + assert current.energy_price_basis == { + "period_type": "fiscal_year", + "period_value": target_year, + } + assert observed.energy_price_basis == basis + + +@pytest.mark.parametrize( + "history", + [ + pytest.param({}, id="empty-history"), + pytest.param({"2028-01-01": None, "2034-01-01": None}, id="all-null-history"), + ], +) +def test_declared_energy_projection_rejects_unusable_index_history( + monkeypatch, history +): + from policyengine_uk.data import economic_assumptions + + parameters = _controlled_energy_history(monkeypatch, history) + observed = energy_dataset(DECLARATION) + current = observed.copy() + current.time_period = "2026" + + with pytest.raises(ValueError, match="controlled_energy_index"): + economic_assumptions.uprate_energy(current, observed, parameters) + assert observed.energy_price_basis == DECLARATION + + +@pytest.mark.parametrize("energy_kind", ["unlabelled", "absent-energy"]) +@pytest.mark.parametrize( + "history", + [ + pytest.param({}, id="empty-history"), + pytest.param({"2034-01-01": None}, id="all-null-history"), + pytest.param({"2034-07-01": 100}, id="later-support"), + ], +) +def test_unlabelled_or_absent_energy_does_not_require_index_history( + monkeypatch, energy_kind, history +): + from policyengine_uk.data import economic_assumptions + + parameters = _controlled_energy_history(monkeypatch, history) + observed = energy_dataset(None if energy_kind == "unlabelled" else DECLARATION) + if energy_kind == "absent-energy": + observed.household = observed.household.drop(columns=list(ENERGY)) + before = observed.household.copy(deep=True) + for year in (2026, 2032): + current = observed.copy() + current.time_period = str(year) + economic_assumptions.uprate_energy(current, observed, parameters) + pd.testing.assert_frame_equal(current.household, before) + assert current.energy_price_basis == observed.energy_price_basis + pd.testing.assert_frame_equal(observed.household, before) + + +def test_energy_projection_validates_reformed_selected_index_history(monkeypatch): + from policyengine_uk.data import economic_assumptions + + parameters = _controlled_energy_history( + monkeypatch, {"2030-01-01": 100, "2033-01-01": 125} + ) + reformed = parameters.clone() + selected_index = reformed.get_child(economic_assumptions.ENERGY_PRICE_INDEX) + # A parameter reform withdraws the two earliest years of this clean history. + selected_index.update(period="year:2030:2", value=None) + assert ( + min( + item.instant_str + for item in selected_index.values_list + if item.value is not None + ) + == "2032-01-01" + ) + observed = energy_dataset( + {"period_type": "calendar_year", "period_value": 2030}, year=2030 + ) + current = observed.copy() + current.time_period = "2033" + economic_assumptions.uprate_energy(current, observed, parameters) + for variable in ENERGY: + np.testing.assert_array_equal(current.household[variable], [125.0, 0.0]) + + with pytest.raises(ValueError, match="controlled_energy_index"): + economic_assumptions.uprate_energy(current.copy(), observed, reformed) + assert ( + parameters.get_child(economic_assumptions.ENERGY_PRICE_INDEX)("2030-01-01") + == 100 + ) + assert observed.energy_price_basis == { + "period_type": "calendar_year", + "period_value": 2030, + } diff --git a/policyengine_uk/tests/test_energy_reset_coherence.py b/policyengine_uk/tests/test_energy_reset_coherence.py index 83d2b05403..87a60091b8 100644 --- a/policyengine_uk/tests/test_energy_reset_coherence.py +++ b/policyengine_uk/tests/test_energy_reset_coherence.py @@ -413,3 +413,41 @@ def test_energy_reset_controls_preserve_coherent_public_observations( (source[year].person, source[year].benunit, source[year].household), ): pd.testing.assert_frame_equal(before, after, check_dtype=False) + + +@pytest.mark.parametrize("unrelated_location", ["shared", "later-only"]) +def test_reset_energy_group_ignores_widened_generic_exclusions( + monkeypatch, unrelated_location +): + from policyengine_uk.data import economic_assumptions + + unrelated = "food_and_non_alcoholic_beverages_consumption" + monkeypatch.setattr( + economic_assumptions, + "DATASET_UPRATING_EXCLUSIONS", + economic_assumptions.DATASET_UPRATING_EXCLUSIONS | {unrelated}, + ) + base_basis = {"period_type": "calendar_year", "period_value": 2024} + base_amounts = dict(zip(ENERGY, [111.0, 222.0, 333.0])) + if unrelated_location == "shared": + base_amounts[unrelated] = 10.0 + later_amounts = {**dict(zip(ENERGY, [777.0, 888.0, 999.0])), unrelated: 20.0} + source = UKMultiYearDataset( + datasets=[ + _observation(2024, base_amounts, base_basis), + _observation(2026, later_amounts, LATER_BASIS), + ] + ) + before = {year: source[year].household.copy(deep=True) for year in source.years} + + reset = economic_assumptions.reset_growthfactor_uprating(source.copy()) + + # Complete energy replacement must not depend on an unrelated later-only column. + for variable, expected in zip(ENERGY, [111.0, 222.0, 333.0]): + np.testing.assert_array_equal(reset[2026].household[variable], [expected]) + assert reset[2026].energy_price_basis == base_basis + np.testing.assert_array_equal(reset[2026].household[unrelated], [20.0]) + for year in source.years: + pd.testing.assert_frame_equal(source[year].household, before[year]) + assert source[2024].energy_price_basis == base_basis + assert source[2026].energy_price_basis == LATER_BASIS