From 02c9a6d016b355e1694223b6724076487722b4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:49:04 +0100 Subject: [PATCH 1/2] Hold BSP flat after the dataset benefit split --- changelog.d/2381.fixed.md | 1 + docs/engineering/skills/dataset-sources.md | 32 ++ policyengine_uk/data/economic_assumptions.py | 18 +- policyengine_uk/data/uprating_indices.yaml | 9 +- policyengine_uk/simulation.py | 41 ++ .../tests/test_bsp_dataset_projection.py | 401 ++++++++++++++++++ .../tests/test_projection_horizon.py | 12 +- .../variables/gov/dwp/bsp_reported.py | 22 +- 8 files changed, 518 insertions(+), 18 deletions(-) create mode 100644 changelog.d/2381.fixed.md create mode 100644 policyengine_uk/tests/test_bsp_dataset_projection.py diff --git a/changelog.d/2381.fixed.md b/changelog.d/2381.fixed.md new file mode 100644 index 0000000000..d77f69d55a --- /dev/null +++ b/changelog.d/2381.fixed.md @@ -0,0 +1 @@ +- Held reported Bereavement Support Payment flat for datasets declaring the completed benefit split, while preserving CPI uprating for legacy mixed reports and the separate pensions. diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index b30a440cc3..f6ca5c0107 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -121,6 +121,38 @@ matching that receipt. The producer must carry both period kind and year into the H5 artifact; this consumer change does not retrofit that metadata or verify the released private file. +## Bereavement-benefit split + +Bereavement Support Payment rates remain unchanged under the +[2026 up-rating order, article 19](https://www.legislation.gov.uk/uksi/2026/148/article/19/made). +Older datasets combine it with CPI-uprated Widowed Parent's Allowance and +War Widow's Pension in `bsp_reported`. They keep CPI projection until the +producer separates those benefits. + +An explicitly supplied `widowed_parents_allowance_reported` person column +declares the completed split used by +[Microcosm PR #1383](https://github.com/PolicyEngine/microcosm/pull/1383), +including when every value in that column is zero. The automatic projector +then holds `bsp_reported` nominally flat; the separate allowance and +`war_widows_pension_reported` still grow by CPI. A war-widow column alone +does not declare completion. Unclassified off-rate amounts that the producer +retains in `bsp_reported` receive the same nominal treatment. + +Beyond the materialized projection horizon, the model holds BSP flat only +when its latest earlier input has an explicitly stored allowance input for +the same period. A calculated or carried allowance default does not count. +This also preserves the distinction when an external multi-year dataset +changes schema between observations. Supplied annual BSP amounts remain +authoritative. Disabling economic assumptions preserves those amounts when +either observation declares the split, rather than replacing separate BSP +with an older combined report. + +UK single-year files, materialized Core datasets and their projected children +preserve the column. Household situations retain their existing CPI behavior, +even when they supply the allowance separately. The change follows dataset +semantics, without inferring a split from a filename, model version or the +amount received. See [issue #2381](https://github.com/PolicyEngine/policyengine-uk/issues/2381). + ## Tests For dataset source routing, prefer stubbed tests that prove dispatch behavior diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 37ce1c3212..5c00738150 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -342,7 +342,16 @@ def apply_single_year_uprating( for table_name, df in zip(current_year.table_names, current_year.tables): if variable in df.columns: prev_year_value = getattr(previous_year, table_name)[variable] - current_year_value = prev_year_value * (1 + index_rel_change) + split_bsp = ( + variable == "bsp_reported" + and "widowed_parents_allowance_reported" + in previous_year.person.columns + ) + current_year_value = ( + prev_year_value + if split_bsp + else prev_year_value * (1 + index_rel_change) + ) getattr(current_year, table_name)[variable] = current_year_value # Next, apply custom uprating. @@ -841,6 +850,13 @@ def reset_growthfactor_uprating( base_table = getattr(base_year_dataset, table_name) current_table = getattr(current_year_dataset, table_name) for variable in growthfactor_uprated_variables: + # Split BSP amounts do not reflect economic growth. In a + # schema transition, the base amount can include other benefits. + if variable == "bsp_reported" and ( + "widowed_parents_allowance_reported" in base_table.columns + or "widowed_parents_allowance_reported" in current_table.columns + ): + continue if variable in current_table.columns and variable in base_table.columns: current_table[variable] = base_table[variable].values if reset_energy: diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index 17f363f2cd..d6d1bad4ca 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -35,11 +35,10 @@ gov.economic_assumptions.yoy_growth.obr.consumer_price_index: - alcohol_and_tobacco_consumption - bus_fare_spending - bus_fare_spending_reported -# bsp_reported still holds FRS code 9 (War Widow's Pension, CPI-uprated) and -# the Widowed Parent's Allowance in code 6 (CPI-uprated) as well as -# Bereavement Support Payment, whose rates are frozen. It moves off this index -# once datasets put code 9 in war_widows_pension_reported and Widowed -# Parent's Allowance in widowed_parents_allowance_reported (#2381). +# Legacy bsp_reported can include War Widow's Pension and Widowed Parent's +# Allowance, so it retains CPI here. The projector holds it flat when the +# source person table supplies widowed_parents_allowance_reported, declaring +# the completed split. The separate benefits still grow by CPI (#2381). - bsp_reported - carers_allowance_reported - child_benefit_reported diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index f175b7e7b0..3d64c4aac5 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -54,6 +54,7 @@ capital_gains_application_metadata, ) from policyengine_uk.utils.dependencies import get_variable_dependencies +from policyengine_uk.utils.data_source import built_from_data from policyengine_uk.utils.input_dataframe_export import input_dataframe from policyengine_uk.utils.supplied_inputs import ( SUPPLIED_INPUT_VARIABLES, @@ -1359,6 +1360,46 @@ def calculate_all(self, year: int) -> None: fiscal_year=year, ) + def _calculate(self, variable_name: str, period: Period = None): + # Core requires an indexed input to have no formula. Keep the legacy + # CPI declaration, but carry split dataset BSP nominally after the + # materialized horizon, including through Core's monthly conversion. + if ( + variable_name == "bsp_reported" + and period is not None + and period.unit == "year" + and period.size == 1 + and built_from_data(self) + ): + variable = self.tax_benefit_system.get_variable(variable_name) + holder = self.get_holder(variable_name) + inputs = holder.get_input_periods(self.branch_name) + if ( + not variable.formulas + and not variable.is_neutralized + and variable.uprating + == "gov.economic_assumptions.indices.obr.consumer_price_index" + and period not in inputs + ): + source = max( + ( + known + for known in inputs + if known.unit == "year" and known.start < period.start + ), + key=lambda known: (known.start, known.stop), + default=None, + ) + if source is not None: + allowance = self.get_holder("widowed_parents_allowance_reported") + split = source in allowance.get_input_periods(self.branch_name) + # Track the dependency even when the marker is absent. + # A calculated default must not declare a completed split. + self.calculate("widowed_parents_allowance_reported", source) + if split: + return self.calculate(variable_name, source) + return super()._calculate(variable_name, period) + def calculate( self, variable_name: str = None, diff --git a/policyengine_uk/tests/test_bsp_dataset_projection.py b/policyengine_uk/tests/test_bsp_dataset_projection.py new file mode 100644 index 0000000000..20cd2e1646 --- /dev/null +++ b/policyengine_uk/tests/test_bsp_dataset_projection.py @@ -0,0 +1,401 @@ +"""Only a completed survey bereavement split freezes BSP projection (#2381). + +The WPA column is the producer's completed-split signal, including an all-zero +column. BSP rates remain fixed under the Bereavement Support Payment Regulations +2017, regulation 3; the legacy combined survey field retains its CPI treatment. +The controlled 10% annual growth below is a projection oracle, not a forecast. +""" + +import h5py +import numpy as np +import pandas as pd +import pytest +from policyengine_core.data import Dataset +from policyengine_core.reforms import Reform + +from policyengine_uk import CountryTaxBenefitSystem, Simulation +from policyengine_uk.data import UKMultiYearDataset, UKSingleYearDataset +from policyengine_uk.data.economic_assumptions import extend_single_year_dataset +from policyengine_uk.model_api import YEAR, Person, Variable +from policyengine_uk.scenarios import no_economic_assumptions +from policyengine_uk.system import system +from policyengine_uk.utils.scenario import Scenario + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +BSP = "bsp_reported" +WPA = "widowed_parents_allowance_reported" +WAR_WIDOW = "war_widows_pension_reported" +CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" +CPI_GROWTH = "gov.economic_assumptions.yoy_growth.obr.consumer_price_index" +BASE_YEAR = 2024 +AMOUNTS = {BSP: 100.0, WPA: 70.0, WAR_WIDOW: 60.0} + + +def bereavement_dataset(*, wpa=70.0, war_widow=True, year=BASE_YEAR): + """Two unrelated adults, one claimant and one with no reported benefits.""" + person = { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 2], + "age": [40, 50], + BSP: [AMOUNTS[BSP], 0.0], + } + if wpa is not None: + person[WPA] = [wpa, 0.0] + if war_widow: + person[WAR_WIDOW] = [AMOUNTS[WAR_WIDOW], 0.0] + return UKSingleYearDataset( + person=pd.DataFrame(person), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1, 2], + "region": ["LONDON", "WALES"], + "tenure_type": ["OWNED_OUTRIGHT", "OWNED_OUTRIGHT"], + "council_tax": [0.0, 0.0], + "rent": [0.0, 0.0], + "household_weight": [1.0, 2.0], + } + ), + fiscal_year=year, + ) + + +def controlled_changes(): + return { + CPI: { + f"year:{year}-01-01:1": 100 * 1.1 ** (year - BASE_YEAR) + for year in range(BASE_YEAR, 2036) + }, + CPI_GROWTH: {f"year:{year}-01-01:1": 0.1 for year in range(BASE_YEAR, 2036)}, + } + + +@pytest.fixture(scope="module") +def controlled_parameters(): + parameters = CountryTaxBenefitSystem().parameters.clone() + for name, periods in controlled_changes().items(): + for period, value in periods.items(): + parameters.get_child(name).update(period=period, value=value) + return parameters + + +def controlled_scenario(): + return Scenario( + parameter_changes=controlled_changes(), applied_before_data_load=True + ) + + +def assert_reported(simulation, name, year, amount): + np.testing.assert_allclose( + simulation.calculate(name, year), [amount, 0.0], rtol=1e-6 + ) + + +@pytest.mark.parametrize("wpa", [70.0, 0.0], ids=["positive-wpa", "zero-wpa"]) +def test_completed_split_freezes_bsp_in_each_projected_year(controlled_parameters, wpa): + source = bereavement_dataset(wpa=wpa) + before = source.person.copy(deep=True) + projected = extend_single_year_dataset(source, controlled_parameters, end_year=2032) + for year in (2024, 2029, 2032): + np.testing.assert_array_equal(projected[year].person[BSP], [100.0, 0.0]) + factor = 1.1 ** (year - BASE_YEAR) + np.testing.assert_allclose(projected[year].person[WPA], [wpa * factor, 0.0]) + np.testing.assert_allclose( + projected[year].person[WAR_WIDOW], [60.0 * factor, 0.0] + ) + pd.testing.assert_frame_equal(source.person, before) + + +@pytest.mark.parametrize("wpa", [70.0, 0.0], ids=["positive-wpa", "zero-wpa"]) +def test_completed_split_freezes_bsp_after_the_projection_horizon(wpa): + simulation = Simulation( + dataset=bereavement_dataset(wpa=wpa), scenario=controlled_scenario() + ) + for year in (2029, 2032, 2035): + assert_reported(simulation, BSP, year, 100.0) + factor = 1.1 ** (year - BASE_YEAR) + assert_reported(simulation, WPA, year, wpa * factor) + assert_reported(simulation, WAR_WIDOW, year, 60.0 * factor) + + +def test_wpa_column_alone_signals_the_completed_split(): + simulation = Simulation( + dataset=bereavement_dataset(wpa=0.0, war_widow=False), + scenario=controlled_scenario(), + ) + for year in (2029, 2032, 2035): + assert_reported(simulation, BSP, year, 100.0) + + +@pytest.mark.parametrize("war_widow", [False, True], ids=["legacy", "partial-split"]) +def test_missing_wpa_column_preserves_legacy_cpi_uprating(war_widow): + simulation = Simulation( + dataset=bereavement_dataset(wpa=None, war_widow=war_widow), + scenario=controlled_scenario(), + ) + for year in (2024, 2029, 2032, 2035): + factor = 1.1 ** (year - BASE_YEAR) + assert_reported(simulation, BSP, year, 100.0 * factor) + if war_widow: + assert_reported(simulation, WAR_WIDOW, year, 60.0 * factor) + + +def test_calculating_default_wpa_does_not_migrate_a_legacy_dataset(): + simulation = Simulation( + dataset=bereavement_dataset(wpa=None), scenario=controlled_scenario() + ) + assert_reported(simulation, WPA, 2024, 0.0) + assert_reported(simulation, WPA, 2035, 0.0) + assert_reported(simulation, BSP, 2035, 100.0 * 1.1**11) + + +def core_arrays(source): + arrays = {} + for table in source.tables: + for name in table.columns: + values = table[name].to_numpy() + possible_values = getattr(system.variables[name], "possible_values", None) + if possible_values is not None: + values = np.asarray(possible_values.encode(values)) + arrays[name] = {source.time_period: values} + return arrays + + +class InMemoryCore(Dataset): + """An already materialized, fixture-scale Core dataset.""" + + def __init__(self, arrays): + self.arrays = arrays + + def load_dataset(self): + return self.arrays + + +@pytest.mark.parametrize("source_kind", ["single", "uk-h5", "core-h5", "core"]) +@pytest.mark.parametrize("wpa", [None, 0.0], ids=["partial-split", "completed-split"]) +def test_source_conversion_preserves_the_completed_split_signal( + source_kind, wpa, tmp_path +): + source = bereavement_dataset(wpa=wpa) + if source_kind == "uk-h5": + path = tmp_path / "bereavement.h5" + source.save(path) + source = path + elif source_kind == "core-h5": + path = tmp_path / "bereavement-core.h5" + with h5py.File(path, "w") as file: + for name, periods in core_arrays(source).items(): + for period, values in periods.items(): + file.create_dataset(f"{name}/{period}", data=values) + source = path + elif source_kind == "core": + source = InMemoryCore(core_arrays(source)) + simulation = Simulation(dataset=source, scenario=controlled_scenario()) + for year in (2029, 2032, 2035): + expected = 100.0 if wpa is not None else 100.0 * 1.1 ** (year - BASE_YEAR) + assert_reported(simulation, BSP, year, expected) + assert_reported(simulation, WAR_WIDOW, year, 60.0 * 1.1 ** (year - BASE_YEAR)) + + +def test_serialized_projected_child_retains_the_split_for_later_projection( + controlled_parameters, tmp_path +): + original = bereavement_dataset(wpa=0.0) + projected = extend_single_year_dataset( + original, controlled_parameters, end_year=2029 + ) + path = tmp_path / "projected-child.h5" + projected[2029].save(path) + simulation = Simulation(dataset=path, scenario=controlled_scenario()) + for year in (2029, 2032, 2035): + assert_reported(simulation, BSP, year, 100.0) + assert_reported(simulation, WAR_WIDOW, year, 60.0 * 1.1 ** (year - BASE_YEAR)) + np.testing.assert_array_equal(original.person[BSP], [100.0, 0.0]) + + +@pytest.mark.parametrize("split_first", [False, True]) +def test_split_and_legacy_simulations_and_their_clones_are_isolated(split_first): + def build(wpa): + return Simulation( + dataset=bereavement_dataset(wpa=wpa), scenario=controlled_scenario() + ) + + if split_first: + split, legacy = build(0.0), build(None) + else: + legacy, split = build(None), build(0.0) + for simulation in (split, split.clone(), split.get_branch("split-child")): + assert_reported(simulation, BSP, 2035, 100.0) + for simulation in (legacy, legacy.clone(), legacy.get_branch("legacy-child")): + assert_reported(simulation, BSP, 2035, 100.0 * 1.1**11) + + +def test_reform_and_baseline_both_recognize_the_completed_split(): + source = bereavement_dataset() + reformed = Simulation(dataset=source, reform=controlled_changes()) + for year in (2029, 2032, 2035): + assert_reported(reformed, BSP, year, 100.0) + assert_reported(reformed.baseline, BSP, year, 100.0) + assert_reported(reformed, WPA, year, 70.0 * 1.1 ** (year - BASE_YEAR)) + # The two CPI histories differ, so this proves the baseline remains distinct. + assert reformed.calculate(WPA, 2029)[0] != pytest.approx( + reformed.baseline.calculate(WPA, 2029)[0] + ) + + +def test_explicit_multiyear_bsp_values_remain_authoritative(): + first = bereavement_dataset() + later = bereavement_dataset(year=2032) + later.person[BSP] = [777.0, 0.0] + source = UKMultiYearDataset(datasets=[first, later]) + before = later.person.copy(deep=True) + simulation = Simulation(dataset=source, scenario=controlled_scenario()) + assert_reported(simulation, BSP, 2024, 100.0) + assert_reported(simulation, BSP, 2032, 777.0) + assert_reported(simulation, BSP, 2035, 777.0) + pd.testing.assert_frame_equal(later.person, before) + + +@pytest.mark.parametrize( + "first_split,later_split,expected", + [ + (False, False, 100.0), + (False, True, 777.0), + (True, False, 777.0), + (True, True, 777.0), + ], + ids=["both-legacy", "new-split", "later-without-wpa", "both-split"], +) +def test_no_economic_reset_preserves_separate_bsp_observations( + first_split, later_split, expected +): + first = bereavement_dataset(wpa=0.0 if first_split else None) + later = bereavement_dataset(wpa=0.0 if later_split else None, year=2032) + later.person[BSP] = [777.0, 0.0] + source = UKMultiYearDataset(datasets=[first, later]) + simulation = Simulation(dataset=source, scenario=no_economic_assumptions) + assert_reported(simulation, BSP, 2032, expected) + np.testing.assert_array_equal(source[2032].person[BSP], [777.0, 0.0]) + + +def household_situation(wpa): + person = {"age": {2024: 40}, BSP: {2024: 100.0}} + if wpa is not None: + person[WPA] = {2024: wpa} + return { + "people": {"person": person}, + "benunits": {"benunit": {"members": ["person"]}}, + "households": {"household": {"members": ["person"]}}, + } + + +@pytest.mark.parametrize("wpa", [None, 0.0, 70.0]) +def test_household_inputs_keep_cpi_treatment_even_with_explicit_wpa(wpa): + simulation = Simulation( + situation=household_situation(wpa), scenario=controlled_scenario() + ) + for year in (2029, 2032, 2035): + np.testing.assert_allclose( + simulation.calculate(BSP, year), + [100.0 * 1.1 ** (year - BASE_YEAR)], + rtol=1e-6, + ) + + +def test_rebuilding_a_split_clone_as_a_household_drops_dataset_freeze(): + original = Simulation( + dataset=bereavement_dataset(wpa=0.0), scenario=controlled_scenario() + ) + assert_reported(original, BSP, 2035, 100.0) + rebuilt = original.clone() + rebuilt.build_from_situation(household_situation(wpa=0.0)) + np.testing.assert_allclose( + rebuilt.calculate(BSP, 2035), [100.0 * 1.1**11], rtol=1e-6 + ) + assert_reported(original, BSP, 2035, 100.0) + + +def test_rebuilding_a_split_clone_as_legacy_data_drops_the_split_signal(): + original = Simulation( + dataset=bereavement_dataset(wpa=0.0), scenario=controlled_scenario() + ) + assert_reported(original, BSP, 2035, 100.0) + rebuilt = original.clone() + rebuilt.build_from_single_year_dataset(bereavement_dataset(wpa=None)) + assert_reported(rebuilt, BSP, 2035, 100.0 * 1.1**11) + assert_reported(original, BSP, 2035, 100.0) + + +@pytest.mark.parametrize("month", ["2035-01", "2035-07"]) +def test_split_bsp_beyond_the_horizon_is_divided_once_for_monthly_output(month): + simulation = Simulation( + dataset=bereavement_dataset(), scenario=controlled_scenario() + ) + assert_reported(simulation, BSP, month, 100.0 / 12) + assert_reported(simulation, BSP, 2035, 100.0) + assert_reported(simulation, WPA, month, 70.0 * 1.1**11 / 12) + + +@pytest.mark.parametrize("override", ["neutralize", "formula"]) +def test_split_bsp_fallback_respects_structural_variable_overrides(override): + class bsp_reported(Variable): + value_type = float + entity = Person + definition_period = YEAR + label = "BSP (synthetic formula override)" + uprating = None + + def formula(person, period, parameters): + # A synthetic override deliberately pays the separately uprated WPA. + return person("widowed_parents_allowance_reported", period) + + class change_bsp(Reform): + def apply(self): + if override == "neutralize": + self.neutralize_variable(BSP) + else: + self.replace_variable(bsp_reported) + + simulation = Simulation( + dataset=bereavement_dataset(), scenario=controlled_scenario() + ) + assert_reported(simulation, BSP, 2035, 100.0) + simulation.apply_reform(change_bsp) + expected = 0.0 if override == "neutralize" else 70.0 * 1.1**11 + assert_reported(simulation, BSP, 2035, expected) + + +def test_branch_split_declaration_invalidates_previously_calculated_cpi_fallback(): + original = Simulation( + dataset=bereavement_dataset(wpa=None), scenario=controlled_scenario() + ) + assert_reported(original, BSP, 2035, 100.0 * 1.1**11) + branch = original.get_branch("completed-split") + assert_reported(branch, BSP, 2035, 100.0 * 1.1**11) + # Equal to the calculated default, but now explicitly supplied in the BSP + # source year. The provenance change alone declares a completed split. + branch.set_input(WPA, 2032, [0.0, 0.0]) + assert_reported(branch, BSP, 2035, 100.0 * 1.1**8) + assert_reported(original, BSP, 2035, 100.0 * 1.1**11) + + +def test_split_bsp_fallback_works_with_cache_opt_out(): + simulation = Simulation( + dataset=bereavement_dataset(), scenario=controlled_scenario() + ) + simulation.opt_out_cache = True + simulation.tax_benefit_system.cache_blacklist = [BSP, WPA, WAR_WIDOW] + for year in (2035, 2034, 2035): + assert_reported(simulation, BSP, year, 100.0) + assert_reported(simulation, WPA, year, 70.0 * 1.1 ** (year - BASE_YEAR)) + + +def test_a_split_declaration_in_another_year_does_not_change_legacy_source_bsp(): + first = bereavement_dataset(wpa=0.0) + later = bereavement_dataset(wpa=None, year=2032) + later.person[BSP] = [777.0, 0.0] + source = UKMultiYearDataset(datasets=[first, later]) + simulation = Simulation(dataset=source, scenario=controlled_scenario()) + assert_reported(simulation, BSP, 2035, 777.0 * 1.1**3) diff --git a/policyengine_uk/tests/test_projection_horizon.py b/policyengine_uk/tests/test_projection_horizon.py index 10b3bb020c..83e4d1909b 100644 --- a/policyengine_uk/tests/test_projection_horizon.py +++ b/policyengine_uk/tests/test_projection_horizon.py @@ -194,7 +194,8 @@ def test_every_year_to_the_horizon_is_a_dataset_year(projected, sim): def test_each_singly_indexed_input_grows_by_its_index_every_year(projected): """For every variable uprating_indices.yaml lists and every projected year, check its index: residential stocks use the original observation - and cumulative HPI; other inputs grow from the previous year's raw YoY.""" + and cumulative HPI. This fixture stores the completed bereavement split, + so BSP stays flat; other inputs grow from the previous year's raw YoY.""" observed = _dataset() for year in range(BASE_YEAR + 1, PROJECTION_END_YEAR + 1): for variable, index in INDEX_OF.items(): @@ -205,7 +206,9 @@ def test_each_singly_indexed_input_grows_by_its_index_every_year(projected): else: previous = _table(projected[year - 1], variable)[variable].to_numpy() growth = parameters.get_child(index)(year) - expected = previous * (1 + growth) + expected = ( + previous if variable == "bsp_reported" else previous * (1 + growth) + ) assert current == pytest.approx(expected), (variable, year) @@ -302,7 +305,8 @@ def test_growth_ratio_is_one_plus_the_index(amounts, year, variable): """For any positive amounts, any projected year and any singly indexed variable, check its annual growth record by record. Residential ratios come from original-observation cumulative HPI levels at storage precision; - every other variable retains its raw one-plus-YoY growth expectation.""" + split BSP stays flat; every other variable retains its raw one-plus-YoY + growth expectation.""" projected = extend_single_year_dataset( _dataset(np.array(amounts)), parameters, end_year=year ) @@ -315,6 +319,8 @@ def test_growth_ratio_is_one_plus_the_index(amounts, year, variable): assert current == pytest.approx(expected_current) assert previous == pytest.approx(expected_previous) expected_ratio = expected_current / expected_previous + elif variable == "bsp_reported": + expected_ratio = np.ones(len(amounts)) else: index = parameters.get_child(INDEX_OF[variable])(year) expected_ratio = np.full(len(amounts), 1 + index) diff --git a/policyengine_uk/variables/gov/dwp/bsp_reported.py b/policyengine_uk/variables/gov/dwp/bsp_reported.py index 22bfc6c103..7a22632140 100644 --- a/policyengine_uk/variables/gov/dwp/bsp_reported.py +++ b/policyengine_uk/variables/gov/dwp/bsp_reported.py @@ -8,16 +8,20 @@ class bsp_reported(Variable): documentation = ( "Reported Bereavement Support Payment. Its rates have been frozen " "since April 2017 (Bereavement Support Payment Regulations 2017 reg 3, " - "held unchanged by each up-rating order). Datasets built so far fill " - "this from Family Resources Survey benefit codes 6 (Bereavement " - "Support Payment or Widowed Parent's Allowance) and 9 (War Widow's or " - "Widower's Pension). Code 9 is most of the value and is uprated by " - "CPI, as is Widowed Parent's Allowance, so this variable keeps CPI " - "uprating until datasets move code 9 into war_widows_pension_reported " - "and Widowed Parent's Allowance into " - "widowed_parents_allowance_reported. It should then be frozen (#2381)." + "held unchanged by each up-rating order). A dataset that explicitly " + "supplies widowed_parents_allowance_reported in the same source year " + "declares the completed bereavement-benefit split: this input stays " + "nominally flat, while the separate allowance and War Widow's Pension " + "keep CPI uprating. Legacy datasets can combine Family Resources " + "Survey benefit codes 6 and 9 here, so they retain CPI uprating, as do " + "household situations. Unclassified amounts that the producer retains " + "in the split BSP column also stay flat. See the dataset-source guide " + "and #2381." ) definition_period = YEAR unit = GBP - reference = "https://www.legislation.gov.uk/uksi/2017/410/regulation/3" + reference = ( + "https://www.legislation.gov.uk/uksi/2017/410/regulation/3", + "https://www.legislation.gov.uk/uksi/2026/148/article/19/made", + ) uprating = "gov.economic_assumptions.indices.obr.consumer_price_index" From 72daf2817bbbb59d933ec205b5a2246efa89c7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 24 Oct 2026 23:37:34 +0100 Subject: [PATCH 2/2] Fix issues from review: guard completed bereavement split --- changelog.d/2381.fixed.md | 2 +- docs/book/assumptions/growthfactors.md | 6 + .../gov/dwp/bereavement-support-plan.md | 35 ++--- docs/engineering/skills/dataset-sources.md | 24 ++-- policyengine_uk/data/economic_assumptions.py | 17 ++- policyengine_uk/data/uprating_indices.yaml | 4 +- policyengine_uk/simulation.py | 15 ++- .../tests/test_bsp_dataset_projection.py | 125 +++++++++++++++--- .../variables/gov/dwp/bsp_reported.py | 7 +- .../variables/gov/dwp/war_widows_pension.py | 6 +- 10 files changed, 182 insertions(+), 59 deletions(-) diff --git a/changelog.d/2381.fixed.md b/changelog.d/2381.fixed.md index d77f69d55a..34e1516fff 100644 --- a/changelog.d/2381.fixed.md +++ b/changelog.d/2381.fixed.md @@ -1 +1 @@ -- Held reported Bereavement Support Payment flat for datasets declaring the completed benefit split, while preserving CPI uprating for legacy mixed reports and the separate pensions. +Held reported Bereavement Support Payment flat for datasets supplying both separate bereavement-benefit inputs, while preserving CPI uprating for legacy and partial splits and the separate pensions. diff --git a/docs/book/assumptions/growthfactors.md b/docs/book/assumptions/growthfactors.md index 6cef640db0..b26c734edc 100644 --- a/docs/book/assumptions/growthfactors.md +++ b/docs/book/assumptions/growthfactors.md @@ -22,6 +22,12 @@ We apply CPI to these variables: - Other variables– `afcs_reported`, `bsp_reported`, `childcare_expenses`, `free_school_fruit_veg`, `free_school_meals`, `free_school_milk`, `maintenance_expenses`, `statutory_maternity_pay`, `statutory_paternity_pay`, `statutory_sick_pay`, `state_pension`, `war_widows_pension_reported`, `widowed_parents_allowance_reported` +`bsp_reported` stays nominally flat when a dataset explicitly supplies both +`war_widows_pension_reported` and `widowed_parents_allowance_reported` in +the same source year, including all-zero columns. Datasets with either +column missing and household situations retain CPI uprating. See the +[bereavement-benefit split guide](https://github.com/PolicyEngine/policyengine-uk/blob/main/docs/engineering/skills/dataset-sources.md#bereavement-benefit-split). + ## Road fuel volume We use HMRC road-fuel clearances and OBR-implied forecast volumes for petrol and diesel spending, because PolicyEngine converts these spending fields into litres by dividing by pump prices. diff --git a/docs/book/programs/gov/dwp/bereavement-support-plan.md b/docs/book/programs/gov/dwp/bereavement-support-plan.md index 0cb8887915..60a0849804 100644 --- a/docs/book/programs/gov/dwp/bereavement-support-plan.md +++ b/docs/book/programs/gov/dwp/bereavement-support-plan.md @@ -55,15 +55,19 @@ That means: - Since [#2382](https://github.com/PolicyEngine/policyengine-uk/pull/2382) (2.135.7), War Widow's Pension has its own variables, `war_widows_pension_reported` and `war_widows_pension`. `bsp_reported` - keeps CPI uprating only until the datasets move FRS code 9 out of it - and WPA into `widowed_parents_allowance_reported` (code 9 is done on - Microcosm's main branch in microcosm#1333; still open for the - Enhanced FRS in - [policyengine-uk-data#578](https://github.com/PolicyEngine/policyengine-uk-data/issues/578), - and [microcosm#1222](https://github.com/PolicyEngine/microcosm/issues/1222) - is still open); - [#2381](https://github.com/PolicyEngine/policyengine-uk/issues/2381) - tracks freezing it after that. + stays nominally flat for datasets that explicitly supply both + `war_widows_pension_reported` and `widowed_parents_allowance_reported` + in the same source year, including all-zero columns. Datasets with + either column missing retain CPI uprating, as do household situations. + Microcosm's merged + [#1383](https://github.com/PolicyEngine/microcosm/pull/1383) supplies both + columns and closes + [#1222](https://github.com/PolicyEngine/microcosm/issues/1222). Older + releases and the Enhanced FRS retain the compatibility path; + [policyengine-uk-data#578](https://github.com/PolicyEngine/policyengine-uk-data/issues/578) + tracks the Enhanced FRS code-9 move. See the + [dataset-source guide](https://github.com/PolicyEngine/policyengine-uk/blob/main/docs/engineering/skills/dataset-sources.md#bereavement-benefit-split) + for projection, fallback and off-rate residual treatment. - Since [#2572](https://github.com/PolicyEngine/policyengine-uk/issues/2572), WPA also has its own variables, `widowed_parents_allowance_reported` (CPI-uprated) and `widowed_parents_allowance`. Unlike BSP, WPA is taxable @@ -74,8 +78,9 @@ That means: cap and an overlapping benefit for Carer's Allowance and Carer Support Payment. It ends at State Pension age and while the recipient lives with a partner ([SSCBA 1992 s. 39A(4) and (5)][sscba-39a]), so it never reaches - Pension Credit. The datasets still put it in `bsp_reported` until they - split FRS code 6. + Pension Credit. Older datasets still put it in `bsp_reported`; + Microcosm #1383 separates amounts matching its positive WPA rate rule, + while retaining unclassified off-rate amounts in BSP. ## Proposed scope @@ -180,10 +185,10 @@ November 2024 ([DWP, July 2025](https://www.gov.uk/government/publications/bereavement-benefit-remedial-order-uptake-february-2023-to-november-2024)). Existing awards continue while the surviving parent is entitled to Child Benefit and under State Pension age, so some recipients remain. In -the current data they sit inside `bsp_reported` (see above), so they -are already counted in income. Once datasets fill -`widowed_parents_allowance_reported`, they are told apart from BSP; the -gap left is a rule-based amount for reforms. +older datasets they sit inside `bsp_reported` (see above), so they are +already counted in income. Datasets that fill +`widowed_parents_allowance_reported` distinguish those reported amounts +from BSP; the gap left is a rule-based amount for reforms. ### Phase 3: data-side bereavement inputs diff --git a/docs/engineering/skills/dataset-sources.md b/docs/engineering/skills/dataset-sources.md index f6ca5c0107..08ab4e1b5b 100644 --- a/docs/engineering/skills/dataset-sources.md +++ b/docs/engineering/skills/dataset-sources.md @@ -129,18 +129,22 @@ Older datasets combine it with CPI-uprated Widowed Parent's Allowance and War Widow's Pension in `bsp_reported`. They keep CPI projection until the producer separates those benefits. -An explicitly supplied `widowed_parents_allowance_reported` person column -declares the completed split used by +Explicitly supplied `widowed_parents_allowance_reported` and +`war_widows_pension_reported` person columns together declare the completed +split used by the merged [Microcosm PR #1383](https://github.com/PolicyEngine/microcosm/pull/1383), -including when every value in that column is zero. The automatic projector +including when every value in either column is zero. The automatic projector then holds `bsp_reported` nominally flat; the separate allowance and -`war_widows_pension_reported` still grow by CPI. A war-widow column alone -does not declare completion. Unclassified off-rate amounts that the producer -retains in `bsp_reported` receive the same nominal treatment. +war-widow inputs still grow by CPI. Either column alone leaves BSP on the +legacy CPI path. Unclassified off-rate amounts that the producer retains in +`bsp_reported` receive the same nominal treatment. These can include Widowed +Parent's Allowance with an inherited additional pension above its rate; +freezing that CPI-linked component understates it. Beyond the materialized projection horizon, the model holds BSP flat only -when its latest earlier input has an explicitly stored allowance input for -the same period. A calculated or carried allowance default does not count. +when its latest earlier input has both explicitly stored split inputs for +the same period. A calculated or carried default does not count, nor do +inputs supplied in different periods. This also preserves the distinction when an external multi-year dataset changes schema between observations. Supplied annual BSP amounts remain authoritative. Disabling economic assumptions preserves those amounts when @@ -148,8 +152,8 @@ either observation declares the split, rather than replacing separate BSP with an older combined report. UK single-year files, materialized Core datasets and their projected children -preserve the column. Household situations retain their existing CPI behavior, -even when they supply the allowance separately. The change follows dataset +preserve both columns. Household situations retain their existing CPI behavior, +even when they supply the separate benefits. The change follows dataset semantics, without inferring a split from a filename, model version or the amount received. See [issue #2381](https://github.com/PolicyEngine/policyengine-uk/issues/2381). diff --git a/policyengine_uk/data/economic_assumptions.py b/policyengine_uk/data/economic_assumptions.py index 5c00738150..6af1d33aaa 100644 --- a/policyengine_uk/data/economic_assumptions.py +++ b/policyengine_uk/data/economic_assumptions.py @@ -128,6 +128,10 @@ } ) DATASET_UPRATING_EXCLUSIONS = DECLARED_ENERGY_INPUTS +# Both separated benefits must be explicit before BSP can be held nominally flat. +BEREAVEMENT_SPLIT_INPUTS = frozenset( + {"war_widows_pension_reported", "widowed_parents_allowance_reported"} +) # Inputs that describe only the year they are stored for, so are not carried # into projected years. Unused annual allowance comes from the three tax years # before this one (Finance Act 2004 s. 228A), a window that moves each year; a @@ -344,8 +348,9 @@ def apply_single_year_uprating( prev_year_value = getattr(previous_year, table_name)[variable] split_bsp = ( variable == "bsp_reported" - and "widowed_parents_allowance_reported" - in previous_year.person.columns + and BEREAVEMENT_SPLIT_INPUTS.issubset( + previous_year.person.columns + ) ) current_year_value = ( prev_year_value @@ -843,6 +848,9 @@ def reset_growthfactor_uprating( current_year_dataset.energy_price_basis == base_year_dataset.energy_price_basis ) + preserve_bsp = BEREAVEMENT_SPLIT_INPUTS.issubset( + base_year_dataset.person.columns + ) or BEREAVEMENT_SPLIT_INPUTS.issubset(current_year_dataset.person.columns) # One declaration describes every stored energy amount. Keep the # later group intact if a partial reset would mix different bases. @@ -852,10 +860,7 @@ def reset_growthfactor_uprating( for variable in growthfactor_uprated_variables: # Split BSP amounts do not reflect economic growth. In a # schema transition, the base amount can include other benefits. - if variable == "bsp_reported" and ( - "widowed_parents_allowance_reported" in base_table.columns - or "widowed_parents_allowance_reported" in current_table.columns - ): + if variable == "bsp_reported" and preserve_bsp: continue if variable in current_table.columns and variable in base_table.columns: current_table[variable] = base_table[variable].values diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index d6d1bad4ca..b76f305761 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -37,8 +37,8 @@ gov.economic_assumptions.yoy_growth.obr.consumer_price_index: - bus_fare_spending_reported # Legacy bsp_reported can include War Widow's Pension and Widowed Parent's # Allowance, so it retains CPI here. The projector holds it flat when the -# source person table supplies widowed_parents_allowance_reported, declaring -# the completed split. The separate benefits still grow by CPI (#2381). +# source person table supplies both widowed_parents_allowance_reported and +# war_widows_pension_reported. The separate benefits still grow by CPI (#2381). - bsp_reported - carers_allowance_reported - child_benefit_reported diff --git a/policyengine_uk/simulation.py b/policyengine_uk/simulation.py index 3d64c4aac5..f1a359bde9 100644 --- a/policyengine_uk/simulation.py +++ b/policyengine_uk/simulation.py @@ -33,6 +33,7 @@ from policyengine_uk.utils.entity_ids import ENTITY_ID_VARIABLES, check_entity_ids from policyengine_uk.utils.scenario import ConstructionReformState, Scenario from policyengine_uk.data.economic_assumptions import ( + BEREAVEMENT_SPLIT_INPUTS, extend_single_year_dataset, reset_growthfactor_uprating, uprating_input_fingerprint, @@ -1391,11 +1392,15 @@ def _calculate(self, variable_name: str, period: Period = None): default=None, ) if source is not None: - allowance = self.get_holder("widowed_parents_allowance_reported") - split = source in allowance.get_input_periods(self.branch_name) - # Track the dependency even when the marker is absent. - # A calculated default must not declare a completed split. - self.calculate("widowed_parents_allowance_reported", source) + split = all( + source + in self.get_holder(name).get_input_periods(self.branch_name) + for name in BEREAVEMENT_SPLIT_INPUTS + ) + # Track both dependencies even when either input is absent. + # Calculated defaults do not declare a completed split. + for name in BEREAVEMENT_SPLIT_INPUTS: + self.calculate(name, source) if split: return self.calculate(variable_name, source) return super()._calculate(variable_name, period) diff --git a/policyengine_uk/tests/test_bsp_dataset_projection.py b/policyengine_uk/tests/test_bsp_dataset_projection.py index 20cd2e1646..36ae8f2be4 100644 --- a/policyengine_uk/tests/test_bsp_dataset_projection.py +++ b/policyengine_uk/tests/test_bsp_dataset_projection.py @@ -1,7 +1,7 @@ """Only a completed survey bereavement split freezes BSP projection (#2381). -The WPA column is the producer's completed-split signal, including an all-zero -column. BSP rates remain fixed under the Bereavement Support Payment Regulations +The WPA and war-widow columns together declare the producer's completed split, +including all-zero columns. BSP rates remain fixed under the Bereavement Support Payment Regulations 2017, regulation 3; the legacy combined survey field retains its CPI treatment. The controlled 10% annual growth below is a projection oracle, not a forecast. """ @@ -32,7 +32,9 @@ AMOUNTS = {BSP: 100.0, WPA: 70.0, WAR_WIDOW: 60.0} -def bereavement_dataset(*, wpa=70.0, war_widow=True, year=BASE_YEAR): +def bereavement_dataset( + *, wpa=70.0, war_widow=True, war_widow_amount=60.0, year=BASE_YEAR +): """Two unrelated adults, one claimant and one with no reported benefits.""" person = { "person_id": [1, 2], @@ -44,7 +46,7 @@ def bereavement_dataset(*, wpa=70.0, war_widow=True, year=BASE_YEAR): if wpa is not None: person[WPA] = [wpa, 0.0] if war_widow: - person[WAR_WIDOW] = [AMOUNTS[WAR_WIDOW], 0.0] + person[WAR_WIDOW] = [war_widow_amount, 0.0] return UKSingleYearDataset( person=pd.DataFrame(person), benunit=pd.DataFrame({"benunit_id": [1, 2]}), @@ -94,8 +96,11 @@ def assert_reported(simulation, name, year, amount): @pytest.mark.parametrize("wpa", [70.0, 0.0], ids=["positive-wpa", "zero-wpa"]) -def test_completed_split_freezes_bsp_in_each_projected_year(controlled_parameters, wpa): - source = bereavement_dataset(wpa=wpa) +@pytest.mark.parametrize("war_widow_amount", [60.0, 0.0]) +def test_completed_split_freezes_bsp_in_each_projected_year( + controlled_parameters, wpa, war_widow_amount +): + source = bereavement_dataset(wpa=wpa, war_widow_amount=war_widow_amount) before = source.person.copy(deep=True) projected = extend_single_year_dataset(source, controlled_parameters, end_year=2032) for year in (2024, 2029, 2032): @@ -103,30 +108,35 @@ def test_completed_split_freezes_bsp_in_each_projected_year(controlled_parameter factor = 1.1 ** (year - BASE_YEAR) np.testing.assert_allclose(projected[year].person[WPA], [wpa * factor, 0.0]) np.testing.assert_allclose( - projected[year].person[WAR_WIDOW], [60.0 * factor, 0.0] + projected[year].person[WAR_WIDOW], [war_widow_amount * factor, 0.0] ) pd.testing.assert_frame_equal(source.person, before) @pytest.mark.parametrize("wpa", [70.0, 0.0], ids=["positive-wpa", "zero-wpa"]) -def test_completed_split_freezes_bsp_after_the_projection_horizon(wpa): +@pytest.mark.parametrize("war_widow_amount", [60.0, 0.0]) +def test_completed_split_freezes_bsp_after_the_projection_horizon( + wpa, war_widow_amount +): simulation = Simulation( - dataset=bereavement_dataset(wpa=wpa), scenario=controlled_scenario() + dataset=bereavement_dataset(wpa=wpa, war_widow_amount=war_widow_amount), + scenario=controlled_scenario(), ) for year in (2029, 2032, 2035): assert_reported(simulation, BSP, year, 100.0) factor = 1.1 ** (year - BASE_YEAR) assert_reported(simulation, WPA, year, wpa * factor) - assert_reported(simulation, WAR_WIDOW, year, 60.0 * factor) + assert_reported(simulation, WAR_WIDOW, year, war_widow_amount * factor) -def test_wpa_column_alone_signals_the_completed_split(): +@pytest.mark.parametrize("wpa", [70.0, 0.0]) +def test_wpa_column_alone_preserves_legacy_cpi_uprating(wpa): simulation = Simulation( - dataset=bereavement_dataset(wpa=0.0, war_widow=False), + dataset=bereavement_dataset(wpa=wpa, war_widow=False), scenario=controlled_scenario(), ) for year in (2029, 2032, 2035): - assert_reported(simulation, BSP, year, 100.0) + assert_reported(simulation, BSP, year, 100.0 * 1.1 ** (year - BASE_YEAR)) @pytest.mark.parametrize("war_widow", [False, True], ids=["legacy", "partial-split"]) @@ -367,16 +377,23 @@ def apply(self): assert_reported(simulation, BSP, 2035, expected) -def test_branch_split_declaration_invalidates_previously_calculated_cpi_fallback(): +@pytest.mark.parametrize("missing_marker", [WPA, WAR_WIDOW]) +def test_branch_split_declaration_invalidates_previously_calculated_cpi_fallback( + missing_marker, +): original = Simulation( - dataset=bereavement_dataset(wpa=None), scenario=controlled_scenario() + dataset=bereavement_dataset( + wpa=None if missing_marker == WPA else 0.0, + war_widow=missing_marker != WAR_WIDOW, + ), + scenario=controlled_scenario(), ) assert_reported(original, BSP, 2035, 100.0 * 1.1**11) branch = original.get_branch("completed-split") assert_reported(branch, BSP, 2035, 100.0 * 1.1**11) # Equal to the calculated default, but now explicitly supplied in the BSP # source year. The provenance change alone declares a completed split. - branch.set_input(WPA, 2032, [0.0, 0.0]) + branch.set_input(missing_marker, 2032, [0.0, 0.0]) assert_reported(branch, BSP, 2035, 100.0 * 1.1**8) assert_reported(original, BSP, 2035, 100.0 * 1.1**11) @@ -399,3 +416,79 @@ def test_a_split_declaration_in_another_year_does_not_change_legacy_source_bsp() source = UKMultiYearDataset(datasets=[first, later]) simulation = Simulation(dataset=source, scenario=controlled_scenario()) assert_reported(simulation, BSP, 2035, 777.0 * 1.1**3) + + +SCHEMAS = [(None, False), (None, True), (0.0, False), (0.0, True)] +SCHEMA_IDS = ["legacy", "war-widow-only", "wpa-only", "complete"] + + +@pytest.mark.parametrize("wpa,war_widow", SCHEMAS, ids=SCHEMA_IDS) +def test_projection_requires_both_split_columns(controlled_parameters, wpa, war_widow): + source = bereavement_dataset(wpa=wpa, war_widow=war_widow, war_widow_amount=0.0) + projected = extend_single_year_dataset(source, controlled_parameters, end_year=2025) + expected = 100.0 if wpa is not None and war_widow else 110.0 + np.testing.assert_allclose(projected[2025].person[BSP], [expected, 0.0]) + + +@pytest.mark.parametrize("wpa,war_widow", SCHEMAS, ids=SCHEMA_IDS) +def test_latest_bsp_observation_requires_both_split_inputs(wpa, war_widow): + first = bereavement_dataset(wpa=0.0, war_widow_amount=0.0) + later = bereavement_dataset( + wpa=wpa, war_widow=war_widow, war_widow_amount=0.0, year=2032 + ) + later.person[BSP] = [777.0, 0.0] + simulation = Simulation( + dataset=UKMultiYearDataset(datasets=[first, later]), + scenario=controlled_scenario(), + ) + expected = 777.0 if wpa is not None and war_widow else 777.0 * 1.1**3 + assert_reported(simulation, BSP, 2035, expected) + + +@pytest.mark.parametrize("base_schema", SCHEMAS, ids=SCHEMA_IDS) +@pytest.mark.parametrize("later_schema", SCHEMAS, ids=SCHEMA_IDS) +def test_no_economic_reset_requires_a_complete_split_in_one_observation( + base_schema, later_schema +): + first = bereavement_dataset( + wpa=base_schema[0], war_widow=base_schema[1], war_widow_amount=0.0 + ) + later = bereavement_dataset( + wpa=later_schema[0], + war_widow=later_schema[1], + war_widow_amount=0.0, + year=2032, + ) + later.person[BSP] = [777.0, 0.0] + source = UKMultiYearDataset(datasets=[first, later]) + simulation = Simulation(dataset=source, scenario=no_economic_assumptions) + complete = any( + wpa is not None and war_widow for wpa, war_widow in (base_schema, later_schema) + ) + assert_reported(simulation, BSP, 2032, 777.0 if complete else 100.0) + np.testing.assert_array_equal(source[2032].person[BSP], [777.0, 0.0]) + + +def test_calculated_default_war_widow_does_not_complete_the_split(): + simulation = Simulation( + dataset=bereavement_dataset(wpa=0.0, war_widow=False), + scenario=controlled_scenario(), + ) + assert_reported(simulation, WAR_WIDOW, 2032, 0.0) + assert_reported(simulation, WAR_WIDOW, 2035, 0.0) + assert_reported(simulation, BSP, 2035, 100.0 * 1.1**11) + + +@pytest.mark.parametrize("earlier_marker", [WPA, WAR_WIDOW]) +def test_split_markers_from_different_periods_do_not_complete_the_split(earlier_marker): + first = bereavement_dataset(wpa=None, war_widow=False) + later = bereavement_dataset(wpa=None, war_widow=False, year=2032) + first.person[earlier_marker] = 0.0 + later_marker = WAR_WIDOW if earlier_marker == WPA else WPA + later.person[later_marker] = 0.0 + later.person[BSP] = [777.0, 0.0] + simulation = Simulation( + dataset=UKMultiYearDataset(datasets=[first, later]), + scenario=controlled_scenario(), + ) + assert_reported(simulation, BSP, 2035, 777.0 * 1.1**3) diff --git a/policyengine_uk/variables/gov/dwp/bsp_reported.py b/policyengine_uk/variables/gov/dwp/bsp_reported.py index 7a22632140..4e7ab02819 100644 --- a/policyengine_uk/variables/gov/dwp/bsp_reported.py +++ b/policyengine_uk/variables/gov/dwp/bsp_reported.py @@ -9,13 +9,16 @@ class bsp_reported(Variable): "Reported Bereavement Support Payment. Its rates have been frozen " "since April 2017 (Bereavement Support Payment Regulations 2017 reg 3, " "held unchanged by each up-rating order). A dataset that explicitly " - "supplies widowed_parents_allowance_reported in the same source year " + "supplies both widowed_parents_allowance_reported and " + "war_widows_pension_reported in the same source year " "declares the completed bereavement-benefit split: this input stays " "nominally flat, while the separate allowance and War Widow's Pension " "keep CPI uprating. Legacy datasets can combine Family Resources " "Survey benefit codes 6 and 9 here, so they retain CPI uprating, as do " "household situations. Unclassified amounts that the producer retains " - "in the split BSP column also stay flat. See the dataset-source guide " + "in the split BSP column also stay flat, understating any CPI-linked " + "Widowed Parent's Allowance with inherited additional pension retained " + "there. See the dataset-source guide " "and #2381." ) definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dwp/war_widows_pension.py b/policyengine_uk/variables/gov/dwp/war_widows_pension.py index e6987680b1..219ef52bc7 100644 --- a/policyengine_uk/variables/gov/dwp/war_widows_pension.py +++ b/policyengine_uk/variables/gov/dwp/war_widows_pension.py @@ -16,8 +16,10 @@ class war_widows_pension(Variable): "State Pension Credit Regulations 2002 Sch. IV paras 2 to 6 and 12 " "(for example a supplementary pension under article 23(2) of the " "Service Pensions Order 2006) are not separated. Family Resources " - "Survey benefit code 9 belongs here; the datasets still add it to " - "bsp_reported (policyengine-uk-data#578, microcosm#1222)." + "Survey benefit code 9 belongs here. Microcosm separates it into " + "war_widows_pension_reported (microcosm#1333); the Enhanced FRS and " + "older Microcosm releases can still include it in bsp_reported " + "(policyengine-uk-data#578)." ) definition_period = YEAR unit = GBP