From dfef1abe2c7bdc0c651e2750018032b4a6afe078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:48 +0100 Subject: [PATCH 01/28] Keep the ETB VAT expenditure rate a share in [0, 1] (#1164) The full-rate VAT expenditure rate is totvat x (1 - reduced-rate share) / standard rate over expdis - totvat, and the stage refused only a zero denominator. The ETB 2023 donor therefore kept the households whose VAT exceeds their disposable expenditure (negative rates) and those whose rate exceeds one, and the QRF drew both onto recipients: the ETB half of uk-data#572. - The derive step declares rate_bounds [0, 1]: donors with no expenditure after VAT are dropped, the rest capped at one, and a negative totvat fails the stage. A donor_rate receipt in the stage evidence counts both. - The output joins the stage's non-negative outputs. - The committed support bound is the declared range, checked against the donor, so no rounded donor extreme is committed. A disclosure-control item went to the author separately. - The schema admits rate_bounds; the typed mirror, the coverage manifest and the H2 fixture are regenerated. Co-Authored-By: Claude Opus 5.5 --- .../1164-etb-vat-rate-share-bounds.fixed.md | 1 + .../spec_engine/schema/sources.schema.json | 8 ++ .../build/uk/etb_vat_support_bounds.json | 6 +- .../src/microcosm/build/uk/source_stages.json | 12 ++- .../src/microcosm/build/uk/spec/sources.yaml | 8 +- .../src/microcosm/build/uk_runtime/etb_vat.py | 87 +++++++++++++++---- .../uk/test_uk_consumption_resources.py | 5 +- .../tests/engine_free/uk/test_uk_etb_vat.py | 80 ++++++++++++++++- .../parity/uk_spine/sources/fixture.json | 10 ++- tools/build_uk_e6_support_bounds.py | 31 ++++++- 10 files changed, 214 insertions(+), 34 deletions(-) create mode 100644 changelog.d/1164-etb-vat-rate-share-bounds.fixed.md diff --git a/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md b/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md new file mode 100644 index 000000000..e85246fe2 --- /dev/null +++ b/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md @@ -0,0 +1 @@ +The UK `etb_vat` stage keeps `full_rate_vat_expenditure_rate` a share of expenditure after VAT, in [0, 1] (microcosm#1164; the ETB half of uk-data#572). The rate is `totvat × (1 − reduced-rate share) / standard rate` over `expdis − totvat`, and the stage refused only a zero denominator. So the ETB 2023 donor kept the households whose VAT exceeds their disposable expenditure, with negative rates, and the few whose little expenditure after VAT gives a rate above one, and the QRF drew both onto recipients. The derive step now declares `rate_bounds: [0, 1]`; it drops donors with no expenditure after VAT, caps the rest at one, and fails on a negative `totvat`. A `donor_rate` receipt counts the rows dropped and capped. The output is declared non-negative, and the committed support bound is the declared range rather than a rounded donor extreme. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index f775b4895..cf415ad68 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -7084,6 +7084,14 @@ "range": { "type": "string" }, + "rate_bounds": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2, + "maxItems": 2 + }, "reduced_rate_share": { "type": "number" }, diff --git a/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json index a12cb90fa..ef16bd41c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json +++ b/packages/microcosm-build/src/microcosm/build/uk/etb_vat_support_bounds.json @@ -6,12 +6,12 @@ "ukds_study_number": 8856, "doi": "10.5255/UKDA-SN-8856-4", "tab_sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", - "sdc_treatment": "Exact donor min/max values are rounded outward to one significant figure before commit." + "sdc_treatment": "The bounds are the stage's declared [0, 1] share range, which the donor is checked to lie within; no donor value is committed." }, "bounds": { "full_rate_vat_expenditure_rate": [ - -4, - 60 + 0.0, + 1 ] }, "chronicle": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 83f3a125a..89b6973d2 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -4211,7 +4211,11 @@ "annualization_weeks": 52, "standard_rate": 0.2, "reduced_rate_share": 0.025, - "fail_loud_on_missing_rate": true + "fail_loud_on_missing_rate": true, + "rate_bounds": [ + 0.0, + 1.0 + ] }, { "kind": "materialize_rules_engine_predictors", @@ -4254,8 +4258,10 @@ "outputs": [ "full_rate_vat_expenditure_rate" ], - "nonnegative_outputs": [], - "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095)." + "nonnegative_outputs": [ + "full_rate_vat_expenditure_rate" + ], + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095)." }, { "stage": "etb_services", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index e271e3327..b3c8b57ea 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3443,6 +3443,9 @@ stages: standard_rate: 0.2 reduced_rate_share: 0.025 fail_loud_on_missing_rate: true + rate_bounds: + - 0.0 + - 1.0 - kind: materialize_rules_engine_predictors predictors: - is_SP_age @@ -3470,8 +3473,9 @@ stages: gate_coupling: The terminal family gate requires a valid mass-conserving MassChangeRecord carrying exactly this stage-specific reason. outputs: - full_rate_vat_expenditure_rate - nonnegative_outputs: [] - notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth. The recipient is_adult and is_child counts are the household''s FRS claimants and partners and its other members (is_uc_claimant), as the donor''s adults and childs count adults and dependent children, not the engine''s age-18 split (uk-data#486, microcosm#1095).' + nonnegative_outputs: + - full_rate_vat_expenditure_rate + notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household''s FRS claimants and partners and its other members (is_uc_claimant), as the donor''s adults and childs count adults and dependent children, not the engine''s age-18 split (uk-data#486, microcosm#1095).' - stage: etb_services survey: Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index f3b8fa42d..fccbbff21 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -49,26 +49,33 @@ if predictor not in UK_ETB_FAMILY_ROLE_COUNTS ) UK_ETB_VAT_OUTPUT_COLUMNS = ("full_rate_vat_expenditure_rate",) -# The donor-realized support includes negative rates (totvat can exceed expdis -# in the raw ETB accounts), so the output stays out of the nonnegative gate and -# the support gate is the guard — the net_financial_wealth precedent. -UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS: tuple[str, ...] = () +#: The full-rate VAT expenditure rate is a share of expenditure after VAT, so +#: it lies in [0, 1] (uk-data#572). The raw ETB accounts carry households whose +#: VAT equals or exceeds their disposable expenditure, which leave nothing for +#: the rate to be a share of; they are dropped from the donor, and the rates +#: above one, where little expenditure is left after VAT, are capped at one. +UK_ETB_VAT_RATE_BOUNDS = (0.0, 1.0) +UK_ETB_VAT_NONNEGATIVE_OUTPUT_COLUMNS = UK_ETB_VAT_OUTPUT_COLUMNS UK_ETB_VAT_FIT_NAME = "uk_etb_2023_vat:full_rate_vat_expenditure_rate" UK_ETB_VAT_STAGE_NAME = "etb_vat" @dataclass class UKETBVATResult: - """Transformed frame and donor-support clip receipt.""" + """Transformed frame, donor-support clip and donor rate receipts.""" frame: Frame support_clip: UKSupportClipReceipt + donor_rate: dict[str, object] | None = None def evidence(self) -> dict[str, object]: - return { + evidence: dict[str, object] = { "stage": UK_ETB_VAT_STAGE_NAME, "support_clip": self.support_clip.evidence(), } + if self.donor_rate is not None: + evidence["donor_rate"] = dict(self.donor_rate) + return evidence #: The household-mass receipt this stage records (the manifest's @@ -134,6 +141,7 @@ def __call__(self, frame: Frame) -> Frame: self.last_result = UKETBVATResult( frame=result, support_clip=clip_result.receipt, + donor_rate=donor.attrs.get(UK_ETB_VAT_DONOR_RATE_RECEIPT), ) return result @@ -147,12 +155,19 @@ def checkpoint_metadata(self) -> dict[str, object]: return {"evidence": self.last_result.evidence()} +#: ``donor.attrs`` key of the receipt ``clean_etb_vat_table`` writes: the +#: declared rate bounds, the rows dropped for leaving no expenditure after VAT +#: and the rows capped at the upper bound. +UK_ETB_VAT_DONOR_RATE_RECEIPT = "donor_rate" + + def clean_etb_vat_table( raw: pd.DataFrame, *, year: int | None = None, standard_rate: float | None = None, reduced_rate_share: float | None = None, + rate_bounds: tuple[float, float] = UK_ETB_VAT_RATE_BOUNDS, ) -> pd.DataFrame: resource = _load_etb_vat_resource() if year is None: @@ -165,6 +180,7 @@ def clean_etb_vat_table( raise ValueError("VAT standard_rate must be positive and finite.") if not np.isfinite(reduced_rate_share): raise ValueError("VAT reduced_rate_share must be finite.") + lower, upper = _checked_rate_bounds(rate_bounds) data = raw.replace(r"^\s*$", np.nan, regex=True) required = [ "year", @@ -184,23 +200,53 @@ def clean_etb_vat_table( for column in required: data[column] = pd.to_numeric(data[column], errors="coerce") data = data.dropna(subset=required) + # A household whose VAT equals or exceeds its disposable expenditure leaves + # no expenditure for the rate to be a share of (uk-data#572): no donor. + denominator = data["expdis"] - data["totvat"] + no_expenditure_after_vat = denominator <= 0 + data = data[~no_expenditure_after_vat] + denominator = denominator[~no_expenditure_after_vat] train = pd.DataFrame() train["is_adult"] = data["adults"] train["is_child"] = data["childs"] train["is_SP_age"] = data["noretd"] train["household_net_income"] = data["disinc"] * 52 train["weight"] = data["hhold_adj_weight"] - denominator = data["expdis"] - data["totvat"] - if (denominator == 0).any(): + rate = (data["totvat"] * (1 - reduced_rate_share) / standard_rate) / denominator + if not np.isfinite(rate).all(): + raise ValueError("ETB VAT donor produced a non-finite VAT expenditure rate.") + if (rate < lower).any(): raise ValueError( - "ETB VAT donor contains zero disposable-expenditure denominators." + "ETB VAT donor produced a VAT expenditure rate below the declared " + f"lower bound {lower}: totvat is negative." ) - train["full_rate_vat_expenditure_rate"] = ( - data["totvat"] * (1 - reduced_rate_share) / standard_rate - ) / denominator - if not np.isfinite(train["full_rate_vat_expenditure_rate"]).all(): - raise ValueError("ETB VAT donor produced a non-finite VAT expenditure rate.") - return train.dropna() + above = rate > upper + train["full_rate_vat_expenditure_rate"] = rate.clip(upper=upper) + train = train.dropna() + train.attrs[UK_ETB_VAT_DONOR_RATE_RECEIPT] = { + "rate_bounds": [lower, upper], + "rows_dropped_no_expenditure_after_vat": int(no_expenditure_after_vat.sum()), + "rows_capped_at_upper_bound": int(above.sum()), + "donor_rows": int(len(train)), + } + return train + + +def _checked_rate_bounds(bounds: object) -> tuple[float, float]: + """The declared rate bounds, which must be the share-of-expenditure [0, 1].""" + + try: + lower, upper = (float(value) for value in bounds) # type: ignore[union-attr] + except (TypeError, ValueError) as error: + raise ValueError( + f"ETB VAT rate_bounds must be a [lower, upper] pair, got {bounds!r}." + ) from error + if (lower, upper) != UK_ETB_VAT_RATE_BOUNDS: + raise ValueError( + f"ETB VAT rate_bounds must be {list(UK_ETB_VAT_RATE_BOUNDS)}: the rate " + f"is a share of expenditure after VAT; got {[lower, upper]}." + ) + return lower, upper def _load_etb_vat_resource() -> dict: @@ -213,11 +259,11 @@ def _load_etb_vat_resource() -> dict: def etb_vat_configuration( stage: SourceStageSpec | None = None, -) -> dict[str, float | int]: +) -> dict[str, object]: """Load VAT settings from the committed resource and validate the manifest.""" resource = _load_etb_vat_resource()["vat"] - config: dict[str, float | int] = { + config: dict[str, object] = { "year": int(resource["standard_rate"]["period"]), "standard_rate": float(resource["standard_rate"]["value"]), "reduced_rate_share": float(resource["reduced_rate_share"]["value"]), @@ -233,6 +279,13 @@ def etb_vat_configuration( f"ETB VAT manifest {key}={declared!r} disagrees with " f"the committed anchor value {value!r}." ) + bounds = derive.parameters.get("rate_bounds") + if bounds is None: + raise ValueError( + f"{stage.stage}: the derive operation must declare rate_bounds, " + "the range the VAT expenditure rate is a share in." + ) + config["rate_bounds"] = _checked_rate_bounds(bounds) return config diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py index 67711267d..d40591d18 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py @@ -101,7 +101,10 @@ def test_e6_support_bounds_resources_are_sha_bound_and_non_placeholder() -> None assert vat["source"]["tab_sha256"] == ( "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" ) - assert set(vat["bounds"]) == {"full_rate_vat_expenditure_rate"} + # The rate is a share of expenditure after VAT (uk-data#572): its bound is + # the stage's declared range, not a rounded donor extreme. + assert vat["bounds"] == {"full_rate_vat_expenditure_rate": [0.0, 1]} + assert "no donor value is committed" in vat["source"]["sdc_treatment"] assert set(services["bounds"]) == { "dfe_education_spending", "rail_subsidy_spending", diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py index 444150ec7..e7550a8b7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_etb_vat.py @@ -5,11 +5,14 @@ import pytest from microcosm.build.uk_runtime.etb_vat import ( + UK_ETB_VAT_DONOR_RATE_RECEIPT, UK_ETB_VAT_FIT_NAME, + UK_ETB_VAT_RATE_BOUNDS, UKETBVATResult, UKETBVATStageTransform, clean_etb_vat_table, donor_realized_ranges, + etb_vat_configuration, impute_etb_vat, support_clip_to_donor, ) @@ -50,14 +53,78 @@ def test_etb_vat_cleaning_fails_loud_on_missing_rate() -> None: clean_etb_vat_table(_raw_etb(), standard_rate=np.nan) -def test_etb_vat_cleaning_fails_loud_on_zero_denominator() -> None: +def test_etb_vat_cleaning_drops_donors_with_no_expenditure_after_vat() -> None: + # uk-data#572: VAT at or above disposable expenditure leaves nothing for + # the rate to be a share of, so the household is no donor. raw = _raw_etb() raw.loc[1, "expdis"] = raw.loc[1, "totvat"] + raw.loc[2, "expdis"] = raw.loc[2, "totvat"] - 1.0 - with pytest.raises(ValueError, match="zero disposable-expenditure"): + donor = clean_etb_vat_table(raw) + + assert donor.empty + assert donor.attrs[UK_ETB_VAT_DONOR_RATE_RECEIPT] == { + "rate_bounds": [0.0, 1.0], + "rows_dropped_no_expenditure_after_vat": 2, + "rows_capped_at_upper_bound": 0, + "donor_rows": 0, + } + + +def test_etb_vat_cleaning_caps_rates_above_one() -> None: + raw = _raw_etb() + # 20 of VAT leaves 1 of expenditure: a rate far above one. + raw.loc[1, "expdis"] = 21.0 + + donor = clean_etb_vat_table(raw) + + assert donor["full_rate_vat_expenditure_rate"].tolist() == [ + 1.0, + pytest.approx((10.0 * 0.975 / 0.20) / (110.0 - 10.0)), + ] + assert donor["full_rate_vat_expenditure_rate"].between(0.0, 1.0).all() + receipt = donor.attrs[UK_ETB_VAT_DONOR_RATE_RECEIPT] + assert receipt["rows_capped_at_upper_bound"] == 1 + assert receipt["rows_dropped_no_expenditure_after_vat"] == 0 + assert receipt["donor_rows"] == 2 + + +def test_etb_vat_cleaning_refuses_negative_vat() -> None: + raw = _raw_etb() + raw.loc[1, "totvat"] = -1.0 + + with pytest.raises(ValueError, match="below the declared lower bound"): clean_etb_vat_table(raw) +@pytest.mark.parametrize("bounds", [(0.0, 2.0), (-1.0, 1.0), (0.0,), "unit"]) +def test_etb_vat_cleaning_refuses_other_rate_bounds(bounds) -> None: + with pytest.raises(ValueError, match="rate_bounds"): + clean_etb_vat_table(_raw_etb(), rate_bounds=bounds) + + +def test_etb_vat_configuration_requires_declared_rate_bounds() -> None: + from microcosm.build.country_spec import load_country_spec + from microcosm.build.source_manifest import SourceStageSpec + + stage = load_country_spec("uk").sources.stage_map()["etb_vat"] + assert etb_vat_configuration(stage)["rate_bounds"] == UK_ETB_VAT_RATE_BOUNDS + + undeclared = SourceStageSpec.from_mapping( + { + "stage": "etb_vat", + "survey": "test", + "source": "test", + "grain": "household", + "artifacts": [], + "operations": [{"kind": "derive", "year": 2023}], + "outputs": [], + } + ) + with pytest.raises(ValueError, match="must declare rate_bounds"): + etb_vat_configuration(undeclared) + + def test_etb_vat_support_clip_and_ranges() -> None: donor = clean_etb_vat_table(_raw_etb()) draws = pd.DataFrame({"full_rate_vat_expenditure_rate": [-99.0, 99.0]}) @@ -93,6 +160,15 @@ def test_etb_vat_support_clip_and_ranges() -> None: "stage": "etb_vat", "support_clip": clip_result.receipt.evidence(), } + transform.last_result = UKETBVATResult( + frame=object(), + support_clip=clip_result.receipt, + donor_rate=donor.attrs[UK_ETB_VAT_DONOR_RATE_RECEIPT], + ) + assert ( + transform.checkpoint_metadata()["evidence"]["donor_rate"] + == (donor.attrs[UK_ETB_VAT_DONOR_RATE_RECEIPT]) + ) def test_etb_vat_weighted_fit_record(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 29d875fa3..84f7611c5 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -936,13 +936,19 @@ } ], "grain": "household", - "nonnegative_outputs": [], - "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The donor-realized support includes negative rates (4 of 4,199 cleaned 2023 donor rows, minimum -3.4: totvat can exceed expdis in the raw ETB accounts), so the output is deliberately absent from nonnegative_outputs and the support gate is the guard - the net_financial_wealth precedent from was_wealth. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095).", + "nonnegative_outputs": [ + "full_rate_vat_expenditure_rate" + ], + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095).", "operations": [ { "annualization_weeks": 52, "fail_loud_on_missing_rate": true, "kind": "derive", + "rate_bounds": [ + 0, + 1 + ], "reduced_rate_share": 0.025, "standard_rate": 0.2, "year": 2023 diff --git a/tools/build_uk_e6_support_bounds.py b/tools/build_uk_e6_support_bounds.py index f76a01e72..18b2b078c 100644 --- a/tools/build_uk_e6_support_bounds.py +++ b/tools/build_uk_e6_support_bounds.py @@ -18,7 +18,10 @@ from microcosm.build.uk_runtime.etb_services import ( donor_realized_ranges as etb_services_ranges, ) -from microcosm.build.uk_runtime.etb_vat import clean_etb_vat_table +from microcosm.build.uk_runtime.etb_vat import ( + clean_etb_vat_table, + etb_vat_configuration, +) from microcosm.build.uk_runtime.etb_vat import ( donor_realized_ranges as etb_vat_ranges, ) @@ -75,16 +78,34 @@ def build_lcfs_support_bounds( def build_etb_vat_support_bounds(etb_tab: Path) -> dict[str, object]: + """The declared [0, 1] share range, checked against the donor (uk-data#572). + + The rate is a share of expenditure after VAT and the stage declares its + bounds, so the committed bound is that range, not a rounded donor extreme. + """ + sha = _sha256(etb_tab) - donor = clean_etb_vat_table(pd.read_csv(etb_tab, sep="\t", low_memory=False)) + lower, upper = etb_vat_configuration(_committed_stage("etb_vat"))["rate_bounds"] + donor = clean_etb_vat_table( + pd.read_csv(etb_tab, sep="\t", low_memory=False), rate_bounds=(lower, upper) + ) + for column, (low, high) in etb_vat_ranges(donor).items(): + if low < lower or high > upper: + raise SystemExit( + f"ETB VAT donor {column} leaves the declared range {[lower, upper]}." + ) return _payload( source={ "ukds_study_number": 8856, "doi": "10.5255/UKDA-SN-8856-4", "tab_sha256": sha, }, - bounds=etb_vat_ranges(donor), + bounds={column: (lower, upper) for column in etb_vat_ranges(donor)}, label="ETB VAT", + sdc_treatment=( + "The bounds are the stage's declared [0, 1] share range, which the " + "donor is checked to lie within; no donor value is committed." + ), ) @@ -214,6 +235,7 @@ def _payload( source: dict[str, object], bounds: dict[str, tuple[float, float]], label: str, + sdc_treatment: str | None = None, ) -> dict[str, object]: return { "version": 1, @@ -225,7 +247,8 @@ def _payload( ), "source": { **source, - "sdc_treatment": ( + "sdc_treatment": sdc_treatment + or ( "Exact donor min/max values are rounded outward to one " "significant figure before commit." ), From 9ee29f2178e92d25e229e4b09e0826ddd9ed181d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:48 +0100 Subject: [PATCH 02/28] Record counts only in the LCFS donor-floor receipt (#1164) The donor-floor receipt now records, per consumption column and in total, the rows the floor raised and nothing else, which is all the support gate reads, so the stage evidence carries no donor-derived magnitude. The tests and notes that described the floor use synthetic values. A disclosure-control item went to the author separately (UKDS EUL CD137 cl. 8, CD171 s5.2.1). Co-Authored-By: Claude Opus 5.5 --- .../1063-release-cut-blockers.fixed.md | 2 +- experiments/823-uk-consolidation-receipts.md | 2 +- .../build/uk_runtime/lcfs_consumption.py | 25 ++++++++----------- .../uk/test_uk_lcfs_consumption.py | 13 +++------- .../engine_free/uk/test_uk_stage_health.py | 4 +-- 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/changelog.d/1063-release-cut-blockers.fixed.md b/changelog.d/1063-release-cut-blockers.fixed.md index 7bce8c6ba..dc80298c5 100644 --- a/changelog.d/1063-release-cut-blockers.fixed.md +++ b/changelog.d/1063-release-cut-blockers.fixed.md @@ -1 +1 @@ -Clear the UK release-cut certifier's blockers found by the rehearsal on the 2026-09-30 national build (microcosm#1063 c9). The candidate label is `microcosm_uk_2024_25`; the certifier and the full-build gate evidence read the export surface through `uk_export_candidate_columns`, which strips the structural id columns and carries `household.household_weight` as the enhanced-FRS reference does, so ids are no longer reported as unreviewed extras nor the weight as missing. Every declared stage output the incumbent never carried is allow-listed (ruling 2026-10-02): the source and support-channel lineage of the three entities, the council tax family, the reported-benefit inputs the take-up stages read, the SPI channel's `hmrc_spi_*` leaves, the FRS education and housing fields, the two fenced raw FRS codes and `other_investment_income`. The one reviewed export exclusion, `person.incapacity_benefit_reported`, and the five internal disability carriers (`attendance_allowance_reported`, `dla_sc_reported`, `dla_m_reported`, `pip_m_reported`, `pip_dl_reported`, stage-internal inputs the export registers keep off the release) now leave at the release boundary: `uk_release_export_frame` drops it from the written candidate, the graph readback expects the same frame, and the spine keeps the column for the SPI income stage. The FRS 2024-25 vintage's recoded training variable (`TRAIN2`, where 9 is "None of these" and 10 the Kickstart Scheme) is read, so `is_in_approved_training` carries signal again. The LCFS donor's diary consumption is floored at the declared `floor` (0) before the imputation, the support-clip ranges and the rake see it, with a receipt of the rows raised; `uk_stage_lcfs_consumption_support` declares the floor and requires the receipt, no remaining negative donor row and no clip range below it (250 release rows had inherited a minimum of -14,165 in housing, water and electricity consumption). The two FRS education inputs the enhanced-FRS reference barely carries (`adult_ema`, `dfe_education_spending`) enter the per-reference input-mass register, and the sparse charity-gifts leaf re-enters the QRF-tail register, each a signed receipt expiring 2027-04-02. `uk_stage_child_benefit_take_up` declares `population_fact_check` like the Pension Credit gate, so a synthetic smoke build records its failure and continues, and the anchor composition check allows the absolute rounding tolerance the other mass identities use. +Clear the UK release-cut certifier's blockers found by the rehearsal on the 2026-09-30 national build (microcosm#1063 c9). The candidate label is `microcosm_uk_2024_25`; the certifier and the full-build gate evidence read the export surface through `uk_export_candidate_columns`, which strips the structural id columns and carries `household.household_weight` as the enhanced-FRS reference does, so ids are no longer reported as unreviewed extras nor the weight as missing. Every declared stage output the incumbent never carried is allow-listed (ruling 2026-10-02): the source and support-channel lineage of the three entities, the council tax family, the reported-benefit inputs the take-up stages read, the SPI channel's `hmrc_spi_*` leaves, the FRS education and housing fields, the two fenced raw FRS codes and `other_investment_income`. The one reviewed export exclusion, `person.incapacity_benefit_reported`, and the five internal disability carriers (`attendance_allowance_reported`, `dla_sc_reported`, `dla_m_reported`, `pip_m_reported`, `pip_dl_reported`, stage-internal inputs the export registers keep off the release) now leave at the release boundary: `uk_release_export_frame` drops it from the written candidate, the graph readback expects the same frame, and the spine keeps the column for the SPI income stage. The FRS 2024-25 vintage's recoded training variable (`TRAIN2`, where 9 is "None of these" and 10 the Kickstart Scheme) is read, so `is_in_approved_training` carries signal again. The LCFS donor's diary consumption is floored at the declared `floor` (0) before the imputation, the support-clip ranges and the rake see it, with a receipt of the rows raised; `uk_stage_lcfs_consumption_support` declares the floor and requires the receipt, no remaining negative donor row and no clip range below it (250 release rows had inherited negative housing, water and electricity consumption). The two FRS education inputs the enhanced-FRS reference barely carries (`adult_ema`, `dfe_education_spending`) enter the per-reference input-mass register, and the sparse charity-gifts leaf re-enters the QRF-tail register, each a signed receipt expiring 2027-04-02. `uk_stage_child_benefit_take_up` declares `population_fact_check` like the Pension Credit gate, so a synthetic smoke build records its failure and continues, and the anchor composition check allows the absolute rounding tolerance the other mass identities use. diff --git a/experiments/823-uk-consolidation-receipts.md b/experiments/823-uk-consolidation-receipts.md index df618170c..74c0b58ef 100644 --- a/experiments/823-uk-consolidation-receipts.md +++ b/experiments/823-uk-consolidation-receipts.md @@ -144,7 +144,7 @@ parity gates, `uk_qrf_tail_concentration`, `uk_release_family_build_stages`, - *Data findings.* `uk_degenerate_release_surface`: two persisted all-zero columns (`person.incapacity_benefit_reported`, `person.is_in_approved_training`). `uk_nonnegative_columns`: `housing_water_and_electricity_consumption` has 239 - values below zero (minimum −14,165). + values below zero. The evaluation path therefore works end to end and the receipt is certifiable in shape; the cut does not certify today because the release-cut battery had diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 2e0e7b0f3..77c878c0f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -1994,28 +1994,25 @@ def floor_negative_donor_consumption( """Raise negative diary consumption on the donor to ``floor`` (microcosm#1063 c9). An LCFS diary spend nets refunds against purchases, so a handful of donor - rows carry a negative annual total (250 release rows inherited a minimum - of -14,165 in ``housing_water_and_electricity_consumption`` on the - 2026-09-30 build, because the support clip takes its floor from the - donor's realised range). The release surface declares these columns - non-negative, so the donor is floored before the imputation, the clip - ranges and the rake see it. The receipt records, per consumption column, - the rows raised and the (negative) mass they carried; columns without a - negative row are listed with zeros so the receipt names the whole surface. + rows carry a negative annual total, and hundreds of release rows inherited + negative ``housing_water_and_electricity_consumption`` on the 2026-09-30 + build, because the support clip takes its floor from the donor's realised + range. The release surface declares these columns non-negative, so the + donor is floored before the imputation, the clip ranges and the rake see + it. The receipt counts, per consumption column, the rows raised; columns + without a negative row are listed with zeros so the receipt names the + whole surface. It records no donor value and no statistic of the raised + rows, which are few: a stage receipt is UKDS output (CD171 §5.2.1). """ floored = donor.copy() - columns: dict[str, dict[str, float | int]] = {} + columns: dict[str, dict[str, int]] = {} for column in CONSUMPTION_VARIABLE_RENAMES.values(): if column not in floored.columns: raise ValueError(f"LCFS donor is missing consumption column {column!r}.") values = _numeric(floored[column]).to_numpy(dtype=float) negative = np.isfinite(values) & (values < floor) - columns[column] = { - "rows_raised": int(negative.sum()), - "negative_mass": float(values[negative].sum()), - "minimum_before": float(np.nanmin(values)) if values.size else 0.0, - } + columns[column] = {"rows_raised": int(negative.sum())} if negative.any(): floored[column] = np.where(negative, floor, values) receipt = { diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 4753740d9..6ce43aa14 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -154,7 +154,7 @@ def test_negative_donor_consumption_is_floored_with_a_receipt() -> None: columns = list(CONSUMPTION_VARIABLE_RENAMES.values()) donor = pd.DataFrame({column: [10.0, 20.0, 30.0] for column in columns}) - donor.loc[1, "housing_water_and_electricity_consumption"] = -14_165.0 + donor.loc[1, "housing_water_and_electricity_consumption"] = -50.0 donor.loc[2, "housing_water_and_electricity_consumption"] = -1.0 floored, receipt = floor_negative_donor_consumption(donor) @@ -171,15 +171,10 @@ def test_negative_donor_consumption_is_floored_with_a_receipt() -> None: assert receipt["rows_raised"] == 2 assert receipt["remaining_negative_rows"] == 0 housing = receipt["columns"]["housing_water_and_electricity_consumption"] - assert housing == { - "rows_raised": 2, - "negative_mass": -14_166.0, - "minimum_before": -14_165.0, - } + # Counts only: no donor value or statistic of the raised rows. + assert housing == {"rows_raised": 2} assert receipt["columns"]["food_and_non_alcoholic_beverages_consumption"] == { - "rows_raised": 0, - "negative_mass": 0.0, - "minimum_before": 10.0, + "rows_raised": 0 } with pytest.raises(ValueError, match="missing consumption column"): floor_negative_donor_consumption(donor.drop(columns=[columns[0]])) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index 8cb4b266d..06eeca8a4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -722,8 +722,6 @@ def evidence(**floor_changes): "columns": { "housing_water_and_electricity_consumption": { "rows_raised": 250, - "negative_mass": -1.0e6, - "minimum_before": -14_165.0, } }, **floor_changes, @@ -766,7 +764,7 @@ def gate(evidence_payload, **parameter_changes): below = evidence() below["support_clip"]["columns"]["housing_water_and_electricity_consumption"][ "donor_min" - ] = -14_165.0 + ] = -50.0 assert any("below the donor floor" in f for f in gate(below).failures) # Without a declared floor (the WAS clip gate) the receipt is not required. undeclared = {**parameters} From 78797a8fa2da18e4b81382b406408e1eb052a535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:48 +0100 Subject: [PATCH 03/28] Floor the LCFS bus fares and road-fuel shares; bound from the floored donor (#1164) The donor floor (microcosm#1063 c9) raised negative diary spend to 0 on the fifteen consumption columns only: - bus_fare_spending, summed from c73212-c73214, was not floored, so a refund could put a negative regime in its QRF fit (Wales keeps the raw draw); - the road-fuel level's other-fuels share read the c72211-c72213 lines raw; - tools/build_uk_e6_support_bounds.py read donor ranges before the floor, so the committed housing bound started below 0. All three now see the floored donor. The regenerated LCFS bounds differ only in that lower bound, now 0. The floor accessor becomes public for the tool. Co-Authored-By: Claude Opus 5.5 --- .../1164-lcfs-donor-floor-gaps.fixed.md | 1 + .../uk/lcfs_consumption_support_bounds.json | 2 +- .../build/uk_runtime/lcfs_consumption.py | 29 +++++++++++-------- .../uk/test_uk_consumption_resources.py | 4 ++- .../uk/test_uk_lcfs_consumption.py | 18 ++++++++++-- tools/build_uk_e6_support_bounds.py | 7 +++++ 6 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 changelog.d/1164-lcfs-donor-floor-gaps.fixed.md diff --git a/changelog.d/1164-lcfs-donor-floor-gaps.fixed.md b/changelog.d/1164-lcfs-donor-floor-gaps.fixed.md new file mode 100644 index 000000000..71f664abd --- /dev/null +++ b/changelog.d/1164-lcfs-donor-floor-gaps.fixed.md @@ -0,0 +1 @@ +The UK LCFS donor floor now covers every donor spend it reads (microcosm#1164). `bus_fare_spending`, summed from three diary lines a refund can also make negative, is floored with the fifteen diary consumption columns. The road-fuel level's other-fuels share reads the c72211–c72213 lines floored the same way. `tools/build_uk_e6_support_bounds.py` applies the stage's declared floor before reading donor ranges, so the committed housing, water and electricity bound starts at 0 like the stage's support; no other bound moves. diff --git a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json index 1b2b3c870..5bba65bde 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json +++ b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json @@ -39,7 +39,7 @@ 200000 ], "housing_water_and_electricity_consumption": [ - -20000, + 0.0, 300000 ], "miscellaneous_consumption": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 77c878c0f..7f0e0ea90 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -151,6 +151,13 @@ CONSUMPTION_VARIABLE_RENAMES[f"p6{division:02d}"] for division in range(1, 13) ) BUS_FARE_LCFS_CODES = ("c73212", "c73213", "c73214") +#: The donor columns floored at zero before the imputation (microcosm#1063 +#: c9): the diary consumption columns and the bus fares summed from the +#: c73212-c73214 diary lines, which a refund can make negative as well. +UK_LCFS_DONOR_FLOORED_COLUMNS = ( + *CONSUMPTION_VARIABLE_RENAMES.values(), + "bus_fare_spending", +) #: The spend columns a price index carries from the April-to-March diary to #: the calendar base year (microcosm#1113). Energy and road fuel are not here: #: later steps set their levels for that year. @@ -427,7 +434,7 @@ def __call__(self, frame: Frame) -> Frame: donor_rake_iterations=_donor_energy_rake_iterations(self.stage), ) donor, donor_floor_receipt = floor_negative_donor_consumption( - donor, floor=_donor_consumption_floor(self.stage) + donor, floor=donor_consumption_floor(self.stage) ) ice_share, ice_share_receipt = lcfs_ice_share(self.stage) recipient = recipient_predictors(frame, self.engine) @@ -1142,15 +1149,13 @@ def lcfs_other_road_fuel_share(lcfs_household: pd.DataFrame) -> float: household, ("weighta", "c72211", "c72212", *UK_LCFS_OTHER_ROAD_FUEL_CODES) ) weight = _numeric(household["weighta"]).to_numpy(dtype=float) - other = sum( - _numeric(household[code]).to_numpy(dtype=float) - for code in UK_LCFS_OTHER_ROAD_FUEL_CODES - ) - total = ( - _numeric(household["c72211"]).to_numpy(dtype=float) - + _numeric(household["c72212"]).to_numpy(dtype=float) - + other - ) + + def spend(code: str) -> np.ndarray: + # Floored at zero, as the imputation's donor is (microcosm#1063 c9). + return np.maximum(_numeric(household[code]).to_numpy(dtype=float), 0.0) + + other = sum(spend(code) for code in UK_LCFS_OTHER_ROAD_FUEL_CODES) + total = spend("c72211") + spend("c72212") + other denominator = float(np.dot(weight, total)) if not np.isfinite(denominator) or denominator <= 0: raise ValueError("LCFS donor records no COICOP 07.2.2 road-fuel spend.") @@ -1969,7 +1974,7 @@ def clean_lcfs_consumption_table( ].dropna() -def _donor_consumption_floor(stage: SourceStageSpec) -> float: +def donor_consumption_floor(stage: SourceStageSpec) -> float: """The declared floor of the donor's diary consumption columns.""" parameters = _operation_parameters(stage, "derive") @@ -2007,7 +2012,7 @@ def floor_negative_donor_consumption( floored = donor.copy() columns: dict[str, dict[str, int]] = {} - for column in CONSUMPTION_VARIABLE_RENAMES.values(): + for column in UK_LCFS_DONOR_FLOORED_COLUMNS: if column not in floored.columns: raise ValueError(f"LCFS donor is missing consumption column {column!r}.") values = _numeric(floored[column]).to_numpy(dtype=float) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py index d40591d18..36c15cb15 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py @@ -93,10 +93,12 @@ def test_e6_support_bounds_resources_are_sha_bound_and_non_placeholder() -> None assert len(lcfs["bounds"]) == 12 assert "bus_fare_spending" not in lcfs["bounds"] assert "petrol_spending" not in lcfs["bounds"] + # The tool floors the donor as the stage does, so no bound starts below 0. assert lcfs["bounds"]["housing_water_and_electricity_consumption"] == [ - -20000, + 0.0, 300000, ] + assert all(low >= 0 for low, _high in lcfs["bounds"].values()) assert lcfs["bounds"]["transport_consumption"] == [0.0, 200000] assert vat["source"]["tab_sha256"] == ( "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 6ce43aa14..69e195d26 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -149,13 +149,18 @@ def test_negative_donor_consumption_is_floored_with_a_receipt() -> None: the declared floor before the clip ranges and the imputation see it.""" from microcosm.build.uk_runtime.lcfs_consumption import ( CONSUMPTION_VARIABLE_RENAMES, + UK_LCFS_DONOR_FLOORED_COLUMNS, floor_negative_donor_consumption, ) - columns = list(CONSUMPTION_VARIABLE_RENAMES.values()) + # The bus fares, summed from three diary lines, are floored too. + columns = list(UK_LCFS_DONOR_FLOORED_COLUMNS) + assert columns == [*CONSUMPTION_VARIABLE_RENAMES.values(), "bus_fare_spending"] donor = pd.DataFrame({column: [10.0, 20.0, 30.0] for column in columns}) donor.loc[1, "housing_water_and_electricity_consumption"] = -50.0 donor.loc[2, "housing_water_and_electricity_consumption"] = -1.0 + donor.loc[0, "bus_fare_spending"] = -2.0 + raised = {"housing_water_and_electricity_consumption", "bus_fare_spending"} floored, receipt = floor_negative_donor_consumption(donor) @@ -164,20 +169,24 @@ def test_negative_donor_consumption_is_floored_with_a_receipt() -> None: 0.0, 0.0, ] + assert floored["bus_fare_spending"].tolist() == [0.0, 20.0, 30.0] for column in columns: - if column != "housing_water_and_electricity_consumption": + if column not in raised: assert floored[column].tolist() == [10.0, 20.0, 30.0] assert receipt["floor"] == 0.0 - assert receipt["rows_raised"] == 2 + assert receipt["rows_raised"] == 3 assert receipt["remaining_negative_rows"] == 0 housing = receipt["columns"]["housing_water_and_electricity_consumption"] # Counts only: no donor value or statistic of the raised rows. assert housing == {"rows_raised": 2} + assert receipt["columns"]["bus_fare_spending"] == {"rows_raised": 1} assert receipt["columns"]["food_and_non_alcoholic_beverages_consumption"] == { "rows_raised": 0 } with pytest.raises(ValueError, match="missing consumption column"): floor_negative_donor_consumption(donor.drop(columns=[columns[0]])) + with pytest.raises(ValueError, match="bus_fare_spending"): + floor_negative_donor_consumption(donor.drop(columns=["bus_fare_spending"])) def test_support_clip_exempts_raked_energy_columns() -> None: @@ -1273,6 +1282,9 @@ def test_other_road_fuel_share_is_the_donor_weighted_c72213_share() -> None: } ) assert lcfs_other_road_fuel_share(donor) == pytest.approx(1.0 / 26.0) + # A refund netting a diary line below zero is floored, as on the donor. + refunded = donor.assign(C72213=[1.0, -4.0]) + assert lcfs_other_road_fuel_share(refunded) == pytest.approx(1.0 / 26.0) with pytest.raises(ValueError, match="no COICOP 07.2.2"): lcfs_other_road_fuel_share(donor.assign(C72211=0.0, C72212=0.0, C72213=0.0)) with pytest.raises(ValueError, match="c72213"): diff --git a/tools/build_uk_e6_support_bounds.py b/tools/build_uk_e6_support_bounds.py index 18b2b078c..7a232aac0 100644 --- a/tools/build_uk_e6_support_bounds.py +++ b/tools/build_uk_e6_support_bounds.py @@ -27,6 +27,8 @@ ) from microcosm.build.uk_runtime.lcfs_consumption import ( clean_lcfs_consumption_table, + donor_consumption_floor, + floor_negative_donor_consumption, lcfs_donor_uprating, lcfs_energy_pricing, ) @@ -65,6 +67,11 @@ def build_lcfs_support_bounds( uprating=uprating, energy=lcfs_energy_pricing(stage), ) + # The stage floors the donor before it reads clip ranges, so the bounds + # start where the stage's support does. + donor, _floor_receipt = floor_negative_donor_consumption( + donor, floor=donor_consumption_floor(stage) + ) return _payload( source={ "ukds_study_number": 9468, From d06f71a8501aefd673a704b55e0ffdc09fe3e0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 04/28] Refuse unmapped LCFS codes instead of dropping donors (#1164) clean_lcfs_consumption_table mapped gorx, a122 and a121 through the stage's maps and ended in a .dropna(), so a donor with an unmapped code was dropped silently; logged_dropna_row_count was declared but never implemented. A new survey year that recodes one of these variables (LCFS 2024-25 added an equity-release mortgage option) would have shrunk the donor without a trace. mapped_lcfs_codes now refuses any unmapped code, naming the codes and blanks without counting rows (the #1097 pattern), and any other missing donor value stops the stage. The derive step declares unmapped_codes: refuse in place of the unimplemented flag, and the stage requires the declaration. Every LCFS 2023-24 code maps and the final .dropna() dropped no row, so the donor is unchanged; the typed mirror, coverage manifest and H2 fixture move with the declaration. Co-Authored-By: Claude Opus 5.5 --- .../1164-lcfs-unmapped-codes-refused.fixed.md | 1 + .../spec_engine/schema/sources.schema.json | 5 ++ .../src/microcosm/build/uk/source_stages.json | 2 +- .../src/microcosm/build/uk/spec/sources.yaml | 2 +- .../build/uk_runtime/lcfs_consumption.py | 59 +++++++++++++++-- .../uk/test_uk_lcfs_consumption.py | 66 +++++++++++++++++++ .../parity/uk_spine/sources/fixture.json | 4 +- 7 files changed, 128 insertions(+), 11 deletions(-) create mode 100644 changelog.d/1164-lcfs-unmapped-codes-refused.fixed.md diff --git a/changelog.d/1164-lcfs-unmapped-codes-refused.fixed.md b/changelog.d/1164-lcfs-unmapped-codes-refused.fixed.md new file mode 100644 index 000000000..7651a580f --- /dev/null +++ b/changelog.d/1164-lcfs-unmapped-codes-refused.fixed.md @@ -0,0 +1 @@ +The UK `lcfs_consumption` stage refuses an LCFS region (`gorx`), tenure (`a122`) or accommodation (`a121`) code it does not map, instead of dropping the donor (microcosm#1164). The donor table ended in a `.dropna()` that silently lost any household whose code fell outside the stage's maps, and the derive step's `logged_dropna_row_count` was never implemented, so a survey year that recoded one of these variables would have shrunk the donor without a trace. The derive step now declares `unmapped_codes: refuse`; the error names each unknown code, or a blank, without counting rows, and any other missing donor value stops the stage too. Every LCFS 2023-24 code maps, so the donor is unchanged. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index cf415ad68..dbb721b9c 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -7056,6 +7056,11 @@ "logged_dropna_row_count": { "type": "boolean" }, + "unmapped_codes": { + "enum": [ + "refuse" + ] + }, "margins": { "type": "array", "items": { diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 89b6973d2..1da6c0b8f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3582,7 +3582,7 @@ "LCFS tenure 4 and 8 -> RENT_PRIVATELY", "LCFS accommodation 4 and 5 -> FLAT" ], - "logged_dropna_row_count": true, + "unmapped_codes": "refuse", "floor": 0.0 }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index b3c8b57ea..0ef978043 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2952,7 +2952,7 @@ stages: lossy_mappings: - LCFS tenure 4 and 8 -> RENT_PRIVATELY - LCFS accommodation 4 and 5 -> FLAT - logged_dropna_row_count: true + unmapped_codes: refuse floor: 0.0 - kind: uprate_donor_columns from_period: 2023 diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 7f0e0ea90..42b950442 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -433,6 +433,7 @@ def __call__(self, frame: Frame) -> Frame: energy=energy, donor_rake_iterations=_donor_energy_rake_iterations(self.stage), ) + require_refused_unmapped_codes(self.stage) donor, donor_floor_receipt = floor_negative_donor_consumption( donor, floor=donor_consumption_floor(self.stage) ) @@ -1916,13 +1917,15 @@ def clean_lcfs_consumption_table( household = _lowercase(lcfs_household).rename(columns=HOUSEHOLD_LCFS_RENAMES) _require_columns(household, ("case", *HOUSEHOLD_LCFS_RENAMES.values())) _require_columns(person, ("case", *PERSON_LCFS_RENAMES.values())) - household["region"] = _numeric(household["region"]).map(LCFS_REGIONS) + household["region"] = mapped_lcfs_codes( + household["region"], LCFS_REGIONS, column="gorx" + ) household["num_vehicles"] = vehicle_count(household["num_vehicles"]) - household["tenure_type"] = _numeric(_lowercase(lcfs_household)["a122"]).map( - LCFS_TENURE_MAP + household["tenure_type"] = mapped_lcfs_codes( + _lowercase(lcfs_household)["a122"], LCFS_TENURE_MAP, column="a122" ) - household["accommodation_type"] = _numeric(_lowercase(lcfs_household)["a121"]).map( - LCFS_ACCOMM_MAP + household["accommodation_type"] = mapped_lcfs_codes( + _lowercase(lcfs_household)["a121"], LCFS_ACCOMM_MAP, column="a121" ) household = derive_energy_from_lcfs(household) household = household.rename(columns=CONSUMPTION_VARIABLE_RENAMES) @@ -1964,14 +1967,56 @@ def clean_lcfs_consumption_table( household["domestic_energy_consumption"] = ( household["electricity_consumption"] + household["gas_consumption"] ) - return household[ + donor = household[ [ *UK_LCFS_CONSUMPTION_PREDICTORS, *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, "household_gross_income", "household_weight", ] - ].dropna() + ] + # Every donor is kept or the stage stops: a silent drop would lose donors + # whenever a survey year recodes a variable (microcosm#1164). + missing = sorted(donor.columns[donor.isna().any()]) + if missing: + raise ValueError(f"LCFS donor has missing values in {missing}.") + return donor + + +def mapped_lcfs_codes( + raw: pd.Series, mapping: Mapping[int, str], *, column: str +) -> pd.Series: + """Map an LCFS code column, refusing any code the map does not name. + + Declared as ``unmapped_codes: refuse`` on the stage's derive step + (microcosm#1164). The message names each unknown code, and blank or + non-numeric entries, without counting rows: a build log is not an output + table, and a count of a rare code would be a small cell if it were posted. + """ + + codes = pd.to_numeric(raw, errors="coerce") + mapped = codes.map(mapping) + unknown = mapped.isna().to_numpy() + if unknown.any(): + bad = codes[unknown] + labels = [f"{code:g}" for code in sorted(bad.dropna().unique())] + if bad.isna().any(): + labels.append("blank or non-numeric") + raise ValueError( + f"LCFS {column} has code(s) the stage does not map: {', '.join(labels)}." + ) + return mapped + + +def require_refused_unmapped_codes(stage: SourceStageSpec) -> None: + """The derive step must declare the unknown-code refusal the code applies.""" + + declared = _operation_parameters(stage, "derive").get("unmapped_codes") + if declared != "refuse": + raise ValueError( + f"{stage.stage}: the derive operation must declare unmapped_codes: " + f"refuse, which mapped_lcfs_codes applies; got {declared!r}." + ) def donor_consumption_floor(stage: SourceStageSpec) -> float: diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 69e195d26..40a5e027d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -7,6 +7,7 @@ from microcosm.build.uk_runtime.lcfs_consumption import ( BUS_FARE_LCFS_CODES, LCFS_ACCOMM_MAP, + LCFS_REGIONS, LCFS_TENURE_MAP, UK_LCFS_CONSUMPTION_PREDICTORS, UK_LCFS_CONSUMPTION_TARGET_COLUMNS, @@ -97,6 +98,71 @@ def test_lcfs_donor_cleaning_arithmetic_and_lossy_maps() -> None: assert np.isclose(donor.loc[0, "bus_fare_spending"], 3.0 * (365.25 / 7)) +@pytest.mark.parametrize( + ("column", "codes", "named"), + [ + ("gorx", [7, 12, 13, 1], "13"), + ("a122", [4, 8, 9, 7], "9"), + ("a121", [4, 5, 6, 0], "0"), + ("a122", [4, 8, " ", 7], "blank or non-numeric"), + ], +) +def test_lcfs_donor_cleaning_refuses_unmapped_codes(column, codes, named) -> None: + # microcosm#1164: a recoded survey year must stop the stage, not lose donors. + household = _household() + household[column] = codes + + with pytest.raises(ValueError, match=f"LCFS {column} has code") as raised: + clean_lcfs_consumption_table(_person(), household) + + assert named in str(raised.value) + # The refusal names codes, never a count of rows. + assert "row" not in str(raised.value) + + +def test_mapped_lcfs_codes_names_every_unknown_code_once() -> None: + from microcosm.build.uk_runtime.lcfs_consumption import mapped_lcfs_codes + + mapped = mapped_lcfs_codes(pd.Series([1, 12]), LCFS_REGIONS, column="gorx") + assert mapped.tolist() == ["NORTH_EAST", "NORTHERN_IRELAND"] + with pytest.raises(ValueError) as raised: + mapped_lcfs_codes( + pd.Series([14, 13, 13, None, "x"]), LCFS_REGIONS, column="gorx" + ) + assert str(raised.value) == ( + "LCFS gorx has code(s) the stage does not map: 13, 14, blank or non-numeric." + ) + + +def test_stage_declares_the_unmapped_code_refusal() -> None: + import dataclasses + + from microcosm.build.country_spec import load_country_spec + from microcosm.build.uk_runtime.lcfs_consumption import ( + require_refused_unmapped_codes, + ) + + stage = load_country_spec("uk").sources.stage_map()["lcfs_consumption"] + require_refused_unmapped_codes(stage) + operations = tuple( + dataclasses.replace( + operation, + parameters={ + key: value + for key, value in operation.parameters.items() + if key != "unmapped_codes" + }, + ) + if operation.kind == "derive" + else operation + for operation in stage.operations + ) + with pytest.raises(ValueError, match="unmapped_codes"): + require_refused_unmapped_codes( + dataclasses.replace(stage, operations=operations) + ) + + def test_energy_split_exercises_four_cases_fallback_and_clamp() -> None: household = _household() split = derive_energy_from_lcfs(household) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 84f7611c5..4ced82ff0 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3525,7 +3525,6 @@ "donor_weight": "weighta * 1000", "floor": 0, "kind": "derive", - "logged_dropna_row_count": true, "lossy_mappings": [ "LCFS tenure 4 and 8 -> RENT_PRIVATELY", "LCFS accommodation 4 and 5 -> FLAT" @@ -3533,7 +3532,8 @@ "lowercase_columns": true, "mappings": { "num_vehicles": "a124" - } + }, + "unmapped_codes": "refuse" }, { "columns": { From 25de4e6318b7c97677db1950d7bd2d475f38864c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 05/28] Band LCFS recipients for the NEED rake on gross income (#1164) The NEED energy rake bands households by income. The donor bands on LCFS gross income (p344p), NEED's concept, but the recipient read a household_gross_income column no spine stage writes and fell back to its HBAI net income, so recipients sat in too low a band and the rake gave richer households the lower bands' mean kWh. That skews electricity and gas by income, the distribution the GB domestic-electricity zero rate reads (PolicyEngine/policyengine-uk#2189). The stage now materializes the engine's household_gross_income with its four income predictors, for the band alone, as nts_bus_travel does; the fallback is gone. The end-to-end test reads the materialize step by kind, not by position. Co-Authored-By: Claude Opus 5.5 --- .../1164-need-band-gross-income.fixed.md | 1 + .../src/microcosm/build/uk/source_stages.json | 5 +- .../src/microcosm/build/uk/spec/sources.yaml | 3 +- .../build/uk_runtime/lcfs_consumption.py | 11 ++-- .../uk/test_uk_lcfs_consumption.py | 60 ++++++++++++++++++- .../parity/uk_spine/sources/fixture.json | 5 +- 6 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 changelog.d/1164-need-band-gross-income.fixed.md diff --git a/changelog.d/1164-need-band-gross-income.fixed.md b/changelog.d/1164-need-band-gross-income.fixed.md new file mode 100644 index 000000000..af56fda35 --- /dev/null +++ b/changelog.d/1164-need-band-gross-income.fixed.md @@ -0,0 +1 @@ +The UK LCFS stage bands recipient households for the NEED energy rake on the engine's `household_gross_income`, the concept the donor and NEED band on (microcosm#1164). The donor bands on LCFS gross income (p344p), but no spine stage writes `household_gross_income`, so the recipient fell back to its HBAI net income and sat in too low a NEED income band: the rake then gave richer households the lower bands' mean electricity and gas. The stage now materializes `household_gross_income` with its income predictors, for the band alone (it is not a QRF predictor). Electricity and gas by income move on every row; the DESNZ level and the NEED margins still hold. diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 1da6c0b8f..b24c4ee45 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3736,7 +3736,8 @@ "employment_income", "self_employment_income", "private_pension_income", - "hbai_household_net_income" + "hbai_household_net_income", + "household_gross_income" ] }, { @@ -4172,7 +4173,7 @@ "gas_consumption", "has_fuel_consumption" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 0ef978043..af546a8cd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3073,6 +3073,7 @@ stages: - self_employment_income - private_pension_income - hbai_household_net_income + - household_gross_income - kind: fit_weighted_qrf_chain predictors: - household_adult_count @@ -3412,7 +3413,7 @@ stages: - electricity_consumption - gas_consumption - has_fuel_consumption - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households'' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary''s other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households'' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary''s other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 42b950442..ec1b4ef78 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -251,11 +251,16 @@ #: which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). UK_LCFS_HOUSEHOLD_COUNT_ADULT_AGE = 18 UK_LCFS_HOUSEHOLD_COUNT_PREDICTORS = ("household_adult_count", "household_child_count") +#: The engine variables the stage materializes on the recipient: the four +#: QRF income predictors and ``household_gross_income``, which only bands the +#: NEED energy rake, on the concept the donor bands on (LCFS p344p, gross; +#: microcosm#1164). Before it the recipient fell back to its HBAI net income. UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS = ( "employment_income", "self_employment_income", "private_pension_income", "hbai_household_net_income", + "household_gross_income", ) UK_LCFS_CONSUMPTION_PREDICTORS = ( "household_adult_count", @@ -2174,12 +2179,6 @@ def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: "(the was_wealth draw the consumption QRF conditions on)." ) result["num_vehicles"] = vehicle_count(household["num_vehicles"]).to_numpy() - if "household_gross_income" in household: - result["household_gross_income"] = household[ - "household_gross_income" - ].to_numpy() - else: - result["household_gross_income"] = result["hbai_household_net_income"] return result diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 40a5e027d..21e1df8db 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -387,6 +387,52 @@ def licensed(fuel_type: str) -> float: ice_share_from_licensed_cars({**declared, "period_value": 1999}) +def test_recipient_need_band_income_is_the_engines_gross_income() -> None: + # microcosm#1164: the donor bands on gross p344p, so the recipient must not + # fall back to its (lower) HBAI net income. + from types import SimpleNamespace + + from microcosm.build.uk_runtime.lcfs_consumption import recipient_predictors + + class _Engine: + def materialize(self, frame, variables, period): + assert "household_gross_income" in variables + return { + name: np.array([30_000.0, 80_000.0]) + if name == "household_gross_income" + else np.array([20_000.0, 50_000.0]) + for name in variables + } + + def variable_metadata(self, name): + return SimpleNamespace(entity="household") + + frame = uk_national_frame( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [10, 20], + "age": [40.0, 70.0], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [10, 20], + "household_weight": [1.0, 1.0], + "num_vehicles": [1, 0], + } + ), + time_period="2024", + ) + + predictors = recipient_predictors(frame, _Engine()) + + assert predictors["household_gross_income"].tolist() == [30_000.0, 80_000.0] + assert predictors["hbai_household_net_income"].tolist() == [20_000.0, 50_000.0] + + def test_recipient_predictors_read_the_vehicle_count_numerically() -> None: from types import SimpleNamespace @@ -807,6 +853,10 @@ def materialize(self, frame, variables, period): "household_gross_income" ].to_numpy() * 0.8, + # The NEED band reads the engine's gross income (microcosm#1164). + "household_gross_income": frame.table("household")[ + "household_gross_income" + ].to_numpy(), } return {name: values[name] for name in variables} @@ -834,8 +884,14 @@ def materialize(self, frame, variables, period): assert receipt.reason == UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON assert receipt.old_total == receipt.new_total > 0 assert receipt.declared_factor == 1.0 - assert set(UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS) <= set( - transform.stage.operations[4].parameters["predictors"] + materialize = next( + operation + for operation in transform.stage.operations + if operation.kind == "materialize_rules_engine_predictors" + ) + assert ( + tuple(materialize.parameters["predictors"]) + == UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS ) evidence = transform.checkpoint_metadata()["evidence"] assert { diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 4ced82ff0..9d0434d92 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3512,7 +3512,7 @@ "gas_consumption", "has_fuel_consumption" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3686,7 +3686,8 @@ "employment_income", "self_employment_income", "private_pension_income", - "hbai_household_net_income" + "hbai_household_net_income", + "household_gross_income" ] }, { From a6d4f05ac7f98799b4dc6f455763ed1d1a7162d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 06/28] Draw the LCFS prepayment-meter flag; export it once the engine reads it (#1110) policyengine-uk 2.123.3 reads the household input uses_energy_prepayment_meter for the Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098); no Microcosm release carried it, so the discount was zero in population simulations. - An LCFS donor prepays when b489 (its prepayment-meter energy payment) or b490 (that payment's gas part) is nonzero: electricity-only prepayment counts, and a household with no prepayment amount does not. These are the fields the stage already reads to split electricity and gas. - A declared impute_prepayment_meter step fits the flag with a weighted regime-gated QRF on the chain's predictors and draws it at uniforms keyed on the household id, so no other draw moves. Its receipt records the donor's and the frame's weighted shares and the Great Britain households flagged, for the benchmark against DESNZ and Ofgem counts. - The locked policyengine-uk (2.123.0) does not know the variable, so the release boundary drops the column (UK_RELEASE_EXPORT_DROPPED_COLUMNS), and an engine-lane test fails once the lock reaches a release that reads it. - The manifest vocabulary, the schema and the graph's typed cells admit the step and the column; the mirror, the coverage manifest and the H2 fixture are regenerated; the seed reaches the build sidecar. Co-Authored-By: Claude Opus 5.5 --- .../1110-prepayment-meter-input.added.md | 1 + .../src/microcosm/build/source_manifest.py | 1 + .../spec_engine/schema/sources.schema.json | 41 ++++ .../src/microcosm/build/uk/source_stages.json | 20 +- .../src/microcosm/build/uk/spec/sources.yaml | 13 +- .../src/microcosm/build/uk_runtime/graph.py | 1 + .../build/uk_runtime/lcfs_consumption.py | 180 ++++++++++++++++++ .../build/uk_runtime/national_frame.py | 11 +- .../test_uk_prepayment_meter_engine_gate.py | 38 ++++ .../tests/engine_free/uk/test_uk_frs_spine.py | 2 + .../uk/test_uk_lcfs_consumption.py | 173 ++++++++++++++++- .../engine_free/uk/test_uk_source_stages.py | 11 +- .../engine_free/uk/test_uk_terminal_gates.py | 6 + .../parity/uk_spine/sources/fixture.json | 20 +- 14 files changed, 506 insertions(+), 12 deletions(-) create mode 100644 changelog.d/1110-prepayment-meter-input.added.md create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_prepayment_meter_engine_gate.py diff --git a/changelog.d/1110-prepayment-meter-input.added.md b/changelog.d/1110-prepayment-meter-input.added.md new file mode 100644 index 000000000..9b2c68fbb --- /dev/null +++ b/changelog.d/1110-prepayment-meter-input.added.md @@ -0,0 +1 @@ +The UK `lcfs_consumption` stage draws `uses_energy_prepayment_meter`, the household input policyengine-uk reads from 2.123.3 for the Energy Price Guarantee prepayment discount, July 2023 to March 2024 (microcosm#1110; PolicyEngine/policyengine-uk#2098; uk-data#539 tracks the same input). An LCFS donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, the fields the stage already reads to split electricity and gas: electricity-only prepayment counts, and a household with no prepayment amount, including one that did not answer, does not. The declared `impute_prepayment_meter` step fits that flag with a weighted regime-gated QRF on the chain's predictors and draws it at uniforms keyed on the household id, so no other draw moves; its receipt records the donor's and the frame's weighted shares and the Great Britain households flagged. The locked policyengine-uk does not read the variable yet, so the release export drops the column, and an engine-lane test fails once the lock reaches a release that does. diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index d44addaf3..203b29544 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -143,6 +143,7 @@ "impute_retirement_distributions_to_puf_support", "impute_workers_compensation_to_puf_support", "impute_weeks_unemployed_to_puf_support", + "impute_prepayment_meter", "iterative_proportional_fit", "level_road_fuel", "redraw_zero_road_fuel", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index dbb721b9c..4a07aa6e8 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -11504,6 +11504,47 @@ } } }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "output", + "codes", + "rule", + "seed", + "salt", + "n_estimators" + ], + "properties": { + "kind": { + "const": "impute_prepayment_meter" + }, + "output": { + "type": "string" + }, + "codes": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "rule": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "salt": { + "type": "string" + }, + "n_estimators": { + "type": "integer", + "minimum": 1 + } + } + }, { "type": "object", "additionalProperties": false, diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index b24c4ee45..b53a46abd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3792,6 +3792,18 @@ "domestic_energy_consumption" ] }, + { + "kind": "impute_prepayment_meter", + "output": "uses_energy_prepayment_meter", + "codes": [ + "b489", + "b490" + ], + "rule": "b489_or_b490_nonzero", + "seed": 0, + "salt": "lcfs_prepayment_meter", + "n_estimators": 100 + }, { "kind": "iterative_proportional_fit", "columns": [ @@ -4150,7 +4162,8 @@ "domestic_energy_consumption", "electricity_consumption", "gas_consumption", - "has_fuel_consumption" + "has_fuel_consumption", + "uses_energy_prepayment_meter" ], "nonnegative_outputs": [ "food_and_non_alcoholic_beverages_consumption", @@ -4171,9 +4184,10 @@ "domestic_energy_consumption", "electricity_consumption", "gas_consumption", - "has_fuel_consumption" + "has_fuel_consumption", + "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index af546a8cd..d5040989b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3118,6 +3118,15 @@ stages: - electricity_consumption - gas_consumption - domestic_energy_consumption + - kind: impute_prepayment_meter + output: uses_energy_prepayment_meter + codes: + - b489 + - b490 + rule: b489_or_b490_nonzero + seed: 0 + salt: lcfs_prepayment_meter + n_estimators: 100 - kind: iterative_proportional_fit columns: - electricity_consumption @@ -3393,6 +3402,7 @@ stages: - electricity_consumption - gas_consumption - has_fuel_consumption + - uses_energy_prepayment_meter nonnegative_outputs: - food_and_non_alcoholic_beverages_consumption - alcohol_and_tobacco_consumption @@ -3413,7 +3423,8 @@ stages: - electricity_consumption - gas_consumption - has_fuel_consumption - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households'' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary''s other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + - uses_energy_prepayment_meter + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment''s gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households'' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary''s other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py index 52202d7d5..5b9432dd4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/graph.py @@ -787,6 +787,7 @@ def _cells( ), ), _Cell("household", "has_fuel_consumption", "bool"), + _Cell("household", "uses_energy_prepayment_meter", "bool"), ), "etb_vat": (_Cell("household", "full_rate_vat_expenditure_rate", "float64"),), "etb_services": ( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index ec1b4ef78..cf115f0ae 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -222,6 +222,19 @@ #: diesel-only household). UK_LCFS_ROAD_FUEL_TOTAL = "road_fuel_total" UK_LCFS_PETROL_SHARE = "petrol_share_of_road_fuel" +#: The household input policyengine-uk 2.123.3 reads for the Energy Price +#: Guarantee prepayment discount, July 2023 to March 2024 +#: (PolicyEngine/policyengine-uk#2098; microcosm#1110, uk-data#539). +UK_LCFS_PREPAYMENT_METER_COLUMN = "uses_energy_prepayment_meter" +IMPUTE_PREPAYMENT_METER_KIND = "impute_prepayment_meter" +#: LCFS b489 is the household's weekly energy payment through a prepayment +#: meter (electricity, and gas with it when b490 is present) and b490 the gas +#: part; b226 is electricity paid by direct debit or quarterly bill. A +#: household paying either fuel by prepayment meter has b489 or b490 nonzero (a +#: negative entry is a refund through the meter). One with neither pays +#: another way or did not say, and counts as not prepaying. +UK_LCFS_PREPAYMENT_CODES = ("b489", "b490") +UK_LCFS_PREPAYMENT_RULE = "b489_or_b490_nonzero" UK_LCFS_ICE_SHARE_RULE = "one_minus_zero_emission_share" #: How the litres audit benchmarks each fuel: HMRC's all-road-user litres times #: the cars share of that fuel's road use (PolicyEngine/chronicle#322). @@ -328,6 +341,7 @@ UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS = ( *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, "has_fuel_consumption", + UK_LCFS_PREPAYMENT_METER_COLUMN, ) UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS = UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS UK_LCFS_CONSUMPTION_FIT_NAME = "uk_lcfs_2023_24_consumption" @@ -342,6 +356,7 @@ class UKLCFSConsumptionResult: support_clip: UKSupportClipReceipt donor_uprating: Mapping[str, Any] | None = None fuel_flag: Mapping[str, Any] | None = None + prepayment_meter: Mapping[str, Any] | None = None bus_pricing: Mapping[str, Any] | None = None energy_pricing: Mapping[str, Any] | None = None energy_rake: Mapping[str, Any] | None = None @@ -361,6 +376,8 @@ def evidence(self) -> dict[str, object]: evidence["donor_uprating"] = dict(self.donor_uprating) if self.fuel_flag is not None: evidence["has_fuel_consumption"] = dict(self.fuel_flag) + if self.prepayment_meter is not None: + evidence[UK_LCFS_PREPAYMENT_METER_COLUMN] = dict(self.prepayment_meter) if self.bus_pricing is not None: evidence["bus_pricing"] = dict(self.bus_pricing) if self.energy_pricing is not None: @@ -466,6 +483,13 @@ def __call__(self, frame: Frame) -> Frame: imputation.draws, donor, exempt=support_clip_exempt(self.stage) ) household_draws = clip_result.clipped + prepayment = lcfs_prepayment_meter( + self.stage, + donor=donor, + recipient=recipient, + household_ids=frame.table("household")["household_id"].to_numpy(), + weights=weights, + ) # The chain's own draws of the parts later steps re-level, for the # recomposed totals (microcosm#1113). drawn_parts = household_draws[list(UK_LCFS_RECOMPOSED_DRAWN_COLUMNS)].copy() @@ -531,6 +555,8 @@ def __call__(self, frame: Frame) -> Frame: household["has_fuel_consumption"] = recipient["has_fuel_consumption"].to_numpy( dtype=bool ) + if prepayment is not None: + household[UK_LCFS_PREPAYMENT_METER_COLUMN] = prepayment.flags result = uk_national_frame( person=frame.table("person").copy(), benunit=frame.table("benunit").copy(), @@ -549,12 +575,14 @@ def __call__(self, frame: Frame) -> Frame: self.last_fit_weight_records = ( *imputation.fit_weight_records, *incidence.fit_weight_records, + *(() if prepayment is None else prepayment.fit_weight_records), ) self.last_result = UKLCFSConsumptionResult( frame=result, support_clip=clip_result.receipt, donor_uprating=uprating_receipt, fuel_flag=fuel_flag_receipt, + prepayment_meter=None if prepayment is None else prepayment.receipt, bus_pricing=bus_pricing_receipt, energy_pricing=None if energy is None else energy.receipt, energy_rake=energy_rake_receipt, @@ -1093,6 +1121,156 @@ def weighted_mean(values: np.ndarray, mask: np.ndarray) -> float | None: return RoadFuelIncidence(redrawn, receipt, records) +def prepayment_meter_from_lcfs(household: pd.DataFrame) -> np.ndarray: + """1.0 where the LCFS household pays energy through a prepayment meter. + + The rule is ``UK_LCFS_PREPAYMENT_RULE`` (microcosm#1110, the field reading + of uk-data#539): b489 or b490 nonzero. A household paying only electricity + by prepayment meter counts; one with no prepayment amount, which includes + any that did not answer, does not. + """ + + _require_columns(household, UK_LCFS_PREPAYMENT_CODES) + nonzero = np.zeros(len(household), dtype=bool) + for code in UK_LCFS_PREPAYMENT_CODES: + nonzero |= _numeric(household[code]).to_numpy(dtype=float) != 0.0 + return nonzero.astype(float) + + +@dataclass(frozen=True) +class PrepaymentMeterDraw: + """The recipient prepayment flags, their receipt and fit record.""" + + flags: np.ndarray + receipt: Mapping[str, Any] + fit_weight_records: tuple[FitWeightRecord, ...] + + +def prepayment_meter_operation(stage: SourceStageSpec) -> Mapping[str, Any] | None: + """The stage's declared ``impute_prepayment_meter`` parameters, if any.""" + + for operation in stage.operations: + if operation.kind == IMPUTE_PREPAYMENT_METER_KIND: + return {"kind": operation.kind, **operation.parameters} + return None + + +def impute_prepayment_meter( + *, + donor: pd.DataFrame, + recipient: pd.DataFrame, + household_ids: np.ndarray, + weights: Sequence[float], + parameters: Mapping[str, Any], +) -> PrepaymentMeterDraw: + """Draw ``uses_energy_prepayment_meter`` from the LCFS donor (microcosm#1110). + + A weighted regime-gated QRF on the chain's predictors (household counts, + region, incomes, tenure, accommodation, vehicles) fits the donor's + prepayment flag; each recipient's draw takes uniforms keyed on its household + id, so the flag does not depend on row order and no other draw moves. The + flag is the regime the draw lands in: prepaying or not. + """ + + from microcosm.fit.qrf import RegimeGatedQRF + + if parameters.get("output") != UK_LCFS_PREPAYMENT_METER_COLUMN: + raise ValueError( + f"{IMPUTE_PREPAYMENT_METER_KIND} must write " + f"{UK_LCFS_PREPAYMENT_METER_COLUMN!r}." + ) + codes = tuple(str(code) for code in parameters.get("codes", ())) + if codes != UK_LCFS_PREPAYMENT_CODES: + raise ValueError( + f"{IMPUTE_PREPAYMENT_METER_KIND} must read {UK_LCFS_PREPAYMENT_CODES}, " + f"not {codes}." + ) + rule = str(parameters.get("rule") or "") + if rule != UK_LCFS_PREPAYMENT_RULE: + raise ValueError(f"unsupported {IMPUTE_PREPAYMENT_METER_KIND} rule {rule!r}.") + seed = parameters.get("seed") + n_estimators = parameters.get("n_estimators") + salt = parameters.get("salt") + if not isinstance(seed, int) or not isinstance(n_estimators, int): + raise ValueError( + f"{IMPUTE_PREPAYMENT_METER_KIND} needs integer seed and trees." + ) + if not isinstance(salt, str) or not salt: + raise ValueError(f"{IMPUTE_PREPAYMENT_METER_KIND} needs a declared salt.") + donor_encoded, recipient_encoded, predictors = _encode_consumption_predictors( + donor, recipient + ) + target = UK_LCFS_PREPAYMENT_METER_COLUMN + train = donor_encoded[[*predictors, "household_weight"]].reset_index(drop=True) + donor_flag = donor[target].to_numpy(dtype=float) + train[target] = donor_flag + fitted = RegimeGatedQRF(n_estimators=n_estimators, seed=seed).fit( + train, list(predictors), [target], weights="household_weight" + ) + ids = np.asarray(household_ids) + drawn = fitted.predict_from_uniforms( + recipient_encoded[list(predictors)].reset_index(drop=True), + quantiles={target: stable_identity_uniforms(ids, seed=seed, salt=salt)}, + sign_uniforms={ + target: stable_identity_uniforms(ids, seed=seed, salt=f"{salt}:sign") + }, + ) + flags = drawn[target].to_numpy(dtype=float) > 0.5 + donor_weight = donor["household_weight"].to_numpy(dtype=float) + weight = np.asarray(weights, dtype=float) + regions = recipient["region"].astype(str).to_numpy() + great_britain = regions != "NORTHERN_IRELAND" + + def share(values: np.ndarray, mass: np.ndarray) -> float | None: + total = float(mass.sum()) + return float(np.dot(mass, values)) / total if total > 0 else None + + receipt = { + "operation": IMPUTE_PREPAYMENT_METER_KIND, + "output": target, + "codes": list(codes), + "rule": rule, + "seed": seed, + "salt": salt, + "n_estimators": n_estimators, + "regime": str(fitted.regimes()[target]), + "donor_share": share(donor_flag, donor_weight), + "recipient_share": share(flags.astype(float), weight), + "recipient_share_great_britain": share( + flags[great_britain].astype(float), weight[great_britain] + ), + "recipient_households_great_britain": float( + weight[great_britain & flags].sum() + ), + } + records = ( + FitWeightRecord(f"{UK_LCFS_CONSUMPTION_FIT_NAME}:{target}", fitted.weight_kind), + ) + return PrepaymentMeterDraw(flags, receipt, records) + + +def lcfs_prepayment_meter( + stage: SourceStageSpec, + *, + donor: pd.DataFrame, + recipient: pd.DataFrame, + household_ids: np.ndarray, + weights: Sequence[float], +) -> PrepaymentMeterDraw | None: + """Apply the declared ``impute_prepayment_meter`` step (none if undeclared).""" + + parameters = prepayment_meter_operation(stage) + if parameters is None: + return None + return impute_prepayment_meter( + donor=donor, + recipient=recipient, + household_ids=household_ids, + weights=weights, + parameters=parameters, + ) + + def lcfs_road_fuel_incidence( stage: SourceStageSpec, household_draws: pd.DataFrame, @@ -1933,6 +2111,7 @@ def clean_lcfs_consumption_table( _lowercase(lcfs_household)["a121"], LCFS_ACCOMM_MAP, column="a121" ) household = derive_energy_from_lcfs(household) + household[UK_LCFS_PREPAYMENT_METER_COLUMN] = prepayment_meter_from_lcfs(household) household = household.rename(columns=CONSUMPTION_VARIABLE_RENAMES) for code in BUS_FARE_LCFS_CODES: if code not in household: @@ -1976,6 +2155,7 @@ def clean_lcfs_consumption_table( [ *UK_LCFS_CONSUMPTION_PREDICTORS, *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + UK_LCFS_PREPAYMENT_METER_COLUMN, "household_gross_income", "household_weight", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py index 798f5d273..967d68687 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/national_frame.py @@ -475,6 +475,15 @@ def _write_uk_single_year_tables( #: ``other_property_value`` remainder the engine sum leaves out. The spine #: keeps it (the WAS stage's identity gate, donor-range cap and later #: segments read it); the engine derives it from the persisted components. +#: +#: ``household.uses_energy_prepayment_meter`` (microcosm#1110) is the LCFS +#: stage's prepayment-meter flag, the input policyengine-uk reads for the +#: Energy Price Guarantee prepayment discount from 2.123.3 +#: (PolicyEngine/policyengine-uk#2098). The locked engine is older and does +#: not know the variable, so the release keeps it off until the lock reaches +#: a release that does; the engine-lane test +#: ``test_prepayment_meter_is_dropped_until_the_engine_reads_it`` fails then, +#: and the bump removes it here. UK_RELEASE_EXPORT_DROPPED_COLUMNS: Mapping[str, tuple[str, ...]] = { "person": ( "incapacity_benefit_reported", @@ -493,7 +502,7 @@ def _write_uk_single_year_tables( # no engine input. "reports_rent_from_other_property", ), - "household": ("property_wealth",), + "household": ("property_wealth", "uses_energy_prepayment_meter"), } diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_prepayment_meter_engine_gate.py b/packages/microcosm-build/tests/engine/uk/test_uk_prepayment_meter_engine_gate.py new file mode 100644 index 000000000..317e92624 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_prepayment_meter_engine_gate.py @@ -0,0 +1,38 @@ +"""The LCFS prepayment-meter flag waits for the engine (microcosm#1110). + +policyengine-uk reads the household input ``uses_energy_prepayment_meter`` +from 2.123.3 (PolicyEngine/policyengine-uk#2098), for the Energy Price +Guarantee prepayment discount. The LCFS stage draws it, and the release +boundary drops it while the locked engine does not know the variable. Once +the lock reaches a release that reads it, this test fails: export the column +by removing it from ``UK_RELEASE_EXPORT_DROPPED_COLUMNS``. +""" + +from __future__ import annotations + +from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_PREPAYMENT_METER_COLUMN, +) +from microcosm.build.uk_runtime.national_frame import ( + UK_RELEASE_EXPORT_DROPPED_COLUMNS, +) + + +def test_prepayment_meter_is_dropped_until_the_engine_reads_it() -> None: + from policyengine_uk import CountryTaxBenefitSystem + + engine_reads_it = ( + UK_LCFS_PREPAYMENT_METER_COLUMN in CountryTaxBenefitSystem().variables + ) + dropped = ( + UK_LCFS_PREPAYMENT_METER_COLUMN + in (UK_RELEASE_EXPORT_DROPPED_COLUMNS["household"]) + ) + assert dropped != engine_reads_it, ( + "The locked policyengine-uk now reads uses_energy_prepayment_meter: " + "remove it from UK_RELEASE_EXPORT_DROPPED_COLUMNS so the release " + "exports the LCFS stage's flag (microcosm#1110)." + if engine_reads_it + else "The release must drop uses_energy_prepayment_meter while the " + "locked policyengine-uk does not read it." + ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py index 75618b0dc..a06b467a3 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_frs_spine.py @@ -2862,6 +2862,8 @@ def test_e8_manifest_seeds_all_reach_the_build_sidecar_harvester() -> None: "has_fuel_consumption": 0, "lcfs_consumption": 0, "gas_disconnection": 0, + # The prepayment-meter flag (microcosm#1110). + "uses_energy_prepayment_meter": 0, } assert declared["uc_capital_coherence"] == {"frs_benunit_capital": 0} assert declared["uc_deduction_attributes"] == { diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 21e1df8db..42fe37f1a 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -871,11 +871,16 @@ def materialize(self, frame, variables, period): out = result.table("household") assert set(UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS) <= set(out.columns) - # The 18 chain targets, then the road-fuel redraw's total and petrol share. - assert len(transform.fit_weight_records) == 20 - assert transform.fit_weight_records[-1].fit_name.endswith( + # The 18 chain targets, the road-fuel redraw's total and petrol share, + # then the prepayment-meter flag (microcosm#1110). + assert len(transform.fit_weight_records) == 21 + assert transform.fit_weight_records[-2].fit_name.endswith( ":petrol_share_of_road_fuel" ) + assert transform.fit_weight_records[-1].fit_name.endswith( + ":uses_energy_prepayment_meter" + ) + assert out["uses_energy_prepayment_meter"].dtype == bool from microcosm.build.uk_runtime.lcfs_consumption import ( UK_LCFS_CONSUMPTION_MASS_CONSERVATION_REASON, ) @@ -1890,3 +1895,165 @@ def materialize(self, frame, variables, period): assert result["household_adult_count"].tolist() == [2.0, 1.0] assert result["household_child_count"].tolist() == [1.0, 1.0] assert not {"is_adult", "is_child"} & set(requested) + + +def test_prepayment_meter_donor_rule_reads_b489_or_b490() -> None: + # microcosm#1110: b489 is the prepayment-meter energy payment and b490 its + # gas part; electricity-only prepayment and refunds through the meter count. + from microcosm.build.uk_runtime.lcfs_consumption import ( + prepayment_meter_from_lcfs, + ) + + household = pd.DataFrame( + { + "b489": [9.0, 0.0, -2.0, 0.0, None], + "b490": [4.0, 3.0, 0.0, 0.0, None], + } + ) + assert prepayment_meter_from_lcfs(household).tolist() == [1.0, 1.0, 1.0, 0.0, 0.0] + with pytest.raises(ValueError, match="b490"): + prepayment_meter_from_lcfs(household.drop(columns="b490")) + + +def _prepayment_donor_and_recipient() -> tuple[pd.DataFrame, pd.DataFrame]: + rng = np.random.default_rng(7) + n = 400 + tenure = rng.choice(["RENT_FROM_COUNCIL", "RENT_PRIVATELY", "OWNED_OUTRIGHT"], n) + donor = pd.DataFrame( + { + "household_adult_count": rng.integers(1, 4, n), + "household_child_count": rng.integers(0, 3, n), + "region": rng.choice(["LONDON", "WALES", "NORTHERN_IRELAND"], n), + "employment_income": rng.uniform(0, 6e4, n), + "self_employment_income": rng.uniform(0, 1e4, n), + "private_pension_income": rng.uniform(0, 2e4, n), + "hbai_household_net_income": rng.uniform(5e3, 8e4, n), + "tenure_type": tenure, + "accommodation_type": rng.choice(["FLAT", "HOUSE_TERRACED"], n), + "num_vehicles": rng.integers(0, 3, n), + "household_weight": rng.uniform(500, 1500, n), + } + ) + # Renters prepay more often, so the flag depends on the predictors. + renter = tenure != "OWNED_OUTRIGHT" + donor["uses_energy_prepayment_meter"] = ( + rng.random(n) < np.where(renter, 0.4, 0.05) + ).astype(float) + recipient = donor.drop( + columns=["household_weight", "uses_energy_prepayment_meter"] + ).iloc[:120] + return donor, recipient.reset_index(drop=True) + + +def _prepayment_parameters(**changes) -> dict: + return { + "kind": "impute_prepayment_meter", + "output": "uses_energy_prepayment_meter", + "codes": ["b489", "b490"], + "rule": "b489_or_b490_nonzero", + "seed": 0, + "salt": "lcfs_prepayment_meter", + "n_estimators": 8, + **changes, + } + + +def test_prepayment_meter_draw_is_keyed_on_the_household() -> None: + from microcosm.build.uk_runtime.lcfs_consumption import impute_prepayment_meter + + donor, recipient = _prepayment_donor_and_recipient() + ids = np.arange(1001, 1001 + len(recipient)) + weights = np.full(len(recipient), 250.0) + + first = impute_prepayment_meter( + donor=donor, + recipient=recipient, + household_ids=ids, + weights=weights, + parameters=_prepayment_parameters(), + ) + order = np.random.default_rng(3).permutation(len(recipient)) + permuted = impute_prepayment_meter( + donor=donor, + recipient=recipient.iloc[order].reset_index(drop=True), + household_ids=ids[order], + weights=weights[order], + parameters=_prepayment_parameters(), + ) + + assert first.flags.dtype == bool + assert 0 < first.flags.sum() < len(recipient) + assert (permuted.flags == first.flags[order]).all() + receipt = first.receipt + assert receipt["operation"] == "impute_prepayment_meter" + assert receipt["donor_share"] == pytest.approx( + np.average( + donor["uses_energy_prepayment_meter"], weights=donor["household_weight"] + ) + ) + assert receipt["recipient_share"] == pytest.approx(first.flags.mean()) + gb = recipient["region"].to_numpy() != "NORTHERN_IRELAND" + assert receipt["recipient_share_great_britain"] == pytest.approx( + first.flags[gb].mean() + ) + assert receipt["recipient_households_great_britain"] == pytest.approx( + 250.0 * first.flags[gb].sum() + ) + assert first.fit_weight_records[0].fit_name.endswith( + ":uses_energy_prepayment_meter" + ) + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ({"output": "prepays"}, "must write"), + ({"codes": ["b489"]}, "must read"), + ({"rule": "b489_only"}, "unsupported"), + ({"salt": ""}, "declared salt"), + ({"seed": "0"}, "integer seed"), + ], +) +def test_prepayment_meter_step_refuses_a_different_declaration( + changes, message +) -> None: + from microcosm.build.uk_runtime.lcfs_consumption import impute_prepayment_meter + + donor, recipient = _prepayment_donor_and_recipient() + with pytest.raises(ValueError, match=message): + impute_prepayment_meter( + donor=donor, + recipient=recipient, + household_ids=np.arange(len(recipient)), + weights=np.ones(len(recipient)), + parameters=_prepayment_parameters(**changes), + ) + + +def test_prepayment_meter_leaves_the_release_until_the_engine_reads_it() -> None: + # policyengine-uk reads uses_energy_prepayment_meter from 2.123.3; until the + # lock reaches it the release boundary drops the column (microcosm#1110). + from microcosm.build.uk_runtime.national_frame import ( + UK_RELEASE_EXPORT_DROPPED_COLUMNS, + uk_release_export_frame, + ) + + assert ( + "uses_energy_prepayment_meter" in UK_RELEASE_EXPORT_DROPPED_COLUMNS["household"] + ) + frame = uk_national_frame( + person=pd.DataFrame( + {"person_id": [1], "person_benunit_id": [1], "person_household_id": [10]} + ), + benunit=pd.DataFrame({"benunit_id": [1]}), + household=pd.DataFrame( + { + "household_id": [10], + "household_weight": [1.0], + "uses_energy_prepayment_meter": [True], + } + ), + time_period="2024", + ) + exported = uk_release_export_frame(frame) + assert "uses_energy_prepayment_meter" not in exported.table("household") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py index 3e9801e17..3317b1502 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_source_stages.py @@ -619,6 +619,7 @@ def test_e3_operation_kinds_are_declared_in_order(self) -> None: "materialize_rules_engine_predictors", "fit_weighted_qrf_chain", "support_clip", + "impute_prepayment_meter", "iterative_proportional_fit", "price_domestic_energy", "price_bus_journeys", @@ -759,6 +760,7 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS, UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS, UK_LCFS_CONSUMPTION_PREDICTORS, + UK_LCFS_CONSUMPTION_TARGET_COLUMNS, ) from microcosm.build.uk_runtime.uc_reporter_redraw import ( UC_REPORTER_AGGREGATES, @@ -831,7 +833,12 @@ def test_engine_predictor_and_rewrite_constants_match_manifest(self) -> None: ) assert ( tuple(lcfs_ops["fit_weighted_qrf_chain"].parameters["targets"]) - == UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS[:-1] + == UK_LCFS_CONSUMPTION_TARGET_COLUMNS + ) + assert UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS == ( + *UK_LCFS_CONSUMPTION_TARGET_COLUMNS, + "has_fuel_consumption", + "uses_energy_prepayment_meter", ) assert ( tuple(stages["etb_vat"].operations[1].parameters["predictors"]) @@ -974,6 +981,8 @@ def test_e6_declared_seed_lockstep(self) -> None: "price_domestic_energy": 0, # The road-fuel incidence redraw (microcosm#1113). "redraw_zero_road_fuel": 0, + # The prepayment-meter flag (microcosm#1110). + "impute_prepayment_meter": 0, } assert stages["etb_vat"].operations[2].parameters["seed"] == 0 assert stages["etb_services"].operations[3].parameters["seed"] == 0 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py index 934ba203a..2ddbea535 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_terminal_gates.py @@ -417,6 +417,9 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: UK_INTERNAL_DISABILITY_REPORTED_COLUMNS, ) from microcosm.build.uk_runtime.frs_spine import FRS_LANDLORD_CARRIER_COLUMN + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_PREPAYMENT_METER_COLUMN, + ) from microcosm.build.uk_runtime.salary_sacrifice import ( SALSAC_PRE_CONVERSION_PAY_COLUMN, ) @@ -431,6 +434,9 @@ def test_export_candidate_columns_strip_ids_and_carry_the_weight() -> None: | {f"person.{column}" for column in UK_INTERNAL_DISABILITY_REPORTED_COLUMNS} | {f"person.{SALSAC_PRE_CONVERSION_PAY_COLUMN}"} | {f"person.{FRS_LANDLORD_CARRIER_COLUMN}"} + # The LCFS prepayment-meter flag waits for an engine that reads it + # (microcosm#1110); it is no reference column, so no reviewed exclusion. + | {f"household.{UK_LCFS_PREPAYMENT_METER_COLUMN}"} ) # A frame without the column passes through untouched. assert uk_release_export_frame(exported) is exported diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 9d0434d92..df151666e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3510,9 +3510,10 @@ "domestic_energy_consumption", "electricity_consumption", "gas_consumption", - "has_fuel_consumption" + "has_fuel_consumption", + "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3742,6 +3743,18 @@ "kind": "support_clip", "range": "donor_realized" }, + { + "codes": [ + "b489", + "b490" + ], + "kind": "impute_prepayment_meter", + "n_estimators": 100, + "output": "uses_energy_prepayment_meter", + "rule": "b489_or_b490_nonzero", + "salt": "lcfs_prepayment_meter", + "seed": 0 + }, { "columns": [ "electricity_consumption", @@ -4100,7 +4113,8 @@ "domestic_energy_consumption", "electricity_consumption", "gas_consumption", - "has_fuel_consumption" + "has_fuel_consumption", + "uses_energy_prepayment_meter" ], "rewrites": [], "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices.", From 1685d0e06df2b51860e371abb48b312fe549807f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 07/28] Read negative LCFS energy payments as zero in the energy split (#1164) derive_energy_from_lcfs chooses one of four cases from b226 (electricity by direct debit or bill), b489 (the prepayment-meter payment) and b490 (its gas part). A refund recorded as a negative amount - a negative b489, or a negative b490 beside a positive b489 - matched none of them, so the donor lost all its electricity and gas. The three amounts are now read floored at zero, so every household falls in the case its remaining payments describe. Co-Authored-By: Claude Opus 5.5 --- ...cfs-energy-split-negative-payments.fixed.md | 1 + .../build/uk_runtime/lcfs_consumption.py | 15 +++++++++++---- .../engine_free/uk/test_uk_lcfs_consumption.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md diff --git a/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md b/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md new file mode 100644 index 000000000..0032fd352 --- /dev/null +++ b/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md @@ -0,0 +1 @@ +The UK LCFS stage's electricity and gas split reads the payment amounts it splits on (b226 electricity by direct debit or bill, b489 the prepayment-meter payment, b490 its gas part) floored at zero (microcosm#1164). A refund recorded as a negative amount, a negative b489 or a negative b490 beside a positive b489, matched none of the derivation's four cases, so the donor household lost all its electricity and gas; on the LCFS 2023-24 donor that was about twenty diaries. They now fall in the case their remaining payments describe. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index cf115f0ae..59b7739ef 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -2265,15 +2265,22 @@ def floor_negative_donor_consumption( def derive_energy_from_lcfs(household: pd.DataFrame) -> pd.DataFrame: - """Split LCFS domestic energy into electricity and gas weekly amounts.""" + """Split LCFS domestic energy into electricity and gas weekly amounts. + + b226 is electricity paid by direct debit or quarterly bill, b489 the + prepayment-meter energy payment and b490 its gas part. A refund can make + any of them negative; they are read floored at zero, so every household + falls in one of the four cases below (microcosm#1164: a negative b489, or a + negative b490 beside a positive b489, matched none and lost its energy). + """ for column in ("p537", "b226", "b489", "b490"): if column not in household: raise ValueError(f"LCFS household donor is missing {column!r}.") p537 = _numeric(household["p537"]) - b226 = _numeric(household["b226"]) - b489 = _numeric(household["b489"]) - b490 = _numeric(household["b490"]) + b226 = _numeric(household["b226"]).clip(lower=0.0) + b489 = _numeric(household["b489"]).clip(lower=0.0) + b490 = _numeric(household["b490"]).clip(lower=0.0) dd_mask = (b226 > 0) & (p537 > 0) mean_elec_share = (b226[dd_mask] / p537[dd_mask]).clip(0, 1).mean() if np.isnan(mean_elec_share): diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 42fe37f1a..7f47d5c94 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -183,6 +183,24 @@ def test_energy_split_exercises_four_cases_fallback_and_clamp() -> None: np.testing.assert_allclose(split["gas_consumption"], [4.8, 9.6, 14.4, 19.2]) +def test_energy_split_reads_negative_payments_as_zero() -> None: + # microcosm#1164: a refund through the meter (negative b489) or a negative + # gas part beside a positive b489 matched none of the four cases, so the + # household lost its electricity and gas. + household = _household() + household["b226"] = [0.0, 0.0, 0.0, -3.0] + household["b489"] = [-5.0, 9.0, 0.0, 0.0] + household["b490"] = [0.0, -1.0, 0.0, 0.0] + household["p537"] = [10.0, 20.0, 30.0, 40.0] + + split = derive_energy_from_lcfs(household) + + energy = split["electricity_consumption"] + split["gas_consumption"] + # Row 0 (b489 refund) and rows 2-3 split p537; row 1 splits b489 alone. + np.testing.assert_allclose(energy, [10.0, 9.0, 30.0, 40.0]) + assert (split["electricity_consumption"] > 0).all() + + def test_recipient_has_fuel_is_conditioned_on_vehicle_count_and_deterministic() -> None: frame = uk_national_frame( person=pd.DataFrame( From 2f31ed72633fb7d73b75a62f5d4aa5a12cbf490e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 08/28] Prepare the LCFS re-pin: study pins in the spec, same-year diary uprating (#1164) - The LCFS tabs carry ukds_study_number and doi in the spec, as the NTS and WAS tabs do, and tools/build_uk_e6_support_bounds.py reads them from there instead of hard-coding SN 9468; the regenerated bounds are byte-identical. - donor_uprating records the following year's index beside base. For a diary year that starts in the target year (LCFS 2024-25 against calendar 2024) base is 0.75 of the starting year's index plus 0.25 of the next, not of ``after``, so the receipt now carries what base was computed from; such a diary takes a factor below one, a quarter back. A unit test covers it, with 2025 added to the fake indices. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/uk/source_stages.json | 4 +++ .../src/microcosm/build/uk/spec/sources.yaml | 4 +++ .../build/uk_runtime/donor_uprating.py | 11 ++++-- .../engine_free/uk/test_uk_donor_uprating.py | 34 +++++++++++++++++++ .../parity/uk_spine/sources/fixture.json | 4 +++ .../microcosm_build/uk_donor_uprating.py | 3 +- tools/build_uk_e6_support_bounds.py | 20 +++++++++-- 7 files changed, 75 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index b53a46abd..5bc190fd7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3477,6 +3477,8 @@ { "role": "lcfs_household_tab", "kind": "private_microdata", + "ukds_study_number": 9468, + "doi": "10.5255/UKDA-SN-9468-3", "format": "tab", "vintage": "2023_24", "locator": "dvhh_ukanon_v2_2023.tab", @@ -3488,6 +3490,8 @@ { "role": "lcfs_person_tab", "kind": "private_microdata", + "ukds_study_number": 9468, + "doi": "10.5255/UKDA-SN-9468-3", "format": "tab", "vintage": "2023_24", "locator": "dvper_ukanon_202324_2023.tab", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index d5040989b..67e5d0cdd 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2878,6 +2878,8 @@ stages: artifacts: - role: lcfs_household_tab kind: private_microdata + ukds_study_number: 9468 + doi: 10.5255/UKDA-SN-9468-3 format: tab vintage: '2023_24' locator: dvhh_ukanon_v2_2023.tab @@ -2887,6 +2889,8 @@ stages: filename: dvhh_ukanon_v2_2023.tab - role: lcfs_person_tab kind: private_microdata + ukds_study_number: 9468 + doi: 10.5255/UKDA-SN-9468-3 format: tab vintage: '2023_24' locator: dvper_ukanon_202324_2023.tab diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py index 3167854a5..297dc3693 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py @@ -14,7 +14,9 @@ year beginning in ``from_period``, placed on the calendar-year index by the operation's ``fiscal_year_bridge`` weights (three quarters in the starting year, one in the next), and ends at calendar ``to_period`` (microcosm#1113: - a full step would carry a 2023-24 diary a quarter past calendar 2024); + a full step would carry a 2023-24 diary a quarter past calendar 2024). A + diary year that starts in ``to_period`` takes a factor below one, a quarter + back to the calendar year; * ``vendored_litre_proxy`` — publisher outturn for road fuel: the DESNZ annual pump-price ratio times the HMRC fiscal-year litres ratio over the ONS population ratio, all read from the vendored Chronicle facts (María's ruling @@ -139,7 +141,12 @@ def donor_uprating_factors( _require_positive(column, before, following) base = bridge[0] * before + bridge[1] * following factor = after / base - detail.update({"from_window": window, "base": base}) + # The following year's index is recorded so base can be + # recomputed when it is not ``after`` (a diary year that starts + # in the target year, as LCFS 2024-25 does for 2024). + detail.update( + {"from_window": window, "base": base, "following": following} + ) elif basis == BASIS_VENDORED_LITRE_PROXY: factor, detail = _litre_proxy_factor(spec, from_period, to_period) audit_path = spec.get("audit_parameter_path") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py index 36f6731e6..e44a86e01 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py @@ -64,6 +64,40 @@ def test_committed_json_mirror_matches_the_yaml_declaration() -> None: ) +def test_a_diary_year_starting_in_the_target_year_steps_a_quarter_back() -> None: + # microcosm#1164: the April 2024 to March 2025 diary sits a quarter past + # calendar 2024, so its window factor is below one; a column without the + # window, an income predictor, takes 1. + factors, receipt = donor_uprating_factors( + { + "from_period": 2024, + "to_period": 2024, + "fiscal_year_bridge": {"start_year_weight": 0.75, "next_year_weight": 0.25}, + "columns": { + "food": { + "basis": "engine_parameter", + "parameter_path": CPI, + "from_window": "fiscal_year", + }, + "pay": {"basis": "engine_parameter", "parameter_path": CPI}, + }, + }, + parameter_reader=_fake_reader, + ) + base = 0.75 * 1.55763 + 0.25 * 1.60100 + assert factors["food"] == pytest.approx(1.55763 / base) + assert factors["food"] < 1.0 + assert factors["pay"] == 1.0 + food = receipt["columns"]["food"] + assert food["base"] == pytest.approx(base) + # The following year's index is recorded: base is not 0.75 before + 0.25 + # after when the diary starts in the target year. + assert food["following"] == 1.60100 + assert food["base"] == pytest.approx( + 0.75 * food["before"] + 0.25 * food["following"] + ) + + def test_engine_parameter_basis_is_the_ratio_of_the_two_instants() -> None: factors, receipt = donor_uprating_factors( { diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index df151666e..fb269dc9e 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3402,6 +3402,7 @@ "lcfs_consumption": { "artifacts": [ { + "doi": "10.5255/UKDA-SN-9468-3", "filename": "dvhh_ukanon_v2_2023.tab", "format": "tab", "kind": "private_microdata", @@ -3410,9 +3411,11 @@ "runtime_sha256_required": true, "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", "size_bytes": 22812887, + "ukds_study_number": 9468, "vintage": "2023_24" }, { + "doi": "10.5255/UKDA-SN-9468-3", "filename": "dvper_ukanon_202324_2023.tab", "format": "tab", "kind": "private_microdata", @@ -3421,6 +3424,7 @@ "runtime_sha256_required": true, "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", "size_bytes": 6545146, + "ukds_study_number": 9468, "vintage": "2023_24" }, { diff --git a/test_support/microcosm_build/uk_donor_uprating.py b/test_support/microcosm_build/uk_donor_uprating.py index fe1e6e937..f7b1c2c48 100644 --- a/test_support/microcosm_build/uk_donor_uprating.py +++ b/test_support/microcosm_build/uk_donor_uprating.py @@ -36,7 +36,8 @@ CPI = "gov.economic_assumptions.indices.obr.consumer_price_index" FAKE_INDICES = { - CPI: {2023: 1.51964, 2024: 1.55763}, + # 2025 serves a diary year that starts in the target year (LCFS 2024-25). + CPI: {2023: 1.51964, 2024: 1.55763, 2025: 1.60100}, "gov.economic_assumptions.indices.obr.average_earnings": {2023: 1.0, 2024: 1.051}, "gov.economic_assumptions.indices.obr.per_capita.mixed_income": { 2023: 2.0, diff --git a/tools/build_uk_e6_support_bounds.py b/tools/build_uk_e6_support_bounds.py index 7a232aac0..0fae95029 100644 --- a/tools/build_uk_e6_support_bounds.py +++ b/tools/build_uk_e6_support_bounds.py @@ -72,10 +72,10 @@ def build_lcfs_support_bounds( donor, _floor_receipt = floor_negative_donor_consumption( donor, floor=donor_consumption_floor(stage) ) + study = _lcfs_study(stage) return _payload( source={ - "ukds_study_number": 9468, - "doi": "10.5255/UKDA-SN-9468-3", + **study, "household_tab_sha256": hh_sha, "person_tab_sha256": person_sha, }, @@ -84,6 +84,22 @@ def build_lcfs_support_bounds( ) +def _lcfs_study(stage) -> dict[str, object]: + """The LCFS study number and DOI the committed stage pins on both tabs.""" + + studies = { + (artifact.get("ukds_study_number"), artifact.get("doi")) + for artifact in stage.artifacts + if artifact.get("role") in ("lcfs_household_tab", "lcfs_person_tab") + } + if len(studies) != 1 or None in next(iter(studies)): + raise SystemExit( + "The LCFS stage must pin one ukds_study_number and doi on both tabs." + ) + number, doi = next(iter(studies)) + return {"ukds_study_number": int(number), "doi": str(doi)} + + def build_etb_vat_support_bounds(etb_tab: Path) -> dict[str, object]: """The declared [0, 1] share range, checked against the donor (uk-data#572). From b963bb86a96b574024e736a54d108a1db7d164a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 09/28] Read LCFS fuel payments by their labelled codes: energy split, prepayment (#1164, #1110) The LCF User Guide's derived-variable labels (2024-25 Volume F, Part 2) and its P537 definition (Part 3) contradict how the electricity and gas split, ported from policyengine-uk-data, read the payment codes. b226 is the gas component of a combined utilities bill (b227 the electricity one), not electricity by direct debit; b489 and b490 are the electricity and gas account payments less rebates (b175 less b178, b170 less b173 on 99 to 100 percent of account payers in both years), not prepayment-meter payments. The split therefore gave combined-bill households, most of the donor, their gas as electricity, subtracted gas from electricity for two-account households, split an electricity account between the fuels, and folded oil and solid fuels into the drawn energy. - derive_energy_from_lcfs sums each fuel's labelled parts: the account payment less its rebate, the prepayment key or meter payment, its part of a combined bill or meter payment, and the diary's account and slot-meter lines. A combined payment left unsplit goes by the household's own electricity share, else the dual-fuel mean. Oil, bottled gas, solid fuels and heat stay in the housing total (#1207's liquid and solid fuels item). Negative amounts are read as zero, which supersedes the previous commit's floors on the old codes. - A donor_energy_split receipt records the donor's electricity, gas and other fuels against p537. - The prepayment flag (#1110) reads what the household last put on an electricity, gas or combined key or meter (b1751, b1701, b233, asked only of prepayment-meter households) and the diary's slot-meter lines, not the account payments the previous commit used. - The synthetic donors and the H2 fixture carry the labelled codes; the mirror, coverage manifest and fixture are regenerated. Co-Authored-By: Claude Opus 5.5 --- .../1110-prepayment-meter-input.added.md | 2 +- ...-lcfs-energy-split-labelled-codes.fixed.md | 1 + ...fs-energy-split-negative-payments.fixed.md | 1 - .../src/microcosm/build/uk/source_stages.json | 11 +- .../src/microcosm/build/uk/spec/sources.yaml | 11 +- .../build/uk_runtime/lcfs_consumption.py | 224 +++++++++++++----- .../uk/test_uk_lcfs_consumption.py | 161 +++++++++---- .../parity/uk_spine/sources/fixture.json | 11 +- .../uk_spine/sources/lcfs_household.csv | 130 +++++----- tools/graph_uk_spine_fixture.py | 22 +- 10 files changed, 392 insertions(+), 182 deletions(-) create mode 100644 changelog.d/1164-lcfs-energy-split-labelled-codes.fixed.md delete mode 100644 changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md diff --git a/changelog.d/1110-prepayment-meter-input.added.md b/changelog.d/1110-prepayment-meter-input.added.md index 9b2c68fbb..061b6dc08 100644 --- a/changelog.d/1110-prepayment-meter-input.added.md +++ b/changelog.d/1110-prepayment-meter-input.added.md @@ -1 +1 @@ -The UK `lcfs_consumption` stage draws `uses_energy_prepayment_meter`, the household input policyengine-uk reads from 2.123.3 for the Energy Price Guarantee prepayment discount, July 2023 to March 2024 (microcosm#1110; PolicyEngine/policyengine-uk#2098; uk-data#539 tracks the same input). An LCFS donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, the fields the stage already reads to split electricity and gas: electricity-only prepayment counts, and a household with no prepayment amount, including one that did not answer, does not. The declared `impute_prepayment_meter` step fits that flag with a weighted regime-gated QRF on the chain's predictors and draws it at uniforms keyed on the household id, so no other draw moves; its receipt records the donor's and the frame's weighted shares and the Great Britain households flagged. The locked policyengine-uk does not read the variable yet, so the release export drops the column, and an engine-lane test fails once the lock reaches a release that does. +The UK `lcfs_consumption` stage draws `uses_energy_prepayment_meter`, the household input policyengine-uk reads from 2.123.3 for the Energy Price Guarantee prepayment discount, July 2023 to March 2024 (microcosm#1110; PolicyEngine/policyengine-uk#2098; uk-data#539 tracks the same input). An LCFS donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214): electricity-only prepayment counts, and a household with no meter payment, including one that did not answer, does not. The declared `impute_prepayment_meter` step fits that flag with a weighted regime-gated QRF on the chain's predictors and draws it at uniforms keyed on the household id, so no other draw moves; its receipt records the donor's and the frame's weighted shares and the Great Britain households flagged. The locked policyengine-uk does not read the variable yet, so the release export drops the column, and an engine-lane test fails once the lock reaches a release that does. diff --git a/changelog.d/1164-lcfs-energy-split-labelled-codes.fixed.md b/changelog.d/1164-lcfs-energy-split-labelled-codes.fixed.md new file mode 100644 index 000000000..9241cf213 --- /dev/null +++ b/changelog.d/1164-lcfs-energy-split-labelled-codes.fixed.md @@ -0,0 +1 @@ +The UK LCFS stage splits the diary's fuel, light and power (p537) into electricity and gas by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164). The derivation it replaces, ported from policyengine-uk-data, read b226 as electricity paid by direct debit and b489 and b490 as prepayment-meter payments. The labels make b226 the gas component of a combined utilities bill (b227 is the electricity one) and b489 and b490 the electricity and gas account payments less rebates (b175 less b178, b170 less b173), so it gave combined-bill households, most of the donor, their gas as electricity; took gas away from electricity for households with two accounts; split an electricity account between the fuels; and folded oil and solid fuels into the drawn energy. Each fuel is now its account payment less its rebate, its prepayment key or meter payment, its part of a combined bill or meter payment, and the diary's account and slot-meter lines; a combined payment left unsplit goes by the household's own electricity share. Oil, bottled gas, solid fuels and heat stay in the housing total. A `donor_energy_split` receipt records the donor's electricity, gas and other fuels against p537. Negative amounts (refunds) are read as zero. The NEED rake and the DESNZ level still set each fuel's level, but which households carry electricity and which gas now follows what they reported. diff --git a/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md b/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md deleted file mode 100644 index 0032fd352..000000000 --- a/changelog.d/1164-lcfs-energy-split-negative-payments.fixed.md +++ /dev/null @@ -1 +0,0 @@ -The UK LCFS stage's electricity and gas split reads the payment amounts it splits on (b226 electricity by direct debit or bill, b489 the prepayment-meter payment, b490 its gas part) floored at zero (microcosm#1164). A refund recorded as a negative amount, a negative b489 or a negative b490 beside a positive b489, matched none of the derivation's four cases, so the donor household lost all its electricity and gas; on the LCFS 2023-24 donor that was about twenty diaries. They now fall in the case their remaining payments describe. diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 5bc190fd7..95cff7319 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3800,10 +3800,13 @@ "kind": "impute_prepayment_meter", "output": "uses_energy_prepayment_meter", "codes": [ - "b489", - "b490" + "b1751", + "b1701", + "b233", + "c45114", + "c45214" ], - "rule": "b489_or_b490_nonzero", + "rule": "key_or_meter_payment_positive", "seed": 0, "salt": "lcfs_prepayment_meter", "n_estimators": 100 @@ -4191,7 +4194,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 67e5d0cdd..1e0f208cf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3125,9 +3125,12 @@ stages: - kind: impute_prepayment_meter output: uses_energy_prepayment_meter codes: - - b489 - - b490 - rule: b489_or_b490_nonzero + - b1751 + - b1701 + - b233 + - c45114 + - c45214 + rule: key_or_meter_payment_positive seed: 0 salt: lcfs_prepayment_meter n_estimators: 100 @@ -3428,7 +3431,7 @@ stages: - gas_consumption - has_fuel_consumption - uses_energy_prepayment_meter - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment''s gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households'' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary''s other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 59b7739ef..85c9658f7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -151,6 +151,63 @@ CONSUMPTION_VARIABLE_RENAMES[f"p6{division:02d}"] for division in range(1, 13) ) BUS_FARE_LCFS_CODES = ("c73212", "c73213", "c73214") +#: LCFS fuel, light and power (p537) by fuel, from the LCF User Guide's +#: derived-variable labels (2024-25 Volume F, Part 2) and its P537 definition +#: (Part 3): the diary's COICOP 04.5 lines plus the questionnaire's account, +#: meter and combined payments, less the gas and electricity rebates. Weekly +#: amounts (microcosm#1164). +#: The last account payment and its rebate: b175/b178 electricity, b170/b173 gas. +UK_LCFS_ELECTRICITY_ACCOUNT = ("b175", "b178") +UK_LCFS_GAS_ACCOUNT = ("b170", "b173") +#: The rest of each fuel: the separate prepayment-meter payment (b1751, b1701), +#: the fuel's component of a combined utilities bill (b227 electricity, b226 +#: gas), its part of a combined key/meter payment (b2332, b2331) and the +#: diary's account and slot-meter lines (c45112/c45114, c45212/c45214). +UK_LCFS_ELECTRICITY_PARTS = ("b1751", "b227", "b2332", "c45112", "c45114") +UK_LCFS_GAS_PARTS = ("b1701", "b226", "b2331", "c45212", "c45214") +#: Combined gas-and-electricity payments (b231 bill, b233 key/meter) and the +#: parts the respondent split them into. +UK_LCFS_COMBINED_PAYMENTS: tuple[tuple[str, tuple[str, str]], ...] = ( + ("b231", ("b226", "b227")), + ("b233", ("b2331", "b2332")), +) +#: The rest of p537, which no output column carries and the housing total +#: keeps: central-heating oil (b017; diary c45312, with paraffin), bottled gas +#: (b018; diary c45222), coal and coke (c45411), wood and peat (c45412), and +#: hot water, steam and ice (c45511). +UK_LCFS_OTHER_FUEL_PARTS = ( + "b017", + "b018", + "c45222", + "c45312", + "c45411", + "c45412", + "c45511", +) +UK_LCFS_ENERGY_CODES = tuple( + dict.fromkeys( + ( + "p537", + *UK_LCFS_ELECTRICITY_ACCOUNT, + *UK_LCFS_GAS_ACCOUNT, + *UK_LCFS_ELECTRICITY_PARTS, + *UK_LCFS_GAS_PARTS, + *( + code + for total, parts in UK_LCFS_COMBINED_PAYMENTS + for code in (total, *parts) + ), + *UK_LCFS_OTHER_FUEL_PARTS, + ) + ) +) +#: The electricity share of an unsplit combined payment when the donor has no +#: dual-fuel household to take a mean from (a synthetic donor only). +UK_LCFS_FALLBACK_ELECTRICITY_SHARE = 0.6 +#: The donor scratch column for the other fuels, read by the split receipt. +UK_LCFS_OTHER_FUELS_COLUMN = "lcfs_other_fuels" +#: ``donor.attrs`` key of the donor energy-split receipt. +UK_LCFS_ENERGY_SPLIT_RECEIPT = "donor_energy_split" #: The donor columns floored at zero before the imputation (microcosm#1063 #: c9): the diary consumption columns and the bus fares summed from the #: c73212-c73214 diary lines, which a refund can make negative as well. @@ -227,14 +284,14 @@ #: (PolicyEngine/policyengine-uk#2098; microcosm#1110, uk-data#539). UK_LCFS_PREPAYMENT_METER_COLUMN = "uses_energy_prepayment_meter" IMPUTE_PREPAYMENT_METER_KIND = "impute_prepayment_meter" -#: LCFS b489 is the household's weekly energy payment through a prepayment -#: meter (electricity, and gas with it when b490 is present) and b490 the gas -#: part; b226 is electricity paid by direct debit or quarterly bill. A -#: household paying either fuel by prepayment meter has b489 or b490 nonzero (a -#: negative entry is a refund through the meter). One with neither pays -#: another way or did not say, and counts as not prepaying. -UK_LCFS_PREPAYMENT_CODES = ("b489", "b490") -UK_LCFS_PREPAYMENT_RULE = "b489_or_b490_nonzero" +#: What a household last put on a prepayment key or meter (LCF questions +#: ELECMETER, GASMETER and GEMETER, asked only of households that pay by +#: prepayment meter): b1751 electricity, b1701 gas, b233 a combined gas and +#: electricity meter, and the diary's slot-meter lines c45114 (electricity) +#: and c45214 (gas). A household with any of them positive prepays; one with +#: none pays by account or direct debit, or did not say (microcosm#1110). +UK_LCFS_PREPAYMENT_CODES = ("b1751", "b1701", "b233", "c45114", "c45214") +UK_LCFS_PREPAYMENT_RULE = "key_or_meter_payment_positive" UK_LCFS_ICE_SHARE_RULE = "one_minus_zero_emission_share" #: How the litres audit benchmarks each fuel: HMRC's all-road-user litres times #: the cars share of that fuel's road use (PolicyEngine/chronicle#322). @@ -313,11 +370,9 @@ #: remainder, and the levelled columns added back. The chain draws each part #: after its total, so the split is the chain's own. Housing (p604) nets the #: chain's domestic-energy draw, which is the electricity and gas -#: ``derive_energy_from_lcfs`` makes of the diary. That derivation splits all of -#: ``p537`` (fuel, light and power) where the household reports no combined -#: payment, so those households' liquid and solid fuels leave with the drawn -#: energy; where it reads the combined payment, whatever else ``p537`` holds -#: stays in the housing remainder. Transport (p607) nets the drawn petrol and +#: ``derive_energy_from_lcfs`` makes of the diary's fuel, light and power +#: (``p537``) by its labelled parts, so the diary's oil, bottled gas, solid +#: fuels and heat stay in the housing remainder (microcosm#1164). Transport (p607) nets the drawn petrol and #: diesel (c72211, c72212) and the drawn bus fares (which the pricing step #: replaces with journeys times yield), and keeps other motor fuels. UK_LCFS_RECOMPOSED_PARENTS: Mapping[str, tuple[tuple[str, ...], tuple[str, ...]]] = { @@ -357,6 +412,7 @@ class UKLCFSConsumptionResult: donor_uprating: Mapping[str, Any] | None = None fuel_flag: Mapping[str, Any] | None = None prepayment_meter: Mapping[str, Any] | None = None + donor_energy_split: Mapping[str, Any] | None = None bus_pricing: Mapping[str, Any] | None = None energy_pricing: Mapping[str, Any] | None = None energy_rake: Mapping[str, Any] | None = None @@ -378,6 +434,8 @@ def evidence(self) -> dict[str, object]: evidence["has_fuel_consumption"] = dict(self.fuel_flag) if self.prepayment_meter is not None: evidence[UK_LCFS_PREPAYMENT_METER_COLUMN] = dict(self.prepayment_meter) + if self.donor_energy_split is not None: + evidence[UK_LCFS_ENERGY_SPLIT_RECEIPT] = dict(self.donor_energy_split) if self.bus_pricing is not None: evidence["bus_pricing"] = dict(self.bus_pricing) if self.energy_pricing is not None: @@ -583,6 +641,7 @@ def __call__(self, frame: Frame) -> Frame: donor_uprating=uprating_receipt, fuel_flag=fuel_flag_receipt, prepayment_meter=None if prepayment is None else prepayment.receipt, + donor_energy_split=donor.attrs.get(UK_LCFS_ENERGY_SPLIT_RECEIPT), bus_pricing=bus_pricing_receipt, energy_pricing=None if energy is None else energy.receipt, energy_rake=energy_rake_receipt, @@ -1125,16 +1184,16 @@ def prepayment_meter_from_lcfs(household: pd.DataFrame) -> np.ndarray: """1.0 where the LCFS household pays energy through a prepayment meter. The rule is ``UK_LCFS_PREPAYMENT_RULE`` (microcosm#1110, the field reading - of uk-data#539): b489 or b490 nonzero. A household paying only electricity - by prepayment meter counts; one with no prepayment amount, which includes - any that did not answer, does not. + of uk-data#539): any key or meter payment positive. Electricity-only + prepayment counts; a household with no meter payment does not, whether it + pays by account (b175/b170, b489/b490 net of rebates) or did not answer. """ _require_columns(household, UK_LCFS_PREPAYMENT_CODES) - nonzero = np.zeros(len(household), dtype=bool) + prepays = np.zeros(len(household), dtype=bool) for code in UK_LCFS_PREPAYMENT_CODES: - nonzero |= _numeric(household[code]).to_numpy(dtype=float) != 0.0 - return nonzero.astype(float) + prepays |= _numeric(household[code]).to_numpy(dtype=float) > 0.0 + return prepays.astype(float) @dataclass(frozen=True) @@ -1505,9 +1564,9 @@ def uncarried_spend(parameters: Mapping[str, Any]) -> dict[str, Any]: ONS Consumer Trends classes read from the vendored rows the declaration names: the liquid and solid fuels of COICOP 04.5, which have no column of - their own; part of them stays inside the housing remainder (the chain's - draw less the diary energy), and the receipt records their published - spend. + their own and stay inside the housing remainder (the chain's draw less the + diary's electricity and gas; microcosm#1164), and the receipt records their + published spend. """ spec = parameters.get("uncarried") @@ -2111,6 +2170,7 @@ def clean_lcfs_consumption_table( _lowercase(lcfs_household)["a121"], LCFS_ACCOMM_MAP, column="a121" ) household = derive_energy_from_lcfs(household) + energy_split = donor_energy_split_receipt(household) household[UK_LCFS_PREPAYMENT_METER_COLUMN] = prepayment_meter_from_lcfs(household) household = household.rename(columns=CONSUMPTION_VARIABLE_RENAMES) for code in BUS_FARE_LCFS_CODES: @@ -2165,9 +2225,45 @@ def clean_lcfs_consumption_table( missing = sorted(donor.columns[donor.isna().any()]) if missing: raise ValueError(f"LCFS donor has missing values in {missing}.") + donor.attrs[UK_LCFS_ENERGY_SPLIT_RECEIPT] = energy_split return donor +def donor_energy_split_receipt(household: pd.DataFrame) -> dict[str, Any]: + """The donor's fuel, light and power by fuel: weighted annual diary totals. + + Electricity and gas are what ``derive_energy_from_lcfs`` gives the chain; + the other fuels stay in the housing total. Their sum against ``p537`` + shows how much of the published definition the labelled parts reach + (microcosm#1164). Totals over every donor, at diary prices. + """ + + weight = _numeric(household["household_weight"]).to_numpy(dtype=float) * 1_000 + annual = weight * WEEKS_IN_YEAR + + def total(column: str) -> float: + return float(np.dot(annual, _numeric(household[column]).to_numpy(dtype=float))) + + electricity = total("electricity_consumption") + gas = total("gas_consumption") + other = total(UK_LCFS_OTHER_FUELS_COLUMN) + fuel_light_power = total("p537") + return { + "electricity": electricity, + "gas": gas, + "other_fuels": other, + "fuel_light_and_power": fuel_light_power, + "parts_over_fuel_light_and_power": ( + (electricity + gas + other) / fuel_light_power + if fuel_light_power > 0 + else None + ), + "electricity_share_of_electricity_and_gas": ( + electricity / (electricity + gas) if electricity + gas > 0 else None + ), + } + + def mapped_lcfs_codes( raw: pd.Series, mapping: Mapping[int, str], *, column: str ) -> pd.Series: @@ -2265,43 +2361,59 @@ def floor_negative_donor_consumption( def derive_energy_from_lcfs(household: pd.DataFrame) -> pd.DataFrame: - """Split LCFS domestic energy into electricity and gas weekly amounts. - - b226 is electricity paid by direct debit or quarterly bill, b489 the - prepayment-meter energy payment and b490 its gas part. A refund can make - any of them negative; they are read floored at zero, so every household - falls in one of the four cases below (microcosm#1164: a negative b489, or a - negative b490 beside a positive b489, matched none and lost its energy). + """Split LCFS fuel, light and power into electricity and gas (weekly amounts). + + Each fuel is the sum of its labelled parts (``UK_LCFS_ELECTRICITY_*``, + ``UK_LCFS_GAS_*``; microcosm#1164): the last account payment less its + rebate, the prepayment-meter payment, the fuel's component of a combined + utilities bill and of a combined key/meter payment, and the diary's + account and slot-meter lines. What a combined payment leaves unsplit is + shared by the household's own electricity share, or the donor's mean share + of dual-fuel households where it reports neither fuel. Oil, bottled gas, + solid fuels and heat (``UK_LCFS_OTHER_FUEL_PARTS``) are neither: they stay + in the housing total. Negative amounts (refunds) are read as zero. + + The derivation this replaces read b226 as electricity paid by direct debit + and b489/b490 as prepayment payments; the LCF labels make b226 the gas + component of a combined bill and b489/b490 the electricity and gas account + payments less rebates, so it gave combined-bill households their gas as + electricity and folded other fuels into gas. """ - for column in ("p537", "b226", "b489", "b490"): - if column not in household: - raise ValueError(f"LCFS household donor is missing {column!r}.") - p537 = _numeric(household["p537"]) - b226 = _numeric(household["b226"]).clip(lower=0.0) - b489 = _numeric(household["b489"]).clip(lower=0.0) - b490 = _numeric(household["b490"]).clip(lower=0.0) - dd_mask = (b226 > 0) & (p537 > 0) - mean_elec_share = (b226[dd_mask] / p537[dd_mask]).clip(0, 1).mean() - if np.isnan(mean_elec_share): - mean_elec_share = 0.52 - electricity = np.zeros(len(household)) - gas = np.zeros(len(household)) - mask1 = b226 > 0 - electricity[mask1] = b226[mask1] - gas[mask1] = np.maximum(p537[mask1] - b226[mask1], 0) - mask2 = (~mask1) & (b489 > 0) & (b490 > 0) - electricity[mask2] = np.maximum(b489[mask2] - b490[mask2], 0) - gas[mask2] = b490[mask2] - mask3 = (~mask1) & (b489 > 0) & (b490 == 0) - electricity[mask3] = b489[mask3] * mean_elec_share - gas[mask3] = b489[mask3] * (1 - mean_elec_share) - mask4 = (~mask1) & (b489 == 0) - electricity[mask4] = p537[mask4] * mean_elec_share - gas[mask4] = p537[mask4] * (1 - mean_elec_share) + _require_columns(household, UK_LCFS_ENERGY_CODES) + + def part(code: str) -> pd.Series: + return _numeric(household[code]).clip(lower=0.0) + + def fuel(account: tuple[str, str], parts: tuple[str, ...]) -> pd.Series: + paid, rebate = account + total = (_numeric(household[paid]) - _numeric(household[rebate])).clip( + lower=0.0 + ) + for code in parts: + total = total + part(code) + return total + + electricity = fuel(UK_LCFS_ELECTRICITY_ACCOUNT, UK_LCFS_ELECTRICITY_PARTS) + gas = fuel(UK_LCFS_GAS_ACCOUNT, UK_LCFS_GAS_PARTS) + unsplit = pd.Series(0.0, index=household.index) + for total, parts in UK_LCFS_COMBINED_PAYMENTS: + split = sum(part(code) for code in parts) + unsplit = unsplit + (part(total) - split).clip(lower=0.0) + both = (electricity > 0) & (gas > 0) + mean_share = ( + float((electricity[both] / (electricity[both] + gas[both])).mean()) + if both.any() + else UK_LCFS_FALLBACK_ELECTRICITY_SHARE + ) + known = electricity + gas + share = np.where(known > 0, electricity / known.where(known > 0, 1.0), mean_share) result = household.copy() - result["electricity_consumption"] = np.maximum(electricity, 0.0) - result["gas_consumption"] = np.maximum(gas, 0.0) + result["electricity_consumption"] = electricity + unsplit * share + result["gas_consumption"] = gas + unsplit * (1.0 - share) + result[UK_LCFS_OTHER_FUELS_COLUMN] = sum( + part(code) for code in UK_LCFS_OTHER_FUEL_PARTS + ) return result diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 7f47d5c94..5ca62c3d2 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -22,6 +22,16 @@ from microcosm.build.uk_runtime.national_frame import uk_national_frame +def _energy_codes(n: int, **values) -> dict[str, np.ndarray]: + """Every labelled LCFS energy code at zero, with the given ones set.""" + + from microcosm.build.uk_runtime.lcfs_consumption import UK_LCFS_ENERGY_CODES + + codes = {code: np.zeros(n) for code in UK_LCFS_ENERGY_CODES if code != "p537"} + codes.update({code: np.asarray(v, dtype=float) for code, v in values.items()}) + return codes + + def _household() -> pd.DataFrame: base = { "case": [1, 2, 3, 4], @@ -34,10 +44,19 @@ def _household() -> pd.DataFrame: "weighta": [1.5, 2.0, 2.5, 3.0], "a122": [4, 8, 5, 7], "a121": [4, 5, 6, 7], - "b226": [6.0, 0.0, 0.0, 0.0], - "b489": [0.0, 9.0, 8.0, 0.0], - "b490": [0.0, 4.0, 0.0, 0.0], - "p537": [10.0, 20.0, 30.0, -1.0], + **_energy_codes( + 4, + b231=[10.0, 0.0, 0.0, 0.0], + b226=[4.0, 0.0, 0.0, 0.0], + b227=[6.0, 0.0, 0.0, 0.0], + b175=[0.0, 9.0, 0.0, 0.0], + b178=[0.0, 1.0, 0.0, 0.0], + b170=[0.0, 5.0, 0.0, 0.0], + b1751=[0.0, 0.0, 7.0, 0.0], + b017=[0.0, 0.0, 3.0, 0.0], + b233=[0.0, 0.0, 0.0, 10.0], + ), + "p537": [10.0, 13.0, 10.0, 10.0], } for source in ( "p601", @@ -163,42 +182,72 @@ def test_stage_declares_the_unmapped_code_refusal() -> None: ) -def test_energy_split_exercises_four_cases_fallback_and_clamp() -> None: - household = _household() - split = derive_energy_from_lcfs(household) +def test_energy_split_sums_each_fuels_labelled_parts() -> None: + # microcosm#1164: b226 is the gas component of a combined bill and b227 the + # electricity one; b175 less b178 is the electricity account, b170 less b173 + # the gas account; b1751 the electricity prepayment meter. + split = derive_energy_from_lcfs(_household()) - assert split["electricity_consumption"].tolist() == [6.0, 5.0, 4.8, 0.0] - assert split["gas_consumption"].tolist() == [4.0, 4.0, 3.2, 0.0] + electricity = split["electricity_consumption"].to_numpy() + gas = split["gas_consumption"].to_numpy() + np.testing.assert_allclose(electricity[:3], [6.0, 8.0, 7.0]) + np.testing.assert_allclose(gas[:3], [4.0, 5.0, 0.0]) + # The unsplit combined key/meter payment takes the dual-fuel mean share. + mean_share = np.mean([6.0 / 10.0, 8.0 / 13.0]) + np.testing.assert_allclose(electricity[3], 10.0 * mean_share) + np.testing.assert_allclose(gas[3], 10.0 * (1.0 - mean_share)) + # Oil is neither: it stays in the housing total. + np.testing.assert_allclose(split["lcfs_other_fuels"], [0.0, 0.0, 3.0, 0.0]) - fallback = household.copy() - fallback["b226"] = 0.0 - fallback["b489"] = 0.0 - fallback["p537"] = [10.0, 20.0, 30.0, 40.0] - split = derive_energy_from_lcfs(fallback) +def test_energy_split_shares_an_unsplit_payment_by_the_households_own_split() -> None: + household = _household() + # A combined bill only partly split: 2 of electricity and 2 of gas are + # known, 6 are not, and go 50:50. + household.loc[0, ["b231", "b227", "b226"]] = [10.0, 2.0, 2.0] - np.testing.assert_allclose( - split["electricity_consumption"], [5.2, 10.4, 15.6, 20.8] - ) - np.testing.assert_allclose(split["gas_consumption"], [4.8, 9.6, 14.4, 19.2]) + split = derive_energy_from_lcfs(household) + + assert split.loc[0, "electricity_consumption"] == pytest.approx(5.0) + assert split.loc[0, "gas_consumption"] == pytest.approx(5.0) def test_energy_split_reads_negative_payments_as_zero() -> None: - # microcosm#1164: a refund through the meter (negative b489) or a negative - # gas part beside a positive b489 matched none of the four cases, so the - # household lost its electricity and gas. household = _household() - household["b226"] = [0.0, 0.0, 0.0, -3.0] - household["b489"] = [-5.0, 9.0, 0.0, 0.0] - household["b490"] = [0.0, -1.0, 0.0, 0.0] - household["p537"] = [10.0, 20.0, 30.0, 40.0] + # A refund larger than the account, and a negative meter payment. + household.loc[1, ["b175", "b178"]] = [2.0, 5.0] + household.loc[2, "b1751"] = -4.0 split = derive_energy_from_lcfs(household) - energy = split["electricity_consumption"] + split["gas_consumption"] - # Row 0 (b489 refund) and rows 2-3 split p537; row 1 splits b489 alone. - np.testing.assert_allclose(energy, [10.0, 9.0, 30.0, 40.0]) - assert (split["electricity_consumption"] > 0).all() + assert split.loc[1, "electricity_consumption"] == 0.0 + assert split.loc[1, "gas_consumption"] == pytest.approx(5.0) + assert split.loc[2, "electricity_consumption"] == 0.0 + assert (split[["electricity_consumption", "gas_consumption"]] >= 0).all().all() + + +def test_energy_split_refuses_a_missing_labelled_code() -> None: + with pytest.raises(ValueError, match="b2332"): + derive_energy_from_lcfs(_household().drop(columns="b2332")) + + +def test_donor_records_its_fuel_light_and_power_split() -> None: + from microcosm.build.uk_runtime.lcfs_consumption import ( + UK_LCFS_ENERGY_SPLIT_RECEIPT, + ) + + donor = clean_lcfs_consumption_table(_person(), _household()) + receipt = donor.attrs[UK_LCFS_ENERGY_SPLIT_RECEIPT] + weights = np.array([1.5, 2.0, 2.5, 3.0]) * 1_000 * (365.25 / 7) + mean_share = np.mean([6.0 / 10.0, 8.0 / 13.0]) + electricity = np.array([6.0, 8.0, 7.0, 10.0 * mean_share]) + assert receipt["electricity"] == pytest.approx(float(weights @ electricity)) + assert receipt["other_fuels"] == pytest.approx(float(weights[2] * 3.0)) + assert receipt["fuel_light_and_power"] == pytest.approx( + float(weights @ np.array([10.0, 13.0, 10.0, 10.0])) + ) + # The labelled parts reach the whole of p537 on this donor. + assert receipt["parts_over_fuel_light_and_power"] == pytest.approx(1.0) def test_recipient_has_fuel_is_conditioned_on_vehicle_count_and_deterministic() -> None: @@ -768,6 +817,18 @@ def _synthetic_lcfs_donor( household["c72213"] = np.where(rows % 10 == 1, 0.5, 0.0) for code in BUS_FARE_LCFS_CODES: household[code] = np.where(rng.random(n) < 0.55, 0.0, rng.uniform(1.0, 30.0, n)) + # The labelled energy parts (microcosm#1164), deterministic so the draws + # below keep their stream: an electricity account everywhere, a gas account + # on five households in six, oil on a few. + household.update( + _energy_codes( + n, + b175=20.0 + rows % 15, + b170=np.where(rows % 6 == 0, 0.0, 10.0 + rows % 9), + b226=household["b226"], + b017=np.where(rows % 20 == 3, 5.0, 0.0), + ) + ) person = pd.DataFrame( { "case": rows + 1, @@ -1915,22 +1976,34 @@ def materialize(self, frame, variables, period): assert not {"is_adult", "is_child"} & set(requested) -def test_prepayment_meter_donor_rule_reads_b489_or_b490() -> None: - # microcosm#1110: b489 is the prepayment-meter energy payment and b490 its - # gas part; electricity-only prepayment and refunds through the meter count. +def test_prepayment_meter_donor_rule_reads_key_and_meter_payments() -> None: + # microcosm#1110: b1751, b1701 and b233 are what the household last put on an + # electricity, gas or combined key/meter, c45114/c45214 the diary's + # slot-meter lines; an account payment (b489) is not prepayment. from microcosm.build.uk_runtime.lcfs_consumption import ( prepayment_meter_from_lcfs, ) household = pd.DataFrame( - { - "b489": [9.0, 0.0, -2.0, 0.0, None], - "b490": [4.0, 3.0, 0.0, 0.0, None], - } - ) - assert prepayment_meter_from_lcfs(household).tolist() == [1.0, 1.0, 1.0, 0.0, 0.0] - with pytest.raises(ValueError, match="b490"): - prepayment_meter_from_lcfs(household.drop(columns="b490")) + _energy_codes( + 6, + b1751=[9.0, 0.0, 0.0, 0.0, 0.0, 0.0], + b1701=[0.0, 4.0, 0.0, 0.0, 0.0, 0.0], + b233=[0.0, 0.0, 12.0, 0.0, 0.0, 0.0], + c45114=[0.0, 0.0, 0.0, 3.0, 0.0, 0.0], + c45214=[0.0, 0.0, 0.0, 0.0, 0.0, -1.0], + ) + ).assign(b489=[0.0, 0.0, 0.0, 0.0, 20.0, 0.0]) + assert prepayment_meter_from_lcfs(household).tolist() == [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + ] + with pytest.raises(ValueError, match="b233"): + prepayment_meter_from_lcfs(household.drop(columns="b233")) def _prepayment_donor_and_recipient() -> tuple[pd.DataFrame, pd.DataFrame]: @@ -1967,8 +2040,8 @@ def _prepayment_parameters(**changes) -> dict: return { "kind": "impute_prepayment_meter", "output": "uses_energy_prepayment_meter", - "codes": ["b489", "b490"], - "rule": "b489_or_b490_nonzero", + "codes": ["b1751", "b1701", "b233", "c45114", "c45214"], + "rule": "key_or_meter_payment_positive", "seed": 0, "salt": "lcfs_prepayment_meter", "n_estimators": 8, @@ -2026,8 +2099,8 @@ def test_prepayment_meter_draw_is_keyed_on_the_household() -> None: ("changes", "message"), [ ({"output": "prepays"}, "must write"), - ({"codes": ["b489"]}, "must read"), - ({"rule": "b489_only"}, "unsupported"), + ({"codes": ["b489", "b490"]}, "must read"), + ({"rule": "b489_or_b490_nonzero"}, "unsupported"), ({"salt": ""}, "declared salt"), ({"seed": "0"}, "integer seed"), ], diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index fb269dc9e..dae8652a5 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3517,7 +3517,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when its prepayment-meter energy payment (b489) or that payment's gas part (b490) is nonzero, so electricity-only prepayment counts and a household with no prepayment amount, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary: GBP 54.9bn on the donor, against GBP 60.1bn for all of its fuel, light and power. The derivation splits all of that total where the household reports no combined payment (2,978 of the 4,202 diaries), so those households' liquid and solid fuels leave with the drawn energy; where it reads the combined payment, the rest (GBP 5.2bn) stays in the housing remainder. The remainder therefore holds part of the diary's other fuels, beside other energy payments, and no column carries them; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3749,13 +3749,16 @@ }, { "codes": [ - "b489", - "b490" + "b1751", + "b1701", + "b233", + "c45114", + "c45214" ], "kind": "impute_prepayment_meter", "n_estimators": 100, "output": "uses_energy_prepayment_meter", - "rule": "b489_or_b490_nonzero", + "rule": "key_or_meter_payment_positive", "salt": "lcfs_prepayment_meter", "seed": 0 }, diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv index 3de97526b..7c0d4ee9c 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv @@ -1,65 +1,65 @@ -case,g018,g019,gorx,a124,p389p,p344p,weighta,a122,a121,b226,b489,b490,p537,p601,p602,p603,p604,p605,p606,p607,p608,p609,p610,p611,p612,c72211,c72212,c73212,c73213,c73214,c72213 -1,1,0,1,0,100,150,1,1,1,6,2,1,10,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,0,0,0.5 -2,2,1,2,1,110,160,1.1000000000000001,2,2,7,3,2,11,1.1000000000000001,2.1000000000000001,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,0 -3,3,2,3,2,120,170,1.2,3,3,8,4,3,12,1.2,2.2000000000000002,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,0 -4,1,0,4,3,130,180,1.3,4,4,6,5,4,13,1.3,2.2999999999999998,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,0,0,0,0 -5,2,1,5,0,140,190,1.3999999999999999,5,5,7,6,1,14,1.3999999999999999,2.3999999999999999,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,0 -6,3,2,6,1,150,200,1.5,6,6,8,2,2,15,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,0.5 -7,1,0,7,2,160,210,1.6000000000000001,7,7,6,3,3,16,1.6000000000000001,2.6000000000000001,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,0,0,0,0 -8,2,1,8,3,170,220,1,8,8,7,4,4,17,1.7,2.7000000000000002,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,0 -9,3,2,9,0,180,230,1.1000000000000001,1,1,8,5,1,18,1.8,2.7999999999999998,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,0 -10,1,0,10,1,190,240,1.2,2,2,6,6,2,19,1.8999999999999999,2.8999999999999999,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,0,0,0,0 -11,2,1,11,2,200,250,1.3,3,3,7,2,3,20,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,0.5 -12,3,2,12,3,210,260,1.3999999999999999,4,4,8,3,4,21,2.1000000000000001,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,0 -13,1,0,1,0,220,270,1.5,5,5,6,4,1,22,2.2000000000000002,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,0,0,0,0 -14,2,1,2,1,230,280,1.6000000000000001,6,6,7,5,2,23,2.2999999999999998,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,0 -15,3,2,3,2,240,290,1,7,7,8,6,3,24,2.3999999999999999,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,0 -16,1,0,4,3,250,300,1.1000000000000001,8,8,6,2,4,25,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,0,0,0,0.5 -17,2,1,5,0,260,310,1.2,1,1,7,3,1,26,2.6000000000000001,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,0 -18,3,2,6,1,270,320,1.3,2,2,8,4,2,27,2.7000000000000002,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,0 -19,1,0,7,2,280,330,1.3999999999999999,3,3,6,5,3,28,2.7999999999999998,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,0,0,0,0 -20,2,1,8,3,290,340,1.5,4,4,7,6,4,29,2.8999999999999999,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,0 -21,3,2,9,0,300,350,1.6000000000000001,5,5,8,2,1,30,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,0.5 -22,1,0,10,1,310,360,1,6,6,6,3,2,31,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,0,0,0,0 -23,2,1,11,2,320,370,1.1000000000000001,7,7,7,4,3,32,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,0 -24,3,2,12,3,330,380,1.2,8,8,8,5,4,33,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,0 -25,1,0,1,0,340,390,1.3,1,1,6,6,1,34,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,0,0,0,0 -26,2,1,2,1,350,400,1.3999999999999999,2,2,7,2,2,35,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,0.5 -27,3,2,3,2,360,410,1.5,3,3,8,3,3,36,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,0 -28,1,0,4,3,370,420,1.6000000000000001,4,4,6,4,4,37,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,0,0,0,0 -29,2,1,5,0,380,430,1,5,5,7,5,1,38,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,0 -30,3,2,6,1,390,440,1.1000000000000001,6,6,8,6,2,39,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,0 -31,1,0,7,2,400,450,1.2,7,7,6,2,3,40,4,5,6,7,8,9,10,11,12,13,14,15,16,17,0,0,0,0.5 -32,2,1,8,3,410,460,1.3,8,8,7,3,4,41,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,0 -33,3,2,9,0,420,470,1.3999999999999999,1,1,8,4,1,42,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,0 -34,1,0,10,1,430,480,1.5,2,2,6,5,2,43,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,0,0,0,0 -35,2,1,11,2,440,490,1.6000000000000001,3,3,7,6,3,44,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,20.399999999999999,0 -36,3,2,12,3,450,500,1,4,4,8,2,4,45,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,20.5,0.5 -37,1,0,1,0,460,510,1.1000000000000001,5,5,6,3,1,46,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,0,0,0,0 -38,2,1,2,1,470,520,1.2,6,6,7,4,2,47,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,20.699999999999999,0 -39,3,2,3,2,480,530,1.3,7,7,8,5,3,48,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,20.800000000000001,0 -40,1,0,4,3,490,540,1.3999999999999999,8,8,6,6,4,49,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,0,0,0,0 -41,2,1,5,0,500,550,1.5,1,1,7,2,1,50,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,0.5 -42,3,2,6,1,510,560,1.6000000000000001,2,2,8,3,2,51,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,21.100000000000001,0 -43,1,0,7,2,520,570,1,3,3,6,4,3,52,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,0,0,0,0 -44,2,1,8,3,530,580,1.1000000000000001,4,4,7,5,4,53,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,21.300000000000001,0 -45,3,2,9,0,540,590,1.2,5,5,8,6,1,54,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,20.399999999999999,21.399999999999999,0 -46,1,0,10,1,550,600,1.3,6,6,6,2,2,55,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,0,0,0,0.5 -47,2,1,11,2,560,610,1.3999999999999999,7,7,7,3,3,56,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,20.600000000000001,21.600000000000001,0 -48,3,2,12,3,570,620,1.5,8,8,8,4,4,57,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,20.699999999999999,21.699999999999999,0 -49,1,0,1,0,580,630,1.6000000000000001,1,1,6,5,1,58,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,0,0,0,0 -50,2,1,2,1,590,640,1,2,2,7,6,2,59,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,20.899999999999999,21.899999999999999,0 -51,3,2,3,2,600,650,1.1000000000000001,3,3,8,2,3,60,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,0.5 -52,1,0,4,3,610,660,1.2,4,4,6,3,4,61,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,0,0,0,0 -53,2,1,5,0,620,670,1.3,5,5,7,4,1,62,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,21.199999999999999,22.199999999999999,0 -54,3,2,6,1,630,680,1.3999999999999999,6,6,8,5,2,63,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,21.300000000000001,22.300000000000001,0 -55,1,0,7,2,640,690,1.5,7,7,6,6,3,64,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,0,0,0,0 -56,2,1,8,3,650,700,1.6000000000000001,8,8,7,2,4,65,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,20.5,21.5,22.5,0.5 -57,3,2,9,0,660,710,1,1,1,8,3,1,66,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,20.600000000000001,21.600000000000001,22.600000000000001,0 -58,1,0,10,1,670,720,1.1000000000000001,2,2,6,4,2,67,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,0,0,0,0 -59,2,1,11,2,680,730,1.2,3,3,7,5,3,68,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,20.800000000000001,21.800000000000001,22.800000000000001,0 -60,3,2,12,3,690,740,1.3,4,4,8,6,4,69,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,20.899999999999999,21.899999999999999,22.899999999999999,0 -61,1,0,1,0,700,750,1.3999999999999999,5,5,6,2,1,70,7,8,9,10,11,12,13,14,15,16,17,18,19,20,0,0,0,0.5 -62,2,1,2,1,710,760,1.5,6,6,7,3,2,71,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,21.100000000000001,22.100000000000001,23.100000000000001,0 -63,3,2,3,2,720,770,1.6000000000000001,7,7,8,4,3,72,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,21.199999999999999,22.199999999999999,23.199999999999999,0 -64,1,0,4,3,730,780,1,8,8,6,5,4,73,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,0,0,0,0 +case,g018,g019,gorx,a124,p389p,p344p,weighta,a122,a121,p537,b175,b178,b170,b173,b1751,b227,b2332,c45112,c45114,b1701,b226,b2331,c45212,c45214,b231,b233,b017,b018,c45222,c45312,c45411,c45412,c45511,p601,p602,p603,p604,p605,p606,p607,p608,p609,p610,p611,p612,c72211,c72212,c73212,c73213,c73214,c72213 +1,1,0,1,0,100,150,1,1,1,10,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,0,0,0.5 +2,2,1,2,1,110,160,1.1000000000000001,2,2,11,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1.1000000000000001,2.1000000000000001,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,0 +3,3,2,3,2,120,170,1.2,3,3,12,10,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.2,2.2000000000000002,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,0 +4,1,0,4,3,130,180,1.3,4,4,13,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,1.3,2.2999999999999998,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,0,0,0,0 +5,2,1,5,0,140,190,1.3999999999999999,5,5,14,12,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.3999999999999999,2.3999999999999999,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,0 +6,3,2,6,1,150,200,1.5,6,6,15,8,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,0.5 +7,1,0,7,2,160,210,1.6000000000000001,7,7,16,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,1.6000000000000001,2.6000000000000001,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,0,0,0,0 +8,2,1,8,3,170,220,1,8,8,17,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,2.5,0,0,0,0,0,0,1.7,2.7000000000000002,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,0 +9,3,2,9,0,180,230,1.1000000000000001,1,1,18,11,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.8,2.7999999999999998,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,0 +10,1,0,10,1,190,240,1.2,2,2,19,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,1.8999999999999999,2.8999999999999999,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,0,0,0,0 +11,2,1,11,2,200,250,1.3,3,3,20,8,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,0.5 +12,3,2,12,3,210,260,1.3999999999999999,4,4,21,9,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.1000000000000001,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,0 +13,1,0,1,0,220,270,1.5,5,5,22,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,2.2000000000000002,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,0,0,0,0 +14,2,1,2,1,230,280,1.6000000000000001,6,6,23,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,2.2999999999999998,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,0 +15,3,2,3,2,240,290,1,7,7,24,12,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.3999999999999999,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,0 +16,1,0,4,3,250,300,1.1000000000000001,8,8,25,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,0,0,0,0.5 +17,2,1,5,0,260,310,1.2,1,1,26,9,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.6000000000000001,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,0 +18,3,2,6,1,270,320,1.3,2,2,27,10,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,2.5,0,0,0,0,0,0,2.7000000000000002,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,0 +19,1,0,7,2,280,330,1.3999999999999999,3,3,28,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,2.7999999999999998,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,0,0,0,0 +20,2,1,8,3,290,340,1.5,4,4,29,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.8999999999999999,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,0 +21,3,2,9,0,300,350,1.6000000000000001,5,5,30,8,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,0.5 +22,1,0,10,1,310,360,1,6,6,31,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,0,0,0,0 +23,2,1,11,2,320,370,1.1000000000000001,7,7,32,10,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,0 +24,3,2,12,3,330,380,1.2,8,8,33,11,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3.2999999999999998,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,0 +25,1,0,1,0,340,390,1.3,1,1,34,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,3.3999999999999999,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,0,0,0,0 +26,2,1,2,1,350,400,1.3999999999999999,2,2,35,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,3.5,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,0.5 +27,3,2,3,2,360,410,1.5,3,3,36,9,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3.6000000000000001,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,0 +28,1,0,4,3,370,420,1.6000000000000001,4,4,37,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,2.5,0,0,0,0,0,0,3.7000000000000002,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,0,0,0,0 +29,2,1,5,0,380,430,1,5,5,38,11,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3.7999999999999998,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,0 +30,3,2,6,1,390,440,1.1000000000000001,6,6,39,12,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3.8999999999999999,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,0 +31,1,0,7,2,400,450,1.2,7,7,40,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,4,5,6,7,8,9,10,11,12,13,14,15,16,17,0,0,0,0.5 +32,2,1,8,3,410,460,1.3,8,8,41,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,0 +33,3,2,9,0,420,470,1.3999999999999999,1,1,42,10,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,0 +34,1,0,10,1,430,480,1.5,2,2,43,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,4.2999999999999998,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,0,0,0,0 +35,2,1,11,2,440,490,1.6000000000000001,3,3,44,12,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4.4000000000000004,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,20.399999999999999,0 +36,3,2,12,3,450,500,1,4,4,45,8,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4.5,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,20.5,0.5 +37,1,0,1,0,460,510,1.1000000000000001,5,5,46,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,4.5999999999999996,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,0,0,0,0 +38,2,1,2,1,470,520,1.2,6,6,47,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,2.5,0,0,0,0,0,0,4.7000000000000002,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,20.699999999999999,0 +39,3,2,3,2,480,530,1.3,7,7,48,11,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4.7999999999999998,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,20.800000000000001,0 +40,1,0,4,3,490,540,1.3999999999999999,8,8,49,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,4.9000000000000004,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,0,0,0,0 +41,2,1,5,0,500,550,1.5,1,1,50,8,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,0.5 +42,3,2,6,1,510,560,1.6000000000000001,2,2,51,9,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,21.100000000000001,0 +43,1,0,7,2,520,570,1,3,3,52,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,0,0,0,0 +44,2,1,8,3,530,580,1.1000000000000001,4,4,53,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5.2999999999999998,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,21.300000000000001,0 +45,3,2,9,0,540,590,1.2,5,5,54,12,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5.4000000000000004,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,20.399999999999999,21.399999999999999,0 +46,1,0,10,1,550,600,1.3,6,6,55,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,5.5,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,0,0,0,0.5 +47,2,1,11,2,560,610,1.3999999999999999,7,7,56,9,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5.5999999999999996,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,20.600000000000001,21.600000000000001,0 +48,3,2,12,3,570,620,1.5,8,8,57,10,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,2.5,0,0,0,0,0,0,5.7000000000000002,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,20.699999999999999,21.699999999999999,0 +49,1,0,1,0,580,630,1.6000000000000001,1,1,58,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,5.7999999999999998,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,0,0,0,0 +50,2,1,2,1,590,640,1,2,2,59,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,5.9000000000000004,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,20.899999999999999,21.899999999999999,0 +51,3,2,3,2,600,650,1.1000000000000001,3,3,60,8,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,0.5 +52,1,0,4,3,610,660,1.2,4,4,61,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,0,0,0,0 +53,2,1,5,0,620,670,1.3,5,5,62,10,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,21.199999999999999,22.199999999999999,0 +54,3,2,6,1,630,680,1.3999999999999999,6,6,63,11,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.2999999999999998,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,21.300000000000001,22.300000000000001,0 +55,1,0,7,2,640,690,1.5,7,7,64,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,6.4000000000000004,7.4000000000000004,8.4000000000000004,9.4000000000000004,10.4,11.4,12.4,13.4,14.4,15.4,16.399999999999999,17.399999999999999,18.399999999999999,19.399999999999999,0,0,0,0 +56,2,1,8,3,650,700,1.6000000000000001,8,8,65,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.5,7.5,8.5,9.5,10.5,11.5,12.5,13.5,14.5,15.5,16.5,17.5,18.5,19.5,20.5,21.5,22.5,0.5 +57,3,2,9,0,660,710,1,1,1,66,9,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.5999999999999996,7.5999999999999996,8.5999999999999996,9.5999999999999996,10.6,11.6,12.6,13.6,14.6,15.6,16.600000000000001,17.600000000000001,18.600000000000001,19.600000000000001,20.600000000000001,21.600000000000001,22.600000000000001,0 +58,1,0,10,1,670,720,1.1000000000000001,2,2,67,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,2.5,0,0,0,0,0,0,6.7000000000000002,7.7000000000000002,8.6999999999999993,9.6999999999999993,10.699999999999999,11.699999999999999,12.699999999999999,13.699999999999999,14.699999999999999,15.699999999999999,16.699999999999999,17.699999999999999,18.699999999999999,19.699999999999999,0,0,0,0 +59,2,1,11,2,680,730,1.2,3,3,68,11,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.7999999999999998,7.7999999999999998,8.8000000000000007,9.8000000000000007,10.800000000000001,11.800000000000001,12.800000000000001,13.800000000000001,14.800000000000001,15.800000000000001,16.800000000000001,17.800000000000001,18.800000000000001,19.800000000000001,20.800000000000001,21.800000000000001,22.800000000000001,0 +60,3,2,12,3,690,740,1.3,4,4,69,12,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6.9000000000000004,7.9000000000000004,8.9000000000000004,9.9000000000000004,10.9,11.9,12.9,13.9,14.9,15.9,16.899999999999999,17.899999999999999,18.899999999999999,19.899999999999999,20.899999999999999,21.899999999999999,22.899999999999999,0 +61,1,0,1,0,700,750,1.3999999999999999,5,5,70,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,7,8,9,10,11,12,13,14,15,16,17,18,19,20,0,0,0,0.5 +62,2,1,2,1,710,760,1.5,6,6,71,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,18.100000000000001,19.100000000000001,20.100000000000001,21.100000000000001,22.100000000000001,23.100000000000001,0 +63,3,2,3,2,720,770,1.6000000000000001,7,7,72,10,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,18.199999999999999,19.199999999999999,20.199999999999999,21.199999999999999,22.199999999999999,23.199999999999999,0 +64,1,0,4,3,730,780,1,8,8,73,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,7.2999999999999998,8.3000000000000007,9.3000000000000007,10.300000000000001,11.300000000000001,12.300000000000001,13.300000000000001,14.300000000000001,15.300000000000001,16.300000000000001,17.300000000000001,18.300000000000001,19.300000000000001,20.300000000000001,0,0,0,0 diff --git a/tools/graph_uk_spine_fixture.py b/tools/graph_uk_spine_fixture.py index d31643c46..bbab2c7b2 100644 --- a/tools/graph_uk_spine_fixture.py +++ b/tools/graph_uk_spine_fixture.py @@ -846,11 +846,27 @@ def _lcfs_donors() -> tuple[pd.DataFrame, pd.DataFrame]: "weighta": 1.0 + rows % 7 / 10.0, "a122": 1 + rows.astype(int) % 8, "a121": 1 + rows.astype(int) % 8, - "b226": 6.0 + rows % 3, - "b489": 2.0 + rows % 5, - "b490": 1.0 + rows % 4, "p537": 10.0 + rows, } + # The labelled fuel, light and power parts (microcosm#1164): an electricity + # account everywhere, a gas account on most diaries, combined bills on a + # third (gas and electricity components), prepayment meters on a sixth, + # and oil on a few. + from microcosm.build.uk_runtime.lcfs_consumption import UK_LCFS_ENERGY_CODES + + for code in UK_LCFS_ENERGY_CODES: + if code != "p537": + household[code] = np.zeros(_DONOR_ROWS) + combined = rows.astype(int) % 3 == 0 + prepay = rows.astype(int) % 6 == 1 + household["b175"] = np.where(combined | prepay, 0.0, 8.0 + rows % 5) + household["b170"] = np.where(combined | prepay, 0.0, 4.0 + rows % 4) + household["b231"] = np.where(combined, 10.0 + rows % 3, 0.0) + household["b226"] = np.where(combined, 4.0, 0.0) + household["b227"] = np.where(combined, 6.0 + rows % 3, 0.0) + household["b1751"] = np.where(prepay, 5.0 + rows % 2, 0.0) + household["b1701"] = np.where(prepay & (rows.astype(int) % 12 == 1), 3.0, 0.0) + household["b017"] = np.where(rows.astype(int) % 10 == 7, 2.5, 0.0) for position, source in enumerate( ( "p601", From 1d57580540bfddafb8ce15e32970ff4d2f427d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 10/28] Cap household energy at the domestic meter limits after the DESNZ level (#1164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The energy columns are exempt from the donor support clip because the NEED rake and the DESNZ level set them, so a few drawn households sat far above any domestic use, and since #1167 the housing total contained them (#1207). María ruled on 2026-10-12 for the domestic meter definitions behind the DESNZ total (NEED Annex B: gas under 73,200 kWh a year, electricity under 50,000 kWh on profile classes 1 and 2). - price_domestic_energy declares household_kwh_ceiling; the schema admits it. - rake_energy_kwh caps after the level with cap_household_kwh: a household above the ceiling is set to it and the kWh it loses goes to the fuel's other households in proportion to their use, so the levelled total holds; the NEED fit is read after the cap. The receipt counts the households drawn above and the share of the total they gave up, and records no household's value. A frame too small for its level (a synthetic fixture) caps everyone and records the shortfall instead of raising. - The energy_rake gate checks the declared ceilings against the receipt: no household left above, the total kept, the cap feasible. - Synthetic test frames weigh to the UK's households so the level gives each household a plausible kWh. Co-Authored-By: Claude Opus 5.5 --- .../1164-lcfs-household-kwh-ceilings.fixed.md | 1 + .../spec_engine/schema/sources.schema.json | 18 +++ .../src/microcosm/build/uk/source_stages.json | 8 +- .../src/microcosm/build/uk/spec/sources.yaml | 5 +- .../build/uk_runtime/energy_pricing.py | 116 +++++++++++++++ .../build/uk_runtime/lcfs_consumption.py | 6 + .../build/uk_runtime/stage_health.py | 45 ++++++ .../engine_free/uk/test_uk_energy_pricing.py | 140 ++++++++++++++++++ .../uk/test_uk_lcfs_consumption.py | 10 +- .../parity/uk_spine/sources/fixture.json | 6 +- 10 files changed, 350 insertions(+), 5 deletions(-) create mode 100644 changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md diff --git a/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md b/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md new file mode 100644 index 000000000..571a3995e --- /dev/null +++ b/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md @@ -0,0 +1 @@ +The UK LCFS stage no longer leaves a household's electricity or gas above the domestic meter definitions behind the DESNZ total it levels to (microcosm#1164; the extreme energy values item of #1207; María's ruling of 2026-10-12). NEED's quality-assurance annex counts a gas meter as domestic under 73,200 kWh a year and an electricity meter under 50,000 kWh on profile classes 1 and 2. The energy columns are exempt from the donor support clip because the NEED rake and the DESNZ level set them, so a few drawn households sat far above any domestic use, and since #1167 the housing total contained them. The `price_domestic_energy` step declares `household_kwh_ceiling`; after the level, `cap_household_kwh` sets a household above it to the ceiling and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds. The `energy_rake` receipt counts the households capped and the share of the total they gave up, the NEED fit is read after the cap, and the `energy_rake` gate checks the declared ceilings, that no household is left above them and that the total held. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 4a07aa6e8..942d6c57f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -9784,6 +9784,24 @@ "enum": [ "actual_temperature" ] + }, + "household_kwh_ceiling": { + "type": "object", + "additionalProperties": false, + "required": [ + "electricity", + "gas" + ], + "properties": { + "electricity": { + "type": "number", + "exclusiveMinimum": 0 + }, + "gas": { + "type": "number", + "exclusiveMinimum": 0 + } + } } }, "if": { diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 95cff7319..def1430df 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3855,7 +3855,11 @@ "level_resource": "desnz_domestic_energy_facts.json", "level_period_type": "calendar_year", "level_period_value": 2024, - "level_temperature_adjustment": "actual_temperature" + "level_temperature_adjustment": "actual_temperature", + "household_kwh_ceiling": { + "electricity": 50000, + "gas": 73200 + } }, { "kind": "price_bus_journeys", @@ -4194,7 +4198,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 1e0f208cf..dd2f0b5a0 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3172,6 +3172,9 @@ stages: level_period_type: calendar_year level_period_value: 2024 level_temperature_adjustment: actual_temperature + household_kwh_ceiling: + electricity: 50000 + gas: 73200 - kind: price_bus_journeys column: bus_fare_spending trips_columns: @@ -3431,7 +3434,7 @@ stages: - gas_consumption - has_fuel_consumption - uses_energy_prepayment_meter - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py index 32bf45da2..303b9648c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py @@ -877,6 +877,100 @@ def income_band_ids(income: Sequence[float], margins: NeedMargins) -> np.ndarray return result +#: The declared household ceilings (microcosm#1164, #1207): kWh a year above +#: which DESNZ and NEED do not count a meter as domestic (NEED Annex B, Quality +#: Assurance: gas meters under 73,200 kWh a year, electricity meters under +#: 50,000 kWh on profile classes 1 and 2). The DESNZ total the stage levels to +#: counts domestic meters, so no household of the frame is drawn above them. +HOUSEHOLD_KWH_CEILING_KEY = "household_kwh_ceiling" + + +def declared_household_kwh_ceilings( + parameters: Mapping[str, Any], +) -> dict[str, float] | None: + """The declared ``household_kwh_ceiling`` per fuel column, if any.""" + + declared = parameters.get(HOUSEHOLD_KWH_CEILING_KEY) + if declared is None: + return None + if not isinstance(declared, Mapping) or set(declared) != {"electricity", "gas"}: + raise ValueError( + f"{HOUSEHOLD_KWH_CEILING_KEY} must give electricity and gas kWh." + ) + ceilings = { + ELECTRICITY_KWH: float(declared["electricity"]), + GAS_KWH: float(declared["gas"]), + } + if not all(np.isfinite(v) and v > 0 for v in ceilings.values()): + raise ValueError(f"{HOUSEHOLD_KWH_CEILING_KEY} values must be positive.") + return ceilings + + +def cap_household_kwh( + values: np.ndarray, + *, + weights: np.ndarray, + population: np.ndarray, + ceiling: float, + max_rounds: int = 100, +) -> tuple[np.ndarray, dict[str, Any]]: + """Cap one fuel's household kWh at ``ceiling`` and keep its weighted total. + + Households above the ceiling are set to it, and the kWh they lose goes to + the other households of the population in proportion to their own use, so + the levelled total holds; a household lifted over the ceiling by that is + capped in the next round. The receipt counts the households drawn above + the ceiling and the share of the total they gave up, and records no + household's value. + """ + + capped_values = np.asarray(values, dtype=float).copy() + weight = np.asarray(weights, dtype=float) + members = np.asarray(population, dtype=bool) + total = float(np.dot(weight[members], capped_values[members])) + at_ceiling = np.zeros(len(capped_values), dtype=bool) + rounds = 0 + feasible = True + while True: + over = members & (capped_values > ceiling) + if not over.any(): + break + if rounds == max_rounds: + raise ValueError(f"capping at {ceiling} kWh did not settle.") + rounds += 1 + at_ceiling |= over + capped_values[over] = ceiling + free = members & ~at_ceiling & (capped_values > 0) + free_total = float(np.dot(weight[free], capped_values[free])) + shortfall = total - float(np.dot(weight[members], capped_values[members])) + if free_total <= 0: + # No household is left below the ceiling to take the rest: a frame + # whose weights fall far short of the households it is levelled to + # (a synthetic fixture). The total then falls short, the receipt + # says so, and the energy_rake gate refuses it. + feasible = False + break + capped_values[free] *= 1.0 + shortfall / free_total + before = np.asarray(values, dtype=float) + # The outliers are the households drawn above the ceiling; one the + # rescaling lifted to it gave nothing up. + outliers = members & (before > ceiling) + removed = float(np.dot(weight[outliers], before[outliers] - ceiling)) + after = float(np.dot(weight[members], capped_values[members])) + return capped_values, { + "ceiling_kwh": float(ceiling), + "households_capped": int(outliers.sum()), + "households_at_ceiling": int(at_ceiling.sum()), + "capped_share_of_total": removed / total if total > 0 else 0.0, + "rounds": rounds, + "households_above_after": int( + (members & (capped_values > ceiling * (1 + 1e-12))).sum() + ), + "feasible": feasible, + "frame_kwh_after": after, + } + + def rake_energy_kwh( table: pd.DataFrame, *, @@ -890,6 +984,7 @@ def rake_energy_kwh( use_region_margin: bool = False, gas_connected: Sequence[bool] | None = None, level: Mapping[str, float] | None = None, + ceilings: Mapping[str, float] | None = None, ) -> tuple[pd.DataFrame, dict[str, Any]]: """Rake ``electricity_kwh`` and ``gas_kwh`` to the NEED shape, then to the level. @@ -910,6 +1005,10 @@ def rake_energy_kwh( NEED means become the shape (every cell mean moves by the same factor, which the fit block records as ``level_factor`` beside the ``shape_target`` and the levelled ``target``). + + ``ceilings`` (kWh per fuel column, after the level) caps each household + and hands what it loses to the fuel's other households, so the total holds + (``cap_household_kwh``; microcosm#1164). The fit is read after the cap. """ frame = table.copy() @@ -1048,6 +1147,22 @@ def rake_energy_kwh( "factor": factor, "frame_kwh_after": after, } + ceiling_receipt: dict[str, Any] | None = None + if ceilings is not None: + ceiling_receipt = {} + for column in (ELECTRICITY_KWH, GAS_KWH): + population = ( + np.ones(len(frame), dtype=bool) + if column == ELECTRICITY_KWH + else connected + ) + capped, ceiling_receipt[column] = cap_household_kwh( + raked[column].to_numpy(dtype=float), + weights=weight_values, + population=population, + ceiling=float(ceilings[column]), + ) + raked[column] = capped fit = _rake_fit(raked, margins, fitted, connected, weight_values, level_factor) receipt = { "unit": "kwh", @@ -1063,6 +1178,7 @@ def rake_energy_kwh( "sweep_residuals": sweep_residuals, "level_factor": dict(level_factor), "level": level_receipt, + "ceiling": ceiling_receipt, "fit": fit, } return raked.drop(columns=[c for c in scratch if c in raked]), receipt diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 85c9658f7..be24fad34 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -41,6 +41,7 @@ UK_QEP_ENERGY_PRICES_RESOURCE, EnergyPrices, NeedMargins, + declared_household_kwh_ceilings, impose_gas_connection, kwh_to_spend, need_margins_from_facts, @@ -1717,6 +1718,7 @@ class LCFSEnergyPricing: level: Mapping[str, float] receipt: dict[str, Any] disconnect_seed: int = 0 + ceilings: Mapping[str, float] | None = None def lcfs_energy_pricing(stage: SourceStageSpec) -> LCFSEnergyPricing | None: @@ -1765,6 +1767,7 @@ def lcfs_energy_pricing(stage: SourceStageSpec) -> LCFSEnergyPricing | None: f"{DISCONNECT_IDENTITY_UNIFORM_ORDER!r} must declare a non-negative " "integer seed." ) + ceilings = declared_household_kwh_ceilings(parameters) return LCFSEnergyPricing( prices=prices, margins=margins, @@ -1773,11 +1776,13 @@ def lcfs_energy_pricing(stage: SourceStageSpec) -> LCFSEnergyPricing | None: disconnect_rule=disconnect_rule, disconnect_seed=int(disconnect_seed), level=level, + ceilings=ceilings, receipt={ **prices_receipt, "need_margins": margins.receipt, "level": level_receipt, "gas_connection": connection_receipt, + "household_kwh_ceiling": ceilings, }, ) @@ -1872,6 +1877,7 @@ def rake_recipient_energy( use_region_margin=True, gas_connected=connected, level=energy.level, + ceilings=energy.ceilings, ) receipt["gas_connection"] = {"rule": energy.gas_connected, **connection_receipt} receipt["gas_connected_share"] = float( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index b8f3fb57a..931c990fa 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -287,6 +287,7 @@ def _energy_rake_gate( GAS_CONNECTED_PUBLISHED_METER_SHARE, GAS_KWH, PRICE_DOMESTIC_ENERGY_KIND, + declared_household_kwh_ceilings, need_margins_from_facts, pricing_operation, published_energy_level, @@ -407,6 +408,50 @@ def _close(observed: object, expected: float, rtol: float) -> bool: ) factors[fuel] = float(factor) details["level_factor"][fuel] = float(factor) + ceilings = declared_household_kwh_ceilings(declared) + if ceilings is not None: + # No household above its fuel's domestic ceiling, and the cap kept the + # levelled total (microcosm#1164). + ceiling_block = receipt.get("ceiling") + if not isinstance(ceiling_block, Mapping): + failures.append( + f"{stage}: the stage declares household kWh ceilings but the " + "energy_rake receipt carries no ceiling block." + ) + ceiling_block = {} + details["ceiling"] = {} + for fuel, ceiling in ceilings.items(): + entry = ceiling_block.get(fuel) + if not isinstance(entry, Mapping): + if ceiling_block: + failures.append(f"{stage}: ceiling block lacks {fuel}.") + continue + if not _close(entry.get("ceiling_kwh"), ceiling, 1e-12): + failures.append( + f"{stage}: {fuel} was capped at {entry.get('ceiling_kwh')!r}, " + f"not the declared {ceiling}." + ) + if entry.get("feasible") is False: + failures.append( + f"{stage}: no household was left below the {fuel} ceiling to " + "take what the capped households gave up." + ) + if entry.get("households_above_after") != 0: + failures.append( + f"{stage}: {fuel} leaves households above the declared " + f"ceiling of {ceiling} kWh." + ) + if not _close(entry.get("frame_kwh_after"), float(level[fuel]), 1e-6): + failures.append( + f"{stage}: {fuel} total after the cap is " + f"{entry.get('frame_kwh_after')!r}, not the published " + f"{float(level[fuel])}." + ) + details["ceiling"][fuel] = { + "ceiling_kwh": ceiling, + "households_capped": entry.get("households_capped"), + "capped_share_of_total": entry.get("capped_share_of_total"), + } for margin in expected_margins: if margin not in declared_margins: failures.append(f"{stage}: margin {margin!r} was not raked.") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py index e6e0deb3d..b45b1fd6e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py @@ -18,6 +18,8 @@ NEED_PROPERTY_TYPE_IDS_BY_GEOGRAPHY, NEED_TENURE_IDS, SCOTLAND_GEOGRAPHY_ID, + cap_household_kwh, + declared_household_kwh_ceilings, fiscal_year_quarters, impose_gas_connection, income_band_ids, @@ -638,7 +640,10 @@ def _synthetic_rake_receipt(n: int = 1200): accommodation = np.array(["HOUSE_DETACHED", "FLAT"], dtype=object)[ (np.arange(n) // 2) % 2 ] + # Weights summing to about the UK's households, so the DESNZ level gives + # each household a plausible kWh under the declared domestic ceilings. weights = rng.uniform(0.5, 2.0, n) + weights *= 28_500_000.0 / weights.sum() connected, connection = impose_gas_connection( table[GAS_KWH].to_numpy(), frs_region=region, weights=weights, shares=shares ) @@ -655,6 +660,7 @@ def _synthetic_rake_receipt(n: int = 1200): use_region_margin=True, gas_connected=connected, level=level, + ceilings=declared_household_kwh_ceilings(declared), ) receipt["gas_connection"] = {"rule": "published_meter_share", **connection} return margins, level, raked, weights, receipt @@ -1061,3 +1067,137 @@ def test_gas_is_raked_over_connected_rows_and_electricity_over_all() -> None: iterations=1, ) assert unmasked[GAS_KWH].mean() == pytest.approx(cell[GAS_KWH]) + + +def test_cap_household_kwh_keeps_the_total_and_caps_every_household() -> None: + # microcosm#1164: the kWh a capped household loses goes to the others in + # proportion to their use, and a household lifted over is capped in turn. + values = np.array([10.0, 4.0, 2.0, 1.0, 3.0]) + weights = np.array([1.0, 1.0, 2.0, 1.0, 2.0]) + population = np.array([True, True, True, True, False]) + capped, receipt = cap_household_kwh( + values, weights=weights, population=population, ceiling=5.0 + ) + total = float(np.dot(weights[population], values[population])) + assert float(np.dot(weights[population], capped[population])) == pytest.approx( + total + ) + assert (capped[population] <= 5.0 + 1e-9).all() + assert capped[4] == 3.0 # outside the population, untouched + assert receipt["ceiling_kwh"] == 5.0 + assert receipt["households_above_after"] == 0 + # One household was drawn above the ceiling; the rescaling lifts another + # to it, which gives nothing up. + assert receipt["households_capped"] == 1 + assert receipt["households_at_ceiling"] == 2 + assert receipt["capped_share_of_total"] == pytest.approx(5.0 / total) + assert receipt["frame_kwh_after"] == pytest.approx(total) + # Nothing to cap leaves the values as they were. + same, untouched = cap_household_kwh( + values, weights=weights, population=population, ceiling=50.0 + ) + np.testing.assert_array_equal(same, values) + assert untouched["households_capped"] == 0 and untouched["rounds"] == 0 + # A frame too small for its level caps everyone and says the total fell + # short; the energy_rake gate then refuses the receipt. + short, infeasible = cap_household_kwh( + np.array([10.0, 10.0]), + weights=np.ones(2), + population=np.ones(2, dtype=bool), + ceiling=5.0, + ) + assert short.tolist() == [5.0, 5.0] + assert infeasible["feasible"] is False + assert infeasible["frame_kwh_after"] == 10.0 + assert receipt["feasible"] is True + + +def test_declared_household_ceilings_are_the_domestic_meter_limits() -> None: + ceilings = declared_household_kwh_ceilings(_declared()) + assert ceilings == {ELECTRICITY_KWH: 50_000.0, GAS_KWH: 73_200.0} + assert declared_household_kwh_ceilings({}) is None + for bad in ({"electricity": 50_000.0}, {"electricity": 50_000.0, "gas": 0.0}): + with pytest.raises(ValueError, match="household_kwh_ceiling"): + declared_household_kwh_ceilings({"household_kwh_ceiling": bad}) + + +def test_rake_caps_after_the_level_and_keeps_the_published_totals() -> None: + declared = _declared() + margins = need_margins_from_facts(period_value=NEED_PERIOD) + level, _ = published_energy_level(declared) + n = 600 + rng = np.random.default_rng(11) + table = pd.DataFrame( + { + ELECTRICITY_KWH: rng.uniform(1000.0, 5000.0, n), + GAS_KWH: rng.uniform(5000.0, 20000.0, n), + } + ) + # Two outliers far above the domestic limits. + table.loc[0, ELECTRICITY_KWH] = 400_000.0 + table.loc[1, GAS_KWH] = 900_000.0 + weights = np.full(n, 28_500_000.0 / n) + raked, receipt = rake_energy_kwh( + table, + margins=margins, + frs_region=np.array(["LONDON"] * n, dtype=object), + income=rng.uniform(5e3, 2e5, n), + weights=weights, + iterations=1, + level=level, + ceilings=declared_household_kwh_ceilings(declared), + ) + for fuel, ceiling in ((ELECTRICITY_KWH, 50_000.0), (GAS_KWH, 73_200.0)): + values = raked[fuel].to_numpy(dtype=float) + assert values.max() <= ceiling * (1 + 1e-9) + assert float(np.dot(weights, values)) == pytest.approx(level[fuel], rel=1e-9) + block = receipt["ceiling"][fuel] + assert block["households_capped"] >= 1 + assert block["households_above_after"] == 0 + + +def test_the_energy_rake_gate_checks_the_declared_household_ceilings() -> None: + # microcosm#1164: the stage declares the domestic meter limits, so the gate + # needs the ceiling block, no household above it and the total kept. + from microcosm.build.uk_runtime.stage_health import uk_stage_health_gate + + gates = json.loads( + ( + _TEST_PATHS.repository + / "packages/microcosm-build/src/microcosm/build/uk/gates.json" + ).read_text("utf-8") + ) + by_id = {gate["id"]: gate for gate in gates["gates"]} + parameters = { + **by_id["uk_stage_lcfs_consumption_energy_rake"]["parameters"], + "maximum_relative_deviation": 1.0, + } + _, _, _, _, receipt = _synthetic_rake_receipt() + + def run(energy_rake): + return uk_stage_health_gate( + evidence={"stage": "lcfs_consumption", "energy_rake": energy_rake}, + stage="lcfs_consumption", + check="energy_rake", + parameters=parameters, + ) + + passed = run(receipt) + assert passed.passed, passed.failures + assert passed.details["ceiling"][ELECTRICITY_KWH]["ceiling_kwh"] == 50_000.0 + assert passed.details["ceiling"][GAS_KWH]["ceiling_kwh"] == 73_200.0 + missing = copy.deepcopy(receipt) + missing["ceiling"] = None + assert any("no ceiling block" in f for f in run(missing).failures) + above = copy.deepcopy(receipt) + above["ceiling"][GAS_KWH]["households_above_after"] = 3 + assert any("above the declared" in f for f in run(above).failures) + other = copy.deepcopy(receipt) + other["ceiling"][ELECTRICITY_KWH]["ceiling_kwh"] = 25_000.0 + assert any("not the declared" in f for f in run(other).failures) + lost = copy.deepcopy(receipt) + lost["ceiling"][ELECTRICITY_KWH]["frame_kwh_after"] *= 0.9 + lost["ceiling"][ELECTRICITY_KWH]["feasible"] = False + failures = run(lost).failures + assert any("total after the cap" in f for f in failures) + assert any("left below the electricity_kwh ceiling" in f for f in failures) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 5ca62c3d2..d5047f0a8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -21,6 +21,9 @@ ) from microcosm.build.uk_runtime.national_frame import uk_national_frame +#: About the UK's households, for synthetic frames the DESNZ energy level reads. +UK_HOUSEHOLDS = 28_500_000.0 + def _energy_codes(n: int, **values) -> dict[str, np.ndarray]: """Every labelled LCFS energy code at zero, with the given ones set.""" @@ -633,7 +636,10 @@ def test_post_imputation_rake_fits_all_four_need_margins_in_kwh() -> None: tenure = rng.choice(["OWNED_OUTRIGHT", "RENT_PRIVATELY", "RENT_FROM_COUNCIL"], n) accommodation = rng.choice(["HOUSE_DETACHED", "FLAT", "OTHER"], n) region = rng.choice(["LONDON", "WALES", "SCOTLAND", "NORTHERN_IRELAND"], n) + # Weights summing to the UK's households, so the DESNZ level gives each + # household a plausible kWh and the domestic ceilings bind only on outliers. weights = rng.uniform(0.5, 2.0, n) + weights *= UK_HOUSEHOLDS / weights.sum() raked, receipt = rake_recipient_energy( household, @@ -879,7 +885,9 @@ def test_stage_transform_prices_bus_fares_from_journeys() -> None: household = pd.DataFrame( { "household_id": np.arange(1, n + 1), - "household_weight": rng.uniform(0.5, 3.0, n), + # Summing to about the UK's households, so the DESNZ energy level + # gives each household a plausible kWh under the domestic ceilings. + "household_weight": rng.uniform(0.5, 3.0, n) * (UK_HOUSEHOLDS / (1.75 * n)), "region": regions, "tenure_type": np.array( ["OWNED_OUTRIGHT", "RENT_PRIVATELY", "RENT_FROM_COUNCIL"] diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index dae8652a5..9765f0b72 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3517,7 +3517,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; Mar\u00eda's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3793,6 +3793,10 @@ "crosswalk_resource": "ofgem_region_crosswalk.json", "disconnect_rule": "identity_uniform_order", "gas_connected": "published_meter_share", + "household_kwh_ceiling": { + "electricity": 50000, + "gas": 73200 + }, "kind": "price_domestic_energy", "level_period_type": "calendar_year", "level_period_value": 2024, From 87064838fb1046aeaf8962aa160c5bdf684687e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 11/28] Move the LCFS stage to the Living Costs and Food Survey 2024-25 (#1164) LCFS was the only UK survey stage not on its latest microdata. SN 9603 (2nd edition, 9 Sep 2026) replaces SN 9468: - The household tab is dvhh_uk_anon_v3_2024.tab, the 2nd edition's, which carries the annual weight as cal_weight_annual (sum 28,694 thousand households, weightq/4 as weighta was); the 1st edition's v2 tab has no weight. The person tab is dvper_uk_anon_2024.tab. Both are pinned with their study number, DOI, sha256 and size. Every other variable the stage reads keeps its name and coding. - The diary year is the FRS 2024-25 year: uprate_donor_columns runs from 2024 to 2024, so the CPI bridge steps back a quarter (a factor below one) and the income predictors and litre proxies take one. - The fit is named uk_lcfs_2024_25_consumption; the dead filename/sha constants go; the stage notes, gate notes, CLI help and docstrings say 2024-25; the road-fuel target's scope note takes the new diary's 0.46 percent other-fuels share. - The support bounds are cut once from the 2024-25 donor at the final factors; the licensed round trip reproduces them. - The gate-battery digests mirrored in microcosm-data follow gates.json; the mirror, coverage manifest and H2 fixture are regenerated. Co-Authored-By: Claude Opus 5.5 --- changelog.d/1164-lcfs-2024-25.changed.md | 1 + .../src/microcosm/build/uk/gates.json | 6 +-- .../uk/lcfs_consumption_support_bounds.json | 24 ++++++------ .../src/microcosm/build/uk/source_stages.json | 38 +++++++++---------- .../src/microcosm/build/uk/spec/sources.yaml | 38 +++++++++---------- .../build/uk/uk_population_targets.json | 2 +- .../build/uk_runtime/donor_uprating.py | 6 +-- .../build/uk_runtime/lcfs_consumption.py | 32 ++++++++-------- .../microcosm/build/uk_runtime/spine_build.py | 4 +- .../tests/engine/uk/test_uk_donor_uprating.py | 29 +++++++------- .../uk/test_uk_consumption_resources.py | 11 ++++-- .../engine_free/uk/test_uk_donor_uprating.py | 8 +++- .../uk/test_uk_lcfs_consumption.py | 12 +++--- .../parity/uk_spine/sources/fixture.json | 38 +++++++++---------- .../uk_spine/sources/lcfs_household.csv | 2 +- tools/graph_uk_spine_fixture.py | 2 +- 16 files changed, 133 insertions(+), 120 deletions(-) create mode 100644 changelog.d/1164-lcfs-2024-25.changed.md diff --git a/changelog.d/1164-lcfs-2024-25.changed.md b/changelog.d/1164-lcfs-2024-25.changed.md new file mode 100644 index 000000000..203fa583c --- /dev/null +++ b/changelog.d/1164-lcfs-2024-25.changed.md @@ -0,0 +1 @@ +The UK `lcfs_consumption` stage reads the Living Costs and Food Survey 2024-25 (UK Data Service SN 9603, 2nd edition) in place of 2023-24 (SN 9468), so every UK survey stage is now on its latest microdata (microcosm#1164). The diary covers April 2024 to March 2025, the FRS 2024-25 spine's year: the CPI bridge to the calendar-2024 basis steps back a quarter instead of forward three, and the income predictors need no step. The donor has about 5,000 households against about 4,200. The 2nd edition's household tab (`dvhh_uk_anon_v3_2024.tab`) carries the annual weight as `cal_weight_annual`, the name `weighta` had, and its 1st-edition `dvhh_uk_anon_v2_2024.tab` carries no weight; no other variable the stage reads changed name or coding, and COICOP 2018 is not yet adopted. The tabs carry their study number and DOI in the spec, the fit is named `uk_lcfs_2024_25_consumption`, the support bounds are cut from the 2024-25 donor, the road-fuel level's other-fuels share is 0.46 percent of COICOP 07.2.2 on the new diary (0.37 on the old), and the road-fuel target's scope note says so. diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 87b31fc26..3876c5131 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -376,7 +376,7 @@ "maximum_relative_deviation": 1e-09, "maximum_other_fuels_share": 0.05 }, - "notes": "Stage-time fact check on the lcfs road_fuel_level receipt (microcosm#1113): the ONS Consumer Trends COICOP 07.2.2 household spend the stage declares is recomputed from the vendored row and must equal the receipt's, and the prior-weighted petrol plus diesel after the step must equal it less the donor's other-fuels share, with the factor reproducing that from the drawn total. The other-fuels share is the LCFS donor's own c72213 share of 07.2.2, licensed microdata the gate cannot recompute, so it is bounded at five percent instead (0.37 percent on the 2023-24 diary). María's ruling of 2026-10-08 set the stage, not the weights, as what levels household road fuel. The level is recomputed from the same vendored row the stage read, so the check catches a step that did not apply the declared level; it cannot catch an error in the row itself or a later stage moving the columns. The outside reading is the report-only litres audit against HMRC clearances. A population fact check: the synthetic smoke records it without blocking." + "notes": "Stage-time fact check on the lcfs road_fuel_level receipt (microcosm#1113): the ONS Consumer Trends COICOP 07.2.2 household spend the stage declares is recomputed from the vendored row and must equal the receipt's, and the prior-weighted petrol plus diesel after the step must equal it less the donor's other-fuels share, with the factor reproducing that from the drawn total. The other-fuels share is the LCFS donor's own c72213 share of 07.2.2, licensed microdata the gate cannot recompute, so it is bounded at five percent instead (0.46 percent on the 2024-25 diary). María's ruling of 2026-10-08 set the stage, not the weights, as what levels household road fuel. The level is recomputed from the same vendored row the stage read, so the check catches a step that did not apply the declared level; it cannot catch an error in the row itself or a later stage moving the columns. The outside reading is the report-only litres audit against HMRC clearances. A population fact check: the synthetic smoke records it without blocking." }, { "id": "uk_stage_lcfs_consumption_recomposed_totals", @@ -390,7 +390,7 @@ "check": "recomposed_totals", "maximum_relative_deviation": 1e-09 }, - "notes": "Stage-time check on the lcfs recomposed_totals receipt (microcosm#1113): housing_water_and_electricity_consumption and transport_consumption each keep their own chain draw less the chain's draws of the parts the stage re-levels (the drawn electricity and gas, or the drawn petrol, diesel and bus fares; floored at zero where the parts outgrow the total) and add the levelled parts back, so no household's total is below its electricity and gas, or its petrol, diesel and priced bus fares, and no remainder is negative. The declared parts must be the receipt's, the weighted remainder must equal the drawn total less the drawn parts plus the floored mass, and the weighted total the remainder plus the levelled parts, to the declared tolerance. The step floors the remainder and adds the parts, so on a receipt the declared step wrote these are identities: the check guards the declaration and the receipt, not the size of the totals. The committed support bounds do that, on the emitted frame, through the terminal uk_support gate. The ONS 04.5.3 and 04.5.4 liquid and solid fuel spend no frame column carries is recomputed from the vendored rows and must equal the receipt's; the housing remainder holds only part of the diary's other fuels, because the stage's energy derivation splits all of the diary's fuel, light and power into electricity and gas where the household reports no combined payment. A population fact check: the synthetic smoke records it without blocking." + "notes": "Stage-time check on the lcfs recomposed_totals receipt (microcosm#1113): housing_water_and_electricity_consumption and transport_consumption each keep their own chain draw less the chain's draws of the parts the stage re-levels (the drawn electricity and gas, or the drawn petrol, diesel and bus fares; floored at zero where the parts outgrow the total) and add the levelled parts back, so no household's total is below its electricity and gas, or its petrol, diesel and priced bus fares, and no remainder is negative. The declared parts must be the receipt's, the weighted remainder must equal the drawn total less the drawn parts plus the floored mass, and the weighted total the remainder plus the levelled parts, to the declared tolerance. The step floors the remainder and adds the parts, so on a receipt the declared step wrote these are identities: the check guards the declaration and the receipt, not the size of the totals. The committed support bounds do that, on the emitted frame, through the terminal uk_support gate. The ONS 04.5.3 and 04.5.4 liquid and solid fuel spend no frame column carries is recomputed from the vendored rows and must equal the receipt's; the housing remainder holds the diary's other fuels, which the stage's energy derivation leaves out of electricity and gas (microcosm#1164). A population fact check: the synthetic smoke records it without blocking." }, { "id": "uk_stage_lcfs_consumption_basis", @@ -405,7 +405,7 @@ "period_type": "calendar_year", "period_value": 2024 }, - "notes": "Stage-time check that every household spend column lcfs_consumption emits is at calendar-2024 prices as households paid them (microcosm#1113): the H5's year, which policyengine-uk projects every column from with calendar-year growth and the calibration uprates to the calendar-2025 window, so no column needs a basis of its own. The donor uprating must end at 2024 and carry every diary spend column from the April-to-March 2023-24 diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024; a full year's step would leave them a quarter past calendar 2024, at about fiscal 2024-25 prices. The QEP energy prices and the DESNZ volume level must be for calendar 2024, bus fares must be re-priced to 2024 by the fares indices, and the road-fuel level must be ONS 07.2.2 for calendar 2024; a step whose receipt is absent fails. A population fact check: the synthetic smoke records it without blocking." + "notes": "Stage-time check that every household spend column lcfs_consumption emits is at calendar-2024 prices as households paid them (microcosm#1113): the H5's year, which policyengine-uk projects every column from with calendar-year growth and the calibration uprates to the calendar-2025 window, so no column needs a basis of its own. The donor uprating must end at 2024 and carry every diary spend column from the April-to-March diary year, placed on the calendar-year index at three quarters and one quarter: the 2024-25 diary sits a quarter past calendar 2024 and steps back to it (microcosm#1164), and a plain ratio of calendar years would leave it at about fiscal 2024-25 prices. The QEP energy prices and the DESNZ volume level must be for calendar 2024, bus fares must be re-priced to 2024 by the fares indices, and the road-fuel level must be ONS 07.2.2 for calendar 2024; a step whose receipt is absent fails. A population fact check: the synthetic smoke records it without blocking." }, { "id": "uk_stage_etb_vat_support", diff --git a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json index 5bba65bde..b5a9e489a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json +++ b/packages/microcosm-build/src/microcosm/build/uk/lcfs_consumption_support_bounds.json @@ -3,10 +3,10 @@ "country": "uk", "policy": "Disclosure-safe outward-rounded LCFS consumption support bounds generated from pinned licensed donor tabs. Values are rounded outward to one significant figure; exact donor min/max values are not committed.", "source": { - "ukds_study_number": 9468, - "doi": "10.5255/UKDA-SN-9468-3", - "household_tab_sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", - "person_tab_sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "ukds_study_number": 9603, + "doi": "10.5255/UKDA-SN-9603-2", + "household_tab_sha256": "b39b1dd39170b63ca0aa84f747e1d2239213fcfcf197bf8d5c047e146ed8c781", + "person_tab_sha256": "952d128362334388af12d9e2ef7b40db591d2d3e043ae8f6cc0f98fb813f2089", "sdc_treatment": "Exact donor min/max values are rounded outward to one significant figure before commit." }, "bounds": { @@ -16,11 +16,11 @@ ], "clothing_and_footwear_consumption": [ 0.0, - 40000 + 100000 ], "communication_consumption": [ 0.0, - 40000 + 30000 ], "education_consumption": [ 0.0, @@ -28,27 +28,27 @@ ], "food_and_non_alcoholic_beverages_consumption": [ 0.0, - 20000 + 30000 ], "health_consumption": [ 0.0, - 100000 + 90000 ], "household_furnishings_consumption": [ 0.0, - 200000 + 300000 ], "housing_water_and_electricity_consumption": [ 0.0, - 300000 + 80000 ], "miscellaneous_consumption": [ 0.0, - 70000 + 200000 ], "recreation_consumption": [ 0.0, - 300000 + 200000 ], "restaurants_and_hotels_consumption": [ 0.0, diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index def1430df..386b7fa26 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3464,8 +3464,8 @@ }, { "stage": "lcfs_consumption", - "survey": "Living Costs and Food Survey 2023-24", - "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices.", + "survey": "Living Costs and Food Survey 2024-25", + "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices.", "grain": "household", "base_candidate": { "filename": "populace_uk_2023.h5", @@ -3477,28 +3477,28 @@ { "role": "lcfs_household_tab", "kind": "private_microdata", - "ukds_study_number": 9468, - "doi": "10.5255/UKDA-SN-9468-3", + "ukds_study_number": 9603, + "doi": "10.5255/UKDA-SN-9603-2", "format": "tab", - "vintage": "2023_24", - "locator": "dvhh_ukanon_v2_2023.tab", - "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", - "size_bytes": 22812887, + "vintage": "2024_25", + "locator": "dvhh_uk_anon_v3_2024.tab", + "sha256": "b39b1dd39170b63ca0aa84f747e1d2239213fcfcf197bf8d5c047e146ed8c781", + "size_bytes": 25528130, "runtime_sha256_required": true, - "filename": "dvhh_ukanon_v2_2023.tab" + "filename": "dvhh_uk_anon_v3_2024.tab" }, { "role": "lcfs_person_tab", "kind": "private_microdata", - "ukds_study_number": 9468, - "doi": "10.5255/UKDA-SN-9468-3", + "ukds_study_number": 9603, + "doi": "10.5255/UKDA-SN-9603-2", "format": "tab", - "vintage": "2023_24", - "locator": "dvper_ukanon_202324_2023.tab", - "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", - "size_bytes": 6545146, + "vintage": "2024_25", + "locator": "dvper_uk_anon_2024.tab", + "sha256": "952d128362334388af12d9e2ef7b40db591d2d3e043ae8f6cc0f98fb813f2089", + "size_bytes": 7491335, "runtime_sha256_required": true, - "filename": "dvper_ukanon_202324_2023.tab" + "filename": "dvper_uk_anon_2024.tab" }, { "role": "need_energy_facts", @@ -3572,7 +3572,7 @@ "kind": "derive", "lowercase_columns": true, "annualization_weeks": 52.17857142857143, - "donor_weight": "weighta * 1000", + "donor_weight": "cal_weight_annual * 1000", "mappings": { "num_vehicles": "a124" }, @@ -3591,7 +3591,7 @@ }, { "kind": "uprate_donor_columns", - "from_period": 2023, + "from_period": 2024, "to_period": 2024, "instant": "january_first", "fiscal_year_bridge": { @@ -4198,7 +4198,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index dd2f0b5a0..2f10bfa13 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2867,8 +2867,8 @@ stages: - household_local_bus_trips notes: Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, María's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed. - stage: lcfs_consumption - survey: Living Costs and Food Survey 2023-24 - source: UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices. + survey: Living Costs and Food Survey 2024-25 + source: UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices. grain: household base_candidate: filename: populace_uk_2023.h5 @@ -2878,26 +2878,26 @@ stages: artifacts: - role: lcfs_household_tab kind: private_microdata - ukds_study_number: 9468 - doi: 10.5255/UKDA-SN-9468-3 + ukds_study_number: 9603 + doi: 10.5255/UKDA-SN-9603-2 format: tab - vintage: '2023_24' - locator: dvhh_ukanon_v2_2023.tab - sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 - size_bytes: 22812887 + vintage: '2024_25' + locator: dvhh_uk_anon_v3_2024.tab + sha256: b39b1dd39170b63ca0aa84f747e1d2239213fcfcf197bf8d5c047e146ed8c781 + size_bytes: 25528130 runtime_sha256_required: true - filename: dvhh_ukanon_v2_2023.tab + filename: dvhh_uk_anon_v3_2024.tab - role: lcfs_person_tab kind: private_microdata - ukds_study_number: 9468 - doi: 10.5255/UKDA-SN-9468-3 + ukds_study_number: 9603 + doi: 10.5255/UKDA-SN-9603-2 format: tab - vintage: '2023_24' - locator: dvper_ukanon_202324_2023.tab - sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 - size_bytes: 6545146 + vintage: '2024_25' + locator: dvper_uk_anon_2024.tab + sha256: 952d128362334388af12d9e2ef7b40db591d2d3e043ae8f6cc0f98fb813f2089 + size_bytes: 7491335 runtime_sha256_required: true - filename: dvper_ukanon_202324_2023.tab + filename: dvper_uk_anon_2024.tab - role: need_energy_facts resource: need_energy_facts.json kind: public_aggregate_reference @@ -2946,7 +2946,7 @@ stages: - kind: derive lowercase_columns: true annualization_weeks: 52.17857142857143 - donor_weight: weighta * 1000 + donor_weight: cal_weight_annual * 1000 mappings: num_vehicles: a124 clip: @@ -2959,7 +2959,7 @@ stages: unmapped_codes: refuse floor: 0.0 - kind: uprate_donor_columns - from_period: 2023 + from_period: 2024 to_period: 2024 instant: january_first fiscal_year_bridge: @@ -3434,7 +3434,7 @@ stages: - gas_consumption - has_fuel_consumption - uses_energy_prepayment_meter - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year''s step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year''s step on the engine''s earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index aca40d462..81bc81ea3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -15566,7 +15566,7 @@ "policyengine": { "metric_name": "ons/household_road_fuel_expenditure", "value_expression": "petrol_spending + diesel_spending", - "notes": "ONS Consumer Trends COICOP 07.2.2 fuels and lubricants for personal transport equipment, current prices, calendar year, bound on the frame's petrol plus diesel spend (microcosm#1113). The publisher class also carries other motor fuels and lubricants, about 0.4 percent of it on the LCFS 2023-24 diary (c72213), which the measure does not. The lcfs_consumption stage levels petrol plus diesel to the calendar-2024 row less that share, so this row reads about 0.4 percent low from scope alone: a known offset, left in place and not scaled into the target (María's ruling of 2026-10-10). ONS publishes the class on the domestic concept (spending on UK territory; PolicyEngine/chronicle#322): it includes non-residents' fuel bought in the UK, which the residents-only frame does not hold, and leaves out residents' fuel bought abroad. Consumer Trends publishes those two flows for total spending only (for 2024, GBP 64.7bn by non-residents in the UK, 3.8 percent of the domestic total, and GBP 78.9bn by residents abroad) and splits them by no class, so no national bridge is applied and their size for road fuel is not known. Beyond the scope offset, the design-weight gap to this calendar-2025 row is the engine's 2024-to-2025 uprating of the two columns against ONS's own move (37.025bn to 35.099bn, -5.2 percent). policyengine-uk 2.122.2 carries both columns -2.7 percent, so on that version the row reads about +2.3 percent at design weights, scope offset included, and the calibration closes that by moving weight away from fuel buyers. That is a desk figure from the engine's indices; the national remeasure records the measured one. Household-scope road fuel replaces the OBR cars receipts as the bound fuel row: those are a fiscal-year duty total that includes business use of cars, which the frame reaches only through the engine's conversion of spend to litres, so they stay measured as a diagnostic (calibration_measure_exclusions.json)." + "notes": "ONS Consumer Trends COICOP 07.2.2 fuels and lubricants for personal transport equipment, current prices, calendar year, bound on the frame's petrol plus diesel spend (microcosm#1113). The publisher class also carries other motor fuels and lubricants, about 0.46 percent of it on the LCFS 2024-25 diary (c72213), which the measure does not. The lcfs_consumption stage levels petrol plus diesel to the calendar-2024 row less that share, so this row reads about 0.5 percent low from scope alone: a known offset, left in place and not scaled into the target (María's ruling of 2026-10-10). ONS publishes the class on the domestic concept (spending on UK territory; PolicyEngine/chronicle#322): it includes non-residents' fuel bought in the UK, which the residents-only frame does not hold, and leaves out residents' fuel bought abroad. Consumer Trends publishes those two flows for total spending only (for 2024, GBP 64.7bn by non-residents in the UK, 3.8 percent of the domestic total, and GBP 78.9bn by residents abroad) and splits them by no class, so no national bridge is applied and their size for road fuel is not known. Beyond the scope offset, the design-weight gap to this calendar-2025 row is the engine's 2024-to-2025 uprating of the two columns against ONS's own move (37.025bn to 35.099bn, -5.2 percent). policyengine-uk 2.122.2 carries both columns -2.7 percent, so on that version the row reads about +2.3 percent at design weights, scope offset included, and the calibration closes that by moving weight away from fuel buyers. That is a desk figure from the engine's indices; the national remeasure records the measured one. Household-scope road fuel replaces the OBR cars receipts as the bound fuel row: those are a fiscal-year duty total that includes business use of cars, which the frame reaches only through the engine's conversion of spend to litres, so they stay measured as a diagnostic (calibration_measure_exclusions.json)." }, "axiom": { "metric_name": "ons/household_road_fuel_expenditure", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py index 297dc3693..0aa2fa9f3 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/donor_uprating.py @@ -1,9 +1,9 @@ """Declared uprating of UK imputation donors to the base year (microcosm#890 U). -The LCFS 2023-24 and ETB FYE-2024 donors sit before the FRS 2024-25 spine's +The LCFS 2024-25 and ETB FYE-2024 donors sit off the FRS 2024-25 spine's calendar-2024 price basis: the LCFS diary year is placed on the calendar-year -index at three quarters and one quarter (microcosm#1113), the ETB year by a -full step. Each stage declares an ``uprate_donor_columns`` operation naming, per +index at three quarters and one quarter (microcosm#1113), a quarter past +calendar 2024 (microcosm#1164), and the ETB year moves a full step. Each stage declares an ``uprate_donor_columns`` operation naming, per column, the basis that moves it from ``from_period`` to ``to_period``: * ``engine_parameter`` — the ratio of a PolicyEngine UK parameter at the two diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index be24fad34..97cd6e511 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -70,14 +70,6 @@ from microcosm.frame.rules import assert_rules_engine_country WEEKS_IN_YEAR = 365.25 / 7 -LCFS_HOUSEHOLD_FILENAME = "dvhh_ukanon_v2_2023.tab" -LCFS_HOUSEHOLD_SHA256 = ( - "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" -) -LCFS_HOUSEHOLD_SIZE_BYTES = 22_812_887 -LCFS_PERSON_FILENAME = "dvper_ukanon_202324_2023.tab" -LCFS_PERSON_SHA256 = "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" -LCFS_PERSON_SIZE_BYTES = 6_545_146 UK_LCFS_CONSUMPTION_DECLARED_SEEDS = {"lcfs_consumption": 0} @@ -115,6 +107,10 @@ 7: "HOUSE_DETACHED", 8: "OTHER", } +#: The donor's annual household weight, thousands of households: LCFS 2024-25 +#: (2nd edition) carries it as cal_weight_annual, the name weighta had in +#: earlier years (microcosm#1164). +UK_LCFS_DONOR_WEIGHT_COLUMN = "cal_weight_annual" HOUSEHOLD_LCFS_RENAMES = { "g018": "household_adult_count", "g019": "household_child_count", @@ -122,7 +118,7 @@ "a124": "num_vehicles", "p389p": "hbai_household_net_income", "p344p": "household_gross_income", - "weighta": "household_weight", + UK_LCFS_DONOR_WEIGHT_COLUMN: "household_weight", } PERSON_LCFS_RENAMES = { "b303p": "employment_income", @@ -400,7 +396,7 @@ UK_LCFS_PREPAYMENT_METER_COLUMN, ) UK_LCFS_CONSUMPTION_NONNEGATIVE_OUTPUT_COLUMNS = UK_LCFS_CONSUMPTION_OUTPUT_COLUMNS -UK_LCFS_CONSUMPTION_FIT_NAME = "uk_lcfs_2023_24_consumption" +UK_LCFS_CONSUMPTION_FIT_NAME = "uk_lcfs_2024_25_consumption" UK_LCFS_CONSUMPTION_STAGE_NAME = "lcfs_consumption" @@ -1390,9 +1386,15 @@ def lcfs_other_road_fuel_share(lcfs_household: pd.DataFrame) -> float: household = _lowercase(lcfs_household) _require_columns( - household, ("weighta", "c72211", "c72212", *UK_LCFS_OTHER_ROAD_FUEL_CODES) + household, + ( + UK_LCFS_DONOR_WEIGHT_COLUMN, + "c72211", + "c72212", + *UK_LCFS_OTHER_ROAD_FUEL_CODES, + ), ) - weight = _numeric(household["weighta"]).to_numpy(dtype=float) + weight = _numeric(household[UK_LCFS_DONOR_WEIGHT_COLUMN]).to_numpy(dtype=float) def spend(code: str) -> np.ndarray: # Floored at zero, as the imputation's donor is (microcosm#1063 c9). @@ -2150,9 +2152,9 @@ def clean_lcfs_consumption_table( """Return the LCFS donor table with annualized consumption variables. ``uprating`` maps donor columns to the declared factors that move the - 2023-24 diary to the base year (``uprate_donor_columns``): the spend - columns to calendar-2024 prices, the income predictors a full year to the - FRS 2024-25 incomes. It is applied after annualisation and before the + 2024-25 diary to the base year (``uprate_donor_columns``): the spend + columns a quarter back to calendar-2024 prices, the income predictors by + one (the diary year is the FRS 2024-25 year; microcosm#1164). It is applied after annualisation and before the donor-side NEED rake, so the income bands and the support-clip ranges see base-year values. ``energy`` (the declared ``price_domestic_energy``) converts the diary's diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py index 48729b4d0..e91c050c4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/spine_build.py @@ -361,12 +361,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument( "--lcfs-hh-tab", type=Path, - help="Caller-supplied private LCFS 2023-24 household tab for lcfs_consumption.", + help="Caller-supplied private LCFS 2024-25 household tab for lcfs_consumption.", ) parser.add_argument( "--lcfs-person-tab", type=Path, - help="Caller-supplied private LCFS 2023-24 person tab for lcfs_consumption.", + help="Caller-supplied private LCFS 2024-25 person tab for lcfs_consumption.", ) parser.add_argument( "--etb-tab", diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_donor_uprating.py b/packages/microcosm-build/tests/engine/uk/test_uk_donor_uprating.py index 9df23f618..c4e493088 100644 --- a/packages/microcosm-build/tests/engine/uk/test_uk_donor_uprating.py +++ b/packages/microcosm-build/tests/engine/uk/test_uk_donor_uprating.py @@ -10,11 +10,11 @@ def test_declared_factors_lockstep_with_the_installed_engine() -> None: reader = module.engine_parameter_reader() lcfs_factors, lcfs_receipt = lcfs_donor_uprating(stages["lcfs_consumption"]) - cpi_ratio = reader(CPI, 2024) / reader(CPI, 2023) - # The diary columns start from the April-to-March 2023-24 year and end at - # calendar 2024, so they take the bridged step, not the full year's. + # The diary columns start from the April 2024 to March 2025 year and end at + # calendar 2024 (microcosm#1164), so they take the bridged step back a + # quarter: a factor below one. diary_ratio = reader(CPI, 2024) / ( - 0.75 * reader(CPI, 2023) + 0.25 * reader(CPI, 2024) + 0.75 * reader(CPI, 2024) + 0.25 * reader(CPI, 2025) ) cpi_columns = [ column @@ -24,21 +24,22 @@ def test_declared_factors_lockstep_with_the_installed_engine() -> None: assert len(cpi_columns) == 13 for column in cpi_columns: assert lcfs_factors[column] == pytest.approx(diary_ratio), column - assert diary_ratio < cpi_ratio - assert lcfs_factors["employment_income"] == reader( - "gov.economic_assumptions.indices.obr.average_earnings", 2024 - ) / reader("gov.economic_assumptions.indices.obr.average_earnings", 2023) - assert lcfs_factors["petrol_spending"] == pytest.approx(0.9846, abs=5e-4) - # ONS's revised mid-2024 population (the 942a1fa feed) moves both per-head - # fuel factors by 0.036%; diesel crosses its old pin (microcosm#1123). - assert lcfs_factors["diesel_spending"] == pytest.approx(0.8985, abs=5e-4) + assert lcfs_receipt["columns"][column]["following"] == reader(CPI, 2025) + assert diary_ratio < 1.0 + # The diary year is the FRS 2024-25 year: incomes and the litre proxies + # take one; the level step sets petrol and diesel anyway. + assert lcfs_factors["employment_income"] == 1.0 + assert lcfs_factors["petrol_spending"] == 1.0 + assert lcfs_factors["diesel_spending"] == 1.0 audits = { column: lcfs_receipt["columns"][column]["engine_audit"] for column in ("petrol_spending", "diesel_spending") } - assert audits["petrol_spending"]["engine_ratio"] == pytest.approx(0.852, abs=1e-3) - assert audits["diesel_spending"]["engine_ratio"] == pytest.approx(1.032, abs=1e-3) + assert audits["petrol_spending"]["engine_ratio"] == pytest.approx(1.0) + assert audits["diesel_spending"]["engine_ratio"] == pytest.approx(1.0) + # The ETB FYE-2024 donor still moves a full year's step. + cpi_ratio = reader(CPI, 2024) / reader(CPI, 2023) etb_factors, _receipt = etb_donor_uprating(stages["etb_services"]) assert etb_factors["dfe_education_spending"] == cpi_ratio assert etb_factors["rail_subsidy_spending"] == pytest.approx(0.9351, abs=5e-4) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py index 36c15cb15..e3222a0e4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py @@ -81,10 +81,10 @@ def test_e6_support_bounds_resources_are_sha_bound_and_non_placeholder() -> None services = _load("etb_services_support_bounds.json") assert lcfs["source"]["household_tab_sha256"] == ( - "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" + "b39b1dd39170b63ca0aa84f747e1d2239213fcfcf197bf8d5c047e146ed8c781" ) assert lcfs["source"]["person_tab_sha256"] == ( - "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" + "952d128362334388af12d9e2ef7b40db591d2d3e043ae8f6cc0f98fb813f2089" ) # Raked or levelled columns carry no bounds: energy, bus fares since # microcosm#890, and petrol and diesel since microcosm#1113. The housing @@ -96,8 +96,13 @@ def test_e6_support_bounds_resources_are_sha_bound_and_non_placeholder() -> None # The tool floors the donor as the stage does, so no bound starts below 0. assert lcfs["bounds"]["housing_water_and_electricity_consumption"] == [ 0.0, - 300000, + 80000, ] + # LCFS 2024-25, SN 9603 2nd edition (microcosm#1164). + assert (lcfs["source"]["ukds_study_number"], lcfs["source"]["doi"]) == ( + 9603, + "10.5255/UKDA-SN-9603-2", + ) assert all(low >= 0 for low, _high in lcfs["bounds"].values()) assert lcfs["bounds"]["transport_consumption"] == [0.0, 200000] assert vat["source"]["tab_sha256"] == ( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py index e44a86e01..44ebe36b0 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_donor_uprating.py @@ -6,10 +6,13 @@ def test_both_donor_stages_declare_uprating_to_the_base_year() -> None: stages = _stages() + # The LCFS 2024-25 diary starts in the base year (microcosm#1164); the ETB + # FYE-2024 donor is a year behind. + periods = {"lcfs_consumption": (2024, 2024), "etb_services": (2023, 2024)} for name in ("lcfs_consumption", "etb_services"): operation = uprating_operation(stages[name]) assert operation is not None, name - assert (operation["from_period"], operation["to_period"]) == (2023, 2024) + assert (operation["from_period"], operation["to_period"]) == periods[name] assert [op.kind for op in stages[name].operations][:2] == [ "derive", "uprate_donor_columns", @@ -23,7 +26,8 @@ def test_both_donor_stages_declare_uprating_to_the_base_year() -> None: assert lcfs["columns"]["petrol_spending"]["basis"] == "vendored_litre_proxy" assert lcfs["columns"]["diesel_spending"]["basis"] == "vendored_litre_proxy" # Every diary spend column starts from the April-to-March diary year and - # ends at calendar 2024 (microcosm#1113); income predictors move a full year. + # ends at calendar 2024 (microcosm#1113); income predictors take the plain + # ratio, one for the 2024-25 diary. assert lcfs["fiscal_year_bridge"] == { "start_year_weight": 0.75, "next_year_weight": 0.25, diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index d5047f0a8..c131472c4 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -44,7 +44,7 @@ def _household() -> pd.DataFrame: "a124": [0, 1, 2, 9], "p389p": [100.0, 200.0, 300.0, 400.0], "p344p": [150.0, 250.0, 350.0, 450.0], - "weighta": [1.5, 2.0, 2.5, 3.0], + "cal_weight_annual": [1.5, 2.0, 2.5, 3.0], "a122": [4, 8, 5, 7], "a121": [4, 5, 6, 7], **_energy_codes( @@ -790,7 +790,7 @@ def _synthetic_lcfs_donor( "a124": rows % 4, "p389p": rng.uniform(100.0, 1500.0, n), "p344p": rng.uniform(150.0, 2000.0, n), - "weighta": rng.uniform(0.5, 2.0, n), + "cal_weight_annual": rng.uniform(0.5, 2.0, n), "a122": 1 + rows % 8, "a121": 1 + rows % 8, "b226": rng.uniform(0.0, 30.0, n), @@ -1015,7 +1015,7 @@ def materialize(self, frame, variables, period): # less the donor's own other-fuels share (#1113). level = evidence["road_fuel_level"] assert level["coicop"] == "07.2.2" and level["period_value"] == 2024 - weight = donor_household["weighta"].to_numpy() + weight = donor_household["cal_weight_annual"].to_numpy() fuels = donor_household[["c72211", "c72212", "c72213"]].to_numpy() assert level["other_fuels_share"] == pytest.approx( weight @ fuels[:, 2] / (weight @ fuels.sum(axis=1)) @@ -1489,7 +1489,7 @@ def test_other_road_fuel_share_is_the_donor_weighted_c72213_share() -> None: donor = pd.DataFrame( { - "WEIGHTA": [1.0, 3.0], + "CAL_WEIGHT_ANNUAL": [1.0, 3.0], "C72211": [10.0, 0.0], "C72212": [0.0, 5.0], "C72213": [1.0, 0.0], @@ -1637,8 +1637,8 @@ def test_redraw_gives_every_flagged_household_road_fuel() -> None: ) assert receipt["weighted_total_after"] == pytest.approx(weights @ after.sum(axis=1)) assert [record.fit_name for record in result.fit_weight_records] == [ - "uk_lcfs_2023_24_consumption:road_fuel_total", - "uk_lcfs_2023_24_consumption:petrol_share_of_road_fuel", + "uk_lcfs_2024_25_consumption:road_fuel_total", + "uk_lcfs_2024_25_consumption:petrol_share_of_road_fuel", ] diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 9765f0b72..f0eec75de 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3402,30 +3402,30 @@ "lcfs_consumption": { "artifacts": [ { - "doi": "10.5255/UKDA-SN-9468-3", - "filename": "dvhh_ukanon_v2_2023.tab", + "doi": "10.5255/UKDA-SN-9603-2", + "filename": "dvhh_uk_anon_v3_2024.tab", "format": "tab", "kind": "private_microdata", - "locator": "dvhh_ukanon_v2_2023.tab", + "locator": "dvhh_uk_anon_v3_2024.tab", "role": "lcfs_household_tab", "runtime_sha256_required": true, - "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", - "size_bytes": 22812887, - "ukds_study_number": 9468, - "vintage": "2023_24" + "sha256": "b39b1dd39170b63ca0aa84f747e1d2239213fcfcf197bf8d5c047e146ed8c781", + "size_bytes": 25528130, + "ukds_study_number": 9603, + "vintage": "2024_25" }, { - "doi": "10.5255/UKDA-SN-9468-3", - "filename": "dvper_ukanon_202324_2023.tab", + "doi": "10.5255/UKDA-SN-9603-2", + "filename": "dvper_uk_anon_2024.tab", "format": "tab", "kind": "private_microdata", - "locator": "dvper_ukanon_202324_2023.tab", + "locator": "dvper_uk_anon_2024.tab", "role": "lcfs_person_tab", "runtime_sha256_required": true, - "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", - "size_bytes": 6545146, - "ukds_study_number": 9468, - "vintage": "2023_24" + "sha256": "952d128362334388af12d9e2ef7b40db591d2d3e043ae8f6cc0f98fb813f2089", + "size_bytes": 7491335, + "ukds_study_number": 9603, + "vintage": "2024_25" }, { "format": "json", @@ -3517,7 +3517,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; Mar\u00eda's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2023-24 diary is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2023 and one of 2024, to calendar 2024 (microcosm#1113; a full year's step would leave them a quarter past it, at about fiscal 2024-25 prices; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a full year's step on the engine's earnings, mixed-income and pension indices, to the FRS 2024-25 incomes they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; Mar\u00eda's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3527,7 +3527,7 @@ 5 ] }, - "donor_weight": "weighta * 1000", + "donor_weight": "cal_weight_annual * 1000", "floor": 0, "kind": "derive", "lossy_mappings": [ @@ -3653,7 +3653,7 @@ "next_year_weight": 0.25, "start_year_weight": 0.75 }, - "from_period": 2023, + "from_period": 2024, "instant": "january_first", "kind": "uprate_donor_columns", "to_period": 2024 @@ -4128,9 +4128,9 @@ "uses_energy_prepayment_meter" ], "rewrites": [], - "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices.", + "source": "UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices.", "stage": "lcfs_consumption", - "survey": "Living Costs and Food Survey 2023-24" + "survey": "Living Costs and Food Survey 2024-25" }, "nts_bus_travel": { "artifacts": [ diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv index 7c0d4ee9c..e9c25c522 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/lcfs_household.csv @@ -1,4 +1,4 @@ -case,g018,g019,gorx,a124,p389p,p344p,weighta,a122,a121,p537,b175,b178,b170,b173,b1751,b227,b2332,c45112,c45114,b1701,b226,b2331,c45212,c45214,b231,b233,b017,b018,c45222,c45312,c45411,c45412,c45511,p601,p602,p603,p604,p605,p606,p607,p608,p609,p610,p611,p612,c72211,c72212,c73212,c73213,c73214,c72213 +case,g018,g019,gorx,a124,p389p,p344p,cal_weight_annual,a122,a121,p537,b175,b178,b170,b173,b1751,b227,b2332,c45112,c45114,b1701,b226,b2331,c45212,c45214,b231,b233,b017,b018,c45222,c45312,c45411,c45412,c45511,p601,p602,p603,p604,p605,p606,p607,p608,p609,p610,p611,p612,c72211,c72212,c73212,c73213,c73214,c72213 1,1,0,1,0,100,150,1,1,1,10,0,0,0,0,0,6,0,0,0,0,4,0,0,0,10,0,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,0,0,0,0.5 2,2,1,2,1,110,160,1.1000000000000001,2,2,11,0,0,0,0,6,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,1.1000000000000001,2.1000000000000001,3.1000000000000001,4.0999999999999996,5.0999999999999996,6.0999999999999996,7.0999999999999996,8.0999999999999996,9.0999999999999996,10.1,11.1,12.1,13.1,14.1,15.1,16.100000000000001,17.100000000000001,0 3,3,2,3,2,120,170,1.2,3,3,12,10,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.2,2.2000000000000002,3.2000000000000002,4.2000000000000002,5.2000000000000002,6.2000000000000002,7.2000000000000002,8.1999999999999993,9.1999999999999993,10.199999999999999,11.199999999999999,12.199999999999999,13.199999999999999,14.199999999999999,15.199999999999999,16.199999999999999,17.199999999999999,0 diff --git a/tools/graph_uk_spine_fixture.py b/tools/graph_uk_spine_fixture.py index bbab2c7b2..1cb48f029 100644 --- a/tools/graph_uk_spine_fixture.py +++ b/tools/graph_uk_spine_fixture.py @@ -843,7 +843,7 @@ def _lcfs_donors() -> tuple[pd.DataFrame, pd.DataFrame]: "a124": rows.astype(int) % 4, "p389p": 100.0 + rows * 10.0, "p344p": 150.0 + rows * 10.0, - "weighta": 1.0 + rows % 7 / 10.0, + "cal_weight_annual": 1.0 + rows % 7 / 10.0, "a122": 1 + rows.astype(int) % 8, "a121": 1 + rows.astype(int) % 8, "p537": 10.0 + rows, From 4e284ae6676c2bafbe5e8a13ba653473ec6b910f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 12/28] Cite microcosm#1225 for the ETB VAT rate bounds (#1164) The issue filed for the ETB half of uk-data#572 now numbers the changelog fragment and the code and stage notes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/1164-etb-vat-rate-share-bounds.fixed.md | 1 - changelog.d/1225-etb-vat-rate-share-bounds.fixed.md | 1 + .../microcosm-build/src/microcosm/build/uk/source_stages.json | 2 +- .../microcosm-build/src/microcosm/build/uk/spec/sources.yaml | 2 +- .../microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py | 2 +- .../tests/fixtures/parity/uk_spine/sources/fixture.json | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/1164-etb-vat-rate-share-bounds.fixed.md create mode 100644 changelog.d/1225-etb-vat-rate-share-bounds.fixed.md diff --git a/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md b/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md deleted file mode 100644 index e85246fe2..000000000 --- a/changelog.d/1164-etb-vat-rate-share-bounds.fixed.md +++ /dev/null @@ -1 +0,0 @@ -The UK `etb_vat` stage keeps `full_rate_vat_expenditure_rate` a share of expenditure after VAT, in [0, 1] (microcosm#1164; the ETB half of uk-data#572). The rate is `totvat × (1 − reduced-rate share) / standard rate` over `expdis − totvat`, and the stage refused only a zero denominator. So the ETB 2023 donor kept the households whose VAT exceeds their disposable expenditure, with negative rates, and the few whose little expenditure after VAT gives a rate above one, and the QRF drew both onto recipients. The derive step now declares `rate_bounds: [0, 1]`; it drops donors with no expenditure after VAT, caps the rest at one, and fails on a negative `totvat`. A `donor_rate` receipt counts the rows dropped and capped. The output is declared non-negative, and the committed support bound is the declared range rather than a rounded donor extreme. diff --git a/changelog.d/1225-etb-vat-rate-share-bounds.fixed.md b/changelog.d/1225-etb-vat-rate-share-bounds.fixed.md new file mode 100644 index 000000000..7fe69a7ce --- /dev/null +++ b/changelog.d/1225-etb-vat-rate-share-bounds.fixed.md @@ -0,0 +1 @@ +The UK `etb_vat` stage keeps `full_rate_vat_expenditure_rate` a share of expenditure after VAT, in [0, 1] (microcosm#1225, the ETB half of uk-data#572; shipped with microcosm#1164). The rate is `totvat × (1 − reduced-rate share) / standard rate` over `expdis − totvat`, and the stage refused only a zero denominator. So the ETB 2023 donor kept the households whose VAT exceeds their disposable expenditure, with negative rates, and the few whose little expenditure after VAT gives a rate above one, and the QRF drew both onto recipients. The derive step now declares `rate_bounds: [0, 1]`; it drops donors with no expenditure after VAT, caps the rest at one, and fails on a negative `totvat`. A `donor_rate` receipt counts the rows dropped and capped. The output is declared non-negative, and the committed support bound is the declared range rather than a rounded donor extreme. diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 386b7fa26..c38f50dcf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -4287,7 +4287,7 @@ "nonnegative_outputs": [ "full_rate_vat_expenditure_rate" ], - "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095)." + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (microcosm#1225, uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095)." }, { "stage": "etb_services", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 2f10bfa13..2d25ab8f4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3497,7 +3497,7 @@ stages: - full_rate_vat_expenditure_rate nonnegative_outputs: - full_rate_vat_expenditure_rate - notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household''s FRS claimants and partners and its other members (is_uc_claimant), as the donor''s adults and childs count adults and dependent children, not the engine''s age-18 split (uk-data#486, microcosm#1095).' + notes: 'Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (microcosm#1225, uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household''s FRS claimants and partners and its other members (is_uc_claimant), as the donor''s adults and childs count adults and dependent children, not the engine''s age-18 split (uk-data#486, microcosm#1095).' - stage: etb_services survey: Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py index fccbbff21..6b70424a1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/etb_vat.py @@ -50,7 +50,7 @@ ) UK_ETB_VAT_OUTPUT_COLUMNS = ("full_rate_vat_expenditure_rate",) #: The full-rate VAT expenditure rate is a share of expenditure after VAT, so -#: it lies in [0, 1] (uk-data#572). The raw ETB accounts carry households whose +#: it lies in [0, 1] (microcosm#1225, uk-data#572). The raw ETB accounts carry households whose #: VAT equals or exceeds their disposable expenditure, which leave nothing for #: the rate to be a share of; they are dropped from the donor, and the rates #: above one, where little expenditure is left after VAT, are capped at one. diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index f0eec75de..6c6650c2a 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -939,7 +939,7 @@ "nonnegative_outputs": [ "full_rate_vat_expenditure_rate" ], - "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095).", + "notes": "Ports ETB VAT imputation using the 2023 donor year and cited VAT anchors; missing or NaN rates fail loud rather than falling back. The rate is a share of expenditure after VAT, so the derive step declares it in [0, 1] (microcosm#1225, uk-data#572): the raw ETB accounts carry households whose VAT equals or exceeds their disposable expenditure, which leave nothing for the rate to be a share of and are dropped from the donor, and households with little expenditure left after VAT, whose rates exceed one, are capped at one. The donor_rate receipt counts both. A negative totvat fails the stage. The output is declared non-negative and the support gate bounds it. The recipient is_adult and is_child counts are the household's FRS claimants and partners and its other members (is_uc_claimant), as the donor's adults and childs count adults and dependent children, not the engine's age-18 split (uk-data#486, microcosm#1095).", "operations": [ { "annualization_weeks": 52, From 5c37eae1dbde0d1be3ff12273e0502f8a819b6fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 13/28] Refuse a second rounding of a donor extreme the tabs have not changed (#1164) Each committed support bound is a donor extreme rounded outward to one significant figure. Re-derived from the same tabs after a change in the declared uprating, the cleaning or the floor, a bound that rounds to a different value tells a reader, with the first, that the extreme lies between the two thresholds, each divided by its own factor: a narrow interval around one record. The generator now refuses to write a bound that would move while the file's tab digests are unchanged; a lower bound may still move to zero, the floor, which reads no donor value. Co-Authored-By: Claude Opus 5.5 --- ...1164-support-bounds-no-rerounding.fixed.md | 1 + .../uk/test_uk_consumption_resources.py | 39 ++++++++++++++ tools/build_uk_e6_support_bounds.py | 51 ++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 changelog.d/1164-support-bounds-no-rerounding.fixed.md diff --git a/changelog.d/1164-support-bounds-no-rerounding.fixed.md b/changelog.d/1164-support-bounds-no-rerounding.fixed.md new file mode 100644 index 000000000..c342fe9cc --- /dev/null +++ b/changelog.d/1164-support-bounds-no-rerounding.fixed.md @@ -0,0 +1 @@ +`tools/build_uk_e6_support_bounds.py` refuses to move a committed rounded support bound while the file's donor tab digests are unchanged (microcosm#1164). Each bound is a donor extreme rounded outward; re-derived from the same tabs after a change in the declared uprating, the cleaning or the floor, a bound that rounds to a different value narrows the extreme between the two thresholds. A lower bound may still move to zero, the floor, which reads no donor value. diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py index e3222a0e4..feaf7ff30 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_consumption_resources.py @@ -191,3 +191,42 @@ def test_e6_support_bounds_round_trip_against_licensed_tabs() -> None: assert json.dumps(payload, indent=2, sort_keys=False) + "\n" == ( UK_PACKAGE / name ).read_text(encoding="utf-8") + + +def test_support_bound_generator_refuses_a_second_rounding_of_unchanged_tabs( + tmp_path, +) -> None: + # microcosm#1164: two outward roundings of one donor extreme under + # different factors narrow it, so with the tabs unchanged a re-derivation + # may not move a rounded bound; a lower bound may move to the floor (zero). + import importlib.util + + path = ROOT / "tools/build_uk_e6_support_bounds.py" + spec = importlib.util.spec_from_file_location("build_uk_e6_support_bounds", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + + committed = tmp_path / "bounds.json" + committed.write_text( + json.dumps( + { + "source": {"household_tab_sha256": "a" * 64}, + "bounds": {"x": [-20000, 300000], "y": [0.0, 70000]}, + } + ) + ) + + def payload(digest: str, x: list, y: list) -> dict: + return { + "source": {"household_tab_sha256": digest * 64}, + "bounds": {"x": x, "y": y}, + } + + module.refuse_rerounding(committed, payload("a", [0.0, 300000], [0.0, 70000])) + module.refuse_rerounding(committed, payload("b", [0.0, 300000], [0.0, 60000])) + module.refuse_rerounding(tmp_path / "absent.json", payload("a", [0, 1], [0, 1])) + with pytest.raises(SystemExit, match=r"\['y'\] would move"): + module.refuse_rerounding( + committed, payload("a", [-20000, 300000], [0.0, 60000]) + ) diff --git a/tools/build_uk_e6_support_bounds.py b/tools/build_uk_e6_support_bounds.py index 0fae95029..e6b369245 100644 --- a/tools/build_uk_e6_support_bounds.py +++ b/tools/build_uk_e6_support_bounds.py @@ -1,4 +1,9 @@ -"""Build disclosure-safe E6 UK support bounds from pinned donor tabs.""" +"""Build disclosure-safe E6 UK support bounds from pinned donor tabs. + +A bound is derived once per donor tab: while a file's tab digests are +unchanged, writing refuses a re-derivation that moves a rounded bound +(``refuse_rerounding``), since two roundings of one donor extreme narrow it. +""" from __future__ import annotations @@ -249,10 +254,54 @@ def main(argv: list[str] | None = None) -> int: if path.read_text(encoding="utf-8") != rendered: raise SystemExit(f"{path} is stale.") else: + refuse_rerounding(path, payload) path.write_text(rendered, encoding="utf-8") return 0 +def refuse_rerounding(path: Path, payload: dict[str, object]) -> None: + """Refuse to move a committed rounded bound while its donor tabs are unchanged. + + Each committed bound is the donor's extreme rounded outward. Re-derived + from the same tabs after a change in the declared uprating, the cleaning + or the floor, a bound that rounds to a different value tells the reader, + with the first, that the extreme lies between the two thresholds, each + divided by its own factor: a narrow interval around a unit record (UKDS + EUL disclosure control; microcosm#1164). A lower bound may move to zero, + the floor, which reads no donor value; any other move needs new donor tabs + or a bound that reads no donor value either. + """ + + if not path.exists(): + return + committed = json.loads(path.read_text(encoding="utf-8")) + + def digests(source: object) -> dict[str, object]: + if not isinstance(source, dict): + return {} + return {k: v for k, v in source.items() if str(k).endswith("sha256")} + + if not digests(committed.get("source")) or digests( + committed.get("source") + ) != digests(payload.get("source")): + return + old = committed.get("bounds") or {} + new = payload.get("bounds") or {} + moved = sorted( + column + for column in set(old) & set(new) + if list(old[column]) != list(new[column]) + and not (list(old[column])[1] == list(new[column])[1] and new[column][0] == 0) + ) + if moved: + raise SystemExit( + f"{path.name}: the donor tabs are unchanged but the rounded bounds of " + f"{moved} would move. Committing a second rounding of the same donor " + "extremes narrows them (UKDS disclosure control); keep the committed " + "bounds or replace them with ones that read no donor value." + ) + + def _payload( *, source: dict[str, object], From f6107656fd3d82511704a10b13f100af3355e607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 14/28] Hand capped household kWh to the capped households' own NEED cells (#1164) The household ceilings handed what a capped household gave up to every household of the fuel in proportion to its use. Capped households sit in a few NEED cells (detached and high-income homes), so the proportional hand-out pulled those cells off their NEED means: on the K25 spine the gas income cells moved 0.36 points further from target. cap_household_kwh now takes the rake's cells and, each round, re-rakes the households below the ceiling to the levelled cell targets with the capped ones held at it, before the proportional step restores the DESNZ total; the fit then moves 0.06 points. Co-Authored-By: Claude Opus 5.5 --- .../1164-lcfs-household-kwh-ceilings.fixed.md | 2 +- .../build/uk_runtime/energy_pricing.py | 74 ++++++++++++++++++- .../engine_free/uk/test_uk_energy_pricing.py | 37 ++++++++++ 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md b/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md index 571a3995e..762065515 100644 --- a/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md +++ b/changelog.d/1164-lcfs-household-kwh-ceilings.fixed.md @@ -1 +1 @@ -The UK LCFS stage no longer leaves a household's electricity or gas above the domestic meter definitions behind the DESNZ total it levels to (microcosm#1164; the extreme energy values item of #1207; María's ruling of 2026-10-12). NEED's quality-assurance annex counts a gas meter as domestic under 73,200 kWh a year and an electricity meter under 50,000 kWh on profile classes 1 and 2. The energy columns are exempt from the donor support clip because the NEED rake and the DESNZ level set them, so a few drawn households sat far above any domestic use, and since #1167 the housing total contained them. The `price_domestic_energy` step declares `household_kwh_ceiling`; after the level, `cap_household_kwh` sets a household above it to the ceiling and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds. The `energy_rake` receipt counts the households capped and the share of the total they gave up, the NEED fit is read after the cap, and the `energy_rake` gate checks the declared ceilings, that no household is left above them and that the total held. +The UK LCFS stage no longer leaves a household's electricity or gas above the domestic meter definitions behind the DESNZ total it levels to (microcosm#1164; the extreme energy values item of #1207; María's ruling of 2026-10-12). NEED's quality-assurance annex counts a gas meter as domestic under 73,200 kWh a year and an electricity meter under 50,000 kWh on profile classes 1 and 2. The energy columns are exempt from the donor support clip because the NEED rake and the DESNZ level set them, so a few drawn households sat far above any domestic use, and since #1167 the housing total contained them. The `price_domestic_energy` step declares `household_kwh_ceiling`; after the level, `cap_household_kwh` sets a household above it to the ceiling and re-rakes the households below it in its own NEED cells to the levelled targets with the capped ones held, then restores the DESNZ total. Handed to every household in proportion instead, the kWh moved the cells that hold most capped households (detached and high-income homes) off their NEED means. The `energy_rake` receipt counts the households capped and the share of the total they gave up, the NEED fit is read after the cap, and the `energy_rake` gate checks the declared ceilings, that no household is left above them and that the total held. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py index 303b9648c..7317db7a1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/energy_pricing.py @@ -913,6 +913,8 @@ def cap_household_kwh( population: np.ndarray, ceiling: float, max_rounds: int = 100, + cells: Sequence[tuple[np.ndarray, Mapping[str, float]]] = (), + sweeps: int = 0, ) -> tuple[np.ndarray, dict[str, Any]]: """Cap one fuel's household kWh at ``ceiling`` and keep its weighted total. @@ -922,6 +924,14 @@ def cap_household_kwh( capped in the next round. The receipt counts the households drawn above the ceiling and the share of the total they gave up, and records no household's value. + + With ``cells`` (each margin's row labels and levelled cell-mean targets, + as the rake fitted them) and ``sweeps``, the households below the ceiling + are first re-raked to those targets with the capped ones held at it, so + what a capped household gives up goes to its own NEED cells; the + proportional step then only restores the total (microcosm#1164: handed to + every household at once, the kWh moved the cells that hold most capped + households, detached and high-income ones, off their NEED means). """ capped_values = np.asarray(values, dtype=float).copy() @@ -940,6 +950,15 @@ def cap_household_kwh( rounds += 1 at_ceiling |= over capped_values[over] = ceiling + if cells and sweeps: + _rake_below_ceiling( + capped_values, + weights=weight, + population=members, + held=at_ceiling, + cells=cells, + sweeps=sweeps, + ) free = members & ~at_ceiling & (capped_values > 0) free_total = float(np.dot(weight[free], capped_values[free])) shortfall = total - float(np.dot(weight[members], capped_values[members])) @@ -971,6 +990,43 @@ def cap_household_kwh( } +def _rake_below_ceiling( + values: np.ndarray, + *, + weights: np.ndarray, + population: np.ndarray, + held: np.ndarray, + cells: Sequence[tuple[np.ndarray, Mapping[str, float]]], + sweeps: int, +) -> None: + """Re-rake, in place, the rows not ``held`` to the cell-mean targets. + + Each sweep visits the margins in order; within a cell the held rows keep + their value and the others are scaled so the cell's weighted mean over its + population rows is the target, where the held mass leaves room for it. + """ + + margins = [ + [ + (np.flatnonzero(population & (labels == key)), float(target)) + for key, target in targets.items() + ] + for labels, targets in cells + ] + for _ in range(sweeps): + for margin in margins: + for rows, target in margin: + if rows.size == 0: + continue + fixed = held[rows] + wanted = target * float(weights[rows].sum()) + fixed_mass = float(np.dot(weights[rows][fixed], values[rows][fixed])) + free = rows[~fixed] + current = float(np.dot(weights[free], values[free])) + if current > 0 and wanted > fixed_mass: + values[free] *= (wanted - fixed_mass) / current + + def rake_energy_kwh( table: pd.DataFrame, *, @@ -1007,8 +1063,10 @@ def rake_energy_kwh( and the levelled ``target``). ``ceilings`` (kWh per fuel column, after the level) caps each household - and hands what it loses to the fuel's other households, so the total holds - (``cap_household_kwh``; microcosm#1164). The fit is read after the cap. + and hands what it loses to the households below the ceiling in its own + NEED cells, re-raking them to the levelled targets with the capped ones + held, then restores the published total (``cap_household_kwh``; + microcosm#1164). The fit is read after the cap. """ frame = table.copy() @@ -1161,6 +1219,18 @@ def rake_energy_kwh( weights=weight_values, population=population, ceiling=float(ceilings[column]), + cells=[ + ( + frame[spec.column].to_numpy().astype(str), + { + key: float(cell[column]) * level_factor[column] + for key, cell in spec.targets.items() + if key in fitted[margin] + }, + ) + for margin, spec in zip(fitted, specs, strict=True) + ], + sweeps=int(iterations), ) raked[column] = capped fit = _rake_fit(raked, margins, fitted, connected, weight_values, level_factor) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py index b45b1fd6e..4cbd59001 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_energy_pricing.py @@ -1112,6 +1112,43 @@ def test_cap_household_kwh_keeps_the_total_and_caps_every_household() -> None: assert receipt["feasible"] is True +def test_cap_household_kwh_hands_the_excess_to_the_capped_households_cells() -> None: + # microcosm#1164: with the rake's cells, what a capped household gives up + # goes to the households below the ceiling in its own cells, so the cell + # means stay at their targets where proportional handing-out moves them. + band = np.array(["a", "a", "a", "b", "b", "b"]) + values = np.array([9.0, 2.0, 1.0, 2.0, 2.0, 2.0]) + weights = np.ones(6) + population = np.ones(6, dtype=bool) + cells = [(band, {"a": 4.0, "b": 2.0})] + + proportional, _ = cap_household_kwh( + values, weights=weights, population=population, ceiling=6.0 + ) + raked, receipt = cap_household_kwh( + values, + weights=weights, + population=population, + ceiling=6.0, + cells=cells, + sweeps=20, + ) + + def mean(x, key): + return float(x[band == key].mean()) + + # Proportional handing-out moves both cells off their targets ... + assert mean(proportional, "a") < 4.0 - 0.1 and mean(proportional, "b") > 2.0 + # ... the re-rake keeps both, with the capped household at the ceiling. + assert mean(raked, "a") == pytest.approx(4.0) + assert mean(raked, "b") == pytest.approx(2.0) + assert raked[0] == 6.0 and (raked <= 6.0 + 1e-9).all() + assert float(raked.sum()) == pytest.approx(float(values.sum())) + assert receipt["households_capped"] == 1 + assert receipt["households_above_after"] == 0 + assert receipt["feasible"] is True + + def test_declared_household_ceilings_are_the_domestic_meter_limits() -> None: ceilings = declared_household_kwh_ceilings(_declared()) assert ceilings == {ELECTRICITY_KWH: 50_000.0, GAS_KWH: 73_200.0} From 1dc65d153e02160969c3742b0e41f934d9d29087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 15/28] Hold each recomposed total to the donor's maximum of it (#1164) The recomposition writes housing and transport as their draw less the drawn parts plus the levelled parts. A remainder near the top of the diary's range plus a levelled energy bill larger than the drawn one can then leave the committed support bound the terminal uk_support gate holds the emitted totals to. On the 2024-25 diary, whose housing bound is far tighter than 2023-24's, a few households on the K25 spine did. recompose_from_remainder now declares maximum_total: donor_support: each total is held to the donor's realised maximum of it, the support every drawn column is clipped to, never below its parts, with the excess taken from its remainder. The receipt counts the rows held and the weighted mass taken, and the recomposed_totals gate checks the declaration and includes the held mass in its identities. Co-Authored-By: Claude Opus 5.5 --- ...s-recomposed-totals-donor-maximum.fixed.md | 1 + .../spec_engine/schema/sources.schema.json | 6 ++ .../src/microcosm/build/uk/source_stages.json | 3 +- .../src/microcosm/build/uk/spec/sources.yaml | 3 +- .../build/uk_runtime/lcfs_consumption.py | 29 +++++- .../build/uk_runtime/stage_health.py | 27 ++++-- .../uk/test_uk_lcfs_consumption.py | 90 ++++++++++++++++++- .../engine_free/uk/test_uk_stage_health.py | 24 ++++- 8 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 changelog.d/1164-lcfs-recomposed-totals-donor-maximum.fixed.md diff --git a/changelog.d/1164-lcfs-recomposed-totals-donor-maximum.fixed.md b/changelog.d/1164-lcfs-recomposed-totals-donor-maximum.fixed.md new file mode 100644 index 000000000..7aa0b00fc --- /dev/null +++ b/changelog.d/1164-lcfs-recomposed-totals-donor-maximum.fixed.md @@ -0,0 +1 @@ +The UK LCFS stage holds each recomposed total, housing and transport, to the donor's realised maximum of that total, the support every drawn column is clipped to (microcosm#1164). The recomposition writes a total as its draw less the drawn parts plus the levelled parts, so a remainder near the top of the diary's range plus a levelled energy bill larger than the drawn one could leave the committed support bound the terminal `uk_support` gate holds the emitted totals to; on the 2024-25 diary, whose housing bound is far tighter than 2023-24's, some households did. The `recompose_from_remainder` step declares `maximum_total: donor_support`; a held total never falls below its parts, the excess comes out of its remainder, and the receipt counts the rows held and the weighted mass taken, which the `recomposed_totals` gate's identities include. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 942d6c57f..6c0526d65 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -11569,12 +11569,18 @@ "required": [ "kind", "parents", + "maximum_total", "uncarried" ], "properties": { "kind": { "const": "recompose_from_remainder" }, + "maximum_total": { + "enum": [ + "donor_support" + ] + }, "parents": { "type": "object", "minProperties": 1, diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index c38f50dcf..fb1a60775 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -4131,6 +4131,7 @@ ] } }, + "maximum_total": "donor_support", "uncarried": { "resource": "ons_household_expenditure_facts.json", "period_value": 2024, @@ -4198,7 +4199,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor's realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary's range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 2d25ab8f4..e00c9893c 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -3379,6 +3379,7 @@ stages: - petrol_spending - diesel_spending - bus_fare_spending + maximum_total: donor_support uncarried: resource: ons_household_expenditure_facts.json period_value: 2024 @@ -3434,7 +3435,7 @@ stages: - gas_consumption - has_fuel_consumption - uses_energy_prepayment_meter - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor''s realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary''s range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index 97cd6e511..c3a9d97a7 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -382,6 +382,12 @@ ("petrol_spending", "diesel_spending", "bus_fare_spending"), ), } +#: The declared ``maximum_total`` of the recomposition (microcosm#1164): each +#: total is held to the donor's realised maximum of that total, the support +#: every drawn column is clipped to, so a remainder near the top of the diary's +#: range plus levelled parts larger than the drawn ones cannot leave the range +#: the terminal support gate holds the emitted totals to. +UK_LCFS_RECOMPOSED_MAXIMUM_TOTAL = "donor_support" #: The chain draws a recomposition subtracts, kept before any step re-levels them. UK_LCFS_RECOMPOSED_DRAWN_COLUMNS = tuple( dict.fromkeys( @@ -589,6 +595,7 @@ def __call__(self, frame: Frame) -> Frame: household_draws, drawn=drawn_parts, weights=weights, + donor_support=donor_realized_ranges(donor), uprating_factors=uprating_factors, ) uprating = uprating_operation(self.stage) @@ -1560,6 +1567,11 @@ def check_recomposed_parents(parameters: Mapping[str, Any]) -> None: f"{RECOMPOSE_FROM_REMAINDER_KIND} declares {parent!r} as " f"{declared!r}, not its draw less {subtracted} plus {components}." ) + if parameters.get("maximum_total") != UK_LCFS_RECOMPOSED_MAXIMUM_TOTAL: + raise ValueError( + f"{RECOMPOSE_FROM_REMAINDER_KIND} must declare maximum_total: " + f"{UK_LCFS_RECOMPOSED_MAXIMUM_TOTAL}." + ) def uncarried_spend(parameters: Mapping[str, Any]) -> dict[str, Any]: @@ -1619,6 +1631,7 @@ def recompose_parent_totals( drawn: pd.DataFrame, parameters: Mapping[str, Any], weights: Sequence[float], + donor_support: Mapping[str, tuple[float, float]], uprating_factors: Mapping[str, float] | None = None, ) -> tuple[pd.DataFrame, dict[str, Any]]: """Write each recomposed total around the levelled parts the stage set. @@ -1631,6 +1644,11 @@ def recompose_parent_totals( conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport (10 to 13 percent above the donor in-sample). + + The total is then held to the donor's realised maximum of it + (``donor_support``, the declared ``maximum_total``; microcosm#1164), never + below its parts: what it gives up comes out of the remainder, and the + receipt counts the rows held and the weighted mass taken. """ check_recomposed_parents(parameters) @@ -1639,11 +1657,15 @@ def recompose_parent_totals( result = household_draws.copy() parents: dict[str, Any] = {} for parent, (subtracted, components) in UK_LCFS_RECOMPOSED_PARENTS.items(): + if parent not in donor_support: + raise ValueError(f"{parent}: no donor support to hold the total to.") + maximum = float(donor_support[parent][1]) drawn_total = result[parent].to_numpy(dtype=float) drawn_parts = sum(drawn[column].to_numpy(dtype=float) for column in subtracted) rest = np.maximum(drawn_total - drawn_parts, 0.0) parts = sum(result[column].to_numpy(dtype=float) for column in components) - total = rest + parts + unheld = rest + parts + total = np.maximum(np.minimum(unheld, maximum), parts) result[parent] = total # The parent was uprated by its own factor and each netted part by its # own (a drawn part the uprating never touched by 1.0), so the @@ -1674,6 +1696,9 @@ def recompose_parent_totals( ), "weighted_remainder": float(np.dot(weight, rest)), "weighted_components": float(np.dot(weight, parts)), + "maximum_total": UK_LCFS_RECOMPOSED_MAXIMUM_TOTAL, + "rows_held_at_maximum": int((unheld > total).sum()), + "weighted_held_mass": float(np.dot(weight, unheld - total)), "weighted_total": float(np.dot(weight, total)), "minimum_remainder": float(rest.min()) if rest.size else 0.0, "rows_below_components": int((total < parts).sum()), @@ -1692,6 +1717,7 @@ def lcfs_recompose_from_remainder( *, drawn: pd.DataFrame, weights: Sequence[float], + donor_support: Mapping[str, tuple[float, float]], uprating_factors: Mapping[str, float] | None = None, ) -> tuple[pd.DataFrame, dict[str, Any] | None]: """Apply the declared ``recompose_from_remainder`` step (none if undeclared).""" @@ -1704,6 +1730,7 @@ def lcfs_recompose_from_remainder( drawn=drawn, parameters=parameters, weights=weights, + donor_support=donor_support, uprating_factors=uprating_factors, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py index 931c990fa..65043c78f 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/stage_health.py @@ -2855,12 +2855,14 @@ def _recomposed_totals_gate( Checks on the ``recomposed_totals`` receipt (microcosm#1113): each total the stage declares keeps its own draw less the declared drawn parts and - adds the declared levelled parts back; no remainder is negative and no + adds the declared levelled parts back, held to the declared + ``maximum_total`` (microcosm#1164); no remainder is negative and no household's total is below its components; to ``maximum_relative_deviation``, the weighted remainder equals the drawn total less the drawn parts plus the floored mass, and the weighted total - the remainder plus the components; and the uncarried ONS spend is - recomputed from the vendored rows through the stage's own declaration. + the remainder plus the components less the mass held at the maximum; and + the uncarried ONS spend is recomputed from the vendored rows through the + stage's own declaration. """ from microcosm.build.country_spec import load_country_spec @@ -2915,6 +2917,16 @@ def _recomposed_totals_gate( failures.append( f"{stage}: {below!r} households have {parent} below its components." ) + if entry.get("maximum_total") != declared.get("maximum_total"): + failures.append( + f"{stage}: {parent} is held to {entry.get('maximum_total')!r}, not " + f"the declared maximum_total {declared.get('maximum_total')!r}." + ) + held = entry.get("rows_held_at_maximum") + if isinstance(held, bool) or not isinstance(held, int) or held < 0: + failures.append( + f"{stage}: {parent} records {held!r} rows held at the maximum." + ) def number(key: str, parent: str = parent, entry=entry) -> float: return _finite_number(entry.get(key), label=f"{stage}.{parent}.{key}") @@ -2931,11 +2943,16 @@ def number(key: str, parent: str = parent, entry=entry) -> float: f"drawn parts plus the floored mass ({split})." ) total = number("weighted_total") - parts = remainder + number("weighted_components") + held_mass = number("weighted_held_mass") + if held_mass < 0: + failures.append( + f"{stage}: {parent} records a negative held mass ({held_mass})." + ) + parts = remainder + number("weighted_components") - held_mass if abs(total - parts) > tolerance * max(1.0, abs(total)): failures.append( f"{stage}: {parent} weighted total {total} is not its remainder plus " - f"components {parts}." + f"components less the mass held at the maximum ({parts})." ) uncarried = uncarried_spend(declared) observed = _mapping(receipt.get("uncarried"), label=f"{stage}.uncarried") diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index c131472c4..70f56d7d8 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -1733,6 +1733,13 @@ def test_redraw_is_a_no_op_when_undeclared() -> None: assert result.fit_weight_records == () +#: Donor support wide enough that no recomposed total is held at its maximum. +_WIDE_RECOMPOSE_SUPPORT = { + "housing_water_and_electricity_consumption": (0.0, 1e9), + "transport_consumption": (0.0, 1e9), +} + + def _recompose_declaration() -> dict: from microcosm.build.country_spec import load_country_spec from microcosm.build.uk_runtime.lcfs_consumption import ( @@ -1778,7 +1785,11 @@ def test_recompose_writes_each_total_around_its_levelled_parts() -> None: weights = np.array([2.0, 1.0, 3.0]) out, receipt = recompose_parent_totals( - draws, drawn=drawn, parameters=_recompose_declaration(), weights=weights + draws, + drawn=drawn, + parameters=_recompose_declaration(), + weights=weights, + donor_support=_WIDE_RECOMPOSE_SUPPORT, ) # Housing: 7000 - 2000 + 1600; 600 - 900 floors at 0, + 600; 800 + 0. @@ -1808,6 +1819,7 @@ def test_recompose_writes_each_total_around_its_levelled_parts() -> None: drawn=drawn, parameters=_recompose_declaration(), weights=weights, + donor_support=_WIDE_RECOMPOSE_SUPPORT, uprating_factors={ "housing_water_and_electricity_consumption": 1.02, "transport_consumption": 1.02, @@ -1832,6 +1844,9 @@ def test_recompose_writes_each_total_around_its_levelled_parts() -> None: + housing["weighted_floored_mass"] ) assert housing["rows_below_components"] == 0 + assert housing["maximum_total"] == "donor_support" + assert housing["rows_held_at_maximum"] == 0 + assert housing["weighted_held_mass"] == 0.0 transport = receipt["parents"]["transport_consumption"] assert transport["drawn_subtracts"] == [ "petrol_spending", @@ -1886,6 +1901,7 @@ def test_recompose_writes_each_total_around_its_levelled_parts() -> None: lambda d: d["uncarried"]["classes"][0].update(concept="ons.other"), "expected one", ), + (lambda d: d.pop("maximum_total"), "maximum_total"), ], ) def test_recompose_refuses_a_declaration_it_does_not_apply(change, match) -> None: @@ -1907,7 +1923,75 @@ def test_recompose_refuses_a_declaration_it_does_not_apply(change, match) -> Non ) draws = pd.DataFrame({column: [1.0] for column in columns}) with pytest.raises(ValueError, match=match): - recompose_parent_totals(draws, drawn=draws, parameters=declared, weights=[1.0]) + recompose_parent_totals( + draws, + drawn=draws, + parameters=declared, + weights=[1.0], + donor_support=_WIDE_RECOMPOSE_SUPPORT, + ) + + +def test_recompose_holds_each_total_to_the_donor_maximum() -> None: + """A total is held to the donor's maximum of it, never below its parts (#1164).""" + + from microcosm.build.uk_runtime.lcfs_consumption import recompose_parent_totals + + drawn = pd.DataFrame( + { + "domestic_energy_consumption": [2000.0, 100.0, 0.0], + "petrol_spending": [0.0, 0.0, 0.0], + "diesel_spending": [0.0, 0.0, 0.0], + "bus_fare_spending": [0.0, 0.0, 0.0], + } + ) + draws = pd.DataFrame( + { + # Remainders 5800, 4900 and 3000 before the levelled energy. + "housing_water_and_electricity_consumption": [7800.0, 5000.0, 3000.0], + "electricity_consumption": [2500.0, 700.0, 4000.0], + "gas_consumption": [1500.0, 200.0, 3000.0], + "transport_consumption": [10.0, 10.0, 10.0], + "petrol_spending": [0.0, 0.0, 0.0], + "diesel_spending": [0.0, 0.0, 0.0], + "bus_fare_spending": [0.0, 0.0, 0.0], + } + ) + support = {**_WIDE_RECOMPOSE_SUPPORT} + support["housing_water_and_electricity_consumption"] = (0.0, 6500.0) + + out, receipt = recompose_parent_totals( + draws, + drawn=drawn, + parameters=_recompose_declaration(), + weights=np.array([2.0, 1.0, 3.0]), + donor_support=support, + ) + + # 5800 + 4000 is held at 6500; 4900 + 900 is within it; 3000 + 7000 would + # be held at 6500, below its parts, so it keeps the parts alone. + assert out["housing_water_and_electricity_consumption"].tolist() == [ + 6500.0, + 5800.0, + 7000.0, + ] + housing = receipt["parents"]["housing_water_and_electricity_consumption"] + assert housing["rows_held_at_maximum"] == 2 + assert housing["weighted_held_mass"] == 2.0 * 3300.0 + 3.0 * 3000.0 + assert housing["rows_below_components"] == 0 + assert housing["weighted_total"] == pytest.approx( + housing["weighted_remainder"] + + housing["weighted_components"] + - housing["weighted_held_mass"] + ) + with pytest.raises(ValueError, match="no donor support"): + recompose_parent_totals( + draws, + drawn=drawn, + parameters=_recompose_declaration(), + weights=np.array([2.0, 1.0, 3.0]), + donor_support={"transport_consumption": (0.0, 1e9)}, + ) def test_recompose_is_a_no_op_when_undeclared() -> None: @@ -1930,7 +2014,7 @@ def test_recompose_is_a_no_op_when_undeclared() -> None: draws = pd.DataFrame({"transport_consumption": [1.0]}) out, receipt = lcfs_recompose_from_remainder( - stage, draws, drawn=draws, weights=[1.0] + stage, draws, drawn=draws, weights=[1.0], donor_support={} ) assert out is draws and receipt is None diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py index 06eeca8a4..1e1be891c 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_stage_health.py @@ -1822,7 +1822,14 @@ def test_recomposed_totals_gate_holds_totals_above_their_components() -> None: } ) _, receipt = recompose_parent_totals( - draws, drawn=drawn, parameters=declared, weights=np.array([2.0, 1.0]) + draws, + drawn=drawn, + parameters=declared, + weights=np.array([2.0, 1.0]), + donor_support={ + "housing_water_and_electricity_consumption": (0.0, 6000.0), + "transport_consumption": (0.0, 1e9), + }, ) def run(ev): @@ -1846,6 +1853,21 @@ def failing(mutate, fragment: str) -> None: assert passed.passed, passed.failures assert passed.details["uncarried_total"] == receipt["uncarried"]["total"] housing = "housing_water_and_electricity_consumption" + # 7000 - 2000 + 1600 is held at 6000 (microcosm#1164). + assert receipt["parents"][housing]["rows_held_at_maximum"] == 1 + assert receipt["parents"][housing]["weighted_held_mass"] == 2.0 * 600.0 + failing( + lambda r: r["parents"][housing].update(weighted_held_mass=0.0), + "is not its remainder plus components", + ) + failing( + lambda r: r["parents"][housing].update(maximum_total="none"), + "declared maximum_total", + ) + failing( + lambda r: r["parents"][housing].update(rows_held_at_maximum=-1), + "rows held at the maximum", + ) failing( lambda r: r["parents"][housing].update(minimum_remainder=-1.0), "negative remainder", From 0b40a1377ce02be11d7c380568c5e0c201bc5540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 16/28] Band NEED and NTS recipients on HBAI gross income; rule 3.5 % (#1164) The engine's household_gross_income is market income plus household_benefits, and those add capital gains and the imputed value of NHS, education and transport spending, about 15,000 pounds a household on average on the K25 spine. Banding NEED recipients on it put them in too high an income band (a 7.8 % NEED residual); the NTS bus-use stage had banded on it since its port, which put about a fifth of households a band too high, pensioners and families with school-age children above all. The new uk_runtime.household_income helper builds the survey concept both surveys band on, gross cash income: the engine's HBAI net income with the deductions it takes to reach it added back (council tax and domestic rates, income tax, National Insurance, student loan repayments, employee and personal pension contributions, maintenance and child payments, LVT). Both stages declare those engine variables, and an engine-lane test fails if policyengine-uk's HBAI deduction list changes. On the spine its distribution sits close to the LCFS donor's gross income (weighted medians about 46,000 and 49,000 pounds). On that band the NEED rake converges at 2.40 % electricity and 2.82 % gas in the income margin, 2.88 % gas after the household ceilings: a cross-margin gap, since NEED's income-band means imply about 2.5 % more energy for the frame's households than its region means do. HBAI net income converges at 1.4 % but leaves the top bands so thin that the ceilings then leave those cells 3.2 % short. The energy_rake gate's residual rule moves from 2.5 % to 3.5 % on Maria's ruling of 2026-10-12, quoted in its notes. On the NTS chain the recipients' person-weighted income bands move from 6 / 21 / 73 % to 13 / 30 / 57 %; London bus trips rise 1.6 % and other local bus trips fall 1.0 %. Regenerated: source_stages.json, the coverage manifest, the H2 fixture and the gate digests. Co-Authored-By: Claude Opus 5.5 --- .../1164-need-band-gross-income.fixed.md | 2 +- .../1164-nts-income-band-hbai-gross.fixed.md | 1 + .../src/microcosm/build/uk/gates.json | 4 +- .../src/microcosm/build/uk/source_stages.json | 27 ++++++- .../src/microcosm/build/uk/spec/sources.yaml | 27 ++++++- .../build/uk_runtime/household_income.py | 75 +++++++++++++++++++ .../build/uk_runtime/lcfs_consumption.py | 44 +++++------ .../build/uk_runtime/nts_bus_travel.py | 17 +++-- .../uk/test_uk_hbai_gross_income_engine.py | 24 ++++++ .../engine_free/shared/test_country_spec.py | 3 +- .../uk/test_uk_lcfs_consumption.py | 46 ++++++++---- .../engine_free/uk/test_uk_nts_bus_travel.py | 45 ++++++++++- .../parity/uk_spine/sources/fixture.json | 28 ++++++- 13 files changed, 283 insertions(+), 60 deletions(-) create mode 100644 changelog.d/1164-nts-income-band-hbai-gross.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/uk_runtime/household_income.py create mode 100644 packages/microcosm-build/tests/engine/uk/test_uk_hbai_gross_income_engine.py diff --git a/changelog.d/1164-need-band-gross-income.fixed.md b/changelog.d/1164-need-band-gross-income.fixed.md index af56fda35..ba7beab8f 100644 --- a/changelog.d/1164-need-band-gross-income.fixed.md +++ b/changelog.d/1164-need-band-gross-income.fixed.md @@ -1 +1 @@ -The UK LCFS stage bands recipient households for the NEED energy rake on the engine's `household_gross_income`, the concept the donor and NEED band on (microcosm#1164). The donor bands on LCFS gross income (p344p), but no spine stage writes `household_gross_income`, so the recipient fell back to its HBAI net income and sat in too low a NEED income band: the rake then gave richer households the lower bands' mean electricity and gas. The stage now materializes `household_gross_income` with its income predictors, for the band alone (it is not a QRF predictor). Electricity and gas by income move on every row; the DESNZ level and the NEED margins still hold. +The UK LCFS stage bands recipient households for the NEED energy rake on gross cash income, the concept the donor (LCFS p344p) and NEED band on (microcosm#1164). No spine stage wrote a gross income, so the recipient fell back to its HBAI net income and sat in too low a NEED income band. The engine's `household_gross_income` is not the concept either: it adds capital gains and the imputed value of NHS, education and transport spending, about £15,000 a household on average, which put the recipient in too high a band (a 7.8 % NEED residual on the K25 spine). The recipient's gross income is now its HBAI net income with the deductions policyengine-uk takes to reach it added back (council tax and domestic rates, income tax, National Insurance, student loan repayments, employee and personal pension contributions, maintenance and child payments, LVT), from the new `uk_runtime.household_income` helper; the stage materializes those for the band alone, and an engine-lane test fails if the engine's HBAI deduction list changes. On the K25 spine its distribution sits close to the donor's (weighted medians about £46k and £49k). The rake then converges at 2.40 % electricity and 2.82 % gas in the income margin, a cross-margin gap with NEED's other margins, so the `energy_rake` gate's residual rule moves from 2.5 % to 3.5 % on María's ruling of 2026-10-12. Electricity and gas by income move on every row; the DESNZ level holds. diff --git a/changelog.d/1164-nts-income-band-hbai-gross.fixed.md b/changelog.d/1164-nts-income-band-hbai-gross.fixed.md new file mode 100644 index 000000000..c5004f560 --- /dev/null +++ b/changelog.d/1164-nts-income-band-hbai-gross.fixed.md @@ -0,0 +1 @@ +The UK `nts_bus_travel` stage bands recipient households on gross cash income, the concept of the NTS household income band, rather than the engine's `household_gross_income` (microcosm#1164). The engine's figure adds capital gains and the imputed value of NHS, education and transport spending, which put about a fifth of households (pensioners, families with school-age children and capital-gains households above all) a band higher than their cash income, and the NTS shows bus use falling with income. The recipient's band income is now its HBAI net income with policyengine-uk's HBAI deductions added back, the same `uk_runtime.household_income` helper the LCFS NEED band uses, and the stage declares those engine variables. Bus-use bands, trips and the bus fares and bus support the LCFS and ETB services stages price from them move. diff --git a/packages/microcosm-build/src/microcosm/build/uk/gates.json b/packages/microcosm-build/src/microcosm/build/uk/gates.json index 3876c5131..b2476e2ea 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/gates.json +++ b/packages/microcosm-build/src/microcosm/build/uk/gates.json @@ -328,12 +328,12 @@ "region" ], "margins_period_value": 2024, - "maximum_relative_deviation": 0.025, + "maximum_relative_deviation": 0.035, "maximum_connected_share_deviation": 0.005, "convergence_window_sweeps": 10, "maximum_residual_change_over_window": 0.001 }, - "notes": "Stage-time energy check (microcosm#890 F, chronicle#270). María's ruling (2026-09-15, PR #927): 'I'm not sure why we are comparing against NEED as a gate in the first place if that is not what we are calibrating, but I agree we should implement Vahid's recommended fix and move the fix to before calibration.' Fact checks on the lcfs energy_rake receipt, every published value recomputed at gate time from the vendored rows the stage declares: each cell's shape target is the vendored NEED 2024 mean (need_energy_facts.json, margins_period_value), each fuel's level is the vendored DESNZ Energy Trends domestic consumption at actual temperature over the four quarters of calendar 2024 (desnz_domestic_energy_facts.json) with the level factor equal to that total over the frame's pre-level total and the levelled frame total equal to the published one, each cell's target is shape times factor, and every region with a published gas-connected share (DESNZ subnational gas meters over electricity meters, 2024) sits within 0.5 points of it after the imposition unless the draw could not fill it (a receipted shortfall). The residual check holds the maximum absolute relative deviation of any cell mean from its levelled target, per margin (income band, tenure, property type, region) and fuel, electricity over every household of a cell and gas over its gas-connected households, to one fixed 2.5 % tolerance on the IPF's cross-margin residual (region is swept last and fits to machine precision by construction, so the tolerance binds the earlier three; a declared rule set above the observed maxima, not a fact: spine-s3 on NEED 2023 showed income 1.65 %, tenure 1.29 %, accommodation 0.83 %; spine-s5 on the NEED 2024 shape with the published connection showed income 2.16 % electricity / 1.97 % gas, accommodation 1.46 %, tenure 1.31 %, which is why the 2 % rule of spine-s3 moved to 2.5 %). This gate replaced the two NEED mean-spend anchors that uk_aggregate_admin measured on the calibrated frame. The ONS 04.5.1 electricity and 04.5.2 gas rows then held the calibrated frame until microcosm#1113 made them measured diagnostics (María's ruling of 2026-10-08, D1). Since then nothing release-blocking holds domestic energy after calibration: this gate holds its level at prior weights, and the report-only uk_diagnostics.consumption_drift shows how far the weights move it. The gap between the DESNZ-volume-times-QEP-price level the stage sets and the ONS national-accounts spend is the published-source residual PR #927 receipts Part T records, not a gate. The entry is a population_fact_check: a synthetic smoke build (the staging smoke over the H2 fixture) records its outcome without blocking, because the fixture is not the population NEED, DESNZ and the meter counts describe; every licensed build blocks on it. Tolerance governance (Vahid's third pass on PR #927): maximum_relative_deviation is a rule set above the observed maxima, not a fact; the rake runs a fixed number of sweeps and the receipt now records the cross-margin residual after every sweep (sweep_residuals), so the terminal residual is shown converged, not truncated, and a vintage whose converged residual exceeds the rule is a new ruling for María, never a silent re-pin. The gate checks the convergence rather than asserting it (Vahid's review of microcosm#1012): across the last convergence_window_sweeps (10) sweeps each fuel's residual may range, maximum minus minimum, by at most maximum_residual_change_over_window (0.1 point of relative deviation), so a rake oscillating inside the window fails as a truncated one does; the window presupposes the stage declares more sweeps than the window is long, and the stage refuses an undeclared sweep count. On the licensed spines the 50-sweep receipts of main and of the SPI-first reorder were still falling 0.11 to 0.75 points over their last 10 sweeps, while the 200-sweep receipt of microcosm#1012's final head moves 0.014 points. maximum_connected_share_deviation is declared, not derived from the walk: the greedy weight-fitting walk lands within a fraction of one household weight of the published share, and the check is waived only for a region whose draw cannot fill its published share (the shortfall is receipted and reported here), never for a region above it. The regions the crosswalk maps to no subnational area are declared on the stage (connection_fallback_regions, Northern Ireland) and refused if they differ from the crosswalk." + "notes": "Stage-time energy check (microcosm#890 F, chronicle#270). María's ruling (2026-09-15, PR #927): 'I'm not sure why we are comparing against NEED as a gate in the first place if that is not what we are calibrating, but I agree we should implement Vahid's recommended fix and move the fix to before calibration.' Fact checks on the lcfs energy_rake receipt, every published value recomputed at gate time from the vendored rows the stage declares: each cell's shape target is the vendored NEED 2024 mean (need_energy_facts.json, margins_period_value), each fuel's level is the vendored DESNZ Energy Trends domestic consumption at actual temperature over the four quarters of calendar 2024 (desnz_domestic_energy_facts.json) with the level factor equal to that total over the frame's pre-level total and the levelled frame total equal to the published one, each cell's target is shape times factor, and every region with a published gas-connected share (DESNZ subnational gas meters over electricity meters, 2024) sits within 0.5 points of it after the imposition unless the draw could not fill it (a receipted shortfall). The residual check holds the maximum absolute relative deviation of any cell mean from its levelled target, per margin (income band, tenure, property type, region) and fuel, electricity over every household of a cell and gas over its gas-connected households, to one fixed 3.5 % tolerance on the IPF's cross-margin residual (region is swept last and fits to machine precision by construction, so the tolerance binds the earlier three; a declared rule set above the observed maxima, not a fact: spine-s3 on NEED 2023 showed income 1.65 %, tenure 1.29 %, accommodation 0.83 %; spine-s5 on the NEED 2024 shape with the published connection showed income 2.16 % electricity / 1.97 % gas, accommodation 1.46 %, tenure 1.31 %, which is why the 2 % rule of spine-s3 moved to 2.5 %). María's ruling of 2026-10-12 (microcosm#1164, 'Gross band, rule 3.5%') moved it to 3.5 %: with the recipient banded on HBAI gross income, the gross concept NEED and the LCFS donor band on, and the household kWh ceilings, the 2024-25 diary on the K25 spine converged at income 2.40 % electricity and 2.82 % gas before the ceilings and 2.88 % gas after them (accommodation 1.31 %, tenure 1.03 %, region exact). The gap is a cross-margin one: NEED's income-band means imply about 2.5 % more energy for the frame's households than its region means do, while banding on HBAI net income, which converged at 1.4 %, leaves the top income bands so thin that the rake drives about 2 % of gas kWh above the domestic meter limit and the ceilings then leave those cells 3.2 % short. This gate replaced the two NEED mean-spend anchors that uk_aggregate_admin measured on the calibrated frame. The ONS 04.5.1 electricity and 04.5.2 gas rows then held the calibrated frame until microcosm#1113 made them measured diagnostics (María's ruling of 2026-10-08, D1). Since then nothing release-blocking holds domestic energy after calibration: this gate holds its level at prior weights, and the report-only uk_diagnostics.consumption_drift shows how far the weights move it. The gap between the DESNZ-volume-times-QEP-price level the stage sets and the ONS national-accounts spend is the published-source residual PR #927 receipts Part T records, not a gate. The entry is a population_fact_check: a synthetic smoke build (the staging smoke over the H2 fixture) records its outcome without blocking, because the fixture is not the population NEED, DESNZ and the meter counts describe; every licensed build blocks on it. Tolerance governance (Vahid's third pass on PR #927): maximum_relative_deviation is a rule set above the observed maxima, not a fact; the rake runs a fixed number of sweeps and the receipt now records the cross-margin residual after every sweep (sweep_residuals), so the terminal residual is shown converged, not truncated, and a vintage whose converged residual exceeds the rule is a new ruling for María, never a silent re-pin. The gate checks the convergence rather than asserting it (Vahid's review of microcosm#1012): across the last convergence_window_sweeps (10) sweeps each fuel's residual may range, maximum minus minimum, by at most maximum_residual_change_over_window (0.1 point of relative deviation), so a rake oscillating inside the window fails as a truncated one does; the window presupposes the stage declares more sweeps than the window is long, and the stage refuses an undeclared sweep count. On the licensed spines the 50-sweep receipts of main and of the SPI-first reorder were still falling 0.11 to 0.75 points over their last 10 sweeps, while the 200-sweep receipt of microcosm#1012's final head moves 0.014 points. maximum_connected_share_deviation is declared, not derived from the walk: the greedy weight-fitting walk lands within a fraction of one household weight of the published share, and the check is waived only for a region whose draw cannot fill its published share (the shortfall is receipted and reported here), never for a region above it. The regions the crosswalk maps to no subnational area are declared on the stage (connection_fallback_regions, Northern Ireland) and refused if they differ from the crosswalk." }, { "id": "uk_stage_lcfs_consumption_bus_pricing", diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index fb1a60775..69ca18207 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -3263,7 +3263,17 @@ { "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_gross_income" + "hbai_household_net_income", + "council_tax", + "domestic_rates", + "income_tax", + "national_insurance", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", + "LVT" ], "consumed_only": true }, @@ -3460,7 +3470,7 @@ "local_bus_trips", "household_local_bus_trips" ], - "notes": "Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, María's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed." + "notes": "Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income (on the recipient its HBAI net income with the deductions policyengine-uk takes to reach it added back, gross cash income as the NTS band measures it; the engine's household_gross_income adds capital gains and imputed NHS, education and transport spending, which put a fifth of households a band too high; microcosm#1164) and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, María's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed." }, { "stage": "lcfs_consumption", @@ -3741,7 +3751,16 @@ "self_employment_income", "private_pension_income", "hbai_household_net_income", - "household_gross_income" + "council_tax", + "domestic_rates", + "income_tax", + "national_insurance", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", + "LVT" ] }, { @@ -4199,7 +4218,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor's realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary's range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p, cash income before tax, National Insurance, council tax and pension contributions, and the recipient on the same concept, its HBAI net income with the deductions policyengine-uk takes to reach it added back (council tax and domestic rates, income tax, National Insurance, student loan repayments, employee and personal pension contributions, maintenance and child payments, LVT), which the stage materializes for that band alone (microcosm#1164). Before it the recipient fell back to its HBAI net income, which put households in too low a band; the engine's household_gross_income would put them in too high a one, since it adds capital gains and the in-kind NHS, education and transport spending to cash income. The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor's realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary's range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total." }, { "stage": "etb_vat", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index e00c9893c..818eaf6bf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -2723,7 +2723,17 @@ stages: codebook_status: verified 2026-09-22 against the deposited SN 5340 19th-edition lookup tables and the data extract user guide. Frequency is OrdBus2Freq_B01ID (asked since 2019; OrdBusFreq_B01ID ended in 2018), ten codes folded onto the seven NTS0313 bands (codes 1 to 3 are the three-or-more-times-a-week band, 9 and 10 the less-than-once-a-year-or-never band); code -9 (question not applicable, proxy responses) is declared a non-user because the published NTS0313 shares reproduce only with those persons in the base (2024 at-least-yearly share 0.526 without them, 0.505 with them, the published 0.505); code -8 (no answer) is dropped and receipted. Age_B01ID has 21 bands, each mapped to its lower age and then banded on the declared edges, so donor and recipient share one banding (every NTS band lies inside one declared band). Region codes 10 and 11 (Wales, Scotland) occur only before 2013 and never in the declared years. HHIncome2002_B02ID -8 (no answer) is dropped. The trip weight is W5, which carries the diary household weight W2 (the user guide's trips per person per year is 52.14 x sum(JJXSC x W5) over sum(W2) on the diary sample), so each person's counted trips are divided by W2 before the W2-weighted mean; W5xHH is the within-household factor only. W2 is zero for interview-only households, which leave the donor (the fit and the diary means use the diary sample, as the user guide prescribes for trip measures). Stage and ticket tables, verified 2026-09-23. A stage is a boarding (NumBoardings counts the vehicles), StageMode_B04ID carries the trip-mode categories, StageFareCost is the boarding cost paid at the point of use and IndTicketID links a stage to a held ticket; SpecialTicket_B01ID codes 7 to 11 are the concessionary passes (older people, scholars, disabled persons, subsidised tokens, other concessionary), 1 to 6 non-concessionary season tickets and passes. A boarding with no ticket record and a positive cost paid a single, return or day fare at the point of use; with no record and zero cost it travelled free (children, in the main); with no cost recorded its class is unknown. - kind: materialize_rules_engine_predictors predictors: - - household_gross_income + - hbai_household_net_income + - council_tax + - domestic_rates + - income_tax + - national_insurance + - student_loan_repayments + - employee_pension_contributions + - personal_pension_contributions + - maintenance_expenses + - external_child_payments + - LVT consumed_only: true - kind: impute_bus_use_band output: local_bus_use_band @@ -2865,7 +2875,7 @@ stages: - other_local_bus_trips - local_bus_trips - household_local_bus_trips - notes: Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, María's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed. + notes: Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income (on the recipient its HBAI net income with the deductions policyengine-uk takes to reach it added back, gross cash income as the NTS band measures it; the engine's household_gross_income adds capital gains and imputed NHS, education and transport spending, which put a fifth of households a band too high; microcosm#1164) and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, María's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed. - stage: lcfs_consumption survey: Living Costs and Food Survey 2024-25 source: UK Data Service SN 9603 Living Costs and Food Survey 2024-25 (2nd edition) household/person tabs and the vendored Chronicle facts for NEED 2024 mean kWh, DESNZ Energy Trends and QEP average prices paid for calendar 2024, road-fuel outturn (DESNZ, HMRC, ONS Consumer Trends) and road fuel by vehicle type (DESNZ), ONS Consumer Trends household spending by COICOP division, the DfT VEH1103 licensed-car stock, NTS bus use, and the DfT and devolved bus finance tables and fares indices. @@ -3077,7 +3087,16 @@ stages: - self_employment_income - private_pension_income - hbai_household_net_income - - household_gross_income + - council_tax + - domestic_rates + - income_tax + - national_insurance + - student_loan_repayments + - employee_pension_contributions + - personal_pension_contributions + - maintenance_expenses + - external_child_payments + - LVT - kind: fit_weighted_qrf_chain predictors: - household_adult_count @@ -3435,7 +3454,7 @@ stages: - gas_consumption - has_fuel_consumption - uses_energy_prepayment_meter - notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p and the recipient on the engine''s household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor''s realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary''s range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' + notes: 'Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent''s WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain''s predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (María''s rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain''s raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household''s QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; María''s ruling of 2026-10-12 on microcosm#1207''s extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel''s other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED''s concept: the donor on LCFS p344p, cash income before tax, National Insurance, council tax and pension contributions, and the recipient on the same concept, its HBAI net income with the deductions policyengine-uk takes to reach it added back (council tax and domestic rates, income tax, National Insurance, student loan repayments, employee and personal pension contributions, maintenance and child payments, LVT), which the stage materializes for that band alone (microcosm#1164). Before it the recipient fell back to its HBAI net income, which put households in too low a band; the engine''s household_gross_income would put them in too high a one, since it adds capital gains and the in-kind NHS, education and transport spending to cash income. The disconnection walk''s seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED''s Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person''s annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household''s non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area''s price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England''s trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame''s eligible trip share beside the publisher''s concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine''s CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey''s age-18 split) predict on both sides; the recipient counts its members by age, not through the engine''s is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor''s distribution among households with positive road fuel, on the chain''s predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor''s own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary''s 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain''s draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain''s own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain''s domestic-energy target is the electricity and gas the stage derives from the diary''s fuel, light and power (p537) by its labelled parts, from the LCF User Guide''s derived-variable labels and P537 definition (microcosm#1164): each fuel''s last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary''s account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household''s own electricity share, or the donor''s mean share of dual-fuel households. The diary''s other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor''s realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary''s range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5''s year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area''s fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England''s index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland''s calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (María''s rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers'' imputed rent and 12.6.1 FISIM), on ONS''s domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel''s frame litres with HMRC''s clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.' - stage: etb_vat survey: Effects of Taxes and Benefits 1977-2024 source: UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/household_income.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_income.py new file mode 100644 index 000000000..23dc57440 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/household_income.py @@ -0,0 +1,75 @@ +"""Household gross income on the survey concept, from the engine's HBAI income. + +The surveys whose income bands the UK stages match recipients to (LCFS p344p, +NEED's modelled gross household income, the NTS household income band) band +households on gross cash income: earnings, self-employment, investment and +pension income and cash benefits, before income tax, National Insurance, +council tax and pension contributions. policyengine-uk's +``household_gross_income`` is not that concept: it adds capital gains and the +imputed value of NHS, education and transport spending that its +``household_benefits`` counts (microcosm#1164). The engine's HBAI net income +with the deductions it takes to reach it added back is. +""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.frame import Frame + +#: The deductions policyengine-uk takes from HBAI income to reach +#: ``hbai_household_net_income`` (its ``HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS``; +#: an engine-lane test fails if the two lists part). +UK_HBAI_GROSS_INCOME_DEDUCTIONS = ( + "council_tax", + "domestic_rates", + "income_tax", + "national_insurance", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", + "LVT", +) +#: The engine variables a stage materializes to build the gross income. +UK_HBAI_GROSS_INCOME_VARIABLES = ( + "hbai_household_net_income", + *UK_HBAI_GROSS_INCOME_DEDUCTIONS, +) + + +def household_level( + frame: Frame, engine: object, variable: str, values: Any +) -> np.ndarray: + """One materialized engine variable at household grain, persons summed.""" + + household = frame.table("household") + person = frame.table("person") + declared = str(engine.variable_metadata(variable).entity) + array = np.asarray(values) + if declared == "household": + return array.astype(float) + if declared == "person": + summed = ( + pd.Series(array.astype(float)) + .groupby(person["person_household_id"].to_numpy()) + .sum() + ) + return summed.reindex(household["household_id"]).fillna(0.0).to_numpy() + raise ValueError(f"unsupported entity {declared!r} for {variable}.") + + +def hbai_gross_income(frame: Frame, engine: object, materialized: Any) -> np.ndarray: + """Household gross income: HBAI net income plus the engine's HBAI deductions. + + ``materialized`` holds every ``UK_HBAI_GROSS_INCOME_VARIABLES`` entry. + """ + + return sum( + household_level(frame, engine, variable, materialized[variable]) + for variable in UK_HBAI_GROSS_INCOME_VARIABLES + ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py index c3a9d97a7..8f95b18f4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/lcfs_consumption.py @@ -53,6 +53,11 @@ spend_to_kwh, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_DEDUCTIONS, + hbai_gross_income, + household_level, +) from microcosm.build.uk_runtime.ledger_fact_vendoring import vendored_rows from microcosm.build.uk_runtime.national_frame import ( uk_household_mass_conservation_receipt, @@ -319,15 +324,15 @@ UK_LCFS_HOUSEHOLD_COUNT_ADULT_AGE = 18 UK_LCFS_HOUSEHOLD_COUNT_PREDICTORS = ("household_adult_count", "household_child_count") #: The engine variables the stage materializes on the recipient: the four -#: QRF income predictors and ``household_gross_income``, which only bands the -#: NEED energy rake, on the concept the donor bands on (LCFS p344p, gross; -#: microcosm#1164). Before it the recipient fell back to its HBAI net income. +#: QRF income predictors and the HBAI deductions that, added back to the HBAI +#: net income, give the recipient's gross income. That only bands the NEED +#: energy rake, on the concept the donor bands on (LCFS p344p; microcosm#1164). UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS = ( "employment_income", "self_employment_income", "private_pension_income", "hbai_household_net_income", - "household_gross_income", + *UK_HBAI_GROSS_INCOME_DEDUCTIONS, ) UK_LCFS_CONSUMPTION_PREDICTORS = ( "household_adult_count", @@ -2467,7 +2472,12 @@ def assign_recipient_has_fuel(frame: Frame, *, rate: float, seed: int) -> np.nda def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: - """Materialize LCFS recipient predictors at household grain.""" + """Materialize LCFS recipient predictors at household grain. + + ``household_gross_income``, the NEED band's income, is the HBAI net + income with the engine's HBAI deductions added back (``hbai_gross_income``); + the deductions are not kept. + """ materialized = engine.materialize( frame, UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS, uk_time_period(frame) @@ -2476,21 +2486,11 @@ def recipient_predictors(frame: Frame, engine: object) -> pd.DataFrame: person = frame.table("person") result = pd.DataFrame(index=household.index) for predictor in UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS: - declared = str(engine.variable_metadata(predictor).entity) - values = np.asarray(materialized[predictor]) - if declared == "household": - result[predictor] = values - elif declared == "person": - summed = ( - pd.Series(values.astype(float)) - .groupby(person["person_household_id"].to_numpy()) - .sum() - ) - result[predictor] = ( - summed.reindex(household["household_id"]).fillna(0.0).to_numpy() + if predictor not in UK_HBAI_GROSS_INCOME_DEDUCTIONS: + result[predictor] = household_level( + frame, engine, predictor, materialized[predictor] ) - else: - raise ValueError(f"unsupported LCFS predictor entity {declared!r}.") + result["household_gross_income"] = hbai_gross_income(frame, engine, materialized) age = pd.to_numeric(person["age"], errors="raise").to_numpy(dtype=float) adult = age >= UK_LCFS_HOUSEHOLD_COUNT_ADULT_AGE for predictor, members in zip( @@ -2585,8 +2585,10 @@ def donor_realized_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]] Levelled columns carry no bounds. The housing and transport totals keep theirs although the stage writes them back around levelled components - (microcosm#1113): the terminal support check then holds the emitted totals - to the diary's own range, which a stage-time receipt check cannot do. + (microcosm#1113): the recomposition holds each to the donor's maximum of + it (microcosm#1164), and the terminal support check holds the emitted + totals to the diary's own range, which a stage-time receipt check cannot + do. """ ranges: dict[str, tuple[float, float]] = {} diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py index dd60928fa..c7b0eadaf 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/nts_bus_travel.py @@ -47,6 +47,10 @@ nts_band_shares, ) from microcosm.build.uk_runtime.frs_spine import read_pinned_tab +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_VARIABLES, + hbai_gross_income, +) from microcosm.build.uk_runtime.ledger_fact_vendoring import vendored_rows from microcosm.build.uk_runtime.national_frame import ( uk_household_mass_conservation_receipt, @@ -880,7 +884,12 @@ def nts_support_ranges(donor: pd.DataFrame) -> dict[str, tuple[float, float]]: # Recipient predictors # -------------------------------------------------------------------------- -UK_NTS_ENGINE_PREDICTORS: tuple[str, ...] = ("household_gross_income",) +#: The engine variables the recipient's income band is built from: the HBAI +#: net income and the deductions that, added back, give gross cash income, the +#: concept of the NTS household income band (microcosm#1164; the engine's +#: ``household_gross_income`` adds capital gains and imputed NHS, education and +#: transport spending, which put a fifth of households a band too high). +UK_NTS_ENGINE_PREDICTORS: tuple[str, ...] = UK_HBAI_GROSS_INCOME_VARIABLES def _enum_name(value: object) -> str: @@ -929,11 +938,7 @@ def recipient_predictors( f"recipient regions {unknown} are outside the donor's regions; declare a " "region_remap for them." ) - gross = np.asarray(materialized["household_gross_income"], dtype=float) - if len(gross) != len(household): - raise NTSBusTravelError( - "household_gross_income did not materialize at household grain." - ) + gross = hbai_gross_income(frame, engine, materialized) gross_by_household = ( pd.Series(gross, index=by_household.index).reindex(hh_ids).to_numpy() ) diff --git a/packages/microcosm-build/tests/engine/uk/test_uk_hbai_gross_income_engine.py b/packages/microcosm-build/tests/engine/uk/test_uk_hbai_gross_income_engine.py new file mode 100644 index 000000000..54ac10065 --- /dev/null +++ b/packages/microcosm-build/tests/engine/uk/test_uk_hbai_gross_income_engine.py @@ -0,0 +1,24 @@ +"""The survey-concept gross income follows the engine's HBAI definition. + +The LCFS NEED band and the NTS income band read a household's HBAI net income +with the deductions policyengine-uk takes to reach it added back +(microcosm#1164). If the engine changes that list, the gross income must move +with it, so this test fails until ``UK_HBAI_GROSS_INCOME_DEDUCTIONS`` and the +two stages' declared ``materialize_rules_engine_predictors`` match again. +""" + +from __future__ import annotations + +from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_DEDUCTIONS, +) + + +def test_gross_income_deductions_are_the_engines_hbai_subtracts() -> None: + from policyengine_uk.variables.household.income.hbai_household_net_income import ( + HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS, + ) + + assert sorted(UK_HBAI_GROSS_INCOME_DEDUCTIONS) == sorted( + HBAI_HOUSEHOLD_NET_INCOME_SUBTRACTS + ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py index 33235ec5a..2e16cec13 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_country_spec.py @@ -1602,7 +1602,8 @@ def test_thresholds_match_the_schema4_manifest(self, manifest) -> None: "accommodation", "region", ] - assert energy_rake["maximum_relative_deviation"] == 0.025 + # María's ruling of 2026-10-12 (microcosm#1164). + assert energy_rake["maximum_relative_deviation"] == 0.035 assert energy_rake["convergence_window_sweeps"] == 10 assert energy_rake["maximum_residual_change_over_window"] == 0.001 diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py index 70f56d7d8..353b4af9d 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_lcfs_consumption.py @@ -457,22 +457,29 @@ def licensed(fuel_type: str) -> float: ice_share_from_licensed_cars({**declared, "period_value": 1999}) -def test_recipient_need_band_income_is_the_engines_gross_income() -> None: - # microcosm#1164: the donor bands on gross p344p, so the recipient must not - # fall back to its (lower) HBAI net income. +def test_recipient_need_band_income_is_hbai_gross_income() -> None: + # microcosm#1164: the donor bands on gross p344p, so the recipient bands on + # its HBAI net income with the engine's HBAI deductions added back, neither + # the (lower) net income nor the engine's household_gross_income, which + # counts capital gains and in-kind NHS and education spending. from types import SimpleNamespace + from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_DEDUCTIONS, + ) from microcosm.build.uk_runtime.lcfs_consumption import recipient_predictors class _Engine: def materialize(self, frame, variables, period): - assert "household_gross_income" in variables - return { - name: np.array([30_000.0, 80_000.0]) - if name == "household_gross_income" - else np.array([20_000.0, 50_000.0]) - for name in variables + assert "household_gross_income" not in variables + assert set(UK_HBAI_GROSS_INCOME_DEDUCTIONS) <= set(variables) + values = { + "hbai_household_net_income": np.array([20_000.0, 50_000.0]), + "income_tax": np.array([3_000.0, 20_000.0]), + "council_tax": np.array([1_500.0, 2_500.0]), + "national_insurance": np.array([500.0, 7_500.0]), } + return {name: values.get(name, np.array([0.0, 0.0])) for name in variables} def variable_metadata(self, name): return SimpleNamespace(entity="household") @@ -499,13 +506,17 @@ def variable_metadata(self, name): predictors = recipient_predictors(frame, _Engine()) - assert predictors["household_gross_income"].tolist() == [30_000.0, 80_000.0] + assert predictors["household_gross_income"].tolist() == [25_000.0, 80_000.0] assert predictors["hbai_household_net_income"].tolist() == [20_000.0, 50_000.0] + assert not set(UK_HBAI_GROSS_INCOME_DEDUCTIONS) & set(predictors.columns) def test_recipient_predictors_read_the_vehicle_count_numerically() -> None: from types import SimpleNamespace + from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_DEDUCTIONS, + ) from microcosm.build.uk_runtime.lcfs_consumption import ( UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS, recipient_predictors, @@ -546,7 +557,11 @@ def variable_metadata(self, name): assert predictors["num_vehicles"].tolist() == [0, 3, 5] assert predictors["num_vehicles"].dtype == np.int64 - assert set(UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS) <= set(predictors.columns) + # The HBAI deductions only build the NEED band's gross income. + assert ( + set(UK_LCFS_CONSUMPTION_ENGINE_PREDICTORS) + - set(UK_HBAI_GROSS_INCOME_DEDUCTIONS) + ) | {"household_gross_income"} <= set(predictors.columns) without = uk_national_frame( person=frame.table("person"), benunit=frame.table("benunit"), @@ -940,12 +955,13 @@ def materialize(self, frame, variables, period): "household_gross_income" ].to_numpy() * 0.8, - # The NEED band reads the engine's gross income (microcosm#1164). - "household_gross_income": frame.table("household")[ + # The NEED band adds the HBAI deductions back (microcosm#1164). + "income_tax": frame.table("household")[ "household_gross_income" - ].to_numpy(), + ].to_numpy() + * 0.2, } - return {name: values[name] for name in variables} + return {name: values.get(name, np.zeros(rows)) for name in variables} donor_person, donor_household = _synthetic_lcfs_donor() transform = UKLCFSConsumptionStageTransform( diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py index bc3f6a7e3..02208f9bf 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_nts_bus_travel.py @@ -403,9 +403,10 @@ def variable_metadata(self, name): return SimpleNamespace(entity="household") def materialize(self, frame, variables, period): + # The band reads HBAI net income plus the HBAI deductions (#1164). rows = len(frame.table("household")) - values = {"household_gross_income": self.rng.uniform(5e3, 9e4, rows)} - return {name: values[name] for name in variables} + values = {"hbai_household_net_income": self.rng.uniform(5e3, 9e4, rows)} + return {name: values.get(name, np.zeros(rows)) for name in variables} def test_band_draw_is_identity_keyed_rounded_and_within_the_donor_range() -> None: @@ -439,6 +440,46 @@ def test_band_draw_is_identity_keyed_rounded_and_within_the_donor_range() -> Non assert (second.band == first.band[order]).all() +def test_recipient_income_band_is_hbai_gross_income() -> None: + # microcosm#1164: the NTS band is gross cash income, so the recipient adds + # the engine's HBAI deductions back to its HBAI net income; the engine's + # household_gross_income (capital gains, imputed NHS and education) is not + # asked for. + from microcosm.build.uk_runtime.household_income import ( + UK_HBAI_GROSS_INCOME_DEDUCTIONS, + ) + + frame, _ = _recipient_frame() + rows = len(frame.table("household")) + net = np.full(rows, 20_000.0) + net[: rows // 2] = 40_000.0 + requested: list[str] = [] + + class _GrossEngine(_Engine): + def materialize(self, frame, variables, period): + requested.extend(variables) + values = { + "hbai_household_net_income": net, + "income_tax": np.full(rows, 6_000.0), + } + return {name: values.get(name, np.zeros(rows)) for name in variables} + + band_parameters = _operation(_committed_stage(), IMPUTE_BUS_USE_BAND_KIND) + recipient = recipient_predictors( + _recipient_frame()[0], + _GrossEngine(np.random.default_rng(0)), + columns=_columns(), + income_band_edges=band_parameters["income_band_edges"], + region_remap=band_parameters.get("region_remap", {}), + ) + + assert "household_gross_income" not in requested + assert set(UK_HBAI_GROSS_INCOME_DEDUCTIONS) <= set(requested) + # 20,000 + 6,000 and 40,000 + 6,000 both fall in band 2 (25,000 to + # 50,000); on the net income alone the first would be band 1. + assert set(recipient["income_band"]) == {2} + + def test_band_means_and_trip_assignment_split_the_series_by_residence_group() -> None: household, individual, trip = _synthetic_nts() donor = clean_nts_travel_tables(household, individual, trip, columns=_columns()) diff --git a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json index 6c6650c2a..aa8734262 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json +++ b/packages/microcosm-graph/tests/fixtures/parity/uk_spine/sources/fixture.json @@ -3517,7 +3517,7 @@ "has_fuel_consumption", "uses_energy_prepayment_meter" ], - "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; Mar\u00eda's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p and the recipient on the engine's household_gross_income, which the stage materializes for that band alone (microcosm#1164; before it the recipient fell back to its HBAI net income, which put households in too low a band). The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", + "notes": "Ports the incumbent LCFS consumption QRF, including NEED energy raking, with adjudicated weighted fits and an identity-keyed seed-0 fuel flag. The vehicle count is a numeric QRF predictor on both sides (LCFS a124 on the donor, the was_wealth draw of WAS vcarnr8 on the recipient, both clipped to 0-5), which retires the incumbent's WAS-to-LCFS has-fuel bridge (microcosm#890 A). The recipient fuel flag is num_vehicles > 0 times a draw at the fuel-buyer share from the vendored DfT VEH1103 licensed-car stock (one minus the battery-electric and fuel-cell share, 2024, Great Britain; plug-in hybrids count as fuel buyers), replacing the hand-copied NTS share (microcosm#890 E2). The frame holds no fuel type, so the draw, keyed on the household identity, is what makes a vehicle-owning household a fuel buyer or a zero-emission one: it is random with respect to income and every other characteristic, and a household with several vehicles is all one or the other. With the redraw of microcosm#1113 every flagged household buys fuel, so this flag alone sets fuel incidence; who holds the zero-emission cars is a known residual. The household input uses_energy_prepayment_meter, which policyengine-uk reads for the July 2023 to March 2024 Energy Price Guarantee prepayment discount (PolicyEngine/policyengine-uk#2098, from 2.123.3), is drawn by the declared impute_prepayment_meter step (microcosm#1110): the donor prepays when it last put money on an electricity, gas or combined key or meter (b1751, b1701, b233, which the LCF asks only of prepayment-meter households) or its diary records a slot-meter payment (c45114, c45214), so electricity-only prepayment counts and a household with no meter payment, including one that did not answer, does not; a weighted regime-gated QRF on the chain's predictors fits that flag and each recipient draws it at uniforms keyed on its household id, which moves no other draw. Until Microcosm locks a policyengine-uk that reads it, the release export drops the column, as it drops property_wealth. Energy support clipping is exempt because NEED raking governs those columns. Bus fares are clipped to donor support before the pricing step (Mar\u00eda's rulings 2026-09-15 and 2026-09-17), which bounds the region no area prices (Wales keeps the chain's raw draw) while the priced regions move beyond donor support afterwards; for that reason the column carries no committed support bound for the terminal gate. Domestic energy is raked in kWh to the NEED 2024 shape at the DESNZ level and priced at the average prices paid (microcosm#890 F, chronicle#270), where both NEED rakes take their mean-kWh margins from the vendored NEED 2024 tables (England and Wales by income band, tenure, property type and region; Scotland by income band, tenure and property type with the all-dwellings mean as its region margin; Northern Ireland untouched), the declared price_domestic_energy step converts spend to kWh and back at each household's QEP price region through the committed crosswalk using the calendar-2024 average variable unit cost and fixed cost across payment methods, standard metering, including VAT (Northern Ireland gas at the UK average because QEP publishes no Northern Ireland gas row), a gas fixed cost only for connected households, the gas connection imposed on the recipient frame at the published share of each region (DESNZ subnational gas meters over electricity meters, 2024) by disconnecting gas-positive households in an identity-keyed uniform order, skipping a household whose weight would overshoot the published share (Northern Ireland has no subnational row and keeps the positive-gas rule), and after the four-margin NEED rake one factor per fuel scales the kWh so the prior-weighted frame total equals the DESNZ Energy Trends domestic consumption at actual temperature summed over the four quarters of calendar 2024; NEED gas means are per gas-metered household, so the gas kWh margins are raked over connected households only while electricity is raked over every household, and the energy_rake gate recomputes every NEED mean, the DESNZ level and the connection shares from the vendored rows. After the level, no household is left above the domestic meter definitions behind the DESNZ total (NEED Annex B: gas meters under 73,200 kWh a year, electricity meters under 50,000 kWh on profile classes 1 and 2; Mar\u00eda's ruling of 2026-10-12 on microcosm#1207's extreme energy values): the declared household_kwh_ceiling caps a household there and hands the kWh it loses to the fuel's other households in proportion to their use, so the DESNZ total holds; the receipt counts the households capped and the share of the total they gave up, and the energy_rake gate checks both. Both rakes band households on gross income, NEED's concept: the donor on LCFS p344p, cash income before tax, National Insurance, council tax and pension contributions, and the recipient on the same concept, its HBAI net income with the deductions policyengine-uk takes to reach it added back (council tax and domestic rates, income tax, National Insurance, student loan repayments, employee and personal pension contributions, maintenance and child payments, LVT), which the stage materializes for that band alone (microcosm#1164). Before it the recipient fell back to its HBAI net income, which put households in too low a band; the engine's household_gross_income would put them in too high a one, since it adds capital gains and the in-kind NHS, education and transport spending to cash income. The disconnection walk's seed-0 uniform order makes every group of households (dwelling type, income band, support channel) lose the same expected share of its drawn connected mass, so only the level of the drawn connection moves; the smallest-drawn-gas-first order it replaces piled the disconnections onto low-draw flats once the SPI support households drew gas from their own incomes, and NEED's Scottish property-type means then implied a level 2.6 % above the Scottish all-dwellings mean, a margin conflict the rake could not close. Bus fares are priced from journeys (microcosm#930), where the nts_bus_travel stage imputes each person's annual bus-in-London and other-local-bus trips and their concessionary eligibility, and the declared price_bus_journeys step sets bus_fare_spending as the sum over the household's non-eligible persons of trips times boardings per resident trip (the published BUS01 journeys over the published NTS0705a trips per resident times the ONS population) times the yield per fare-paying boarding (BUS05ai receipts over journeys less concessionary journeys), per area; bus-in-London trips take the London area's price for every resident, other-local-bus trips the price of the residence area (England outside London; Scotland and Northern Ireland from their own receipts and journeys with England's trip rate as the declared proxy); Wales is unpriced and keeps the raw draw. The chain still draws bus_fare_spending so later targets condition on the same raw draw; the emitted column is the priced value. The receipt records every factor, the frame-implied boardings beside the published ones and the frame's eligible trip share beside the publisher's concessionary boarding share, so the prior-weight fit of the bound receipts targets is read as a composition check, never a level the stage set; the #890 incidence draw and fare rake are retired. The 2024-25 diary (UKDS SN 9603, 2nd edition, whose household tab carries the annual weight as cal_weight_annual, the name weighta had; microcosm#1164) is moved to the base year by the declared uprate_donor_columns step (microcosm#890). Consumption columns and bus fares take the engine's CPI index from the April-to-March diary year, placed on the calendar-year index at three quarters of 2024 and one of 2025, back a quarter to calendar 2024 (microcosm#1113; the diary sits a quarter past it, so the factor is below one; the fare factor survives only where no fare cell levels the column, i.e. Wales). Income predictors take a factor of one: the diary year is the FRS 2024-25 year they are matched against. Petrol and diesel take the DESNZ pump-price times HMRC litres per head outturn from the vendored road_fuel_anchors.json, which sets only their donor distribution because the level step follows; the energy columns are exempt because the NEED 2024 shape at the DESNZ level and the QEP average prices paid set their level. The household adult and child counts (LCFS G018 and G019, the survey's age-18 split) predict on both sides; the recipient counts its members by age, not through the engine's is_adult and is_child flags, which policyengine-uk#1896 deprecates (uk-data#486, microcosm#1095). Road fuel incidence and level are set by two declared steps (microcosm#1113). The two-week diary leaves about a third of car-owning donors with no fuel purchase, which the chain reproduces among the households the fuel flag marks as fuel buyers, so the redraw_zero_road_fuel step redraws a flagged household that drew zero from the donor's distribution among households with positive road fuel, on the chain's predictors: a weighted regime-gated QRF draws the total, positive by construction, then the petrol share of it, at quantiles keyed on the household id; positive draws are kept. Then the level_road_fuel step, after the fuel flag has zeroed the households without a fuel car, scales petrol and diesel so their prior-weighted total equals ONS Consumer Trends household spending on COICOP 07.2.2 for calendar 2024 (the year of the DESNZ pump prices the donor uprating uses) less the donor's own other-fuels share of 07.2.2 (c72213), keeping the drawn mix. The diary's 07.2.2 has three lines (c72211 petrol, c72212 diesel and c72213, the other motor fuels and lubricants) and none for vehicle electricity. The LCFS diary records about a quarter less road fuel than ONS, and binding the OBR cars receipts had closed that gap by moving weight onto fuel-buying households; petrol and diesel therefore carry no committed support bound, like the other levelled columns. The totals that contain levelled columns are written back around them by the declared recompose_from_remainder step (microcosm#1113): each keeps its own chain draw less the chain's draws of the parts the stage re-levels, kept before any step touches them and floored at zero where the parts outgrow the total, and adds the levelled parts back, so housing_water_and_electricity_consumption is its draw less the drawn electricity and gas plus the levelled electricity and gas, and transport_consumption its draw less the drawn petrol, diesel and bus fares plus the levelled petrol and diesel and the priced bus fares. The chain draws each part after, and conditional on, its total, so the remainder is the chain's own split; a remainder drawn as a chain target of its own over-drew the vehicle-purchase tail of transport, 10 to 13 percent above the donor in-sample. The chain's domestic-energy target is the electricity and gas the stage derives from the diary's fuel, light and power (p537) by its labelled parts, from the LCF User Guide's derived-variable labels and P537 definition (microcosm#1164): each fuel's last account payment less its rebate (b175 less b178, b170 less b173), its prepayment key or meter payment (b1751, b1701), its component of a combined utilities bill (b227 electricity, b226 gas) and of a combined key or meter payment (b2332, b2331), and the diary's account and slot-meter lines (c45112 and c45114, c45212 and c45214); what a combined payment leaves unsplit (b231, b233 less their parts) goes by the household's own electricity share, or the donor's mean share of dual-fuel households. The diary's other fuels (central-heating oil b017 and c45312, bottled gas b018 and c45222, coal and coke c45411, wood and peat c45412, hot water, steam and ice c45511) are neither, so they stay in the housing remainder, and the donor_energy_split receipt records the split against p537. The derivation it replaces read b226 as electricity by direct debit and b489 and b490 as prepayment payments, where the labels make them the gas component of a combined bill and the electricity and gas account payments less rebates. No column carries the other fuels; the receipt records their published ONS 04.5.3 and 04.5.4 spend. Each total is then held to the donor's realised maximum of that total (maximum_total: donor_support, the support every drawn column is clipped to; microcosm#1164), never below its parts, the excess coming out of its remainder: a remainder near the top of the diary's range plus levelled parts larger than the drawn ones would otherwise leave it. The two totals keep their committed support bounds, which the terminal support gate applies to the emitted frame; the recomposed_totals gate checks the declaration and the receipt, whose rows held at the maximum and weighted mass taken enter its identities. Every household spend column is at calendar-2024 prices as households paid them, the H5's year that policyengine-uk projects from and the calibration uprates to the calendar-2025 window (microcosm#1113): the diary columns by the CPI uprating to 2024, road fuel at ONS 07.2.2 for 2024, energy at DESNZ volume and QEP prices for 2024, and bus fares at each area's fiscal-year yield re-priced to 2024 by its fares index (the DfT local bus fares index by quarter for London and England outside London, Northern Ireland on England's index, a declared proxy that carries the January 2025 rise in the English fare cap Northern Ireland did not have, Scotland's calendar-year index); the consumption_basis gate holds every level-setting step to that year. policyengine-uk applies the fiscal 2024-25 rules to model year 2024 (it samples each parameter at 30 April), so every levelled column sits about a quarter before the midpoint of the policy year it is taxed in; the calibration binds calendar-year facts as published and fiscal-year facts at the calendar window (Mar\u00eda's rule of 2026-09-22). The stage receipt also reports, for each of the twelve COICOP divisions, the prior-weighted frame total against ONS Consumer Trends for the same calendar year less the classes a household diary does not record (02.3 narcotics, 04.2 owner-occupiers' imputed rent and 12.6.1 FISIM), on ONS's domestic concept, and the twelve together against the national concept as well; it is report-only (microcosm#1113). The report-only litres audit compares each fuel's frame litres with HMRC's clearances of that fuel scaled by the cars share of its road use, from the vendored DESNZ sub-national road transport fuel consumption by vehicle type and fuel (PolicyEngine/chronicle#322); the OBR cars share of fuel duty receipts, uniform across fuels, stays beside the total.", "operations": [ { "annualization_weeks": 52.17857142857143, @@ -3692,7 +3692,16 @@ "self_employment_income", "private_pension_income", "hbai_household_net_income", - "household_gross_income" + "council_tax", + "domestic_rates", + "income_tax", + "national_insurance", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", + "LVT" ] }, { @@ -4059,6 +4068,7 @@ }, { "kind": "recompose_from_remainder", + "maximum_total": "donor_support", "parents": { "housing_water_and_electricity_consumption": { "components": [ @@ -4221,7 +4231,7 @@ "local_bus_trips", "household_local_bus_trips" ], - "notes": "Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, Mar\u00eda's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed.", + "notes": "Imputes local-bus travel per person from the National Travel Survey microdata (microcosm#930; the structural fix Maria ruled on 2026-09-15 for the rake-then-bind bus fares of microcosm#890). The donor is the NTS End User Licence extract (England residents only since 2013) pooled over the declared survey years for the conditional fit; the artifact pins are the 19th-edition (November 2025) tabs, verified at runtime, and the H2 fixture drives the stage with synthetic tabs in the same codebook. One QRF target, the interview frequency-of-use band (the variable behind NTS0313; 0 is the non-user band), conditioned on age band, sex, cars (the was_wealth draw on the recipient), household size, the three-band gross household income (on the recipient its HBAI net income with the deductions policyengine-uk takes to reach it added back, gross cash income as the NTS band measures it; the engine's household_gross_income adds capital gains and imputed NHS, education and transport spending, which put a fifth of households a band too high; microcosm#1164) and region, drawn from identity-keyed uniforms and rounded to the nearest band; devolved recipients take a declared proxy region for prediction only. Each person's annual local-bus trips are the diary's weighted mean for their band and residence group (London or the rest of England), split into the bus-in-London and other-local-bus series by main mode, W5 x JJXSC over W2 persons annualised by 52.14 weeks; a seven-day diary annualised per person would be right in aggregate and wrong in distribution, so it supplies band means, not per-person counts. If the extract carries no frequency column the band is drawn from the vendored NTS0313/NTS0621 shares and declared band midpoints are scaled to the vendored NTS0303 rate (the receipt names the path). Concessionary eligibility is the declared statutory age rule of the person's area, emitted for the lcfs pricing step to zero-price eligible persons' trips; the donor's eligible share of trips per series is receipted beside the publisher's concessionary boarding share. The stage and ticket tables class every diary bus boarding by how it was paid for (at the point of use, a season ticket or pass, a concessionary pass, free) and give each person the share of fare-paying boardings paid at the point of use for their band and residence group (local_bus_single_fare_share), the boardings a per-journey fare cap binds on, with the fare paid per boarding by residence group and survey year receipted beside it (microcosm#930 C13, Mar\u00eda's ask of 2026-09-23). The stage records the frame's band shares against NTS0313/NTS0621, its trips per person against NTS0303 and by age band against NTS0601, and the frame's car availability against NTS0205, all from vendored rows; nothing derived from the licensed rows is committed.", "operations": [ { "age_band_edges": [ @@ -4346,7 +4356,17 @@ "consumed_only": true, "kind": "materialize_rules_engine_predictors", "predictors": [ - "household_gross_income" + "hbai_household_net_income", + "council_tax", + "domestic_rates", + "income_tax", + "national_insurance", + "student_loan_repayments", + "employee_pension_contributions", + "personal_pension_contributions", + "maintenance_expenses", + "external_child_payments", + "LVT" ] }, { From 3fedbbb8546f5b3cc04f3690d87b36dd87133f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:19:49 +0100 Subject: [PATCH 17/28] Add the #1164 stage-only measurement scripts and outputs docs/evidence/uk-lcfs-2024-25-1164 holds the stage-only runs of the lcfs_consumption stage, and of nts_bus_travel then lcfs_consumption, on one K25 spine for the base tree (#1145's head, LCFS 2023-24) and this branch (LCFS 2024-25), with every lcfs and NTS stage-health gate run on the receipts, the scripts that ran them, and the sanitizer that copies the outputs in as aggregates only: no exact donor extremes, no remainder minima or donor-floor masses, counts under ten written "<10" with the quantities their households carry dropped, and no household dumps. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/uk-lcfs-2024-25-1164/README.md | 71 ++ .../outputs/nts_lcfs_chain_run.control.json | 62 + .../outputs/nts_lcfs_chain_run.treatment.json | 62 + .../stage_dev_run.control-2023-24-final.json | 1071 ++++++++++++++++ ...stage_dev_run.treatment-2024-25-final.json | 1135 +++++++++++++++++ .../uk-lcfs-2024-25-1164/receipts.json | 33 + .../scripts/nts_lcfs_chain_run.py | 173 +++ .../scripts/run_stage_dev_at.sh | 33 + .../scripts/sanitize_outputs.py | 98 ++ .../scripts/stage_dev_run.py | 306 +++++ 10 files changed, 3044 insertions(+) create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/README.md create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/outputs/nts_lcfs_chain_run.control.json create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/outputs/nts_lcfs_chain_run.treatment.json create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/outputs/stage_dev_run.control-2023-24-final.json create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/outputs/stage_dev_run.treatment-2024-25-final.json create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/receipts.json create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/scripts/nts_lcfs_chain_run.py create mode 100755 docs/evidence/uk-lcfs-2024-25-1164/scripts/run_stage_dev_at.sh create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/scripts/sanitize_outputs.py create mode 100644 docs/evidence/uk-lcfs-2024-25-1164/scripts/stage_dev_run.py diff --git a/docs/evidence/uk-lcfs-2024-25-1164/README.md b/docs/evidence/uk-lcfs-2024-25-1164/README.md new file mode 100644 index 000000000..5482c99d8 --- /dev/null +++ b/docs/evidence/uk-lcfs-2024-25-1164/README.md @@ -0,0 +1,71 @@ +# LCFS 2024-25 and the consumption-stage fixes: measurement scripts and outputs + +Evidence for microcosm#1164: the `lcfs_consumption` stage moved from the Living Costs and Food Survey 2023-24 (UKDS SN 9468) to 2024-25 (SN 9603, 2nd edition), with the fixes that ship with it. `receipts.json` carries the digests of the files here and of the spine the runs measured. + +## What was measured + +- **Stage-only runs** (`scripts/stage_dev_run.py`, `scripts/run_stage_dev_at.sh`). Each tree's own `lcfs_consumption` stage, on the LCFS tabs its own spec pins, re-run on one finished K25 spine (about 64,000 households, built on 2026-10-10 with policyengine-uk 2.122.2, before #1145's `property_components` stage), with every lcfs stage-health gate run on its receipts: + - `stage_dev_run.control-2023-24-final.json`: the base tree, #1145's head (63943a76b), on LCFS 2023-24; + - `stage_dev_run.treatment-2024-25-final.json`: this branch, on LCFS 2024-25. +- **The NTS-then-LCFS chain** (`scripts/nts_lcfs_chain_run.py`). `nts_bus_travel` then `lcfs_consumption` on the same spine, because the LCFS stage prices bus fares from the NTS trips: `nts_lcfs_chain_run.control.json` and `nts_lcfs_chain_run.treatment.json`. + +Both arms run the same spine and policyengine-uk 2.123.0, so a run's comparison with the other arm isolates this branch's stage. The spine already holds the stages after the LCFS stage, so compare the arms, not a run with the spine's stored columns (`before_spine`), which mixes in redraw drift. The R1 remeasure on fresh spines follows the merge of #1198 and #1200. + +## Results + +Every LCFS and NTS stage-health gate passes on both arms, and no household is left outside a committed support bound. Prior-weighted annual totals, £bn: + +| Column | Control (LCFS 2023-24) | This branch (LCFS 2024-25) | +|---|---|---| +| `food_and_non_alcoholic_beverages_consumption` | 109.6 | 113.8 | +| `alcohol_and_tobacco_consumption` | 21.8 | 19.7 | +| `clothing_and_footwear_consumption` | 32.2 | 30.1 | +| `housing_water_and_electricity_consumption` | 171.4 | 179.2 | +| `household_furnishings_consumption` | 94.9 | 80.2 | +| `health_consumption` | 19.5 | 23.5 | +| `transport_consumption` | 185.5 | 187.2 | +| `communication_consumption` | 24.6 | 25.5 | +| `recreation_consumption` | 139.5 | 183.0 | +| `education_consumption` | 4.6 | 20.3 | +| `restaurants_and_hotels_consumption` | 83.1 | 85.4 | +| `miscellaneous_consumption` | 84.3 | 97.4 | +| Twelve divisions | 970.8 | 1,045.2 | +| `electricity_consumption` | 29.5 | 29.6 | +| `gas_consumption` | 19.2 | 19.1 | +| `petrol_spending` | 25.5 | 26.1 | +| `diesel_spending` | 11.4 | 10.7 | +| `bus_fare_spending` | 4.1 | 4.1 | + +- **The diary year.** The 2024-25 diary records more spending than 2023-24: its own weighted twelve-division total rises about 7 % at base-year prices, with health (+58 %), education (+74 %) and recreation (+14 %) leading in current prices. The division capture against ONS Consumer Trends 2024 (domestic concept less the classes a diary does not record) rises from 0.73 to 0.78 overall. Furnishings moves from 1.25 to 1.05; recreation from 0.85 to 1.12, above ONS, and education from 0.11 to 0.48. The chain's draws run above the donor's own weighted totals on several divisions (recreation and education about 1.5 times), the over-draw follow-up. +- **The bridge.** The diary columns now step a quarter back to calendar 2024 (factor below one) where they stepped three quarters forward, and income predictors take a factor of one. The recomposition's uprating residue falls from £1.05bn (housing) and £1.92bn (transport) to −£0.46bn and −£0.24bn. +- **Energy.** The labelled energy split reads the donor's electricity at £30.4bn and gas at £20.9bn a year, with £3.2bn of other fuels (oil, bottled gas, solid fuels) left in the housing remainder; its parts sum to 1.4 % above the diary's fuel, light and power (P537). The old split gave gas to 98 % of donors and electricity to 92-94 %; the labelled one gives gas to about 80 % and electricity to 98 %. The DESNZ level holds electricity and gas totals on both arms. +- **Gas connection.** The frame's gas-connected share falls from 84.6 % to 79.2 %. The old split's near-universal gas let the connection walk disconnect down to every region's published DESNZ share; the labelled split's donors pay for gas less often than DESNZ counts meters, so nine of the eleven Great Britain regions now fall short of their share (receipted, as the gate allows), most in the North East, East Midlands, North West and Yorkshire. A follow-up connects the shortfall. +- **The NEED rake.** Recipients band on HBAI gross income. The rake converges at 2.40 % electricity and 2.82 % gas in the income margin, 2.88 % gas after the household ceilings, against 0.6 % on the control's HBAI-net band; the `energy_rake` gate's rule is 3.5 % (María's ruling of 2026-10-12). Under the household ceilings the gas cap binds on about 0.35 % of gas kWh and electricity's on none. +- **The NTS band.** On HBAI gross income the recipients' person-weighted NTS income bands (under £25k, £25-50k, £50k and over) move from 6 / 21 / 73 % to 13 / 30 / 57 %; London bus trips rise 1.6 % and other local bus trips fall 1.0 %, and bus fares are within 1 %. +- **Prepayment meters.** The donor's weighted prepayment share is 11.0 %; the frame's is 11.5 % (10.5 % in Great Britain, about 3.0 million households). +- **Road fuel.** The incidence redraw leaves no flagged household at zero and the level holds ONS 07.2.2 less the donor's other-fuels share. + +## Decisions behind the energy changes + +- **Which income bands the NEED rake.** The engine's `household_gross_income` adds capital gains and the imputed value of NHS, education and transport spending, about £15,000 a household on average, and missed NEED by 7.8 %. HBAI gross income (the HBAI net income with policyengine-uk's HBAI deductions added back) has a distribution close to the LCFS donor's gross income (weighted medians about £46k and £49k). HBAI net income leaves the two top NEED bands thin (about 6 % of households against about 14 % on HBAI gross), so the rake drives about 2 % of gas kWh above the domestic meter limit and the ceilings then leave those cells 3.2 % short. +- **How the ceilings hand out what they take.** Handed to every household in proportion, the kWh capped households give up moved the cells holding most of them off their NEED means (3.2 % gas on the HBAI-gross band); re-raked within the capped households' own cells, it moves the fit 0.06 points. + +## What is not here + +- **Unit records.** The household dumps the runs write (`*.households.parquet`) stay outside the repository, with the diagnostic runs behind the decisions above. +- **Licensed inputs.** The LCFS and NTS tabs and the spine. +- **Small cells and extremes.** `scripts/sanitize_outputs.py` drops exact donor extremes, remainder minima and the donor-floor negative mass, writes a count of one to nine households "<10" and drops the weighted quantity those households carry, and keeps each NEED margin's largest deviation rather than its cells. + +## Re-running + +The runs need the licensed LCFS 2023-24 and 2024-25 tabs, the NTS 2002-2024 tabs and the spine: + +```bash +scripts/run_stage_dev_at.sh