diff --git a/changelog.d/968.added.md b/changelog.d/968.added.md new file mode 100644 index 000000000..7b576aa13 --- /dev/null +++ b/changelog.d/968.added.md @@ -0,0 +1,3 @@ +A `breakdown=local_areas` run now fills `constituency_withheld_codes` and `local_authority_withheld_codes` on its `LocalAreaBreakdownOutput`, fields the contract and the gateway already accept. Each holds, sorted, the ONS codes of that level's areas that the run estimated (the areas with household weight) but disclosure control did not publish. That includes every estimated area when withholding cannot resolve the disclosure findings, and the list is `[]` when none were withheld. An area the run never estimated, such as one with no households in the dataset, is in neither list. + +Executors deployed for older policyengine.py versions keep leaving both fields out. The beta local-area breakdown check now requires both lists, sorted and without a published area. diff --git a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py index c17f1d599..19eb917c9 100644 --- a/projects/policyengine-apis-integ/tests/simulation/test_calculate.py +++ b/projects/policyengine-apis-integ/tests/simulation/test_calculate.py @@ -250,6 +250,15 @@ def assert_congressional_district_results( _UK_AREA_RESULT_KEYS = ("constituency_impact", "local_authority_impact") +# For each UK area level: its records, their code field and its withheld codes. +_UK_AREA_LEVEL_KEYS = ( + ("constituency_impact", "constituency_code", "constituency_withheld_codes"), + ( + "local_authority_impact", + "local_authority_code", + "local_authority_withheld_codes", + ), +) _PUBLISHED_DISCLOSURE_TIERS = {"reported", "approximate"} @@ -267,7 +276,8 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: Both area levels must publish at least one area, and every published area must carry a disclosure tier and no x or y, which the executor no longer - sends. + sends. Each level must list the codes of the areas disclosure control + withheld, sorted and without a published area. """ breakdown = _as_mapping(breakdown_result, label="local-area breakdown") assert breakdown.get("kind") == "local_areas", ( @@ -278,10 +288,11 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: assert dataset.endswith("_dense"), ( f"Expected the local-area dataset, got {dataset!r}" ) - for key in _UK_AREA_RESULT_KEYS: + for key, code_key, withheld_key in _UK_AREA_LEVEL_KEYS: areas = breakdown.get(key) assert isinstance(areas, list), f"Expected {key} to be a list, got {areas!r}" assert areas, f"Expected {key} to publish at least one area" + published_codes: set[object] = set() for area in areas: record = _as_mapping(area, label=f"{key} record") assert record.get("disclosure_tier") in _PUBLISHED_DISCLOSURE_TIERS, ( @@ -291,6 +302,29 @@ def assert_uk_local_area_breakdown(breakdown_result: object) -> None: assert not map_position, ( f"Expected {key} records without x or y, got {record}" ) + published_codes.add(record.get(code_key)) + assert_withheld_area_codes(breakdown, withheld_key, published_codes) + + +def assert_withheld_area_codes( + breakdown: Mapping, withheld_key: str, published_codes: set[object] +) -> None: + """Check one level's withheld codes are sorted strings and none is published.""" + withheld_codes = breakdown.get(withheld_key) + assert isinstance(withheld_codes, list), ( + f"Expected {withheld_key} to be a list, got {withheld_codes!r}" + ) + assert all(isinstance(code, str) for code in withheld_codes), ( + f"Expected {withheld_key} to hold area codes, got {withheld_codes!r}" + ) + assert withheld_codes == sorted(set(withheld_codes)), ( + f"Expected {withheld_key} sorted and without repeats, got {withheld_codes}" + ) + published_and_withheld = published_codes & set(withheld_codes) + assert not published_and_withheld, ( + f"Expected no published area in {withheld_key}, got " + f"{sorted(map(str, published_and_withheld))}" + ) class _GeneratedClientObject: @@ -363,6 +397,8 @@ def test_result_assertions_accept_generated_client_objects() -> None: "data_version": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", "constituency_impact": [_BREAKDOWN_CONSTITUENCY], "local_authority_impact": [_BREAKDOWN_LOCAL_AUTHORITY], + "constituency_withheld_codes": ["E14000530", "E14001172"], + "local_authority_withheld_codes": [], } @@ -396,6 +432,14 @@ def test_generated_client_area_records_have_no_x_or_y(record_model: type) -> Non }, "disclosure_tier", ), + # The executor under test reports the withheld areas of each level. + ({"constituency_withheld_codes": None}, "constituency_withheld_codes"), + ({"local_authority_withheld_codes": None}, "local_authority_withheld_codes"), + ( + {"constituency_withheld_codes": ["E14001172", "E14000530"]}, + "sorted", + ), + ({"local_authority_withheld_codes": ["E06000063"]}, "published"), ( {"constituency_impact": [{**_BREAKDOWN_CONSTITUENCY, "x": 56, "y": -40}]}, "x or y", @@ -854,7 +898,8 @@ def test_calculate_uk_local_area_breakdown( Given a UK-wide request with breakdown=local_areas When the simulation is submitted and polled to completion Then the result is a local-area breakdown with published constituencies - and local authorities, each with a disclosure tier. + and local authorities, each with a disclosure tier, and each level lists + the codes of the areas disclosure control withheld. """ # Given request = SimulationRequest.from_dict( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index e79339525..79aef2f38 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -318,15 +318,23 @@ def record_fields(self) -> dict[str, object]: class UKGeographyDisclosureOutput: """The published areas of each UK level, keyed by area code. - An area the disclosure check withholds is absent. + An area the disclosure check withholds is absent. ``estimated`` holds the + codes of each level's areas that the release estimated, those with + household weight as policyengine.py reads them; each one that is not + published was withheld. """ areas: Mapping[str, Mapping[str, UKAreaIncomeChange]] = field(default_factory=dict) + estimated: Mapping[str, frozenset[str]] = field(default_factory=dict) def level(self, level: str) -> Mapping[str, UKAreaIncomeChange]: """Return the published areas of one level.""" return self.areas.get(level, {}) + def withheld_codes(self, level: str) -> list[str]: + """Return the codes of one level's estimated areas not published, sorted.""" + return sorted(self.estimated.get(level, frozenset()) - set(self.level(level))) + def _require_uk_area_column( household: pd.DataFrame, column: str, breakdown: str @@ -632,13 +640,15 @@ def compute_uk_geography_disclosure_output( public = _resolved_public_table( impact, area_codes, year=year, inside=_uk_areas_inside(household) ) + estimated = {level: frozenset(codes) for level, codes in area_codes.items()} if public is None: - return UKGeographyDisclosureOutput() + return UKGeographyDisclosureOutput(estimated=estimated) return UKGeographyDisclosureOutput( areas={ level: _published_area_changes(public, level, area_codes[level]) for level in UK_DISCLOSURE_AREA_COLUMNS - } + }, + estimated=estimated, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_local_areas.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_local_areas.py index 9647ce091..deec1f22b 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_local_areas.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_local_areas.py @@ -86,7 +86,8 @@ def build(self) -> LocalAreaBreakdownOutput: No national impact analysis runs. Both simulations are computed first, baseline then reform, so a status poll moves through those stages before the outputs stage, and both levels then come from one - disclosure release over the pair. + disclosure release over the pair. Each level also lists the codes of + the areas the release estimated but did not publish. """ self.baseline.ensure() self.reform.ensure() @@ -120,6 +121,9 @@ def build(self) -> LocalAreaBreakdownOutput: if constituency_impact is None or local_authority_impact is None: msg = "UK area outputs returned no constituency or local-authority list" raise ValueError(msg) + if disclosure is None: + msg = "A UK local-area breakdown needs the disclosure-controlled output" + raise ValueError(msg) return LocalAreaBreakdownOutput( kind="local_areas", model_version=output_model_version( @@ -138,6 +142,10 @@ def build(self) -> LocalAreaBreakdownOutput: local_authority_impact=_area_records( local_authority_impact, LocalAuthorityImpactRecord ), + constituency_withheld_codes=disclosure.withheld_codes("constituency"), + local_authority_withheld_codes=disclosure.withheld_codes( + "local_authority" + ), ) def serialize(self) -> dict[str, Any]: diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index 05b27f12b..c411b755e 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -136,7 +136,9 @@ def test_complete_uk_geography_output_rejects_fallback_metadata( # the check then withholds one more seat so that subtraction from the region # or from its local authority cannot recover it. NEGATIVE has a negative total # baseline income, the Cardiff West case: its relative change is undefined, so -# the seat is left out although its average change passes. +# the seat is left out although its average change passes. UNWEIGHTED, when +# requested, has households of zero weight only, so the run estimates nothing +# for it. NORTH_WEST = "E12000002" CUMBERLAND = "E06000063" WESTMORLAND = "E06000064" @@ -149,10 +151,14 @@ def test_complete_uk_geography_output_rejects_fallback_metadata( } THIN = "E14000006" NEGATIVE = "E14000007" +UNWEIGHTED = "E14000008" +ESTIMATED_SEATS = {*SEATS, THIN, NEGATIVE} LAD23 = UKLocalAuthorityMetadata(boundary_version=UKLocalAuthorityBoundaryVersion.LAD23) -def _uk_disclosure_columns(*, with_source_ids: bool = True) -> dict[str, list]: +def _uk_disclosure_columns( + *, with_source_ids: bool = True, unweighted_seat: bool = False +) -> dict[str, list]: """Return the households' columns, with baseline and reformed incomes.""" rng = np.random.default_rng(7) rows: list[tuple[str, str, float, float]] = [] @@ -164,12 +170,16 @@ def _uk_disclosure_columns(*, with_source_ids: bool = True) -> dict[str, list]: for index in range(30): income = -40_000.0 if index < 20 else 30_000.0 rows.append((NEGATIVE, WESTMORLAND, income, -float(rng.uniform(50, 150)))) + if unweighted_seat: + rows.extend((UNWEIGHTED, WESTMORLAND, 30_000.0, -100.0) for _ in range(30)) columns: dict[str, list[object]] = { "household_id": list(range(len(rows))), "constituency_code_oa": [row[0] for row in rows], "la_code_oa": [row[1] for row in rows], "region_code_oa": [NORTH_WEST] * len(rows), - "household_weight": [float(rng.uniform(50, 150)) for _ in rows], + "household_weight": [ + 0.0 if row[0] == UNWEIGHTED else float(rng.uniform(50, 150)) for row in rows + ], } if with_source_ids: columns["source_household_id"] = list(range(len(rows))) @@ -179,9 +189,11 @@ def _uk_disclosure_columns(*, with_source_ids: bool = True) -> dict[str, list]: def _uk_disclosure_simulations( - *, with_source_ids: bool = True + *, with_source_ids: bool = True, unweighted_seat: bool = False ) -> tuple[SimpleNamespace, SimpleNamespace]: - columns = _uk_disclosure_columns(with_source_ids=with_source_ids) + columns = _uk_disclosure_columns( + with_source_ids=with_source_ids, unweighted_seat=unweighted_seat + ) baseline_income = columns.pop("baseline_income") reformed_income = columns.pop("reformed_income") baseline = weighted_entity_frame( @@ -266,6 +278,73 @@ def test_uk_disclosure_population_is_the_weighted_household_count( assert change.population == pytest.approx(expected[code]) +@pytest.mark.parametrize( + ("level", "estimated"), + [ + ("constituency", ESTIMATED_SEATS), + ("local_authority", {CUMBERLAND, WESTMORLAND}), + ], +) +def test_uk_disclosure_lists_each_estimated_area_it_does_not_publish( + uk_disclosure, level, estimated +) -> None: + withheld = uk_disclosure.withheld_codes(level) + published = set(uk_disclosure.level(level)) + + assert withheld == sorted(withheld) + assert not published & set(withheld) + # Every area the run estimated is either published or withheld. + assert published | set(withheld) == estimated + + +def test_uk_disclosure_lists_thin_undefined_and_complementary_seats( + uk_disclosure, +) -> None: + withheld = uk_disclosure.withheld_codes("constituency") + + assert {THIN, NEGATIVE} <= set(withheld) + # With THIN withheld, at least one more seat is withheld to protect it. + assert set(withheld) & set(SEATS) + + +def test_uk_disclosure_lists_no_area_the_run_never_estimated() -> None: + # Every household of UNWEIGHTED has zero weight, so the release has no + # estimate for it: it is neither published nor withheld. + baseline, reform = _uk_disclosure_simulations(unweighted_seat=True) + + output = simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 + ) + + assert output is not None + assert UNWEIGHTED not in output.level("constituency") + assert UNWEIGHTED not in output.withheld_codes("constituency") + published = set(output.level("constituency")) + assert published | set(output.withheld_codes("constituency")) == ESTIMATED_SEATS + + +def test_uk_disclosure_withholds_every_estimated_area_when_findings_remain( + monkeypatch, +) -> None: + # When withholding cells cannot resolve the findings, no area is published. + monkeypatch.setattr( + simulation_output_geographic, + "_resolved_public_table", + lambda *_args, **_kwargs: None, + ) + baseline, reform = _uk_disclosure_simulations() + + output = simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 + ) + + assert output is not None + assert output.level("constituency") == {} + assert output.level("local_authority") == {} + assert output.withheld_codes("constituency") == sorted(ESTIMATED_SEATS) + assert output.withheld_codes("local_authority") == [CUMBERLAND, WESTMORLAND] + + def test_uk_constituency_output_lists_only_published_seats( uk_disclosure, constituency_lookup ) -> None: diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_uk_local_areas.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_uk_local_areas.py index e4f0c4096..bb3e22de5 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_uk_local_areas.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_uk_local_areas.py @@ -686,13 +686,38 @@ def enter(self, stage): self.events.append(stage) -def _breakdown_builder(monkeypatch, *, events, runtime=None): +# The release estimated three seats and published one, the seat the faked +# constituency output lists; it estimated no local authority. +_PUBLISHED_SEAT = constituency_impact_record().constituency_code +_WITHHELD_SEATS = ["E14000529", "E14000531"] +_SEAT_CHANGE = simulation_output_geographic.UKAreaIncomeChange( + tier="reported", + average=10.0, + average_lower=8.0, + average_upper=12.0, + relative=0.01, + relative_lower=0.008, + relative_upper=0.012, + population=1000.0, +) +_BREAKDOWN_DISCLOSURE = simulation_output_geographic.UKGeographyDisclosureOutput( + areas={"constituency": {_PUBLISHED_SEAT: _SEAT_CHANGE}}, + estimated={ + "constituency": frozenset({_PUBLISHED_SEAT, *_WITHHELD_SEATS}), + "local_authority": frozenset(), + }, +) + + +def _breakdown_builder(monkeypatch, *, events, runtime=None, disclosure=None): """Return a UK breakdown builder whose area outputs are faked. The country module's impact analysis fails the test if it is called. - Each fake records into ``events``. + Each fake records into ``events``. The faked release returns + ``disclosure``, by default ``_BREAKDOWN_DISCLOSURE``. """ - disclosure = simulation_output_geographic.UKGeographyDisclosureOutput() + if disclosure is None: + disclosure = _BREAKDOWN_DISCLOSURE def compute(country, baseline_simulation, reform_simulation, *, year): events.append( @@ -762,9 +787,43 @@ def test_a_local_area_breakdown_publishes_both_levels_from_one_release(monkeypat "data_version": _LOCAL_AREA_REVISION, "constituency_impact": [constituency_impact_record().model_dump(mode="json")], "local_authority_impact": [], + "constituency_withheld_codes": _WITHHELD_SEATS, + # The release withheld no local authority, which is reported as such. + "local_authority_withheld_codes": [], } +def test_a_local_area_breakdown_lists_every_estimated_area_when_none_is_published( + monkeypatch, +): + events: list[object] = [] + disclosure = simulation_output_geographic.UKGeographyDisclosureOutput( + estimated={ + "constituency": frozenset({"E14000531", "E14000529"}), + "local_authority": frozenset({"E06000064", "E06000063"}), + } + ) + builder, _ = _breakdown_builder(monkeypatch, events=events, disclosure=disclosure) + + output = builder.serialize() + + assert output["constituency_withheld_codes"] == ["E14000529", "E14000531"] + assert output["local_authority_withheld_codes"] == ["E06000063", "E06000064"] + + +def test_a_local_area_breakdown_needs_the_disclosure_output(monkeypatch): + # Without the release, the breakdown could not say which areas it withheld. + builder, _ = _breakdown_builder(monkeypatch, events=[]) + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + lambda *_args, **_kwargs: None, + ) + + with pytest.raises(ValueError, match="disclosure-controlled"): + builder.build() + + def test_a_local_area_breakdown_enters_outputs_after_both_simulations(monkeypatch): events: list[object] = [] builder, _ = _breakdown_builder(monkeypatch, events=events)