From afee349d2c14de1b121b0b7e4f03b695d03f418c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 08:01:41 -0400 Subject: [PATCH 01/20] Let UK certification keep a local-area release on an accepted model mismatch certify-data refuses a regional release that neither was built with nor claims the pinned policyengine-uk. The only Microcosm local-area cut is on 2.132.6, a new one takes 2-13 hours plus a two-build sign-off, and area receipts are uncertified anyway (the certificate covers the national dataset only, #685). So each national re-cut on a newer policyengine-uk had to drop area regions or wait for a dense rebuild. --accept-regional-model-mismatch REASON (UK only) keeps the regional release routed and records regional_certification in the bundle: basis accepted_model_mismatch, both model versions, the reason and area_receipts: uncertified. It refuses an empty reason, a regional release that already covers the version, and a run without a regional release; the core must still be covered. The national certificate is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../accept-regional-model-mismatch.added.md | 1 + docs/engineering/skills/data-certification.md | 25 +++ scripts/bundle.py | 13 ++ scripts/certify_data_release.py | 10 ++ src/policyengine/provenance/certification.py | 100 ++++++++++++ src/policyengine/provenance/manifest.py | 18 +++ tests/test_certify_data_release.py | 144 ++++++++++++++++++ 7 files changed, 311 insertions(+) create mode 100644 changelog.d/accept-regional-model-mismatch.added.md diff --git a/changelog.d/accept-regional-model-mismatch.added.md b/changelog.d/accept-regional-model-mismatch.added.md new file mode 100644 index 000000000..a60f85002 --- /dev/null +++ b/changelog.d/accept-regional-model-mismatch.added.md @@ -0,0 +1 @@ +`certify-data --accept-regional-model-mismatch REASON` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 23c5d8133..1ce62d882 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -79,6 +79,31 @@ Microcosm national release with one of: claim first. Its `frs_2024_25` is not carried, so keep it loadable with a `dataset_overlays.uk` entry. +- **An older local-area release, on an accepted mismatch.** When the national + cut moves to a new policyengine-uk before a local-area cut on that version + exists, a reviewer can keep the current local-area release routed instead of + dropping area regions: + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "" \ + --accept-regional-model-mismatch "" \ + --model-version "" + ``` + + Certification then records `regional_certification` in the bundle: + `compatibility_basis: accepted_model_mismatch`, both model versions, the + reason and `area_receipts: uncertified`. Area receipts are uncertified in any + case, because the certificate covers the national dataset only (#685). + The cost is real and goes in the pull request. The local-area file was built + and calibrated under the older model. policyengine-core ignores, with a + warning, the columns the newer model no longer defines, and inputs the newer + model added take their defaults in area runs. The flag is UK only. It refuses + an empty reason, a regional release that already covers the version (so no + certificate records a mismatch that is not there) and a run without a + regional release. The core must still be covered. + Either way, certification checks that the regional release serves the pinned model by its own build or compatibility claims (a `--qualified-pair-id` names the primary release only), checks its core claims as for the primary release diff --git a/scripts/bundle.py b/scripts/bundle.py index 53b74a26a..5530a0a72 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -48,6 +48,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-release-kind", args.regional_release_kind]) if args.drop_regional_release: argv.append("--drop-regional-release") + if args.accept_regional_model_mismatch: + argv.extend( + ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] + ) if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -226,6 +230,15 @@ def _parser() -> argparse.ArgumentParser: "Cannot be combined with --regional-manifest-uri." ), ) + certify.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release on another model " + "version, recording this reviewed reason; forwarded to " + "scripts/certify_data_release.py." + ), + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index f3a720ee4..5c3bf6874 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -115,6 +115,15 @@ def main(argv=None) -> int: "qualification of this pair (requires --core-version)." ), ) + parser.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release that does not cover " + "the pinned model, recording this reviewed reason in the bundle " + "instead of refusing. The core must still be covered." + ), + ) parser.add_argument("--no-generate", action="store_true") parser.add_argument("--no-changelog", action="store_true") parser.add_argument( @@ -149,6 +158,7 @@ def main(argv=None) -> int: drop_regional_release=args.drop_regional_release, core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, + regional_model_mismatch_reason=args.accept_regional_model_mismatch, ) print(result.summary()) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index dc773e215..d42c0ca1c 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -410,6 +410,56 @@ def model_compatibility_basis( return basis, [] if warning is None else [warning] +#: A regional release's model coverage accepted by a reviewer, not by the +#: release's own build or claims (``--accept-regional-model-mismatch``). +REGIONAL_BASIS_ACCEPTED_MISMATCH = "accepted_model_mismatch" + + +def accepted_regional_mismatch( + regional_manifest: DataReleaseManifest, + model_package: str, + model_version: str, + reason: str, +) -> tuple[dict, str]: + """Record a reviewed decision to route regions to a release on another model. + + Area runs on such a release are uncertified twice over: the certificate + covers the national dataset only, and the regional file was built and + calibrated under another model version. The record names both versions + and the reason so the bundle, its TRO and every consumer can say so. + Returns ``(record, warning)``. + """ + + reason = reason.strip() + if not reason: + raise CertificationError( + "--accept-regional-model-mismatch needs a reason a reviewer can read." + ) + built = ( + regional_manifest.build.built_with_model_package + if (regional_manifest.build is not None) + else None + ) + built_with = ( + built.version if built is not None and built.name == model_package else None + ) + record = { + "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, + "certified_for_model_version": model_version, + "built_with_model_version": built_with, + "accepted_reason": reason, + "area_receipts": "uncertified", + } + warning = ( + f"regional release: built with {model_package} " + f"{built_with or 'an unrecorded version'} and routed under " + f"{model_version} on an accepted mismatch ({reason}); area runs are " + "uncertified, inputs the newer model added take their defaults there, " + "and columns it dropped are ignored on load" + ) + return record, warning + + def _check_core_compatibility( manifest: DataReleaseManifest, core_version: str, @@ -1107,6 +1157,7 @@ def build_bundle_data_release_payload( regional_manifest_uri: Optional[str] = None, regional_uri_parts: Optional[dict] = None, regional_release_kind: Optional[RegionalReleaseKind] = None, + regional_certification: Optional[dict] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -1126,6 +1177,8 @@ def build_bundle_data_release_payload( ) if regional_release_kind is not None: payload["regional_release_kind"] = regional_release_kind + if regional_certification is not None: + payload["regional_certification"] = regional_certification payload["default_dataset_uri"] = certified_artifact.get("uri") return payload @@ -1149,6 +1202,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1172,6 +1226,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1198,6 +1253,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token @@ -1208,6 +1264,18 @@ def certify( recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} regional_warnings: list[str] = [] + regional_certification: Optional[dict] = None + mismatch_warning: Optional[str] = None + if regional_model_mismatch_reason is not None and regional_manifest_uri is None: + raise CertificationError( + "--accept-regional-model-mismatch applies to a regional release; " + "pass --regional-manifest-uri." + ) + if regional_model_mismatch_reason is not None and country != "uk": + raise CertificationError( + "--accept-regional-model-mismatch is only supported for the UK, " + "whose area receipts are uncertified." + ) if regional_manifest_uri is not None: regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token @@ -1219,6 +1287,27 @@ def certify( _, regional_warnings = model_compatibility_basis( regional_manifest, model_package, model_version ) + except CertificationError as exc: + if regional_model_mismatch_reason is None: + raise CertificationError( + f"Regional release {regional_manifest_uri}: {exc}" + ) from exc + regional_certification, mismatch_warning = accepted_regional_mismatch( + regional_manifest, + model_package, + model_version, + regional_model_mismatch_reason, + ) + regional_warnings = [] + else: + if regional_model_mismatch_reason is not None: + raise CertificationError( + f"Regional release {regional_manifest_uri} serves " + f"{model_package} {model_version} by its own build or " + "claims; drop --accept-regional-model-mismatch so the " + "certificate does not record a mismatch that is not there." + ) + try: if core_version is not None: _check_core_compatibility( regional_manifest, core_version, None, regional_warnings @@ -1230,6 +1319,8 @@ def certify( regional_warnings = [ f"regional release: {warning}" for warning in regional_warnings ] + if mismatch_warning is not None: + regional_warnings.append(mismatch_warning) primary_artifact_names = set(manifest.artifacts) manifest, recorded_release_kind = merge_regional_release_manifest( country, @@ -1323,6 +1414,7 @@ def certify( regional_manifest_uri=regional_manifest_uri, regional_uri_parts=regional_uri_parts, regional_release_kind=recorded_release_kind, + regional_certification=regional_certification, ), manifest_sha256, warnings, @@ -1368,6 +1460,7 @@ def certify_data_release( drop_regional_release: bool = False, core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> CertificationResult: """Certify a data release into the bundle manifest. @@ -1383,6 +1476,12 @@ def certify_data_release( given, by its own build or claims: a qualified pair names the primary release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. + + ``regional_model_mismatch_reason`` (UK only) records a reviewer's + acceptance of a regional release that does not cover the pinned model, + instead of refusing it. The core must still be covered. The bundle then + carries ``regional_certification`` naming both model versions and the + reason; area receipts are uncertified either way. """ certified_country = _certified_country(country) if drop_regional_release and regional_manifest_uri is not None: @@ -1429,6 +1528,7 @@ def certify_data_release( bundle_core_version=( (bundle.get("packages") or {}).get("policyengine-core") or {} ).get("version"), + regional_model_mismatch_reason=regional_model_mismatch_reason, ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f3b15fb5d..328db86e2 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -202,6 +202,21 @@ class CertifiedDataArtifact(BaseModel): RegionalReleaseKind = Literal["local_area", "area_coded_national"] +class RegionalCertification(BaseModel): + """A reviewer's acceptance of a regional release on another model version. + + Written only by ``certify-data --accept-regional-model-mismatch``: the + regional release neither was built with nor claims the certified model, + and area runs on it are uncertified. + """ + + compatibility_basis: Literal["accepted_model_mismatch"] + certified_for_model_version: str + built_with_model_version: Optional[str] = None + accepted_reason: str + area_receipts: Literal["uncertified"] = "uncertified" + + class CountryReleaseManifest(BaseModel): schema_version: int = 1 bundle_id: Optional[str] = None @@ -218,6 +233,9 @@ class CountryReleaseManifest(BaseModel): ``local_area`` for a non-default local-area release, and ``area_coded_national`` for another release's national dataset that carries area codes (and its area weight matrices).""" + regional_certification: Optional[RegionalCertification] = None + """Present only when the regional release was routed on an accepted + model mismatch rather than its own build or claims.""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 092420f61..a8c8b5b21 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1412,6 +1412,150 @@ def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): regional_release_kind="area_coded_national", ) + def _older_dense(self) -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( + self, tmp_path + ): + with pytest.raises(CertificationError, match="Regional release .*2.126.12"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): + bundle_path = _tmp_bundle(tmp_path) + reason = "area receipts are uncertified; the dense cut lags one release" + + result = _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["regional_release_kind"] == "local_area" + assert release["regional_certification"] == { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": UK_MODEL_VERSION, + "built_with_model_version": "2.120.0", + "accepted_reason": reason, + "area_receipts": "uncertified", + } + # The national certificate itself is untouched by the regional record. + assert release["certification"]["compatibility_basis"] == ( + "built_with_model_package" + ) + assert _sim_api_routes_both_area_types(release) + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + assert manifest.regional_certification.built_with_model_version == "2.120.0" + mismatch = [w for w in result.warnings if "accepted mismatch" in w] + assert len(mismatch) == 1 and "2.120.0" in mismatch[0] + + def test__given_a_covering_regional_release__then_an_accepted_mismatch_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="drop --accept-regional"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="not needed", + ) + + @pytest.mark.parametrize("reason", ["", " "]) + def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): + with pytest.raises(CertificationError, match="needs a reason"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + ) + + def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( + self, tmp_path + ): + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.0"} + ] + with pytest.raises( + CertificationError, match="Regional release .*policyengine-core 3.33.0" + ): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(self._older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="area receipts are uncertified", + core_version="3.33.0", + ) + + def test__given_an_accepted_mismatch_without_a_regional_release__then_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="pass --regional-manifest-uri"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + regional_model_mismatch_reason="area receipts are uncertified", + ) + + def test__given_bundle_wrapper_accepted_mismatch__then_forwards_it( + self, monkeypatch + ): + bundle_script = _load_bundle_script(monkeypatch) + captured: dict[str, list[str]] = {} + fake = ModuleType("certify_data_release") + + def fake_main(argv): + captured["argv"] = argv + return 0 + + fake.main = fake_main + monkeypatch.setitem(sys.modules, "certify_data_release", fake) + reason = "area receipts are uncertified" + assert ( + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + reason, + ] + ) + == 0 + ) + argv = captured["argv"] + assert argv[argv.index("--accept-regional-model-mismatch") + 1] == reason + def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path ): From 0bb58755002a33830f097207e31aa2d526ea78fa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 04:34:09 -0400 Subject: [PATCH 02/20] Add bundle-pinned local data release manifests --- changelog.d/703.added.md | 1 + src/policyengine/provenance/manifest.py | 133 ++++- .../main_485dc2330_certification.py.txt | 167 ++++++ tests/test_local_data_manifest.py | 556 ++++++++++++++++++ 4 files changed, 853 insertions(+), 4 deletions(-) create mode 100644 changelog.d/703.added.md create mode 100644 tests/fixtures/local_manifest/main_485dc2330_certification.py.txt create mode 100644 tests/test_local_data_manifest.py diff --git a/changelog.d/703.added.md b/changelog.d/703.added.md new file mode 100644 index 000000000..9c630be9b --- /dev/null +++ b/changelog.d/703.added.md @@ -0,0 +1 @@ +Support opt-in local data release manifests pinned by the installed bundle TRO, with fail-closed verification and a distinct runtime certification basis for offline Budget-day runners. diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f3b15fb5d..0e9b97d37 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -1,6 +1,7 @@ import hashlib import json import os +import re from functools import lru_cache from importlib.resources import files from pathlib import Path @@ -8,7 +9,7 @@ from urllib.parse import quote import requests -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, PrivateAttr from policyengine.provenance.bundle_validation import ( shadowed_dataset_overlays, @@ -140,6 +141,7 @@ def uri(self) -> str: class DataReleaseManifest(BaseModel): + _source_is_local: bool = PrivateAttr(default=False) schema_version: int data_package: PackageVersion dataset_role: Optional[str] = None @@ -187,7 +189,12 @@ class DataCertification(BaseModel): #: Bases whose runtime model legitimately differs from the data's build-time #: model, so the build fingerprint is not expected to match. CLAIM_BASED_COMPATIBILITY = frozenset( - {"legacy_compatible_model_package", "qualified_exact_pair"} + { + "legacy_compatible_model_package", + "qualified_exact_pair", + "legacy_compatible_model_package_local_manifest", + "qualified_exact_pair_local_manifest", + } ) @@ -367,8 +374,84 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: return manifest -@lru_cache +def bundled_data_release_manifest_sha256(country_id: str) -> str: + """Read the manifest's byte digest from the installed country bundle TRO. + + The artifact and its arrangement must describe the release this bundle + serves. A missing or stale sidecar cannot authorise a local manifest. + """ + country_manifest = get_release_manifest(country_id) + tro_path = files("policyengine").joinpath( + "data", "bundle", f"{country_id}.trace.tro.jsonld" + ) + artifact_id = "composition/1/artifact/data_release_manifest" + try: + tro = json.loads(tro_path.read_bytes()) + artifacts = [ + artifact + for node in tro["@graph"] + for artifact in node.get("trov:hasComposition", {}).get( + "trov:hasArtifact", [] + ) + if artifact.get("@id") == artifact_id + ] + locations = [ + location.get("trov:hasLocation") + for node in tro["@graph"] + for arrangement in node.get("trov:hasArrangement", []) + for location in arrangement.get("trov:hasArtifactLocation", []) + if location.get("trov:hasArtifact", {}).get("@id") == artifact_id + ] + digest = artifacts[0].get("trov:sha256") if len(artifacts) == 1 else None + if ( + not isinstance(digest, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", digest) is None + or locations != [https_release_manifest_uri(country_manifest.data_package)] + ): + raise ValueError("missing, ambiguous, malformed, or stale manifest pin") + except (OSError, ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError( + f"Cannot use a local data release manifest for country '{country_id}': " + f"no valid SHA-256 pin in the installed bundle TRO {tro_path}. " + "Expected SHA-256: unavailable; actual SHA-256: unavailable." + ) from exc + return digest.lower() + + def get_data_release_manifest(country_id: str) -> DataReleaseManifest: + """Load an explicitly configured, bundle-pinned file or the cached fetch. + + Local files are read and verified on every call, including after a remote + manifest was cached. Configure the setting before importing country models. + """ + env_name = f"POLICYENGINE_{country_id.upper()}_DATA_MANIFEST_FILE" + local_file = os.environ.get(env_name) + if local_file is None: + return _fetch_data_release_manifest(country_id) + + expected = bundled_data_release_manifest_sha256(country_id) + try: + source_bytes = Path(local_file).expanduser().read_bytes() + except (OSError, ValueError, RuntimeError) as exc: + raise ValueError( + f"Cannot read {env_name}={local_file!r}: {exc}. " + f"Expected SHA-256: {expected}; actual SHA-256: unavailable." + ) from exc + actual = hashlib.sha256(source_bytes).hexdigest() + if actual != expected: + raise ValueError( + f"Local data release manifest {env_name}={local_file!r} " + f"does not match the installed bundle. Expected SHA-256: {expected}; " + f"actual SHA-256: {actual}." + ) + manifest = DataReleaseManifest.model_validate_json(source_bytes) + manifest.source_sha256 = actual + manifest._source_is_local = True + return _apply_release_revision(manifest, get_release_manifest(country_id)) + + +@lru_cache +def _fetch_data_release_manifest(country_id: str) -> DataReleaseManifest: country_manifest = get_release_manifest(country_id) try: @@ -402,6 +485,13 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: if not isinstance(source_bytes, bytes): source_bytes = response.text.encode("utf-8") data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() + return _apply_release_revision(data_release_manifest, country_manifest) + + +def _apply_release_revision( + data_release_manifest: DataReleaseManifest, + country_manifest: CountryReleaseManifest, +) -> DataReleaseManifest: release_revision = country_manifest.data_package.release_manifest_revision if release_revision is not None: for artifact in data_release_manifest.artifacts.values(): @@ -413,6 +503,11 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: return data_release_manifest +# Preserve the existing cache controls for callers and tests of fetched manifests. +get_data_release_manifest.cache_clear = _fetch_data_release_manifest.cache_clear +get_data_release_manifest.cache_info = _fetch_data_release_manifest.cache_info + + def _specifier_matches(version: str, specifier: str) -> bool: """Match a PEP 440 version specifier (``==``, ``>=``, ``<``, ``,``-joined).""" from packaging.specifiers import InvalidSpecifier, SpecifierSet @@ -433,7 +528,10 @@ def _runtime_core_is_certified( tested with one core, so it holds only for that core. """ - if certification.compatibility_basis != "qualified_exact_pair": + if certification.compatibility_basis not in { + "qualified_exact_pair", + "qualified_exact_pair_local_manifest", + }: return True if runtime_core_version is None: from importlib import metadata @@ -492,6 +590,33 @@ def certify_data_release_compatibility( ) raise exc + certification = _certify_available_data_release( + country_id, + country_manifest, + data_release_manifest, + runtime_model_version, + runtime_data_build_fingerprint, + runtime_core_version, + ) + if data_release_manifest._source_is_local: + return certification.model_copy( + update={ + "compatibility_basis": ( + f"{certification.compatibility_basis}_local_manifest" + ) + } + ) + return certification + + +def _certify_available_data_release( + country_id: str, + country_manifest: CountryReleaseManifest, + data_release_manifest: DataReleaseManifest, + runtime_model_version: str, + runtime_data_build_fingerprint: Optional[str], + runtime_core_version: Optional[str], +) -> DataCertification: # A remote build-model, fingerprint or publisher claim cannot broaden an # exact-pair certificate beyond the model, core and recorded data build # that qualification tested. Check them before any acceptance path returns. diff --git a/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt b/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt new file mode 100644 index 000000000..b439117fb --- /dev/null +++ b/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt @@ -0,0 +1,167 @@ +# Unmodified certification function from policyengine.py main 485dc2330. +def certify_data_release_compatibility( + country_id: str, + runtime_model_version: str, + runtime_data_build_fingerprint: Optional[str] = None, + runtime_core_version: Optional[str] = None, +) -> DataCertification: + country_manifest = get_release_manifest(country_id) + try: + data_release_manifest = get_data_release_manifest(country_id) + except DataReleaseManifestUnavailableError as exc: + bundled_certification = country_manifest.certification + if bundled_certification is not None: + if ( + runtime_data_build_fingerprint is not None + and bundled_certification.data_build_fingerprint is not None + and bundled_certification.compatibility_basis + not in CLAIM_BASED_COMPATIBILITY + and runtime_data_build_fingerprint + != bundled_certification.data_build_fingerprint + ): + raise ValueError( + "Runtime data build fingerprint does not match the bundled " + "data certification." + ) + if ( + bundled_certification.certified_for_model_version + == runtime_model_version + and _runtime_core_is_certified( + bundled_certification, runtime_core_version + ) + ): + return bundled_certification + return DataCertification( + compatibility_basis="unverified_data_release_manifest_unavailable", + certified_for_model_version=runtime_model_version, + data_build_id=bundled_certification.data_build_id, + built_with_model_version=( + bundled_certification.built_with_model_version + ), + built_with_model_git_sha=( + bundled_certification.built_with_model_git_sha + ), + data_build_fingerprint=(bundled_certification.data_build_fingerprint), + certified_by=bundled_certification.certified_by, + ) + raise exc + + # A remote build-model, fingerprint or publisher claim cannot broaden an + # exact-pair certificate beyond the model, core and recorded data build + # that qualification tested. Check them before any acceptance path returns. + bundled_certification = country_manifest.certification + # Named as certification names it: the cut tag when the release has one. + remote_build_id = release_build_id(data_release_manifest.build) + if ( + bundled_certification is not None + and bundled_certification.compatibility_basis == "qualified_exact_pair" + ): + if bundled_certification.certified_for_model_version != runtime_model_version: + raise ValueError( + "Data release manifest is not certified for the runtime model " + f"version {runtime_model_version} in country '{country_id}'; " + "the exact pair requires " + f"{bundled_certification.certified_for_model_version}." + ) + if not _runtime_core_is_certified(bundled_certification, runtime_core_version): + raise ValueError( + "Data release manifest is not certified for the runtime core " + f"version {runtime_core_version} in country '{country_id}'; " + "the exact pair requires " + f"{bundled_certification.certified_for_core_version}." + ) + if ( + bundled_certification.data_build_id is not None + and bundled_certification.data_build_id != remote_build_id + ): + raise ValueError( + "Data release manifest is not certified for the remote data " + f"build {remote_build_id} in country '{country_id}'; the exact " + f"pair requires {bundled_certification.data_build_id}." + ) + built_with_model = ( + data_release_manifest.build.built_with_model_package + if data_release_manifest.build is not None + else None + ) + + if ( + built_with_model is not None + and built_with_model.name != country_manifest.model_package.name + ): + raise ValueError( + "Data release manifest was built with a different model package: " + f"expected {country_manifest.model_package.name}, " + f"got {built_with_model.name}." + ) + + if ( + built_with_model is not None + and built_with_model.version == runtime_model_version + ): + return DataCertification( + compatibility_basis="exact_build_model_version", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=built_with_model.version, + built_with_model_git_sha=built_with_model.git_sha, + data_build_fingerprint=built_with_model.data_build_fingerprint, + ) + + if ( + built_with_model is not None + and built_with_model.data_build_fingerprint is not None + and runtime_data_build_fingerprint is not None + and built_with_model.data_build_fingerprint == runtime_data_build_fingerprint + ): + return DataCertification( + compatibility_basis="matching_data_build_fingerprint", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=built_with_model.version, + built_with_model_git_sha=built_with_model.git_sha, + data_build_fingerprint=built_with_model.data_build_fingerprint, + ) + + for compatible_model_package in data_release_manifest.compatible_model_packages: + if compatible_model_package.name != country_manifest.model_package.name: + continue + if _specifier_matches( + version=runtime_model_version, + specifier=compatible_model_package.specifier, + ): + return DataCertification( + compatibility_basis="legacy_compatible_model_package", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=( + built_with_model.version if built_with_model is not None else None + ), + built_with_model_git_sha=( + built_with_model.git_sha if built_with_model is not None else None + ), + data_build_fingerprint=( + built_with_model.data_build_fingerprint + if built_with_model is not None + else None + ), + ) + + # An automated qualification of exactly this model and data build: the + # bundled certificate names the pair, and the ledger evidence for it is + # enforced when the bundle is built (tests/test_certification_evidence.py). + bundled_certification = country_manifest.certification + if ( + bundled_certification is not None + and bundled_certification.compatibility_basis == "qualified_exact_pair" + and bundled_certification.certified_for_model_version == runtime_model_version + and bundled_certification.qualification_pair_id is not None + and bundled_certification.data_build_id == remote_build_id + and _runtime_core_is_certified(bundled_certification, runtime_core_version) + ): + return bundled_certification + + raise ValueError( + "Data release manifest is not certified for the runtime model version " + f"{runtime_model_version} in country '{country_id}'." + ) diff --git a/tests/test_local_data_manifest.py b/tests/test_local_data_manifest.py new file mode 100644 index 000000000..112599dac --- /dev/null +++ b/tests/test_local_data_manifest.py @@ -0,0 +1,556 @@ +"""The local manifest is an opt-in transport for the bundle's exact bytes. + +The unset cases characterize certification on main (485dc2330), including +fetched, bundled-offline, and refused pairs. No credentials or network are used. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +import requests +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine.provenance import manifest as manifests + +ENVIRONMENT_VARIABLE = "POLICYENGINE_UK_DATA_MANIFEST_FILE" +MODEL = "2.130.1" +OTHER_MODEL = "2.130.2" +CORE = "3.33.0" +BUILD = "test-data-build" +FINGERPRINT = "sha256:test-build" +_DEFAULT = object() + + +def _payload(): + return { + "schema_version": 1, + "data_package": {"name": "policyengine-uk-data", "version": "test-release"}, + "build": { + "build_id": BUILD, + "built_with_core_package": {"name": "policyengine-core", "version": CORE}, + "built_with_model_package": { + "name": "policyengine-uk", + "version": MODEL, + "git_sha": "abc123", + "data_build_fingerprint": FINGERPRINT, + }, + }, + "default_datasets": {"national": "test_dataset"}, + "artifacts": { + "test_dataset": { + "kind": "microdata", + "path": "test_dataset.h5", + "repo_id": "policyengine/test-data", + "revision": "test-release", + "sha256": "d" * 64, + } + }, + } + + +def _manifest_bytes(payload): + # Whitespace and the final newline are deliberately part of the pin. + return (json.dumps(payload, indent=2) + "\n").encode("utf-8") + + +def _qualified_certification(): + return { + "compatibility_basis": "qualified_exact_pair", + "certified_for_model_version": MODEL, + "certified_for_core_version": CORE, + "data_build_id": BUILD, + "built_with_model_version": MODEL, + "built_with_model_git_sha": "abc123", + "data_build_fingerprint": FINGERPRINT, + "qualification_pair_id": "a" * 64, + "certified_by": "test-bundle", + } + + +def _install_bundle(tmp_path, monkeypatch, payload=None, certification=_DEFAULT): + """Install synthetic package resources, exercising the real TRO pin reader.""" + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + payload = payload if payload is not None else _payload() + raw_bytes = _manifest_bytes(payload) + local_path = tmp_path / "release_manifest.json" + local_path.write_bytes(raw_bytes) + resource_root = tmp_path / "package" + bundle_dir = resource_root / "data" / "bundle" + bundle_dir.mkdir(parents=True, exist_ok=True) + country = { + "schema_version": 1, + "country_id": "uk", + "policyengine_version": "6.10.1", + "model_package": {"name": "policyengine-uk", "version": MODEL}, + "data_package": { + "name": "policyengine-uk-data", + "version": "test-release", + "repo_id": "policyengine/test-data", + "release_manifest_revision": "f" * 40, + }, + "default_dataset": "test_dataset", + "certification": ( + _qualified_certification() if certification is _DEFAULT else certification + ), + } + (bundle_dir / "manifest.json").write_text( + json.dumps({"data_releases": {"uk": country}}) + ) + tro = { + "@graph": [ + { + "@id": "tro", + "trov:hasArrangement": [ + { + "trov:hasArtifactLocation": [ + { + "trov:hasArtifact": { + "@id": "composition/1/artifact/data_release_manifest" + }, + "trov:hasLocation": ( + "https://huggingface.co/policyengine/test-data/resolve/" + f"{'f' * 40}/release_manifest.json" + ), + } + ] + } + ], + "trov:hasComposition": { + "@id": "composition/1", + "trov:hasArtifact": [ + { + "@id": "composition/1/artifact/data_release_manifest", + "trov:sha256": hashlib.sha256(raw_bytes).hexdigest(), + } + ], + }, + } + ] + } + trace_path = bundle_dir / "uk.trace.tro.jsonld" + trace_path.write_text(json.dumps(tro)) + monkeypatch.setattr(manifests, "files", lambda package: resource_root) + return local_path, trace_path, raw_bytes + + +@pytest.fixture(autouse=True) +def _isolated_manifest_settings(monkeypatch): + monkeypatch.delenv(ENVIRONMENT_VARIABLE, raising=False) + monkeypatch.delenv("POLICYENGINE_US_DATA_MANIFEST_FILE", raising=False) + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + yield + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + + +def _certify(**kwargs): + implementation = kwargs.pop( + "_implementation", manifests.certify_data_release_compatibility + ) + return implementation( + "uk", + runtime_model_version=kwargs.pop("runtime_model_version", MODEL), + runtime_core_version=kwargs.pop("runtime_core_version", CORE), + **kwargs, + ) + + +def test_matching_local_manifest_certifies_without_a_network_call( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object( + manifests.requests, "get", side_effect=requests.ConnectionError("HF is down") + ) as network: + certification = _certify() + loaded = manifests.get_data_release_manifest("uk") + + network.assert_not_called() + assert ( + certification.compatibility_basis == "exact_build_model_version_local_manifest" + ) + assert certification.certified_for_model_version == MODEL + assert certification.data_build_id == BUILD + assert certification.data_build_fingerprint == FINGERPRINT + assert loaded.source_sha256 == hashlib.sha256(raw_bytes).hexdigest() + assert loaded.artifacts["test_dataset"].revision == "f" * 40 + # Source information stays out of public manifest JSON round trips. + expected = manifests.DataReleaseManifest.model_validate_json(raw_bytes) + expected.artifacts["test_dataset"].revision = "f" * 40 + assert loaded.model_dump(mode="json") == expected.model_dump(mode="json") + + +@settings( + max_examples=80, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + position=st.integers(min_value=0, max_value=len(_manifest_bytes(_payload())) - 1) +) +def test_any_one_byte_mutation_is_refused(tmp_path, monkeypatch, position): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + altered = bytearray(raw_bytes) + altered[position] ^= 1 + local_path.write_bytes(altered) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError) as error: + _certify() + + network.assert_not_called() + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert hashlib.sha256(altered).hexdigest() in str(error.value) + + +def test_another_release_is_refused_even_with_an_environment_digest( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + another_release = _payload() + another_release["build"]["build_id"] = "another-release" + altered_bytes = _manifest_bytes(another_release) + local_path.write_bytes(altered_bytes) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + monkeypatch.setenv( + "POLICYENGINE_UK_DATA_MANIFEST_SHA256", + hashlib.sha256(altered_bytes).hexdigest(), + ) + + with pytest.raises(ValueError) as error: + _certify() + + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert hashlib.sha256(altered_bytes).hexdigest() in str(error.value) + + +@pytest.mark.parametrize("failure", ["missing", "unreadable", "directory", "empty"]) +def test_bad_local_path_is_a_hard_error(tmp_path, monkeypatch, failure): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + setting = str(local_path) + if failure == "missing": + local_path.unlink() + elif failure == "unreadable": + read_bytes = Path.read_bytes + + def read_unless_local(path): + if path == local_path: + raise PermissionError("test permission denied") + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_unless_local) + elif failure == "directory": + setting = str(tmp_path) + else: + setting = "" + monkeypatch.setenv(ENVIRONMENT_VARIABLE, setting) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError) as error: + _certify() + + network.assert_not_called() + assert ENVIRONMENT_VARIABLE in str(error.value) + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert "actual" in str(error.value).lower() + + +@pytest.mark.parametrize( + "failure", + [ + "absent-trace", + "absent-pin", + "invalid-pin", + "other-artifact", + "absent-location", + "stale-release-location", + "duplicate-pin", + "duplicate-location", + "malformed-graph", + "malformed-composition", + ], +) +def test_local_manifest_requires_the_bundled_manifest_artifact_pin( + tmp_path, monkeypatch, failure +): + local_path, trace_path, _ = _install_bundle(tmp_path, monkeypatch) + if failure == "absent-trace": + trace_path.unlink() + else: + tro = json.loads(trace_path.read_text()) + artifact = tro["@graph"][0]["trov:hasComposition"]["trov:hasArtifact"][0] + if failure == "absent-pin": + del artifact["trov:sha256"] + elif failure == "invalid-pin": + artifact["trov:sha256"] = "not-a-sha256" + elif failure == "other-artifact": + artifact["@id"] = "composition/1/artifact/dataset" + elif failure == "absent-location": + del tro["@graph"][0]["trov:hasArrangement"] + elif failure == "stale-release-location": + location = tro["@graph"][0]["trov:hasArrangement"][0][ + "trov:hasArtifactLocation" + ][0] + location["trov:hasLocation"] = ( + "https://huggingface.co/policyengine/test-data/resolve/" + "old-release/release_manifest.json" + ) + elif failure == "duplicate-pin": + tro["@graph"][0]["trov:hasComposition"]["trov:hasArtifact"].append( + copy.deepcopy(artifact) + ) + elif failure == "duplicate-location": + locations = tro["@graph"][0]["trov:hasArrangement"][0][ + "trov:hasArtifactLocation" + ] + locations.append(copy.deepcopy(locations[0])) + elif failure == "malformed-graph": + tro["@graph"] = None + else: + tro["@graph"][0]["trov:hasComposition"] = [] + trace_path.write_text(json.dumps(tro)) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError, match="(?i)(bundle|pin|digest|sha256)"): + _certify() + + network.assert_not_called() + + +@pytest.mark.parametrize("difference", ["model", "core", "build", "model-package"]) +def test_a_valid_digest_cannot_widen_exact_pair_certification( + tmp_path, monkeypatch, difference +): + payload = _payload() + runtime = {} + if difference == "model": + runtime["runtime_model_version"] = OTHER_MODEL + elif difference == "core": + runtime["runtime_core_version"] = "99.0.0" + elif difference == "build": + payload["build"]["build_id"] = "another-build" + else: + payload["build"]["built_with_model_package"]["name"] = "another-model" + local_path, _, _ = _install_bundle(tmp_path, monkeypatch, payload) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with pytest.raises(ValueError, match="not certified|different model package"): + _certify(**runtime) + + +@pytest.mark.parametrize( + ("mode", "expected_basis"), + [ + ("fingerprint", "matching_data_build_fingerprint_local_manifest"), + ("publisher-claim", "legacy_compatible_model_package_local_manifest"), + ("qualified", "qualified_exact_pair_local_manifest"), + ], +) +def test_local_transport_retains_existing_certification_bases( + tmp_path, monkeypatch, mode, expected_basis +): + payload = _payload() + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + runtime = {} + if mode == "fingerprint": + runtime["runtime_data_build_fingerprint"] = FINGERPRINT + elif mode == "publisher-claim": + payload["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f"=={MODEL}"} + ] + local_path, _, _ = _install_bundle(tmp_path, monkeypatch, payload) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + assert _certify(**runtime).compatibility_basis == expected_basis + + +def test_local_fingerprint_mismatch_does_not_create_a_new_acceptance_path( + tmp_path, monkeypatch +): + payload = _payload() + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + local_path, _, _ = _install_bundle( + tmp_path, monkeypatch, payload, certification=None + ) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with pytest.raises(ValueError, match="not certified"): + _certify(runtime_data_build_fingerprint="sha256:different") + + +def _response(raw_bytes): + response = Mock() + response.status_code = 200 + response.content = raw_bytes + response.text = raw_bytes.decode("utf-8") + return response + + +def test_setting_local_manifest_after_a_cached_fetch_still_checks_its_pin( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + assert _certify().compatibility_basis == "exact_build_model_version" + local_path.write_bytes(raw_bytes + b" ") + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError): + _certify() + + network.assert_not_called() + + +@pytest.mark.parametrize("change", ["mutation", "deletion"]) +def test_a_successful_local_load_does_not_cache_unchecked_bytes( + tmp_path, monkeypatch, change +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get") as network: + assert ( + _certify().compatibility_basis == "exact_build_model_version_local_manifest" + ) + if change == "mutation": + local_path.write_bytes(raw_bytes + b" ") + else: + local_path.unlink() + with pytest.raises(ValueError): + _certify() + network.assert_not_called() + + +@pytest.fixture(scope="module") +def main_certification(): + """Keep the exact main implementation as a differential test oracle.""" + snapshot = ( + Path(__file__).parent + / "fixtures" + / "local_manifest" + / "main_485dc2330_certification.py.txt" + ) + namespace = vars(manifests).copy() + exec(compile(snapshot.read_text(), str(snapshot), "exec"), namespace) + return namespace["certify_data_release_compatibility"] + + +@pytest.mark.parametrize( + "case", + [ + "fetched-exact", + "fetched-fingerprint", + "fetched-claim", + "fetched-qualified", + "fetched-model-mismatch", + "fetched-core-mismatch", + "fetched-build-mismatch", + "offline-bundled", + "offline-model-mismatch", + "offline-fingerprint-mismatch", + ], +) +def test_unset_setting_matches_main_certification_cases( + tmp_path, monkeypatch, main_certification, case +): + """Golden differential of existing release/qualified-pair test scenarios. + + Compare complete results or exact exception types/messages with an unchanged + main 485dc2330 implementation, then check the characterized expected result. + Synthetic resources isolate the comparison from future country release pins. + """ + payload = _payload() + bundled = _qualified_certification() + runtime = {} + expected_error = None + expected = manifests.DataCertification( + compatibility_basis="exact_build_model_version", + certified_for_model_version=MODEL, + data_build_id=BUILD, + built_with_model_version=MODEL, + built_with_model_git_sha="abc123", + data_build_fingerprint=FINGERPRINT, + ) + if case in {"fetched-fingerprint", "fetched-claim", "fetched-qualified"}: + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + expected.built_with_model_version = OTHER_MODEL + if case == "fetched-fingerprint": + runtime["runtime_data_build_fingerprint"] = FINGERPRINT + expected.compatibility_basis = "matching_data_build_fingerprint" + elif case == "fetched-claim": + payload["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f">={MODEL},<3.0.0"} + ] + expected.compatibility_basis = "legacy_compatible_model_package" + elif case in {"fetched-qualified", "offline-bundled"}: + expected = manifests.DataCertification.model_validate(bundled) + elif case == "fetched-model-mismatch": + runtime["runtime_model_version"] = OTHER_MODEL + expected_error = "Data release manifest is not certified for the runtime model" + elif case == "fetched-core-mismatch": + runtime["runtime_core_version"] = "99.0.0" + expected_error = "Data release manifest is not certified for the runtime core" + elif case == "fetched-build-mismatch": + payload["build"]["build_id"] = "another-build" + expected_error = ( + "Data release manifest is not certified for the remote data build" + ) + elif case == "offline-model-mismatch": + runtime["runtime_model_version"] = OTHER_MODEL + expected = manifests.DataCertification( + compatibility_basis="unverified_data_release_manifest_unavailable", + certified_for_model_version=OTHER_MODEL, + data_build_id=BUILD, + built_with_model_version=MODEL, + built_with_model_git_sha="abc123", + data_build_fingerprint=FINGERPRINT, + certified_by="test-bundle", + ) + elif case == "offline-fingerprint-mismatch": + bundled["compatibility_basis"] = "matching_data_build_fingerprint" + runtime["runtime_data_build_fingerprint"] = "sha256:different" + expected_error = "Runtime data build fingerprint does not match the bundled" + _, _, raw_bytes = _install_bundle( + tmp_path, monkeypatch, payload, certification=copy.deepcopy(bundled) + ) + assert ENVIRONMENT_VARIABLE not in manifests.os.environ + network_settings = ( + {"side_effect": requests.Timeout("offline")} + if case.startswith("offline-") + else {"return_value": _response(raw_bytes)} + ) + + def outcome(implementation): + try: + return ( + "ok", + _certify(_implementation=implementation, **runtime).model_dump(), + ) + except ValueError as error: + return ("error", type(error), str(error)) + + with patch.object(manifests.requests, "get", **network_settings) as network: + main_outcome = outcome(main_certification) + manifests.get_data_release_manifest.cache_clear() + current_outcome = outcome(manifests.certify_data_release_compatibility) + + assert current_outcome == main_outcome + if expected_error: + assert current_outcome[0] == "error" + assert expected_error in current_outcome[2] + else: + assert current_outcome == ("ok", expected.model_dump()) + assert network.call_count == 2 From 78e09ed63c3805b8739bc9ea229a0d393c213e46 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 04:38:39 -0400 Subject: [PATCH 03/20] Record local manifest certification in execution receipts and runner docs --- docs/bundles.md | 63 +++++++ docs/receipts.md | 19 ++ .../schemas/execution_receipt.v1.schema.json | 13 ++ src/policyengine/provenance/receipt.py | 41 +++- tests/test_local_manifest_receipt.py | 176 ++++++++++++++++++ 5 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 tests/test_local_manifest_receipt.py diff --git a/docs/bundles.md b/docs/bundles.md index 21bcd37a9..c715ff6de 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -116,6 +116,69 @@ policyengine bundle manifest 4.19.1 `install` targeted. Use `--venv` or `--python` only to inspect a different target explicitly. +## Local release manifests for Budget-day runners + +Set `POLICYENGINE_UK_DATA_MANIFEST_FILE` before starting a generation process to +read the UK release manifest from that file. The same setting is available as +`POLICYENGINE__DATA_MANIFEST_FILE` for other bundled countries. When set, +it always selects the local file, avoiding a Hugging Face request and its timeout. +An unset setting preserves the existing fetch and certification behavior. + +The trust anchor is the `trov:sha256` on +`composition/1/artifact/data_release_manifest` in the installed country's +`data/bundle/{country}.trace.tro.jsonld`. The file's **raw bytes** must match that +pin. A missing pin, unreadable file, altered bytes, or certification mismatch +stops the process. The model, core, build ID and fingerprint checks still apply; +selecting a local file cannot extend the bundle's certification. + +At G0 each morning, while Hugging Face is available, use the exact installed +release that the runner will use and download and verify the manifest before +enabling the setting. Keep the Hugging Face token in `HUGGING_FACE_TOKEN`, +`HF_TOKEN` or `HUGGINGFACE_HUB_TOKEN`: + +```bash +env -u POLICYENGINE_UK_DATA_MANIFEST_FILE POLICYENGINE_SKIP_COUNTRY_IMPORTS=1 python - <<'PY' +import hashlib +from pathlib import Path + +import requests +from policyengine.provenance.manifest import ( + bundled_data_release_manifest_sha256, get_release_manifest, + hugging_face_auth_headers, https_release_manifest_uri, +) + +expected = bundled_data_release_manifest_sha256("uk") +response = requests.get( + https_release_manifest_uri(get_release_manifest("uk").data_package), + headers=hugging_face_auth_headers(), timeout=30, +) +response.raise_for_status() +actual = hashlib.sha256(response.content).hexdigest() +if actual != expected: + raise RuntimeError(f"Manifest SHA-256: expected {expected}; actual {actual}") +target = Path("data/uk-release-manifest.json") +target.parent.mkdir(parents=True, exist_ok=True) +staged = target.with_suffix(".json.tmp") +staged.write_bytes(response.content) +staged.replace(target) +print(f"Verified {target}: {actual}") +PY +export POLICYENGINE_UK_DATA_MANIFEST_FILE="$PWD/data/uk-release-manifest.json" +``` + +Start a fresh runner process with that setting and record its receipt at G0. +Runtime certification and TRACE output use a distinct basis ending in +`_local_manifest`, such as `exact_build_model_version_local_manifest`; packaged +execution receipts record it as `runtime.data_certification_basis`. The +receipt's certificate keeps the bundle's published basis so certificate +verification remains reproducible. See [execution receipts](receipts.md). +Refresh and verify the file whenever the installed certified release changes. + +For Autumn Budget deployment, ship this support in the same `policyengine` +release as María's weekend Microcosm cut certification. Fetch the manifest for +that released bundle at G0; an earlier cut's manifest will fail its new pin. +Use the normal changelog-fragment release workflow for both changes. + ## Bundle-only PRs Run: diff --git a/docs/receipts.md b/docs/receipts.md index a082f449b..48686f6e8 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -47,6 +47,25 @@ receipt_json = receipt.to_canonical_json() For a country `Microsimulation` built with `managed_microsimulation()`, call `build_execution_receipt_for_microsimulation(sim, years=..., reform=..., baseline=...)`. It reads the dataset that was loaded from `sim.policyengine_bundle`. The file must still hash to the SHA-256 verified when it was loaded, and the receipt must name the dataset, URI, revision and package recorded then; otherwise it is refused, because it would describe bytes that did not run. A simulation loaded from an unmanaged file or URI recorded no load-time hash, so it cannot be receipted. The microsimulation does not record its reform, so pass the reform you built it with. +With `POLICYENGINE_UK_DATA_MANIFEST_FILE` enabled, a packaged-bundle receipt +rechecks runtime certification using the local file and records its distinct +`*_local_manifest` basis in `runtime.data_certification_basis`. Local-file +verification errors stop receipt creation. The certificate's +`compatibility_basis` stays the published bundle's assertion, which +`verify_certificate` can reproduce. Like the other runtime context, this source +annotation is excluded from execution and receipt digests. Caller-supplied +bundles do not claim this packaged-bundle runtime annotation. + +The execution-receipt schema accepts the annotation as an optional string; +compatibility bases in the certificate and TRACE TRO are also strings. The UK +Budget dashboard's receipt adapter uses this package's receipt model and +`verify_certificate`, so it needs no basis allowlist change. When adopting this +release, refresh the dashboard's vendored +`tests/fixtures/w4/execution_receipt.v1.schema.json`: its +`tests/test_released_receipt.py` checks exact equality against the installed +schema. Older consumers that forbid extra runtime fields need the updated +schema before reading these receipts. + There are two ways to bind a policyengine-uk `Scenario`: - A scenario built from `parameter_changes` is bound by content with `scenario_reform_identity(scenario)`. - `Scenario.from_reform(d)` wraps `d` in a closure that cannot be bound by its code, so bind `d` itself with `reform_identity(d, "policyengine-uk.reform_dict/v1")`. diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index 12fc69861..d10d1955b 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -473,6 +473,19 @@ "default": null, "title": "Ci Run Url" }, + "data_certification_basis": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Runtime certification basis when an opt-in local data manifest was verified against the packaged bundle TRACE TRO. The certificate's compatibility_basis remains the published bundle's assertion.", + "title": "Data Certification Basis" + }, "platform": { "title": "Platform", "type": "string" diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 12bcbcdbd..1c7b43e04 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -76,6 +76,7 @@ import json import math import operator +import os import platform import re import struct @@ -113,7 +114,14 @@ from urllib.request import url2pathname import psutil -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_serializer, + model_validator, +) from policyengine.provenance.bytecode import code_shape as _code_shape from policyengine.provenance.bytecode import compile_source @@ -488,6 +496,21 @@ class RuntimeContext(_Contract): python_version: str platform: str ci_run_url: Optional[str] = None + data_certification_basis: Optional[str] = Field( + default=None, + description=( + "Runtime certification basis when an opt-in local data manifest was " + "verified against the packaged bundle TRACE TRO. The certificate's " + "compatibility_basis remains the published bundle's assertion." + ), + ) + + @model_serializer(mode="wrap") + def _omit_absent_local_basis(self, handler): + payload = handler(self) + if self.data_certification_basis is None: + payload.pop("data_certification_basis", None) + return payload class ExecutionReceipt(_Contract): @@ -3871,6 +3894,21 @@ def identity(name: str) -> dict[str, Any]: actual_sha256 = sha256_file(dataset_path) data = _data_identity(country_id, record, bundle, dataset_name, actual_sha256) + runtime_basis = None + if source == "packaged" and ( + f"POLICYENGINE_{country_id.upper()}_DATA_MANIFEST_FILE" in os.environ + ): + from policyengine.provenance.manifest import ( + certify_data_release_compatibility, + ) + + runtime_basis = certify_data_release_compatibility( + country_id, + model.version, + runtime_data_build_fingerprint=data.build_fingerprint, + runtime_core_version=core.version, + ).compatibility_basis + fields = _certificate_fields(country_id, record, bundle) certified_artifact = record.get("certified_data_artifact") or {} certificate = None @@ -3920,6 +3958,7 @@ def identity(name: str) -> dict[str, Any]: python_version=platform.python_version(), platform=sys.platform, ci_run_url=ci_run_url, + data_certification_basis=runtime_basis, ), ) diff --git a/tests/test_local_manifest_receipt.py b/tests/test_local_manifest_receipt.py new file mode 100644 index 000000000..f89a9bbf0 --- /dev/null +++ b/tests/test_local_manifest_receipt.py @@ -0,0 +1,176 @@ +"""Local-manifest runtime evidence preserves the bundle certificate's identity.""" + +import hashlib +import json +from pathlib import Path + +import jsonschema +import pytest + +from policyengine.provenance import certification, manifest, receipt + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ( + REPO_ROOT / "src/policyengine/data/schemas/execution_receipt.v1.schema.json" +) +LOCAL_BASIS = "exact_build_model_version_local_manifest" + + +@pytest.fixture +def packaged_receipt(monkeypatch, tmp_path): + bundle = json.loads( + (REPO_ROOT / "src/policyengine/data/bundle/manifest.json").read_text() + ) + record = bundle["data_releases"]["uk"] + dataset_path = tmp_path / "dataset.h5" + dataset_path.write_bytes(b"synthetic certified dataset") + digest = hashlib.sha256(dataset_path.read_bytes()).hexdigest() + record["datasets"][record["default_dataset"]]["sha256"] = digest + record["certified_data_artifact"]["sha256"] = digest + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text(json.dumps(bundle)) + versions = { + "policyengine": bundle["policyengine_version"], + "policyengine-uk": record["certification"]["certified_for_model_version"], + "policyengine-core": bundle["packages"]["policyengine-core"]["version"], + } + monkeypatch.setattr( + certification, "default_bundle_source_path", lambda: bundle_path + ) + monkeypatch.setattr( + receipt, + "installed_package_identity", + lambda name, **kwargs: receipt.PackageIdentity( + name=name, version=versions[name] + ), + ) + monkeypatch.setattr(receipt, "loaded_versions", lambda: {}) + monkeypatch.setattr( + receipt, "_refuse_package_changed_after_process_start", lambda name: None + ) + monkeypatch.setattr(receipt, "_projected_years", lambda *args: (2025, 2026)) + monkeypatch.delenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", raising=False) + + def build(*, caller_supplied=False): + current_law = receipt.ReformIdentity(kind="current_law") + return receipt.build_execution_receipt( + country_id="uk", + dataset_path=dataset_path, + years=[2026], + reform=current_law, + baseline=current_law, + bundle=bundle if caller_supplied else None, + ) + + return build, bundle + + +def test_packaged_receipt_records_local_basis_without_changing_certificate( + packaged_receipt, monkeypatch +): + build, bundle = packaged_receipt + calls = [] + + def certify(country_id, model_version, **kwargs): + calls.append((country_id, model_version, kwargs)) + return manifest.DataCertification( + compatibility_basis=LOCAL_BASIS, + certified_for_model_version=model_version, + ) + + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + monkeypatch.setattr(manifest, "certify_data_release_compatibility", certify) + result = build() + assert result.runtime.data_certification_basis == LOCAL_BASIS + assert calls == [ + ( + "uk", + result.model.version, + { + "runtime_core_version": result.core.version, + "runtime_data_build_fingerprint": result.data.build_fingerprint, + }, + ) + ] + assert ( + result.certificate.compatibility_basis + == (bundle["data_releases"]["uk"]["certification"]["compatibility_basis"]) + ) + assert receipt.verify_certificate(result, bundle) == [] + payload = json.loads(result.model_dump_json()) + jsonschema.Draft202012Validator(json.loads(SCHEMA_PATH.read_text())).validate( + payload + ) + assert ( + receipt.ExecutionReceipt.model_validate_json(result.model_dump_json()) == result + ) + + +def test_local_manifest_errors_stop_receipt_building(packaged_receipt, monkeypatch): + build, _ = packaged_receipt + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/missing/manifest.json") + + def refuse(*args, **kwargs): + raise ValueError("local manifest: expected SHA-256 aaa; actual SHA-256 bbb") + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", refuse) + with pytest.raises(ValueError, match="expected SHA-256 aaa; actual SHA-256 bbb"): + build() + + +@pytest.mark.parametrize("caller_supplied", [False, True]) +def test_unset_setting_keeps_receipt_building_offline( + packaged_receipt, monkeypatch, caller_supplied +): + build, _ = packaged_receipt + + def unexpected(*args, **kwargs): + raise AssertionError( + "unset local manifest must not cause certification or fetches" + ) + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", unexpected) + monkeypatch.setattr(manifest.requests, "get", unexpected) + result = build(caller_supplied=caller_supplied) + assert result.runtime.data_certification_basis is None + assert ( + "data_certification_basis" + not in json.loads(result.model_dump_json())["runtime"] + ) + + +def test_local_runtime_context_does_not_change_certificate_or_result_digests( + packaged_receipt, monkeypatch +): + build, bundle = packaged_receipt + original = build() + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + monkeypatch.setattr( + manifest, + "certify_data_release_compatibility", + lambda *args, **kwargs: manifest.DataCertification( + compatibility_basis=LOCAL_BASIS, + certified_for_model_version=original.model.version, + ), + ) + local = build() + assert local.runtime.data_certification_basis == LOCAL_BASIS + assert local.certificate == original.certificate + assert local.execution_identity_sha256 == original.execution_identity_sha256 + assert local.receipt_sha256 == original.receipt_sha256 + assert receipt.verify_certificate(local, bundle) == [] + + +def test_caller_supplied_bundle_does_not_claim_packaged_manifest_provenance( + packaged_receipt, monkeypatch +): + build, _ = packaged_receipt + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + + def unexpected(*args, **kwargs): + raise AssertionError("a caller's bundle is not the local file's trust anchor") + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", unexpected) + result = build(caller_supplied=True) + assert result.certificate.certificate_source == "caller_supplied" + assert result.runtime.data_certification_basis is None From fd4e099ed3ff768041f3a537e6e38527c07b0bbf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 04:43:26 -0400 Subject: [PATCH 04/20] Document local manifest validation and hub handoff --- .job/HANDOFF.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .job/HANDOFF.md diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md new file mode 100644 index 000000000..d2a91e2b6 --- /dev/null +++ b/.job/HANDOFF.md @@ -0,0 +1,47 @@ +# Local manifest fallback handoff + +- Draft PR: https://github.com/PolicyEngine/policyengine.py/pull/704 +- Issue: https://github.com/PolicyEngine/policyengine.py/issues/703 +- Branch: `budget-local-data-manifest`, based on `origin/main` `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. +- Reviewed implementation head: `78e09ed63c3805b8739bc9ea229a0d393c213e46` (this handoff is committed immediately afterwards). +- Commits: `0bb58755` local manifest/certification/tests; `78e09ed6` receipt/schema/docs/tests. + +## Design + +`POLICYENGINE__DATA_MANIFEST_FILE` always selects the local file when present. This avoids outage latency and gives deterministic behavior. Unset keeps the original cached network fetch and certification results. + +The pin is the raw-byte `trov:sha256` for `composition/1/artifact/data_release_manifest` in the installed country TRACE TRO. Its arrangement must name the installed release's manifest URI. Missing, malformed, duplicate, or stale pins refuse. The current installed UK pin was checked as `49f9fa820db2e6544ff5cdceeb2d868a504843a6a1c1c06b37188621e700066f`. No environment digest is accepted. Local bytes are re-read and hashed on every call, before JSON parsing/revision rewriting, so cached remote/local loads cannot bypass verification. Invalid paths and digest errors fail closed with digest context. + +Local bytes run through the unchanged certification acceptance logic. All four successful runtime bases gain `_local_manifest`; the claim-based set includes the corresponding local variants. Exact-pair model/core/build restrictions and fingerprint acceptance remain the same. TRACE accepts the basis as a string. Packaged receipts recheck local certification and record it in optional `runtime.data_certification_basis`, keeping the published bundle certificate basis and both digest domains unchanged. Unset receipts omit the new annotation. The schema is regenerated and checked. Caller-supplied bundles do not claim packaged provenance. + +## Validation + +- 38 local-manifest tests passed, including 80 Hypothesis byte mutations and 10 direct main-snapshot differential scenarios. +- 146 existing certification tests passed. +- 96 bundle/bootstrap/tooling tests passed. +- Broad provenance suite: 2,337 passed, 6 skipped, including all 6 new receipt tests. +- Wheel/package-pin tests: 96 passed. +- Total across these final focused runs: **2,713 passed, 6 skipped**. Skips cover private data, Python/model-version-specific cases, and disabled top-level country imports in the broad run. +- `make format`, `make lint`, `git diff --check`, and `scripts/generate_receipt_schemas.py --check` passed. +- Every pytest invocation checked load1 <=110 and `os.statvfs` available bytes >=55,000,000,000. Observed load was below 37 and free disk above 132 GB. No pytest workers were used. +- Local fixtures are synthetic; no private manifest or microdata is committed. + +Detailed parent-run logs are local files under `.job/*tests.log`. + +## Coordination and remaining work + +- Leave PR draft; the hub reviews and merges it. +- Ship in the same policyengine.py release as María's weekend Microcosm cut certification. No versions or country/data pins were changed. `changelog.d/703.added.md` follows the release workflow. +- The G0 morning fetch/verify/atomic-refresh instructions are in `docs/bundles.md`. Refresh the local manifest after adopting the newly certified release. +- Dashboard follow-up: refresh `tests/fixtures/w4/execution_receipt.v1.schema.json` in `PolicyEngine/uk-autumn-budget-dashboard-2026` when upgrading; its receipt adapter has no basis allowlist, but its schema equality test requires the new schema. See `docs/receipts.md`. + +## Workspace Git metadata + +The assigned worktree's protected `.git` points into the caller's repository, where metadata writes are sandbox-blocked. All work and independent Git metadata stayed in this workspace. To inspect the actual branch here: + +```bash +GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git status +GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -3 --oneline +``` + +The branch is pushed to the canonical repository. A portable `.job/local-data-manifest.bundle` is also left for harvesting; it contains `budget-local-data-manifest` relative to the main base. The original protected `.git` pointer was not changed. No caller files were written. From 91aa875ee3ad202883e316ec2f5743c9acb32fd6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 05:37:30 -0400 Subject: [PATCH 05/20] Reject conflicting TRO pins and bundle artifact identities --- src/policyengine/provenance/certification.py | 22 +- src/policyengine/provenance/manifest.py | 141 ++++++++++++- tests/test_local_data_manifest.py | 199 +++++++++++++++++++ tests/test_local_manifest_tro_pins.py | 174 ++++++++++++++++ tests/test_release_manifests.py | 124 +++++++++--- tests/test_release_tro_generation.py | 2 +- 6 files changed, 614 insertions(+), 48 deletions(-) create mode 100644 tests/test_local_manifest_tro_pins.py diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index dc773e215..381c1e707 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -62,6 +62,12 @@ fetch_pypi_wheel_metadata, release_build_id, ) +from policyengine.provenance.manifest import ( + RELEASE_SCOPED_ARTIFACT_KINDS as RELEASE_SCOPED_ARTIFACT_KINDS, +) +from policyengine.provenance.manifest import ( + artifact_path_for_country_manifest as artifact_path_for_country_manifest, +) MANIFEST_URI_PATTERN = re.compile( r"^hf://(?Pdataset|model)/(?P[^@]+)@(?P[^/]+)" @@ -113,9 +119,6 @@ CONSTITUENCY_ASSET_SPEC.weight_matrix_filename, LOCAL_AUTHORITY_ASSET_SPEC.weight_matrix_filename, ) -# Artifact kinds published beside the release manifest rather than at the repo -# root, so their manifest paths are relative to the release directory. -RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) US_STATE_CODES = ( "AL", "AK", @@ -526,19 +529,6 @@ def head_release_file( return response.status_code == 200 -def artifact_path_for_country_manifest(artifact, uri_parts: dict) -> str: - path = artifact.path - release_dir = release_manifest_dir(uri_parts) - if ( - artifact.kind in RELEASE_SCOPED_ARTIFACT_KINDS - and artifact.repo_id == uri_parts["repo_id"] - and artifact.revision == uri_parts["revision"] - and not path.startswith(f"{release_dir}/") - ): - return f"{release_dir}/{path}" - return path - - def artifact_reference_url( reference: dict, uri_parts: dict, prefix: str = "datasets/" ) -> str: diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 0e9b97d37..f06c4bc12 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -5,7 +5,7 @@ from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Literal, Optional +from typing import Literal, NoReturn, Optional from urllib.parse import quote import requests @@ -18,6 +18,8 @@ HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 +# Producer paths for these artifacts are relative to the release directory. +RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) class DataReleaseManifestUnavailableError(ValueError): @@ -374,11 +376,27 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: return manifest +def _tro_nodes_with_id(value: object, node_id: str) -> list[dict]: + """Collect node definitions and references wherever JSON-LD nests them.""" + nodes: list[dict] = [] + if isinstance(value, dict): + if value.get("@id") == node_id: + nodes.append(value) + for child in value.values(): + nodes.extend(_tro_nodes_with_id(child, node_id)) + elif isinstance(value, list): + for child in value: + nodes.extend(_tro_nodes_with_id(child, node_id)) + return nodes + + def bundled_data_release_manifest_sha256(country_id: str) -> str: """Read the manifest's byte digest from the installed country bundle TRO. The artifact and its arrangement must describe the release this bundle - serves. A missing or stale sidecar cannot authorise a local manifest. + serves. All assertions for the artifact's JSON-LD identifier must agree, + including definitions outside the inline composition. A missing or stale + sidecar cannot authorise a local manifest. """ country_manifest = get_release_manifest(country_id) tro_path = files("policyengine").joinpath( @@ -409,6 +427,19 @@ def bundled_data_release_manifest_sha256(country_id: str) -> str: or locations != [https_release_manifest_uri(country_manifest.data_package)] ): raise ValueError("missing, ambiguous, malformed, or stale manifest pin") + asserted_digests = set() + for node in _tro_nodes_with_id(tro, artifact_id): + if "trov:sha256" not in node: + continue + assertion = node["trov:sha256"] + if ( + not isinstance(assertion, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", assertion) is None + ): + raise ValueError("malformed manifest pin assertion") + asserted_digests.add(assertion.lower()) + if asserted_digests != {digest.lower()}: + raise ValueError("conflicting manifest pin assertions") except (OSError, ValueError, KeyError, TypeError, AttributeError) as exc: raise ValueError( f"Cannot use a local data release manifest for country '{country_id}': " @@ -418,7 +449,7 @@ def bundled_data_release_manifest_sha256(country_id: str) -> str: return digest.lower() -def get_data_release_manifest(country_id: str) -> DataReleaseManifest: +def _load_data_release_manifest(country_id: str) -> DataReleaseManifest: """Load an explicitly configured, bundle-pinned file or the cached fetch. Local files are read and verified on every call, including after a remote @@ -447,7 +478,10 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: manifest = DataReleaseManifest.model_validate_json(source_bytes) manifest.source_sha256 = actual manifest._source_is_local = True - return _apply_release_revision(manifest, get_release_manifest(country_id)) + country_manifest = get_release_manifest(country_id) + _apply_release_revision(manifest, country_manifest) + _validate_bundle_artifacts(manifest, country_manifest) + return manifest @lru_cache @@ -485,7 +519,9 @@ def _fetch_data_release_manifest(country_id: str) -> DataReleaseManifest: if not isinstance(source_bytes, bytes): source_bytes = response.text.encode("utf-8") data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() - return _apply_release_revision(data_release_manifest, country_manifest) + _apply_release_revision(data_release_manifest, country_manifest) + _validate_bundle_artifacts(data_release_manifest, country_manifest) + return data_release_manifest def _apply_release_revision( @@ -503,9 +539,98 @@ def _apply_release_revision( return data_release_manifest -# Preserve the existing cache controls for callers and tests of fetched manifests. -get_data_release_manifest.cache_clear = _fetch_data_release_manifest.cache_clear -get_data_release_manifest.cache_info = _fetch_data_release_manifest.cache_info +def artifact_path_for_country_manifest( + artifact: DataReleaseArtifact, uri_parts: dict +) -> str: + """Resolve producer-relative evidence paths as bundle certification does.""" + path = artifact.path + release_dir = uri_parts["path"].rsplit("/", 1)[0] + if ( + artifact.kind in RELEASE_SCOPED_ARTIFACT_KINDS + and artifact.repo_id == uri_parts["repo_id"] + and artifact.revision == uri_parts["revision"] + and not path.startswith(f"{release_dir}/") + ): + return f"{release_dir}/{path}" + return path + + +def _validate_bundle_artifacts( + manifest: DataReleaseManifest, country: CountryReleaseManifest +) -> None: + """Bind a loaded manifest to the artifacts this installed bundle serves. + + Run after revision rewriting. Regional releases and additive overlays may + supply records absent from the primary manifest; every overlapping record + must agree, and the certified/default artifact must always be present. + """ + + def refuse(dataset: str, detail: str) -> NoReturn: + raise ValueError( + f"Data release manifest artifact {dataset!r} does not match the " + f"installed bundle for country {country.country_id!r}: {detail}." + ) + + default = country.default_dataset + declared_default = manifest.default_datasets.get("national") + if declared_default is not None and declared_default != default: + refuse(default, f"manifest declares default {declared_default!r}") + if default not in manifest.artifacts: + refuse(default, "missing default dataset artifact") + certified = country.certified_data_artifact + if certified is not None: + artifact = manifest.artifacts.get(certified.dataset) + if artifact is None: + refuse(certified.dataset, "missing certified dataset artifact") + if artifact.uri != certified.uri: + refuse(certified.dataset, f"expected {certified.uri}, got {artifact.uri}") + if certified.sha256 is not None and artifact.sha256 != certified.sha256: + refuse(certified.dataset, "certified SHA-256 pin differs") + if default not in country.datasets and ( + certified is None or certified.dataset != default + ): + refuse(default, "missing bundled default artifact record") + + uri_parts = { + "repo_id": country.data_package.repo_id, + "revision": _artifact_revision(country.data_package), + "path": country.data_package.release_manifest_path, + } + for name, artifact in manifest.artifacts.items(): + reference = country.datasets.get(name) + if reference is None: + continue + expected_uri = build_hf_uri( + repo_id=reference.repo_id or country.data_package.repo_id, + path_in_repo=reference.path, + revision=reference.revision or _artifact_revision(country.data_package), + ) + actual_uri = build_hf_uri( + repo_id=artifact.repo_id, + path_in_repo=artifact_path_for_country_manifest(artifact, uri_parts), + revision=artifact.revision, + ) + if actual_uri != expected_uri: + refuse(name, f"expected {expected_uri}, got {actual_uri}") + if (artifact.repo_type or country.data_package.repo_type) != ( + reference.repo_type or country.data_package.repo_type + ): + refuse(name, "repository type differs") + if reference.sha256 is not None and artifact.sha256 != reference.sha256: + refuse(name, "managed SHA-256 pin differs") + + +class _DataReleaseManifestLoader: + """Callable loader retaining the fetched manifest's typed cache controls.""" + + def __call__(self, country_id: str) -> DataReleaseManifest: + return _load_data_release_manifest(country_id) + + cache_clear = staticmethod(_fetch_data_release_manifest.cache_clear) + cache_info = staticmethod(_fetch_data_release_manifest.cache_info) + + +get_data_release_manifest = _DataReleaseManifestLoader() def _specifier_matches(version: str, specifier: str) -> bool: diff --git a/tests/test_local_data_manifest.py b/tests/test_local_data_manifest.py index 112599dac..9b27e2938 100644 --- a/tests/test_local_data_manifest.py +++ b/tests/test_local_data_manifest.py @@ -97,6 +97,18 @@ def _install_bundle(tmp_path, monkeypatch, payload=None, certification=_DEFAULT) "release_manifest_revision": "f" * 40, }, "default_dataset": "test_dataset", + "certified_data_artifact": { + "dataset": "test_dataset", + "uri": f"hf://policyengine/test-data/test_dataset.h5@{'f' * 40}", + "sha256": "d" * 64, + }, + "datasets": { + "test_dataset": { + "path": "test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + }, "certification": ( _qualified_certification() if certification is _DEFAULT else certification ), @@ -399,6 +411,193 @@ def _response(raw_bytes): return response +def _edit_bundle(trace_path, edit): + bundle_path = trace_path.with_name("manifest.json") + bundle = json.loads(bundle_path.read_text()) + edit(bundle["data_releases"]["uk"]) + bundle_path.write_text(json.dumps(bundle)) + manifests.get_release_manifest.cache_clear() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("repo_id", "adversary/elsewhere"), + ("path", "elsewhere.h5"), + ("revision", "unqualified-revision"), + ("sha256", "0" * 64), + ("sha256", None), + ("repo_type", "dataset"), + ("location", "adversary/elsewhere/elsewhere.h5"), + ], +) +def test_pinned_manifest_cannot_change_the_bundled_default_artifact( + tmp_path, monkeypatch, transport, field, value +): + payload = _payload() + artifact = payload["artifacts"]["test_dataset"] + if field == "location": + artifact.update(repo_id="adversary/elsewhere", path="elsewhere.h5") + else: + artifact[field] = value + # Install a TRO matching these exact conflicting bytes while keeping the + # bundle's original certified URI, build, model and fingerprint unchanged. + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object( + manifests.requests, "get", return_value=_response(raw_bytes) + ) as network: + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + assert network.call_count == (transport == "fetched") + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("failure", ["missing-artifact", "other-default"]) +def test_manifest_requires_the_bundled_default_dataset( + tmp_path, monkeypatch, transport, failure +): + payload = _payload() + if failure == "missing-artifact": + payload["artifacts"].clear() + else: + payload["default_datasets"]["national"] = "another_dataset" + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("certified_record", [True, False]) +def test_consistent_default_uses_rewritten_revision( + tmp_path, monkeypatch, transport, certified_record +): + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch) + if not certified_record: + _edit_bundle(trace_path, lambda country: country.pop("certified_data_artifact")) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + assert _certify().compatibility_basis == ( + "exact_build_model_version" + + ("_local_manifest" if transport == "local" else "") + ) + assert ( + manifests.get_data_release_manifest("uk").artifacts["test_dataset"].revision + == "f" * 40 + ) + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +def test_a_different_certified_artifact_cannot_leave_the_default_unpinned( + tmp_path, monkeypatch, transport +): + payload = _payload() + payload["artifacts"]["certified_other"] = dict(payload["artifacts"]["test_dataset"]) + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + country["datasets"].clear() + country["certified_data_artifact"]["dataset"] = "certified_other" + + _edit_bundle(trace_path, edit) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + with pytest.raises(ValueError, match="missing bundled default artifact record"): + _certify() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize( + "difference", ["none", "repo_id", "path", "revision", "sha256", "repo_type"] +) +def test_every_overlapping_managed_artifact_must_match( + tmp_path, monkeypatch, transport, difference +): + payload = _payload() + payload["artifacts"]["supplement"] = { + "kind": "microdata", + "path": "inherited.h5", + "repo_id": "policyengine/inherited-data", + "revision": "inherited-release", + "repo_type": "dataset", + "sha256": "b" * 64, + } + reference = dict(payload["artifacts"]["supplement"]) + reference.pop("kind") + if difference != "none": + payload["artifacts"]["supplement"][difference] = ( + "model" if difference == "repo_type" else "conflicting-pin" + ) + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + _edit_bundle( + trace_path, + lambda country: country["datasets"].update( + supplement=reference, + regional_only={"path": "regional.h5", "revision": "regional-release"}, + ), + ) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + if difference == "none": + _certify() + else: + with pytest.raises( + ValueError, match="artifact 'supplement'.*installed bundle" + ): + _certify() + + +@pytest.mark.parametrize("kind", ["diagnostics", "evidence"]) +@pytest.mark.parametrize("prefixed", [False, True]) +def test_release_relative_paths_match_certification_normalization( + tmp_path, monkeypatch, kind, prefixed +): + payload = _payload() + relative_path = "report.json" + bundled_path = "releases/test-release/report.json" + payload["artifacts"]["report"] = { + "kind": kind, + "path": bundled_path if prefixed else relative_path, + "repo_id": "policyengine/test-data", + "revision": "test-release", + "sha256": "b" * 64, + } + local_path, trace_path, _ = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + country["data_package"]["release_manifest_path"] = ( + "releases/test-release/release_manifest.json" + ) + country["datasets"]["report"] = { + "path": bundled_path, + "revision": "f" * 40, + "sha256": "b" * 64, + } + + _edit_bundle(trace_path, edit) + tro = json.loads(trace_path.read_text()) + tro["@graph"][0]["trov:hasArrangement"][0]["trov:hasArtifactLocation"][0][ + "trov:hasLocation" + ] = ( + "https://huggingface.co/policyengine/test-data/resolve/" + f"{'f' * 40}/releases/test-release/release_manifest.json" + ) + trace_path.write_text(json.dumps(tro)) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get") as network: + _certify() + network.assert_not_called() + + def test_setting_local_manifest_after_a_cached_fetch_still_checks_its_pin( tmp_path, monkeypatch ): diff --git a/tests/test_local_manifest_tro_pins.py b/tests/test_local_manifest_tro_pins.py new file mode 100644 index 000000000..fed5a9786 --- /dev/null +++ b/tests/test_local_manifest_tro_pins.py @@ -0,0 +1,174 @@ +"""A TRO's manifest pin includes every assertion for the JSON-LD node ID.""" + +from __future__ import annotations + +import json + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine.provenance import manifest as manifests + +ARTIFACT_ID = "composition/1/artifact/data_release_manifest" +PIN = "a" * 64 + + +def _tro(): + return { + "@graph": [ + { + "@id": "tro", + "trov:hasComposition": { + "trov:hasArtifact": [{"@id": ARTIFACT_ID, "trov:sha256": PIN}] + }, + "trov:hasArrangement": [ + { + "trov:hasArtifactLocation": [ + { + "trov:hasArtifact": {"@id": ARTIFACT_ID}, + "trov:hasLocation": ( + "https://huggingface.co/policyengine/test-data/" + "resolve/test-release/release_manifest.json" + ), + } + ] + } + ], + } + ] + } + + +@pytest.fixture +def read_pin(tmp_path, monkeypatch): + bundle_dir = tmp_path / "data" / "bundle" + bundle_dir.mkdir(parents=True) + country_manifest = manifests.CountryReleaseManifest( + country_id="uk", + policyengine_version="6.10.1", + model_package=manifests.PackageVersion(name="policyengine-uk", version="test"), + data_package=manifests.DataPackageVersion( + name="policyengine-uk-data", + version="test-release", + repo_id="policyengine/test-data", + ), + default_dataset="test_dataset", + ) + monkeypatch.setattr(manifests, "files", lambda package: tmp_path) + monkeypatch.setattr( + manifests, "get_release_manifest", lambda country: country_manifest + ) + + def read(tro): + (bundle_dir / "uk.trace.tro.jsonld").write_text(json.dumps(tro)) + return manifests.bundled_data_release_manifest_sha256("uk") + + return read + + +@pytest.mark.parametrize("before_inline_definition", [False, True]) +def test_direct_graph_conflicting_manifest_pin_is_refused( + read_pin, before_inline_definition +): + tro = _tro() + duplicate = {"@id": ARTIFACT_ID, "trov:sha256": "0" * 64} + if before_inline_definition: + tro["@graph"].insert(0, duplicate) + else: + tro["@graph"].append(duplicate) + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +@pytest.mark.parametrize( + "assertion", + [ + [PIN], + [PIN, "0" * 64], + {"@value": PIN}, + {"@value": "0" * 64}, + "not-a-sha256", + None, + 42, + ], +) +def test_noncanonical_duplicate_hash_assertions_fail_closed(read_pin, assertion): + tro = _tro() + tro["@graph"].append({"@id": ARTIFACT_ID, "trov:sha256": assertion}) + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +def test_identical_external_definitions_and_references_do_not_add_a_hash(read_pin): + tro = _tro() + tro["@graph"].extend( + [ + {"@id": ARTIFACT_ID, "trov:sha256": PIN.upper()}, + {"extension": {"@id": ARTIFACT_ID, "trov:sha256": PIN}}, + {"@id": ARTIFACT_ID}, + ] + ) + + assert read_pin(tro) == PIN + + +def test_hash_on_an_existing_artifact_reference_is_checked(read_pin): + tro = _tro() + reference = tro["@graph"][0]["trov:hasArrangement"][0]["trov:hasArtifactLocation"][ + 0 + ]["trov:hasArtifact"] + reference["trov:sha256"] = "0" * 64 + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +@settings( + max_examples=80, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + placement=st.sampled_from( + ["graph", "document", "tro", "composition", "arrangement", "location"] + ), + wrappers=st.lists( + st.sampled_from(["list", "property", "graph"]), min_size=0, max_size=8 + ), + conflicting_hash=st.binary(min_size=32, max_size=32) + .map(bytes.hex) + .filter(lambda digest: digest != PIN), +) +def test_conflicting_pin_is_refused_at_any_duplicate_placement( + read_pin, placement, wrappers, conflicting_hash +): + tro = _tro() + duplicate = {"@id": ARTIFACT_ID, "trov:sha256": conflicting_hash} + for wrapper in wrappers: + if wrapper == "list": + duplicate = [duplicate] + elif wrapper == "graph": + duplicate = {"@graph": [duplicate]} + else: + duplicate = {"extension": duplicate} + + root = tro["@graph"][0] + if placement == "graph": + # Keep the surrounding graph canonical so refusal must inspect the + # nested assertion, rather than fail on an unrelated malformed node. + tro["@graph"].append({"extension": duplicate}) + else: + target = { + "document": tro, + "tro": root, + "composition": root["trov:hasComposition"], + "arrangement": root["trov:hasArrangement"][0], + "location": root["trov:hasArrangement"][0]["trov:hasArtifactLocation"][0], + }[placement] + target["extension"] = duplicate + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index aff21577a..006070e2a 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -154,6 +154,52 @@ def _response_with_json(payload: dict) -> MagicMock: return response +def _legacy_us_country_manifest( + version: str = "1.73.0", package_name: str = "policyengine-us-data" +) -> CountryReleaseManifest: + """Pin the legacy fixture independently of the installed Populace bundle.""" + return CountryReleaseManifest.model_validate( + { + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "model_package": {"name": "policyengine-us", "version": "1.653.3"}, + "data_package": { + "name": package_name, + "version": version, + "repo_id": "policyengine/policyengine-us-data", + }, + "default_dataset": "enhanced_cps_2024", + "datasets": { + "enhanced_cps_2024": { + "path": "enhanced_cps_2024.h5", + "revision": version, + "sha256": "abc", + } + }, + "certified_data_artifact": { + "dataset": "enhanced_cps_2024", + "uri": ( + "hf://policyengine/policyengine-us-data/" + f"enhanced_cps_2024.h5@{version}" + ), + "sha256": "abc", + }, + } + ) + + +def _legacy_us_artifacts(version: str = "1.73.0") -> dict: + return { + "enhanced_cps_2024": { + "kind": "microdata", + "path": "enhanced_cps_2024.h5", + "repo_id": "policyengine/policyengine-us-data", + "revision": version, + "sha256": "abc", + } + } + + def _country_modules_with_microsimulation( name: str, microsimulation: MagicMock, @@ -535,10 +581,16 @@ def test__given_country__then_can_fetch_data_release_manifest(self): }, } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), - ) as mock_get: + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest("1.78.2"), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ) as mock_get, + ): manifest = get_data_release_manifest("us") assert manifest.schema_version == 1 @@ -561,8 +613,8 @@ def test__given_country__then_can_fetch_data_release_manifest(self): ) mock_get.assert_called_once() assert mock_get.call_args.args[0] == ( - "https://huggingface.co/datasets/policyengine/populace-us/resolve/" - f"{US_DATA_RELEASE_REVISION}/{US_DATA_RELEASE_PATH}" + "https://huggingface.co/policyengine/policyengine-us-data/resolve/" + "1.78.2/release_manifest.json" ) def test__given_explicit_manifest_revision__then_builds_manifest_url(self): @@ -589,7 +641,9 @@ def test__given_release_manifest_artifact_uses_version_tag__then_rewrites_to_com "path": "populace_us_2024.h5", "repo_id": "policyengine/populace-us", "revision": US_DATA_RELEASE_VERSION, - "sha256": "abc", + "sha256": ( + "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + ), "size_bytes": 123, } }, @@ -651,12 +705,18 @@ def test__given_range_specifier__then_certification_accepts_compatible_version( } ], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -685,12 +745,18 @@ def test__given_matching_fingerprint__then_certification_allows_reuse(self): }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -720,12 +786,18 @@ def test__given_release_with_cut_tag__then_cut_tag_is_the_data_build_id(self): }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts("0.1.0"), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest("0.1.0", "microcosm-data"), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -914,12 +986,18 @@ def test__given_mismatched_version_and_fingerprint__then_certification_fails(sel }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): try: certify_data_release_compatibility( diff --git a/tests/test_release_tro_generation.py b/tests/test_release_tro_generation.py index d0aa9410d..fb4b6b33c 100644 --- a/tests/test_release_tro_generation.py +++ b/tests/test_release_tro_generation.py @@ -253,7 +253,7 @@ def test_strict_missing_remote_dataset_never_writes_limited_output(release, caps data = json.loads(release[1].content) del data["artifacts"][data["default_datasets"]["national"]] release[1]._content = json.dumps(data).encode() - with pytest.raises(ValueError, match="Default dataset"): + with pytest.raises(ValueError, match="missing default dataset artifact"): generator.regenerate_all(strict=True) assert path.read_bytes() == before assert "limited TRO" not in capsys.readouterr().err From 24bfd47d21d76f05d9daf6d7cd972a656b1c12e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 05:37:30 -0400 Subject: [PATCH 06/20] Make local manifest activation conditional on verification --- changelog.d/703.added.md | 2 +- docs/bundles.md | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/changelog.d/703.added.md b/changelog.d/703.added.md index 9c630be9b..821b167b8 100644 --- a/changelog.d/703.added.md +++ b/changelog.d/703.added.md @@ -1 +1 @@ -Support opt-in local data release manifests pinned by the installed bundle TRO, with fail-closed verification and a distinct runtime certification basis for offline Budget-day runners. +Support opt-in local data release manifests pinned by the installed bundle TRO, with fail-closed verification and a distinct runtime certification basis for offline Budget-day runners. Reject conflicting TRO hash assertions and require local and fetched manifest artifacts to agree with the installed bundle. diff --git a/docs/bundles.md b/docs/bundles.md index c715ff6de..55fff09be 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -128,7 +128,11 @@ The trust anchor is the `trov:sha256` on `composition/1/artifact/data_release_manifest` in the installed country's `data/bundle/{country}.trace.tro.jsonld`. The file's **raw bytes** must match that pin. A missing pin, unreadable file, altered bytes, or certification mismatch -stops the process. The model, core, build ID and fingerprint checks still apply; +stops the process. Every definition of the artifact ID in the TRO must agree on +one digest. After revision normalization, the manifest's certified/default +artifact and all artifacts shared with the bundle must agree with the bundle's +locations and pinned hashes. This artifact check also applies to fetched +manifests. The model, core, build ID and fingerprint checks still apply; selecting a local file cannot extend the bundle's certification. At G0 each morning, while Hugging Face is available, use the exact installed @@ -137,7 +141,7 @@ enabling the setting. Keep the Hugging Face token in `HUGGING_FACE_TOKEN`, `HF_TOKEN` or `HUGGINGFACE_HUB_TOKEN`: ```bash -env -u POLICYENGINE_UK_DATA_MANIFEST_FILE POLICYENGINE_SKIP_COUNTRY_IMPORTS=1 python - <<'PY' +env -u POLICYENGINE_UK_DATA_MANIFEST_FILE POLICYENGINE_SKIP_COUNTRY_IMPORTS=1 python - <<'PY' && import hashlib from pathlib import Path @@ -174,11 +178,6 @@ receipt's certificate keeps the bundle's published basis so certificate verification remains reproducible. See [execution receipts](receipts.md). Refresh and verify the file whenever the installed certified release changes. -For Autumn Budget deployment, ship this support in the same `policyengine` -release as María's weekend Microcosm cut certification. Fetch the manifest for -that released bundle at G0; an earlier cut's manifest will fail its new pin. -Use the normal changelog-fragment release workflow for both changes. - ## Bundle-only PRs Run: From 0d7bf1a641d61b8951f47ab12d8d4867f57df140 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 05:49:34 -0400 Subject: [PATCH 07/20] Compare artifact identity fields without URI ambiguity --- src/policyengine/provenance/manifest.py | 37 ++++++++++++++++-------- tests/test_local_data_manifest.py | 38 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f06c4bc12..7729313ef 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -582,8 +582,21 @@ def refuse(dataset: str, detail: str) -> NoReturn: artifact = manifest.artifacts.get(certified.dataset) if artifact is None: refuse(certified.dataset, "missing certified dataset artifact") - if artifact.uri != certified.uri: - refuse(certified.dataset, f"expected {certified.uri}, got {artifact.uri}") + try: + location, revision = certified.uri.removeprefix("hf://").rsplit("@", 1) + owner, repo, path = location.split("/", 2) + except ValueError: + refuse(certified.dataset, "invalid bundled certified artifact URI") + expected_identity = (f"{owner}/{repo}", path, revision) + actual_identity = (artifact.repo_id, artifact.path, artifact.revision) + if ( + not certified.uri.startswith("hf://") + or actual_identity != expected_identity + ): + refuse( + certified.dataset, + f"expected {expected_identity!r}, got {actual_identity!r}", + ) if certified.sha256 is not None and artifact.sha256 != certified.sha256: refuse(certified.dataset, "certified SHA-256 pin differs") if default not in country.datasets and ( @@ -600,18 +613,18 @@ def refuse(dataset: str, detail: str) -> NoReturn: reference = country.datasets.get(name) if reference is None: continue - expected_uri = build_hf_uri( - repo_id=reference.repo_id or country.data_package.repo_id, - path_in_repo=reference.path, - revision=reference.revision or _artifact_revision(country.data_package), + expected_identity = ( + reference.repo_id or country.data_package.repo_id, + reference.path, + reference.revision or _artifact_revision(country.data_package), ) - actual_uri = build_hf_uri( - repo_id=artifact.repo_id, - path_in_repo=artifact_path_for_country_manifest(artifact, uri_parts), - revision=artifact.revision, + actual_identity = ( + artifact.repo_id, + artifact_path_for_country_manifest(artifact, uri_parts), + artifact.revision, ) - if actual_uri != expected_uri: - refuse(name, f"expected {expected_uri}, got {actual_uri}") + if actual_identity != expected_identity: + refuse(name, f"expected {expected_identity!r}, got {actual_identity!r}") if (artifact.repo_type or country.data_package.repo_type) != ( reference.repo_type or country.data_package.repo_type ): diff --git a/tests/test_local_data_manifest.py b/tests/test_local_data_manifest.py index 9b27e2938..b63cbdf14 100644 --- a/tests/test_local_data_manifest.py +++ b/tests/test_local_data_manifest.py @@ -556,6 +556,44 @@ def test_every_overlapping_managed_artifact_must_match( _certify() +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("record", ["certified", "managed"]) +def test_artifact_repository_and_path_are_distinct_identity_fields( + tmp_path, monkeypatch, transport, record +): + payload = _payload() + name = "test_dataset" if record == "certified" else "supplement" + payload["artifacts"][name] = { + "kind": "microdata", + "repo_id": "policyengine/test-data/nested", + "path": "test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + if record == "certified": + country["datasets"].clear() + country["certified_data_artifact"]["uri"] = ( + f"hf://policyengine/test-data/nested/test_dataset.h5@{'f' * 40}" + ) + else: + country["datasets"][name] = { + "path": "nested/test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + + _edit_bundle(trace_path, edit) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + # Concatenated URI strings agree, but the asserted repo/path do not. + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + + @pytest.mark.parametrize("kind", ["diagnostics", "evidence"]) @pytest.mark.parametrize("prefixed", [False, True]) def test_release_relative_paths_match_certification_normalization( From da2ae8fb8e97145093e35366686df99c1e38449e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 05:50:51 -0400 Subject: [PATCH 08/20] Record PR 704 review fixes and validation handoff --- .job/HANDOFF.md | 57 +++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md index d2a91e2b6..d0029679a 100644 --- a/.job/HANDOFF.md +++ b/.job/HANDOFF.md @@ -1,47 +1,44 @@ -# Local manifest fallback handoff +# PR #704 review-fix handoff - Draft PR: https://github.com/PolicyEngine/policyengine.py/pull/704 -- Issue: https://github.com/PolicyEngine/policyengine.py/issues/703 -- Branch: `budget-local-data-manifest`, based on `origin/main` `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. -- Reviewed implementation head: `78e09ed63c3805b8739bc9ea229a0d393c213e46` (this handoff is committed immediately afterwards). -- Commits: `0bb58755` local manifest/certification/tests; `78e09ed6` receipt/schema/docs/tests. +- Branch: `budget-local-data-manifest`; confirmed reviewed remote head `fd4e099ed3ff768041f3a537e6e38527c07b0bbf`. +- Validated implementation head SHA: `0d7bf1a641d61b8951f47ab12d8d4867f57df140`. +- The following commit updates only this handoff. Its exact final head is recorded in `.job/HEAD` and `.job/local-data-manifest.bundle`. -## Design +## Fixes -`POLICYENGINE__DATA_MANIFEST_FILE` always selects the local file when present. This avoids outage latency and gives deterministic behavior. Unset keeps the original cached network fetch and certification results. - -The pin is the raw-byte `trov:sha256` for `composition/1/artifact/data_release_manifest` in the installed country TRACE TRO. Its arrangement must name the installed release's manifest URI. Missing, malformed, duplicate, or stale pins refuse. The current installed UK pin was checked as `49f9fa820db2e6544ff5cdceeb2d868a504843a6a1c1c06b37188621e700066f`. No environment digest is accepted. Local bytes are re-read and hashed on every call, before JSON parsing/revision rewriting, so cached remote/local loads cannot bypass verification. Invalid paths and digest errors fail closed with digest context. - -Local bytes run through the unchanged certification acceptance logic. All four successful runtime bases gain `_local_manifest`; the claim-based set includes the corresponding local variants. Exact-pair model/core/build restrictions and fingerprint acceptance remain the same. TRACE accepts the basis as a string. Packaged receipts recheck local certification and record it in optional `runtime.data_certification_basis`, keeping the published bundle certificate basis and both digest domains unchanged. Unset receipts omit the new annotation. The schema is regenerated and checked. Caller-supplied bundles do not claim packaged provenance. +- Scan every TRO definition of the manifest artifact ID, including direct graph nodes, nested definitions and artifact references; require one distinct valid SHA-256 assertion. Preserve canonical inline/location checks. +- Validate both local and fetched manifest artifacts after revision rewriting. The certified/default artifact must be present and anchored; every overlapping managed artifact must agree with bundled repo/path/revision, repository type and pinned SHA-256. Compare identity fields separately to prevent ambiguous URI concatenation. Primary manifests may omit separately pinned regional artifacts and overlays. +- Share the existing release-relative diagnostics/evidence normalization with certification. Exact-pair model/core/build and fingerprint acceptance logic remains unchanged; unset fetch/cache/offline behavior is preserved for consistent bundles. +- Replace function attributes with a typed callable cache facade. +- Make the documented export conditional on successful download/verification/atomic replacement; remove release coordination from permanent public docs. +- PR body records the invariants and both downstream actions. No other repository was edited. ## Validation -- 38 local-manifest tests passed, including 80 Hypothesis byte mutations and 10 direct main-snapshot differential scenarios. -- 146 existing certification tests passed. -- 96 bundle/bootstrap/tooling tests passed. -- Broad provenance suite: 2,337 passed, 6 skipped, including all 6 new receipt tests. -- Wheel/package-pin tests: 96 passed. -- Total across these final focused runs: **2,713 passed, 6 skipped**. Skips cover private data, Python/model-version-specific cases, and disabled top-level country imports in the broad run. -- `make format`, `make lint`, `git diff --check`, and `scripts/generate_receipt_schemas.py --check` passed. -- Every pytest invocation checked load1 <=110 and `os.statvfs` available bytes >=55,000,000,000. Observed load was below 37 and free disk above 132 GB. No pytest workers were used. -- Local fixtures are synthetic; no private manifest or microdata is committed. +**3,322 unique cases passed; five skipped.** -Detailed parent-run logs are local files under `.job/*tests.log`. +- Initial local/TRO-pin run: 88 passed, including two 80-example Hypothesis properties (byte mutation and duplicate placement). +- Broad 56-module provenance/certification/disclosure run: 3,256 passed, five skipped, one failure from the old missing-default error-message assertion. It now refuses earlier in the shared validator; the corrected complete TRO regeneration module passed all 40 cases on rerun. +- Release/bootstrap-focused run: 189 passed, including 59 cases outside the broad selection and two new default-anchor cases. +- Final affected manifest/certification run: 280 passed, including all 82 local-manifest cases, all 12 TRO-pin cases and four new repo/path-boundary cases. No failures remain in pytest's failure cache. +- `make format`, `make lint`, focused `mypy --follow-imports=silent src/policyengine/provenance/manifest.py`, `git diff --check`, and generated receipt schema checks passed. Documentation shell block passes `bash -n`. +- Every pytest invocation checked load1 <=110 and available disk >=55,000,000,000 bytes; no pytest workers. Observed load1 9.04–19.02; available disk 144.3–146.9 GB decimal. Tests used Python 3.13.9. +- Detailed logs and selected modules remain under `.job/`. No private manifests or microdata were committed. -## Coordination and remaining work +## Downstream actions -- Leave PR draft; the hub reviews and merges it. -- Ship in the same policyengine.py release as María's weekend Microcosm cut certification. No versions or country/data pins were changed. `changelog.d/703.added.md` follows the release workflow. -- The G0 morning fetch/verify/atomic-refresh instructions are in `docs/bundles.md`. Refresh the local manifest after adopting the newly certified release. -- Dashboard follow-up: refresh `tests/fixtures/w4/execution_receipt.v1.schema.json` in `PolicyEngine/uk-autumn-budget-dashboard-2026` when upgrading; its receipt adapter has no basis allowlist, but its schema equality test requires the new schema. See `docs/receipts.md`. +- Dashboard: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` when upgrading. +- Scorecard: `pipeline/uk_replay_variants.py:730` retains the setting in offline isolation, while `:750` requires exact runtime bundle equality. Clear `POLICYENGINE_*_DATA_MANIFEST_FILE` during isolation. +- Leave the PR draft; do not merge. No versions, country pins, data or certified releases changed. -## Workspace Git metadata +## Workspace Git metadata and delivery -The assigned worktree's protected `.git` points into the caller's repository, where metadata writes are sandbox-blocked. All work and independent Git metadata stayed in this workspace. To inspect the actual branch here: +The protected workspace `.git` points into the caller's repository. All mutable Git metadata is instead inside this workspace: ```bash GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git status -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -3 --oneline +GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -4 --oneline ``` -The branch is pushed to the canonical repository. A portable `.job/local-data-manifest.bundle` is also left for harvesting; it contains `budget-local-data-manifest` relative to the main base. The original protected `.git` pointer was not changed. No caller files were written. +The original pointer and caller checkout were not changed. `.job/local-data-manifest.bundle` is portable relative to main `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. The single final push is attempted after committing this handoff; its outcome is recorded in `.job/PUSH_RESULT.txt`. `.job/HEAD` contains the final branch SHA. No history was rewritten and nothing was pushed to main or master. From 80f54cadd6c35229d14ec1d3b72878553d58941b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:23:28 -0400 Subject: [PATCH 09/20] Test bound regional mismatch input evidence and reversal --- tests/test_regional_input_evidence.py | 237 ++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 tests/test_regional_input_evidence.py diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py new file mode 100644 index 000000000..d0f6a31b3 --- /dev/null +++ b/tests/test_regional_input_evidence.py @@ -0,0 +1,237 @@ +"""A regional model exception requires an audit of the exact dense bytes.""" + +import copy +import hashlib +import json + +import pytest + +from policyengine.provenance.certification import CertificationError +from policyengine.provenance.manifest import CountryReleaseManifest +from tests.test_certify_data_release import ( + UK_DENSE_URI, + UK_MODEL_VERSION, + _certify_uk, + _microcosm_uk_dense_payload, + _microcosm_uk_national_payload, + _response, + _tmp_bundle, +) + +REASON = "Budget accepts the audited dense model gap; area runs remain uncertified" +VAT_LIMIT = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) +DEFAULTED_INPUTS = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", +] + + +def older_dense() -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + +def input_evidence(dense: dict | None = None) -> dict: + dense = older_dense() if dense is None else dense + name = "microcosm_uk_2024_25_dense" + artifact = dense["artifacts"][name] + return { + "schema_version": 1, + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": hashlib.sha256( + json.dumps(dense).encode() + ).hexdigest(), + "model_package": "policyengine-uk", + "model_version": UK_MODEL_VERSION, + "artifacts": { + name: { + key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") + } + | {"columns": ["employment_income"]} + }, + "model_variables": ["employment_income", *DEFAULTED_INPUTS], + "model_inputs": ["employment_income", *DEFAULTED_INPUTS], + "dropped_columns": [], + "defaulted_inputs": DEFAULTED_INPUTS.copy(), + "audited_by": "Budget regional input-surface review", + "limitations": [VAT_LIMIT], + } + + +def certify_with_evidence(bundle_path, evidence: dict, dense: dict | None = None): + dense = older_dense() if dense is None else dense + return _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + +def test_accepted_mismatch_without_audited_evidence_refuses(tmp_path): + with pytest.raises(CertificationError, match="[Ee]vidence|[Aa]udit"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + ) + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("regional_manifest_uri", UK_DENSE_URI.replace("dense", "other-dense")), + ("regional_manifest_sha256", "0" * 64), + ("model_package", "policyengine-us"), + ("model_version", "2.147.1"), + ], +) +def test_evidence_for_another_cut_bytes_or_model_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence[field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("repo_id", "policyengine/another-dense"), + ("path", "different_dense.h5"), + ("revision", "different-cut"), + ("sha256", "0" * 64), + ], +) +def test_evidence_for_other_artifact_bytes_or_identity_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"][field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( + "removed_model_variable" + ) + evidence["dropped_columns"] = ["removed_model_variable"] + with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize("field", ["dropped_columns", "defaulted_inputs"]) +def test_the_declared_comparison_must_match_the_measured_surfaces(tmp_path, field): + evidence = input_evidence() + evidence[field] = ["invented_variable"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_missing_artifact_measurement_refuses(tmp_path): + evidence = input_evidence() + evidence["artifacts"] = {} + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + evidence = input_evidence() + certify_with_evidence(bundle_path, evidence) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + record = manifest.regional_certification.model_dump(mode="json") + assert record["defaulted_inputs"] == DEFAULTED_INPUTS + assert record["dropped_columns"] == [] + assert record["limitations"] == [VAT_LIMIT] + assert record["evidence"] == evidence + assert record["area_receipts"] == "uncertified" + assert record["accepted_reason"] == REASON + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + rebuilt = CountryReleaseManifest.model_validate_json(manifest.model_dump_json()) + assert rebuilt.regional_certification == manifest.regional_certification + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_evidence_digest_binds_original_manifest_bytes(tmp_path): + dense = older_dense() + response = _response(dense) + response.content = json.dumps(dense, indent=2).encode() + evidence = input_evidence(dense) + with pytest.raises(CertificationError): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + evidence["regional_manifest_sha256"] = hashlib.sha256(response.content).hexdigest() + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + + +def test_drop_regional_release_removes_acceptance_and_preserves_national_cert(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text())["data_releases"]["uk"] + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + ) + after = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert after["certification"] == before["certification"] + assert "regional_certification" not in after + assert "regional_source_manifest_sha256" not in after + assert "regional_source_manifest_uri" not in after + assert "regional_release_kind" not in after + assert "microcosm_uk_2024_25_dense" not in after["datasets"] + assert set(after["region_datasets"]) == {"national"} + + +def test_limitations_and_evidence_are_in_hashed_bundle_material(tmp_path): + from policyengine.provenance.canonical import canonical_sha256 + + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text()) + after = copy.deepcopy(before) + after["data_releases"]["uk"]["regional_certification"]["limitations"].append( + "An additional reviewed regional limitation" + ) + + def digest(bundle): + manifest = CountryReleaseManifest.model_validate(bundle["data_releases"]["uk"]) + return canonical_sha256(manifest.model_dump(mode="json")) + + assert digest(before) != digest(after) From 17c68b4d8f5774fd8936bf822bf680c812e1a6ec Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:27:23 -0400 Subject: [PATCH 10/20] Document audited UK regional mismatch acceptance --- changelog.d/699.fixed.md | 1 + docs/engineering/skills/data-certification.md | 70 +++++++++++++++---- 2 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 changelog.d/699.fixed.md diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md new file mode 100644 index 000000000..55eb27701 --- /dev/null +++ b/changelog.d/699.fixed.md @@ -0,0 +1 @@ +Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 1ce62d882..91c2dc2ec 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -81,28 +81,68 @@ Microcosm national release with one of: - **An older local-area release, on an accepted mismatch.** When the national cut moves to a new policyengine-uk before a local-area cut on that version - exists, a reviewer can keep the current local-area release routed instead of - dropping area regions: + exists, a reviewer can accept retaining the older regional route. A reason + and a byte-bound input-surface audit are both required. First audit the + actual local H5 with the exact target model installed: ```bash - python scripts/bundle.py certify-data --country uk --data-producer populace \ + .venv/bin/python scripts/audit_regional_input_surface.py \ + --regional-manifest-uri "" \ + --artifact "microcosm_uk_2024_25_dense=/local/dense.h5" \ + --model-version 2.147.2 \ + --audited-by "" \ + --limitation "the dense cut's VAT predates the policyengine-uk VAT factor fix (#2187, 2.135.7); the dashboard holds a VAT limit" \ + --output regional-input-surface.json + + .venv/bin/python scripts/bundle.py certify-data --country uk --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "" \ --accept-regional-model-mismatch "" \ - --model-version "" + --regional-input-surface-evidence regional-input-surface.json \ + --model-version 2.147.2 ``` - Certification then records `regional_certification` in the bundle: - `compatibility_basis: accepted_model_mismatch`, both model versions, the - reason and `area_receipts: uncertified`. Area receipts are uncertified in any - case, because the certificate covers the national dataset only (#685). - The cost is real and goes in the pull request. The local-area file was built - and calibrated under the older model. policyengine-core ignores, with a - warning, the columns the newer model no longer defines, and inputs the newer - model added take their defaults in area runs. The flag is UK only. It refuses - an empty reason, a regional release that already covers the version (so no - certificate records a mismatch that is not there) and a run without a - regional release. The core must still be covered. + The comparison is ported from Microcosm #1348's + `tools/certify_uk_cut_in_policyengine_py.py`, and strengthened to inspect the + persisted entity-table columns, including each period and country-loader + aliases. Comparing the old model's input inventory alone cannot prove which + columns the file carries. The audit binds the regional release identity, + original manifest bytes, actual H5 bytes and exact target model version. + Certification refuses missing or mismatched evidence and **any dropped + column**. It records every defaulted input and the free-text limitations in + the validated `regional_certification`; the bundle hash binds that record. + + The national certificate never widens. Area status remains `uncertified`: + the certificate still covers the national dataset only (#685). TRACE and + execution receipts expose the regional identity, build model, acceptance, + reason, input-surface result and limitations separately from the national + certification. The flag remains UK only; an empty reason, a release that + already covers the target model, or a missing regional release refuses. + The regional core checks still apply. Re-certifying with + `--drop-regional-release` removes the regional record, routes and artifact. + + For the Budget route, the measured **model-inventory comparison** from + 2.132.6 to 2.147.2 has no dropped names and these 16 new pure inputs: + `additional_residential_property_purchased`, `defined_benefit_pension_share`, + `directly_held_shares`, `early_years_school_start_deferred`, + `is_full_year_category_a_director`, `local_bus_uncapped_single_fare`, + `non_residential_property_purchased`, `non_residential_property_secured_debt`, + `other_residential_property_secured_debt`, + `pension_annual_allowance_taxable_lump_sum_death_benefits`, + `pension_contributions_via_salary_sacrifice_pre_2015`, `pip_dl_points`, + `pip_m_points`, `salary_sacrifice_returned_to_income`, + `unit_and_investment_trusts`, and `war_widows_pension_reported`. + Both `salary_sacrifice_returned_to_income` and + `local_bus_uncapped_single_fare` default in that comparison. This is not an + audit of cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`: + its private H5 was unavailable to this implementation job without + authentication. Certification must use the complete list measured from the + actual cut by the audit above, including any further missing stored inputs. + Record the pre-fix VAT limitation shown above for this dense cut. + + After the receipt schema changes, refresh the Budget dashboard's vendored + `tests/fixtures/w4/execution_receipt.v1.schema.json` from + `src/policyengine/data/schemas/execution_receipt.v1.schema.json`. Either way, certification checks that the regional release serves the pinned model by its own build or compatibility claims (a `--qualified-pair-id` names From 451f106c798f30462243d453e2dce64be920c589 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:28:08 -0400 Subject: [PATCH 11/20] Test measurement of regional H5 inputs against the exact UK model --- tests/test_measure_regional_inputs.py | 345 ++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 tests/test_measure_regional_inputs.py diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py new file mode 100644 index 000000000..8cf4006af --- /dev/null +++ b/tests/test_measure_regional_inputs.py @@ -0,0 +1,345 @@ +"""Measure the actual dense H5 surface, never an older model's inventory.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from policyengine.provenance.manifest import RegionalInputArtifactEvidence + +MODEL_VERSION = "2.147.2" +URI = ( + "hf://dataset/policyengine/regional@dense-cut/releases/dense/release_manifest.json" +) +LIMITATION = "Reviewed dense VAT limitation; dashboard holds a VAT limit" +DEFAULTS = ["local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income"] + + +class FakeSimulation: + def __init__(self): + self.move_values("employment_income", "employment_income_before_lsr") + + def move_values(self, source, target): + pass + + +class FakeVariable: + def __init__(self, is_input): + self.is_input = is_input + + def is_input_variable(self): + return self.is_input + + +class FakeCountryTaxBenefitSystem: + def __init__(self): + inputs = { + "employment_income_before_lsr", + "household_weight", + "benunit_id", + *DEFAULTS, + } + self.variables = { + name: FakeVariable(name in inputs) + for name in sorted(inputs | {"employment_income"}) + } + + +def _module(): + return importlib.import_module("policyengine.provenance.regional_input_surface") + + +def _h5(path, *, periods=None, fmt="fixed", missing=None): + with pd.HDFStore(path, mode="w") as store: + if periods is None: + store.put("time_period", pd.Series([2025])) + periods = {None: True} + for period, has_income in periods.items(): + for entity, columns in { + "person": {"employment_income": [1.0]} if has_income else {}, + "benunit": {"benunit_id": [0]}, + "household": {"household_weight": [1.0]}, + }.items(): + if (period, entity) == missing: + continue + key = entity if period is None else f"{entity}/{period}" + if not columns: + columns = {"benunit_id": [0]} + store.put(key, pd.DataFrame(columns), format=fmt) + return path + + +def _response(artifact_paths): + payload = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "1"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "artifacts": { + name: { + "kind": "microdata", + "path": f"{name}.h5", + "repo_id": "policyengine/regional", + "revision": "dense-cut", + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for name, path in artifact_paths.items() + }, + } + response = MagicMock(status_code=200) + response.content = json.dumps(payload, indent=2).encode() + return response + + +@pytest.fixture +def model(monkeypatch): + module = _module() + monkeypatch.setattr(module, "version", lambda name: MODEL_VERSION) + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace( + CountryTaxBenefitSystem=FakeCountryTaxBenefitSystem, + Simulation=FakeSimulation, + ), + ) + return module + + +def _measure(module, paths, response=None, **kwargs): + response = _response(paths) if response is None else response + with patch( + "policyengine.provenance.certification.requests.get", return_value=response + ): + return module.measure_regional_input_surface( + URI, + paths, + MODEL_VERSION, + "Budget operator", + [LIMITATION], + **kwargs, + ) + + +@pytest.mark.parametrize("fmt", ["fixed", "table"]) +def test_actual_h5_columns_and_aliases_produce_bound_audit(tmp_path, model, fmt): + paths = {"dense": _h5(tmp_path / "dense.h5", fmt=fmt)} + response = _response(paths) + result = _measure(model, paths, response) + assert result.model_version == MODEL_VERSION + assert result.model_package == "policyengine-uk" + assert result.regional_manifest_uri == URI + assert ( + result.regional_manifest_sha256 == hashlib.sha256(response.content).hexdigest() + ) + artifact = result.artifacts["dense"] + assert artifact.sha256 == hashlib.sha256(paths["dense"].read_bytes()).hexdigest() + assert artifact.repo_type == "dataset" + assert artifact.columns == ["benunit_id", "employment_income", "household_weight"] + assert artifact.columns_by_period == {"2025": artifact.columns} + assert result.input_aliases == {"employment_income": "employment_income_before_lsr"} + assert result.defaulted_inputs == DEFAULTS + assert result.dropped_columns == [] + assert result.limitations == [LIMITATION] + assert result.audited_by == "Budget operator" + + +def test_a_default_in_one_period_is_not_hidden_by_another_period(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5", periods={"2025": True, "2026": False})} + result = _measure(model, paths) + assert result.defaulted_inputs == [ + "employment_income_before_lsr", + *DEFAULTS, + ] + assert "employment_income" in result.artifacts["dense"].columns_by_period["2025"] + assert ( + "employment_income" not in result.artifacts["dense"].columns_by_period["2026"] + ) + + +@pytest.mark.parametrize("entity", ["person", "benunit", "household"]) +def test_each_period_requires_every_entity(tmp_path, model, entity): + paths = { + "dense": _h5( + tmp_path / "dense.h5", + periods={"2025": True, "2026": True}, + missing=("2026", entity), + ) + } + with pytest.raises(ValueError, match="2026|entity"): + _measure(model, paths) + + +def test_single_year_layout_requires_a_time_period(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.remove("time_period") + with pytest.raises(ValueError, match="time_period|period"): + _measure(model, {"dense": path}) + + +def test_extra_metadata_is_not_mistaken_for_a_model_column(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("metadata", pd.Series(["producer build notes"])) + assert _measure(model, {"dense": path}).dropped_columns == [] + + +def test_an_actual_dropped_column_blocks_audit(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("person", pd.DataFrame({"removed_column": [1]})) + with pytest.raises(ValueError, match="[Dd]ropped|removed_column"): + _measure(model, {"dense": path}) + + +def test_installed_model_must_be_the_requested_exact_version( + tmp_path, model, monkeypatch +): + monkeypatch.setattr(model, "version", lambda name: "2.146.4") + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="2.147.2|2.146.4"): + _measure(model, paths) + + +def test_artifact_bytes_must_match_the_manifest(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + payload["artifacts"]["dense"]["sha256"] = "0" * 64 + response.content = json.dumps(payload).encode() + with pytest.raises(ValueError, match="sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +def test_file_changing_during_metadata_read_refuses(tmp_path, model, monkeypatch): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + original = pd.HDFStore.select + mutated = False + + def select_and_modify(store, *args, **kwargs): + nonlocal mutated + result = original(store, *args, **kwargs) + if not mutated and args[0] != "/time_period": + with paths["dense"].open("ab") as file: + file.write(b"changed while inspecting metadata") + mutated = True + return result + + monkeypatch.setattr(pd.HDFStore, "select", select_and_modify) + with pytest.raises(ValueError, match="chang|sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +@pytest.mark.parametrize("names", [[], ["other"], ["dense", "other"]]) +def test_every_routed_h5_requires_exactly_one_supplied_path(tmp_path, model, names): + path = _h5(tmp_path / "dense.h5") + response = _response({"dense": path}) + with pytest.raises(ValueError, match="artifact|[Hh]5"): + _measure(model, {name: path for name in names}, response) + + +def test_comparison_uses_each_artifact_and_every_period(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["income", "weight"], + columns_by_period={"2025": ["income", "weight"], "2026": ["weight"]}, + ) + result = module.compare_input_surface( + {"dense": artifact}, + ["income", "income_reported", "weight", "new_input"], + ["income_reported", "weight", "new_input"], + {"income": "income_reported"}, + ) + assert result == { + "dropped_columns": [], + "defaulted_inputs": ["income_reported", "new_input"], + } + + +def test_comparison_refuses_a_period_union_that_hides_columns(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["weight"], + columns_by_period={"2025": ["weight", "removed"]}, + ) + with pytest.raises(ValueError, match="columns|union|period"): + module.compare_input_surface({"dense": artifact}, ["weight"], ["weight"]) + + +def test_conditional_alias_cannot_silently_claim_a_supplied_input( + tmp_path, model, monkeypatch +): + class ConditionalSimulation: + def __init__(self): + if self.condition: + self.move_values("employment_income", "employment_income_before_lsr") + + monkeypatch.setattr( + sys.modules["policyengine_uk"], "Simulation", ConditionalSimulation + ) + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="alias|move_values|conditional"): + _measure(model, paths) + + +def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch): + script_path = ( + Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + ) + spec = importlib.util.spec_from_file_location( + "audit_regional_input_surface", script_path + ) + script = importlib.util.module_from_spec(spec) + spec.loader.exec_module(script) + path = _h5(tmp_path / "dense.h5") + result = _measure(model, {"dense": path}) + captured = {} + + def measure(**kwargs): + captured.update(kwargs) + return result + + monkeypatch.setattr(script, "measure_regional_input_surface", measure) + output = tmp_path / "evidence.json" + assert ( + script.main( + [ + "--regional-manifest-uri", + URI, + "--artifact", + f"dense={path}", + "--model-version", + MODEL_VERSION, + "--audited-by", + "Budget operator", + "--limitation", + LIMITATION, + "--output", + str(output), + ] + ) + == 0 + ) + assert captured["artifact_paths"] == {"dense": path} + assert captured["limitations"] == [LIMITATION] + assert json.loads(output.read_text()) == result.model_dump(mode="json") From 97ce735ebfffd146bea5182ed4fa06a56ff34c95 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:28:17 -0400 Subject: [PATCH 12/20] Expose audited regional acceptance in TRACE and execution receipts --- .../schemas/execution_receipt.v1.schema.json | 287 +++++++++++++++++- src/policyengine/provenance/receipt.py | 78 ++++- src/policyengine/provenance/trace.py | 6 + tests/test_execution_receipt.py | 162 ++++++++++ 4 files changed, 522 insertions(+), 11 deletions(-) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index d10d1955b..e8a449b87 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -457,6 +457,279 @@ "title": "ReformIdentity", "type": "object" }, + "RegionalCertification": { + "description": "A reviewer's acceptance of a regional release on another model version.\n\nWritten only by ``certify-data --accept-regional-model-mismatch``: the\nregional release neither was built with nor claims the certified model,\nand area runs on it are uncertified.", + "properties": { + "accepted_reason": { + "title": "Accepted Reason", + "type": "string" + }, + "area_receipts": { + "const": "uncertified", + "default": "uncertified", + "title": "Area Receipts", + "type": "string" + }, + "built_with_model_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Built With Model Version" + }, + "certified_for_model_version": { + "title": "Certified For Model Version", + "type": "string" + }, + "compatibility_basis": { + "const": "accepted_model_mismatch", + "title": "Compatibility Basis", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "evidence": { + "$ref": "#/$defs/RegionalInputSurfaceEvidence" + }, + "limitations": { + "items": { + "type": "string" + }, + "title": "Limitations", + "type": "array" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "title": "Regional Manifest Uri", + "type": "string" + } + }, + "required": [ + "compatibility_basis", + "certified_for_model_version", + "accepted_reason", + "regional_manifest_uri", + "regional_manifest_sha256", + "defaulted_inputs", + "dropped_columns", + "limitations", + "evidence" + ], + "title": "RegionalCertification", + "type": "object" + }, + "RegionalInputArtifactEvidence": { + "additionalProperties": false, + "description": "Measured columns of one byte-verified regional H5 artifact.", + "properties": { + "columns": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Columns", + "type": "array" + }, + "columns_by_period": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Columns By Period" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "repo_id": { + "minLength": 1, + "title": "Repo Id", + "type": "string" + }, + "repo_type": { + "anyOf": [ + { + "enum": [ + "model", + "dataset" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repo Type" + }, + "revision": { + "minLength": 1, + "title": "Revision", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "repo_id", + "path", + "revision", + "sha256", + "columns" + ], + "title": "RegionalInputArtifactEvidence", + "type": "object" + }, + "RegionalInputSurfaceEvidence": { + "additionalProperties": false, + "description": "An operator's audit of actual regional columns against an exact model.\n\nSurfaces are retained so certification recomputes the comparison rather\nthan trusting a declared zero-loss result. The artifact hashes identify\nthe bytes inspected by the auditor, not merely a model's possible inputs.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/$defs/RegionalInputArtifactEvidence" + }, + "minProperties": 1, + "title": "Artifacts", + "type": "object" + }, + "audited_by": { + "minLength": 1, + "pattern": "\\S", + "title": "Audited By", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "input_aliases": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Aliases" + }, + "limitations": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "model_inputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Inputs", + "type": "array" + }, + "model_package": { + "minLength": 1, + "title": "Model Package", + "type": "string" + }, + "model_variables": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Variables", + "type": "array" + }, + "model_version": { + "minLength": 1, + "title": "Model Version", + "type": "string" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "minLength": 1, + "title": "Regional Manifest Uri", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "regional_manifest_uri", + "regional_manifest_sha256", + "model_package", + "model_version", + "artifacts", + "model_variables", + "model_inputs", + "dropped_columns", + "defaulted_inputs", + "audited_by", + "limitations" + ], + "title": "RegionalInputSurfaceEvidence", + "type": "object" + }, "RuntimeContext": { "additionalProperties": false, "description": "Where the execution ran. Part of no digest.", @@ -655,6 +928,18 @@ "reform": { "$ref": "#/$defs/ReformIdentity" }, + "regional_certification": { + "anyOf": [ + { + "$ref": "#/$defs/RegionalCertification" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The bundle's accepted regional model mismatch, including its exact regional manifest and artifact identity, measured input surface, acceptance reason and limitations. This records an exception for uncertified area runs and does not widen the national certificate." + }, "runtime": { "anyOf": [ { @@ -711,7 +996,7 @@ "x-policyengine-digests": { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "outputs_sha256": "the canonical output payload the receipt binds", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself" diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 1c7b43e04..2492e706f 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -133,6 +133,7 @@ drop_null_members, ) from policyengine.provenance.loaded import loaded_versions +from policyengine.provenance.manifest import RegionalCertification from policyengine.utils.hashing import sha256_file __all__ = [ @@ -223,7 +224,7 @@ #: How each digest is computed. DIGEST_RULES: dict[str, str] = { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", "reform.reform_sha256": 'RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {"$nonfinite": "Infinity"} or {"$nonfinite": "-Infinity"}, and a payload may not use the key $nonfinite itself', @@ -529,6 +530,15 @@ class ExecutionReceipt(_Contract): data: DataIdentity certification_status: Literal["certified", "uncertified"] certificate: Optional[CertificateIdentity] = None + regional_certification: Optional[RegionalCertification] = Field( + default=None, + description=( + "The bundle's accepted regional model mismatch, including its exact " + "regional manifest and artifact identity, measured input surface, " + "acceptance reason and limitations. This records an exception for " + "uncertified area runs and does not widen the national certificate." + ), + ) reform: ReformIdentity baseline: ReformIdentity fiscal_year: FiscalYearConvention @@ -549,6 +559,13 @@ class ExecutionReceipt(_Contract): ) runtime: Optional[RuntimeContext] = None + @model_serializer(mode="wrap") + def _omit_absent_regional_certification(self, handler): + payload = handler(self) + if self.regional_certification is None: + payload.pop("regional_certification", None) + return payload + @field_validator("schema_version", mode="before") @classmethod def _schema_version_is_an_integer(cls, value: Any) -> Any: @@ -627,6 +644,10 @@ def execution_identity(self) -> dict[str, Any]: for key in CertificateIdentity.IDENTITY_FIELDS if key in payload } + if self.regional_certification is not None: + identity["regional_certification"] = self.regional_certification.model_dump( + mode="json", exclude_none=True + ) return identity @property @@ -3811,12 +3832,25 @@ def _data_identity( datasets = record.get("datasets") or {} entry = datasets.get(dataset_name) primary = datasets.get(record.get("default_dataset")) or {} - # A regional release's artifacts are certified beside the primary release - # but are not part of it, so only entries at the primary release's own - # revision take its build and manifest identity. - release = entry is not None and entry.get("revision") in ( - None, - primary.get("revision"), + # A regional release's artifacts keep their own build provenance, even + # when acceptance allows them to run beside another national model. + regional_certification = manifest.regional_certification + regional = regional_certification is not None and any( + artifact.repo_id == spec.repo_id + and artifact.path == spec.path + and artifact.revision == spec.revision + and artifact.sha256 == spec.sha256 + and artifact.repo_type in (None, spec.repo_type) + for artifact in regional_certification.evidence.artifacts.values() + ) + release = ( + not regional + and entry is not None + and entry.get("revision") + in ( + None, + primary.get("revision"), + ) ) data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} @@ -3832,9 +3866,13 @@ def _data_identity( build_fingerprint=certification.get("data_build_fingerprint") if release else None, - build_model_version=certification.get("built_with_model_version") - if release - else None, + build_model_version=( + regional_certification.built_with_model_version + if regional and regional_certification is not None + else certification.get("built_with_model_version") + if release + else None + ), release_manifest_revision=data_package.get("release_manifest_revision") if release else None, @@ -3946,6 +3984,7 @@ def identity(name: str) -> dict[str, Any]: data=data, certification_status=status, certificate=certificate, + regional_certification=record.get("regional_certification"), reform=reform, baseline=baseline, fiscal_year=uk_fiscal_year_convention(), @@ -3993,6 +4032,25 @@ def verify_certificate( if not isinstance(record, Mapping): return [f"the bundle has no {receipt.country_id} data release"] problems = [] + regional_record = record.get("regional_certification") + try: + expected_regional = ( + RegionalCertification.model_validate(regional_record).model_dump( + mode="json", exclude_none=True + ) + if regional_record is not None + else None + ) + except ValueError as exc: + problems.append(f"the bundle cannot determine regional_certification: {exc}") + else: + claimed_regional = ( + receipt.regional_certification.model_dump(mode="json", exclude_none=True) + if receipt.regional_certification is not None + else None + ) + if claimed_regional != expected_regional: + problems.append("regional_certification differs from the bundle's record") for package in (receipt.wrapper, receipt.model, receipt.core): if not _package_can_be_certified(package): problems.append( diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index bcffda78a..4a3128ceb 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -556,6 +556,12 @@ def _build_bundle_performance( performance["pe:dataBuildId"] = certification.data_build_id if certification.certified_by is not None: performance["pe:certifiedBy"] = certification.certified_by + if country_manifest.regional_certification is not None: + performance["pe:regionalCertification"] = ( + country_manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ) + ) performance.update(dict(emission_context or _emission_context())) return performance diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 66fe4d220..b7f03a3bc 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -433,6 +433,168 @@ def test_a_regional_release_dataset_keeps_its_own_identity(dataset_file): assert receipt.data.release_manifest_revision is None +def _bundle_with_regional_acceptance(dataset_file): + """A synthetic, audited area exception beside a national certificate.""" + + bundle = _bundle_for(dataset_file.read_bytes()) + record = bundle["data_releases"]["uk"] + artifact = { + "path": "area_file.h5", + "repo_id": record["data_package"]["repo_id"], + "revision": "regional-release-cut", + "sha256": hashlib.sha256(dataset_file.read_bytes()).hexdigest(), + } + record["datasets"]["area_file"] = artifact + manifest_uri = ( + "hf://dataset/policyengine/populace-uk-private@regional-release-cut/" + "releases/dense/release_manifest.json" + ) + defaulted = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", + ] + limitations = [ + "The dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit." + ] + record["regional_certification"] = { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": INSTALLED_UK, + "built_with_model_version": "2.132.6", + "accepted_reason": "UK hub accepted this measured input surface.", + "area_receipts": "uncertified", + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "defaulted_inputs": defaulted, + "dropped_columns": [], + "limitations": limitations, + "evidence": { + "model_package": "policyengine-uk", + "model_version": INSTALLED_UK, + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "artifacts": { + "area_file": { + **artifact, + "columns": ["employment_income", "household_weight"], + } + }, + "model_variables": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "model_inputs": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "dropped_columns": [], + "defaulted_inputs": defaulted, + "audited_by": "UK hub review", + "limitations": limitations, + }, + } + return bundle + + +@pytest.mark.parametrize("area", [False, True], ids=["national", "area"]) +def test_receipt_exposes_regional_acceptance_without_widening_certification( + dataset_file, schema_validator, area +): + from policyengine.provenance.manifest import RegionalCertification + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + + receipt = _build(dataset_file, bundle, dataset="area_file" if area else None) + + payload = json.loads(receipt.model_dump_json(exclude_none=True)) + assert payload["regional_certification"] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert receipt.certification_status == ("uncertified" if area else "certified") + assert (receipt.certificate is None) is area + assert receipt.data.build_model_version == ( + "2.132.6" if area else record["certification"]["built_with_model_version"] + ) + schema_validator.validate(payload) + restored = ExecutionReceipt.model_validate_json(receipt.model_dump_json()) + assert restored.regional_certification == receipt.regional_certification + assert restored.receipt_sha256 == receipt.receipt_sha256 + + +def test_regional_limitations_are_bound_to_receipt_identity(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle, dataset="area_file") + changed_bundle = copy.deepcopy(bundle) + record = changed_bundle["data_releases"]["uk"]["regional_certification"] + record["limitations"].append("Additional accepted area limitation.") + record["evidence"]["limitations"].append("Additional accepted area limitation.") + + changed = _build(dataset_file, changed_bundle, dataset="area_file") + + assert changed.execution_identity_sha256 != receipt.execution_identity_sha256 + assert changed.receipt_sha256 != receipt.receipt_sha256 + + +def test_certificate_verification_checks_the_exposed_regional_record(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle) + payload = receipt.model_dump(mode="json") + payload["regional_certification"]["accepted_reason"] = "A different acceptance." + changed = ExecutionReceipt.model_validate(payload) + + assert any( + "regional_certification" in issue + for issue in verify_certificate(changed, bundle) + ) + + +def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file): + from policyengine.provenance.manifest import ( + CountryReleaseManifest, + RegionalCertification, + ) + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + performance = tro["@graph"][0]["trov:hasPerformance"] + + assert performance[ + "pe:regionalCertification" + ] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert ( + performance["pe:compatibilityBasis"] + == record["certification"]["compatibility_basis"] + ) + assert performance["pe:regionalCertification"]["area_receipts"] == "uncertified" + + +def test_removing_regional_acceptance_removes_downstream_record(dataset_file): + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + del record["regional_certification"] + del record["datasets"]["area_file"] + + receipt = _build(dataset_file, bundle) + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + + assert "regional_certification" not in receipt.canonical_payload() + assert "pe:regionalCertification" not in tro["@graph"][0]["trov:hasPerformance"] + assert receipt.certification_status == "certified" + + def test_an_omitted_artifact_revision_is_the_release_manifest_revision(dataset_file): """Round-4 review M5: the receipt names what the materializer downloads.""" From 68cf966da7930f131acd792a6223bb136f589bba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:31:42 -0400 Subject: [PATCH 13/20] Measure byte-bound regional inputs against the installed UK model --- scripts/audit_regional_input_surface.py | 54 +++ .../provenance/regional_input_surface.py | 345 ++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 scripts/audit_regional_input_surface.py create mode 100644 src/policyengine/provenance/regional_input_surface.py diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py new file mode 100644 index 000000000..635a610da --- /dev/null +++ b/scripts/audit_regional_input_surface.py @@ -0,0 +1,54 @@ +"""Write a byte-bound audit of regional H5 columns for an installed UK model. + +Run in an environment with the exact requested policyengine-uk version and +PyTables installed. Supply the H5 files from the immutable regional manifest; +the audit verifies their hashes without installing models or downloading data. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from policyengine.provenance.regional_input_surface import ( # noqa: E402 + measure_regional_input_surface, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--regional-manifest-uri", required=True) + parser.add_argument( + "--artifact", required=True, action="append", metavar="NAME=PATH" + ) + parser.add_argument("--model-version", required=True) + parser.add_argument("--audited-by", required=True) + parser.add_argument("--limitation", required=True, action="append") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + paths: dict[str, Path] = {} + for entry in args.artifact: + name, separator, path = entry.partition("=") + if not separator or not name.strip() or not path.strip() or name in paths: + parser.error("Each --artifact must be a unique nonblank NAME=PATH.") + paths[name] = Path(path) + evidence = measure_regional_input_surface( + regional_manifest_uri=args.regional_manifest_uri, + artifact_paths=paths, + model_version=args.model_version, + audited_by=args.audited_by, + limitations=args.limitation, + token=os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN"), + ) + args.output.write_text(evidence.model_dump_json(indent=2) + "\n") + print(f"Saved {len(evidence.artifacts)} artifact input audits to {args.output}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py new file mode 100644 index 000000000..d156d64c0 --- /dev/null +++ b/src/policyengine/provenance/regional_input_surface.py @@ -0,0 +1,345 @@ +"""Audit regional H5 bytes against the exact installed UK country model. + +The set comparison ports Microcosm PR #1348's handoff comparison, replacing +its older-model inventory proxy with the columns actually stored in every +artifact and period. Country-model input moves are read from that installed +model's constructor, so an alias cannot make a supplied input look defaulted. +No household values are included in the resulting audit. +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import re +import textwrap +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from policyengine.provenance.manifest import ( + RegionalInputArtifactEvidence, + RegionalInputSurfaceEvidence, +) + +UK_ENTITIES = ("person", "benunit", "household") + + +class RegionalInputSurfaceError(ValueError): + """A complete, byte-bound regional surface cannot be established.""" + + +def _names(values: list[str], label: str) -> set[str]: + if not values or any( + not isinstance(value, str) or not value.strip() or value != value.strip() + for value in values + ): + raise RegionalInputSurfaceError(f"{label} must contain nonblank names.") + names = set(values) + if len(names) != len(values): + raise RegionalInputSurfaceError(f"{label} contains duplicate names.") + return names + + +def compare_input_surface( + artifacts: dict[str, RegionalInputArtifactEvidence], + model_variables: list[str], + model_inputs: list[str], + input_aliases: dict[str, str] | None = None, +) -> dict[str, list[str]]: + """Recompute losses and every default, including defaults in just one year. + + Inputs supplied by the model's unconditional ``move_values`` calls use the + destination name after the move. Raw columns still have to be variables + defined by the target model. Every artifact and period must pass: the + union of supplied columns cannot hide a missing input in another period. + """ + variables = _names(model_variables, "model_variables") + inputs = _names(model_inputs, "model_inputs") + if not inputs <= variables: + raise RegionalInputSurfaceError("model_inputs must be model_variables.") + aliases = input_aliases or {} + for donor, target in aliases.items(): + if ( + donor not in variables + or target not in inputs + or donor == target + or target in aliases + ): + raise RegionalInputSurfaceError( + f"Unsupported input alias {donor!r} -> {target!r}." + ) + if not artifacts: + raise RegionalInputSurfaceError("No regional artifact columns measured.") + dropped: set[str] = set() + defaulted: set[str] = set() + for name, artifact in artifacts.items(): + columns = _names(artifact.columns, f"{name} columns") + if not artifact.columns_by_period: + raise RegionalInputSurfaceError(f"{name} has no measured periods.") + periods = [] + for period, period_columns in artifact.columns_by_period.items(): + if not re.fullmatch(r"[0-9]{4}", period): + raise RegionalInputSurfaceError( + f"{name} has an invalid period {period!r}." + ) + periods.append(_names(period_columns, f"{name}/{period} columns")) + if set().union(*periods) != columns: + raise RegionalInputSurfaceError( + f"{name} columns must equal the union of columns_by_period." + ) + for supplied in periods: + dropped.update(supplied - variables) + effective = set(supplied) + for donor, target in aliases.items(): + if donor in effective: + effective.remove(donor) + effective.add(target) + defaulted.update(inputs - effective) + return { + "dropped_columns": sorted(dropped), + "defaulted_inputs": sorted(defaulted), + } + + +def _input_aliases(simulation_class: Any) -> dict[str, str]: + """Read unconditional literal moves; refuse unsupported dynamic semantics. + + policyengine-uk has no public alias inventory. Its public Simulation + constructor declares the input moves. Reading that source avoids running + a simulation or encoding one version's aliases as a permanent allowlist. + """ + try: + tree = ast.parse(textwrap.dedent(inspect.getsource(simulation_class.__init__))) + except (OSError, TypeError, SyntaxError) as exc: + raise RegionalInputSurfaceError( + "Cannot inspect the installed UK Simulation input aliases." + ) from exc + constructor = tree.body[0] + if not isinstance(constructor, (ast.FunctionDef, ast.AsyncFunctionDef)): + raise RegionalInputSurfaceError("Cannot locate the UK Simulation constructor.") + unconditional = { + id(statement.value) + for statement in constructor.body + if isinstance(statement, ast.Expr) + } + aliases: dict[str, str] = {} + for node in ast.walk(constructor): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "self" + and node.func.attr == "move_values" + ): + continue + if ( + id(node) not in unconditional + or len(node.args) != 2 + or node.keywords + or any( + not isinstance(argument, ast.Constant) + or not isinstance(argument.value, str) + for argument in node.args + ) + ): + raise RegionalInputSurfaceError( + "UK Simulation has a conditional or dynamic move_values input alias; " + "an explicit audit of its semantics is required." + ) + donor = str(node.args[0].value) # type: ignore[attr-defined] + target = str(node.args[1].value) # type: ignore[attr-defined] + if donor in aliases: + raise RegionalInputSurfaceError(f"Duplicate move_values alias {donor!r}.") + aliases[donor] = target + return aliases + + +def _model_surface(model_version: str) -> tuple[list[str], list[str], dict[str, str]]: + try: + installed = version("policyengine-uk") + except PackageNotFoundError as exc: + raise RegionalInputSurfaceError( + "Install the exact policyengine-uk target first." + ) from exc + if installed != model_version: + raise RegionalInputSurfaceError( + f"Installed policyengine-uk {installed} differs from requested {model_version}." + ) + # Keep country-model loading out of certification's evidence-validation path. + import policyengine_uk + + system = policyengine_uk.CountryTaxBenefitSystem() + variables = sorted(system.variables) + inputs = sorted( + name + for name, variable in system.variables.items() + if variable.is_input_variable() + ) + return variables, inputs, _input_aliases(policyengine_uk.Simulation) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _hdf_columns(path: Path) -> dict[str, list[str]]: + import pandas as pd + + try: + with pd.HDFStore(path, mode="r") as store: + keys = set(store.keys()) + flat = {f"/{entity}" for entity in UK_ENTITIES} & keys + nested = { + key + for key in keys + if any(key.startswith(f"/{entity}/") for entity in UK_ENTITIES) + } + if flat and nested: + raise RegionalInputSurfaceError( + "H5 mixes single-year and multi-year entity keys." + ) + entity_keys: dict[str, dict[str, str]] = {} + if flat: + if "/time_period" not in keys: + raise RegionalInputSurfaceError( + "Single-year H5 requires /time_period." + ) + metadata = store.select("/time_period") + values = metadata.to_numpy().ravel().tolist() + if len(values) != 1 or not re.fullmatch(r"[0-9]{4}", str(values[0])): + raise RegionalInputSurfaceError( + "H5 /time_period must name one year." + ) + entity_keys[str(values[0])] = { + entity: f"/{entity}" + for entity in UK_ENTITIES + if f"/{entity}" in flat + } + else: + for key in sorted(nested): + parts = key.strip("/").split("/") + if len(parts) != 2 or not re.fullmatch(r"[0-9]{4}", parts[1]): + raise RegionalInputSurfaceError( + f"Invalid H5 entity period key {key!r}." + ) + entity_keys.setdefault(parts[1], {})[parts[0]] = key + if not entity_keys: + raise RegionalInputSurfaceError("H5 has no UK entity periods.") + columns_by_period = {} + for period, entities in sorted(entity_keys.items()): + missing = set(UK_ENTITIES) - set(entities) + if missing: + raise RegionalInputSurfaceError( + f"H5 period {period} is missing entity tables: {sorted(missing)}." + ) + columns: set[str] = set() + for entity in UK_ENTITIES: + # This obtains the same column metadata for fixed and table + # stores without selecting household or person rows. + frame = store.select(entities[entity], start=0, stop=0) + columns.update( + _names(list(frame.columns), f"{entity}/{period} columns") + ) + columns_by_period[period] = sorted(columns) + return columns_by_period + except RegionalInputSurfaceError: + raise + except (OSError, KeyError, ValueError, ImportError) as exc: + raise RegionalInputSurfaceError( + f"Cannot read H5 entity metadata from {path.name}." + ) from exc + + +def measure_regional_input_surface( + regional_manifest_uri: str, + artifact_paths: dict[str, Path], + model_version: str, + audited_by: str, + limitations: list[str], + token: str | None = None, +) -> RegionalInputSurfaceEvidence: + """Create an audit from actual H5 columns and an exact installed UK model. + + The operator supplies local files named by the immutable regional release. + Every microdata H5 must be measured and match its manifest's SHA-256 both + before and after metadata inspection. This reads no unit values for output. + """ + # Imported lazily: the certification gate reuses compare_input_surface. + from policyengine.provenance.certification import fetch_release_manifest + + if ( + not audited_by.strip() + or not limitations + or any(not item.strip() for item in limitations) + ): + raise RegionalInputSurfaceError( + "An auditor and nonblank limitations are required." + ) + manifest, manifest_sha256, parts = fetch_release_manifest( + regional_manifest_uri, token=token + ) + expected = { + name: artifact + for name, artifact in manifest.artifacts.items() + if artifact.kind == "microdata" and Path(artifact.path).suffix.lower() == ".h5" + } + if not expected or set(artifact_paths) != set(expected): + raise RegionalInputSurfaceError( + "Supplied artifact paths must exactly cover the regional microdata H5 artifacts: " + f"{sorted(expected)}." + ) + variables, inputs, aliases = _model_surface(model_version) + artifacts = {} + for name, artifact in sorted(expected.items()): + path = Path(artifact_paths[name]) + if not artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} has no manifest sha256." + ) + before = _sha256_file(path) + if before != artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes differ from its manifest sha256." + ) + periods = _hdf_columns(path) + after = _sha256_file(path) + if after != before: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes changed during metadata inspection." + ) + artifacts[name] = RegionalInputArtifactEvidence( + repo_id=artifact.repo_id, + repo_type=artifact.repo_type or parts["repo_type"], + path=artifact.path, + revision=artifact.revision, + sha256=after, + columns=sorted( + {column for columns in periods.values() for column in columns} + ), + columns_by_period=periods, + ) + comparison = compare_input_surface(artifacts, variables, inputs, aliases) + if comparison["dropped_columns"]: + raise RegionalInputSurfaceError( + f"Dropped regional input columns refuse acceptance: {comparison['dropped_columns']}." + ) + return RegionalInputSurfaceEvidence( + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=manifest_sha256, + model_package="policyengine-uk", + model_version=model_version, + artifacts=artifacts, + model_variables=variables, + model_inputs=inputs, + input_aliases=aliases, + dropped_columns=comparison["dropped_columns"], + defaulted_inputs=comparison["defaulted_inputs"], + audited_by=audited_by, + limitations=limitations, + ) From 355fd34ba1f8f891ffdcc440b67a97e72871a5e4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:33:06 -0400 Subject: [PATCH 14/20] Preserve regional audit evidence through JSON-LD expansion --- .../schemas/execution_receipt.v1.schema.json | 26 +++++------ src/policyengine/provenance/trace.py | 11 +++-- tests/test_execution_receipt.py | 44 +++++++++++++++++-- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index e8a449b87..f2e467cc3 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -552,22 +552,15 @@ "type": "array" }, "columns_by_period": { - "anyOf": [ - { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "type": "object" + "additionalProperties": { + "items": { + "type": "string" }, - { - "type": "null" - } - ], - "default": null, - "title": "Columns By Period" + "type": "array" + }, + "minProperties": 1, + "title": "Columns By Period", + "type": "object" }, "path": { "minLength": 1, @@ -611,7 +604,8 @@ "path", "revision", "sha256", - "columns" + "columns", + "columns_by_period" ], "title": "RegionalInputArtifactEvidence", "type": "object" diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index 4a3128ceb..0e79b34e8 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -557,11 +557,14 @@ def _build_bundle_performance( if certification.certified_by is not None: performance["pe:certifiedBy"] = certification.certified_by if country_manifest.regional_certification is not None: - performance["pe:regionalCertification"] = ( - country_manifest.regional_certification.model_dump( + # An explicit JSON literal preserves evidence member names through + # JSON-LD expansion; they are JSON contract keys, not RDF predicates. + performance["pe:regionalCertification"] = { + "@type": "@json", + "@value": country_manifest.regional_certification.model_dump( mode="json", exclude_none=True - ) - ) + ), + } performance.update(dict(emission_context or _emission_context())) return performance diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index b7f03a3bc..6589bff47 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -477,6 +477,9 @@ def _bundle_with_regional_acceptance(dataset_file): "area_file": { **artifact, "columns": ["employment_income", "household_weight"], + "columns_by_period": { + "2025": ["employment_income", "household_weight"] + }, } }, "model_variables": sorted( @@ -562,8 +565,9 @@ def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file ) performance = tro["@graph"][0]["trov:hasPerformance"] - assert performance[ - "pe:regionalCertification" + assert performance["pe:regionalCertification"]["@type"] == "@json" + assert performance["pe:regionalCertification"][ + "@value" ] == RegionalCertification.model_validate( record["regional_certification"] ).model_dump(mode="json", exclude_none=True) @@ -571,7 +575,41 @@ def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file performance["pe:compatibilityBasis"] == record["certification"]["compatibility_basis"] ) - assert performance["pe:regionalCertification"]["area_receipts"] == "uncertified" + assert ( + performance["pe:regionalCertification"]["@value"]["area_receipts"] + == "uncertified" + ) + + +def test_trace_regional_acceptance_survives_jsonld_expansion(dataset_file): + rdflib = pytest.importorskip("rdflib") + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import ( + POLICYENGINE_TRACE_NAMESPACE, + build_trace_tro_from_release_bundle, + ) + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(record) + tro = build_trace_tro_from_release_bundle( + manifest, + None, + fetch_pypi=lambda name, version: {}, + ) + + graph = rdflib.Graph().parse(data=json.dumps(tro), format="json-ld") + values = list( + graph.objects( + None, rdflib.URIRef(POLICYENGINE_TRACE_NAMESPACE + "regionalCertification") + ) + ) + + assert len(values) == 1 + assert isinstance(values[0], rdflib.Literal) + assert values[0].datatype == rdflib.RDF.JSON + assert json.loads(str(values[0])) == manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ) def test_removing_regional_acceptance_removes_downstream_record(dataset_file): From 04e0450a32e16a156cb8c2e2fa8e725061367202 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:41:29 -0400 Subject: [PATCH 15/20] Require bound zero-loss evidence for UK regional model exceptions --- scripts/bundle.py | 8 + scripts/certify_data_release.py | 14 +- src/policyengine/provenance/certification.py | 161 +++++++++++++++++-- src/policyengine/provenance/manifest.py | 47 ++++++ tests/test_certify_data_release.py | 19 +++ tests/test_regional_input_evidence.py | 52 +++++- 6 files changed, 289 insertions(+), 12 deletions(-) diff --git a/scripts/bundle.py b/scripts/bundle.py index 5530a0a72..125635cd6 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -52,6 +52,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend( ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] ) + if args.regional_input_surface_evidence: + argv.extend( + ["--regional-input-surface-evidence", args.regional_input_surface_evidence] + ) if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -239,6 +243,10 @@ def _parser() -> argparse.ArgumentParser: "scripts/certify_data_release.py." ), ) + certify.add_argument( + "--regional-input-surface-evidence", + help="Audited regional input-surface JSON; required for an accepted mismatch.", + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 5c3bf6874..7cdc1a5a1 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -24,6 +24,7 @@ from __future__ import annotations import argparse +import json import os import sys from pathlib import Path @@ -121,10 +122,16 @@ def main(argv=None) -> int: help=( "UK only: route regions to a regional release that does not cover " "the pinned model, recording this reviewed reason in the bundle " - "instead of refusing. The core must still be covered." + "instead of refusing. Requires --regional-input-surface-evidence; " + "dropped columns refuse. The core must still be covered." ), ) parser.add_argument("--no-generate", action="store_true") + parser.add_argument( + "--regional-input-surface-evidence", + type=Path, + help="Audited JSON evidence bound to the regional bytes and exact target model.", + ) parser.add_argument("--no-changelog", action="store_true") parser.add_argument( "--skip-artifact-check", @@ -159,6 +166,11 @@ def main(argv=None) -> int: core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, regional_model_mismatch_reason=args.accept_regional_model_mismatch, + regional_input_surface_evidence=( + json.loads(args.regional_input_surface_evidence.read_text()) + if args.regional_input_surface_evidence is not None + else None + ), ) print(result.summary()) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index edae62391..e773ba900 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -47,6 +47,8 @@ from typing import Literal, Optional, cast, get_args import requests +from packaging.version import InvalidVersion, Version +from pydantic import ValidationError from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, @@ -57,6 +59,7 @@ HF_REQUEST_TIMEOUT_SECONDS, DataReleaseArtifact, DataReleaseManifest, + RegionalInputSurfaceEvidence, RegionalReleaseKind, _specifier_matches, fetch_pypi_wheel_metadata, @@ -416,6 +419,10 @@ def model_compatibility_basis( #: A regional release's model coverage accepted by a reviewer, not by the #: release's own build or claims (``--accept-regional-model-mismatch``). REGIONAL_BASIS_ACCEPTED_MISMATCH = "accepted_model_mismatch" +REGIONAL_VAT_LIMITATION = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) def accepted_regional_mismatch( @@ -423,6 +430,10 @@ def accepted_regional_mismatch( model_package: str, model_version: str, reason: str, + *, + regional_manifest_uri: str, + regional_manifest_sha256: str, + evidence: Optional[RegionalInputSurfaceEvidence], ) -> tuple[dict, str]: """Record a reviewed decision to route regions to a release on another model. @@ -438,6 +449,99 @@ def accepted_regional_mismatch( raise CertificationError( "--accept-regional-model-mismatch needs a reason a reviewer can read." ) + if evidence is None: + raise CertificationError( + "An accepted regional model mismatch requires audited input-surface " + "evidence (--regional-input-surface-evidence)." + ) + try: + audit = RegionalInputSurfaceEvidence.model_validate(evidence) + except ValidationError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.regional_manifest_uri != regional_manifest_uri + or audit.regional_manifest_sha256 != regional_manifest_sha256 + or audit.model_package != model_package + or audit.model_version != model_version + ): + raise CertificationError( + "Regional input-surface evidence must bind the exact regional cut, " + "raw manifest bytes and target model package/version." + ) + expected_artifacts = { + name: artifact + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "microdata" and artifact.path.endswith(".h5") + } + if not expected_artifacts or set(audit.artifacts) != set(expected_artifacts): + raise CertificationError( + "Evidence must measure every regional microdata artifact." + ) + for name, artifact in expected_artifacts.items(): + measured = audit.artifacts[name] + if any( + getattr(measured, key) != getattr(artifact, key) + for key in ("repo_id", "path", "revision", "sha256") + ) or ( + measured.repo_type is not None + and measured.repo_type + != ( + artifact.repo_type + or parse_manifest_uri(regional_manifest_uri)["repo_type"] + ) + ): + raise CertificationError( + f"Regional evidence artifact {name!r} does not match its identity/bytes." + ) + if not all(column.strip() for column in measured.columns): + raise CertificationError("Regional evidence has an empty column name.") + if measured.columns_by_period is not None: + if ( + not measured.columns_by_period + or any(not columns for columns in measured.columns_by_period.values()) + or set(measured.columns) + != { + column + for columns in measured.columns_by_period.values() + for column in columns + } + ): + raise CertificationError( + "Regional evidence has inconsistent period columns." + ) + if not set(audit.model_inputs).issubset(audit.model_variables): + raise CertificationError( + "Evidence model inputs must belong to its variable surface." + ) + if not all(item.strip() for item in audit.limitations): + raise CertificationError("Regional evidence limitations must not be blank.") + from policyengine.provenance.regional_input_surface import compare_input_surface + + try: + comparison = compare_input_surface( + audit.artifacts, + audit.model_variables, + audit.model_inputs, + audit.input_aliases, + ) + except ValueError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.dropped_columns != comparison["dropped_columns"] + or audit.defaulted_inputs != comparison["defaulted_inputs"] + ): + raise CertificationError( + "Regional evidence comparison disagrees with its measured surfaces." + ) + if comparison["dropped_columns"]: + raise CertificationError( + "Dropped regional input columns are refused: " + + ", ".join(comparison["dropped_columns"]) + ) built = ( regional_manifest.build.built_with_model_package if (regional_manifest.build is not None) @@ -446,19 +550,37 @@ def accepted_regional_mismatch( built_with = ( built.version if built is not None and built.name == model_package else None ) + limitations = list(audit.limitations) + try: + predates_vat_fix = built_with is not None and Version(built_with) < Version( + "2.135.7" + ) <= Version(model_version) + except InvalidVersion as exc: + raise CertificationError( + "Regional model versions must be valid versions." + ) from exc + if predates_vat_fix and REGIONAL_VAT_LIMITATION not in limitations: + limitations.append(REGIONAL_VAT_LIMITATION) record = { "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, "certified_for_model_version": model_version, "built_with_model_version": built_with, "accepted_reason": reason, "area_receipts": "uncertified", + "regional_manifest_uri": regional_manifest_uri, + "regional_manifest_sha256": regional_manifest_sha256, + "defaulted_inputs": comparison["defaulted_inputs"], + "dropped_columns": comparison["dropped_columns"], + "limitations": limitations, + "evidence": audit.model_dump(mode="json", exclude_none=True), } warning = ( f"regional release: built with {model_package} " f"{built_with or 'an unrecorded version'} and routed under " f"{model_version} on an accepted mismatch ({reason}); area runs are " - "uncertified, inputs the newer model added take their defaults there, " - "and columns it dropped are ignored on load" + f"uncertified; audited dropped columns: none; defaulted inputs: " + f"{', '.join(comparison['defaulted_inputs']) or 'none'}. " + + " ".join(limitations) ) return record, warning @@ -1148,6 +1270,7 @@ def build_bundle_data_release_payload( regional_uri_parts: Optional[dict] = None, regional_release_kind: Optional[RegionalReleaseKind] = None, regional_certification: Optional[dict] = None, + regional_manifest_sha256: Optional[str] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -1161,6 +1284,8 @@ def build_bundle_data_release_payload( payload["release_manifest_uri"] = https_manifest_url(uri_parts) if regional_manifest_uri is not None: payload["regional_source_manifest_uri"] = regional_manifest_uri + if regional_manifest_sha256 is not None: + payload["regional_source_manifest_sha256"] = regional_manifest_sha256 if regional_uri_parts is not None: payload["regional_release_manifest_uri"] = https_manifest_url( regional_uri_parts @@ -1193,6 +1318,7 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1217,6 +1343,7 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1244,12 +1371,14 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) primary_manifest = manifest regional_manifest = None + regional_manifest_sha256 = None regional_uri_parts = None recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} @@ -1267,8 +1396,8 @@ def certify( "whose area receipts are uncertified." ) if regional_manifest_uri is not None: - regional_manifest, _, regional_uri_parts = fetch_release_manifest( - regional_manifest_uri, token=token + regional_manifest, regional_manifest_sha256, regional_uri_parts = ( + fetch_release_manifest(regional_manifest_uri, token=token) ) # The regional release runs on the same model and core, so it must # cover the pinned versions by its own build or claims. A qualified @@ -1287,6 +1416,9 @@ def certify( model_package, model_version, regional_model_mismatch_reason, + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=regional_manifest_sha256, + evidence=regional_input_surface_evidence, ) regional_warnings = [] else: @@ -1405,6 +1537,7 @@ def certify( regional_uri_parts=regional_uri_parts, regional_release_kind=recorded_release_kind, regional_certification=regional_certification, + regional_manifest_sha256=regional_manifest_sha256, ), manifest_sha256, warnings, @@ -1451,6 +1584,7 @@ def certify_data_release( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> CertificationResult: """Certify a data release into the bundle manifest. @@ -1467,13 +1601,21 @@ def certify_data_release( release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. - ``regional_model_mismatch_reason`` (UK only) records a reviewer's - acceptance of a regional release that does not cover the pinned model, - instead of refusing it. The core must still be covered. The bundle then - carries ``regional_certification`` naming both model versions and the - reason; area receipts are uncertified either way. + ``regional_model_mismatch_reason`` (UK only) requires + ``regional_input_surface_evidence`` audited against the exact target model + and regional manifest/artifact bytes. Dropped columns are always refused; + every defaulted input and limitation is recorded in + ``regional_certification`` alongside the reason and both model versions. + The core must still be covered; area receipts remain uncertified. """ certified_country = _certified_country(country) + if ( + regional_input_surface_evidence is not None + and regional_model_mismatch_reason is None + ): + raise CertificationError( + "Regional input-surface evidence requires an accepted mismatch reason." + ) if drop_regional_release and regional_manifest_uri is not None: raise CertificationError( "--drop-regional-release removes the bundle's regional routing, so it " @@ -1519,6 +1661,7 @@ def certify_data_release( (bundle.get("packages") or {}).get("policyengine-core") or {} ).get("version"), regional_model_mismatch_reason=regional_model_mismatch_reason, + regional_input_surface_evidence=regional_input_surface_evidence, ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f309636cd..1e00f5fce 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -211,6 +211,45 @@ class CertifiedDataArtifact(BaseModel): RegionalReleaseKind = Literal["local_area", "area_coded_national"] +class RegionalInputArtifactEvidence(BaseModel): + """Measured columns of one byte-verified regional H5 artifact.""" + + model_config = {"extra": "forbid"} + + repo_id: str = Field(min_length=1) + path: str = Field(min_length=1) + revision: str = Field(min_length=1) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + repo_type: Optional[Literal["model", "dataset"]] = None + columns: list[str] = Field(min_length=1) + columns_by_period: dict[str, list[str]] = Field(min_length=1) + + +class RegionalInputSurfaceEvidence(BaseModel): + """An operator's audit of actual regional columns against an exact model. + + Surfaces are retained so certification recomputes the comparison rather + than trusting a declared zero-loss result. The artifact hashes identify + the bytes inspected by the auditor, not merely a model's possible inputs. + """ + + model_config = {"extra": "forbid"} + + schema_version: Literal[1] = 1 + regional_manifest_uri: str = Field(min_length=1) + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + model_package: str = Field(min_length=1) + model_version: str = Field(min_length=1) + artifacts: dict[str, RegionalInputArtifactEvidence] = Field(min_length=1) + model_variables: list[str] = Field(min_length=1) + model_inputs: list[str] = Field(min_length=1) + input_aliases: Optional[dict[str, str]] = None + dropped_columns: list[str] + defaulted_inputs: list[str] + audited_by: str = Field(min_length=1, pattern=r"\S") + limitations: list[str] = Field(min_length=1) + + class RegionalCertification(BaseModel): """A reviewer's acceptance of a regional release on another model version. @@ -224,6 +263,12 @@ class RegionalCertification(BaseModel): built_with_model_version: Optional[str] = None accepted_reason: str area_receipts: Literal["uncertified"] = "uncertified" + regional_manifest_uri: str + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + defaulted_inputs: list[str] + dropped_columns: list[str] + limitations: list[str] + evidence: RegionalInputSurfaceEvidence class CountryReleaseManifest(BaseModel): @@ -245,6 +290,8 @@ class CountryReleaseManifest(BaseModel): regional_certification: Optional[RegionalCertification] = None """Present only when the regional release was routed on an accepted model mismatch rather than its own build or claims.""" + regional_source_manifest_sha256: Optional[str] = None + """SHA-256 of the original regional release manifest bytes.""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index a8c8b5b21..b00893fca 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1436,6 +1436,9 @@ def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): bundle_path = _tmp_bundle(tmp_path) reason = "area receipts are uncertified; the dense cut lags one release" + from tests.test_regional_input_evidence import input_evidence + + evidence = input_evidence(self._older_dense()) result = _certify_uk( bundle_path, @@ -1445,6 +1448,7 @@ def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): ], regional_manifest_uri=UK_DENSE_URI, regional_model_mismatch_reason=reason, + regional_input_surface_evidence=evidence, ) release = json.loads(bundle_path.read_text())["data_releases"]["uk"] @@ -1455,6 +1459,12 @@ def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): "built_with_model_version": "2.120.0", "accepted_reason": reason, "area_receipts": "uncertified", + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": evidence["regional_manifest_sha256"], + "defaulted_inputs": evidence["defaulted_inputs"], + "dropped_columns": [], + "limitations": evidence["limitations"], + "evidence": evidence, } # The national certificate itself is untouched by the regional record. assert release["certification"]["compatibility_basis"] == ( @@ -1497,6 +1507,8 @@ def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( self, tmp_path ): + from tests.test_regional_input_evidence import input_evidence + national = _microcosm_uk_national_payload() national["compatible_core_packages"] = [ {"name": "policyengine-core", "specifier": "==3.33.0"} @@ -1509,6 +1521,7 @@ def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( [_response(national), _response(self._older_dense())], regional_manifest_uri=UK_DENSE_URI, regional_model_mismatch_reason="area receipts are uncertified", + regional_input_surface_evidence=input_evidence(self._older_dense()), core_version="3.33.0", ) @@ -1549,12 +1562,18 @@ def fake_main(argv): UK_DENSE_URI, "--accept-regional-model-mismatch", reason, + "--regional-input-surface-evidence", + "regional-audit.json", ] ) == 0 ) argv = captured["argv"] assert argv[argv.index("--accept-regional-model-mismatch") + 1] == reason + assert ( + argv[argv.index("--regional-input-surface-evidence") + 1] + == "regional-audit.json" + ) def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index d0f6a31b3..5efdd99a0 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -54,7 +54,10 @@ def input_evidence(dense: dict | None = None) -> dict: name: { key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") } - | {"columns": ["employment_income"]} + | { + "columns": ["employment_income"], + "columns_by_period": {"2025": ["employment_income"]}, + } }, "model_variables": ["employment_income", *DEFAULTED_INPUTS], "model_inputs": ["employment_income", *DEFAULTED_INPUTS], @@ -126,6 +129,9 @@ def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( "removed_model_variable" ) + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"][ + "2025" + ].append("removed_model_variable") evidence["dropped_columns"] = ["removed_model_variable"] with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): certify_with_evidence(_tmp_bundle(tmp_path), evidence) @@ -154,7 +160,7 @@ def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path release = json.loads(bundle_path.read_text())["data_releases"]["uk"] manifest = CountryReleaseManifest.model_validate(release) assert manifest.regional_certification is not None - record = manifest.regional_certification.model_dump(mode="json") + record = manifest.regional_certification.model_dump(mode="json", exclude_none=True) assert record["defaulted_inputs"] == DEFAULTED_INPUTS assert record["dropped_columns"] == [] assert record["limitations"] == [VAT_LIMIT] @@ -235,3 +241,45 @@ def digest(bundle): return canonical_sha256(manifest.model_dump(mode="json")) assert digest(before) != digest(after) + + +def test_missing_period_measurements_refuse(tmp_path): + evidence = input_evidence() + del evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_budget_target_records_vat_limit_and_both_named_defaults(tmp_path, monkeypatch): + from tests import test_certify_data_release as fixtures + + monkeypatch.setattr(fixtures, "UK_MODEL_VERSION", "2.147.2") + dense = older_dense() + dense["build"]["built_with_model_package"]["version"] = "2.132.6" + dense["compatible_model_packages"][0]["specifier"] = "==2.132.6" + audit = input_evidence(dense) + audit["model_version"] = "2.147.2" + audit["limitations"] = ["The dense cut was calibrated under an older model."] + path = _tmp_bundle(tmp_path) + certify_with_evidence(path, audit, dense) + release = json.loads(path.read_text())["data_releases"]["uk"] + record = release["regional_certification"] + assert record["certified_for_model_version"] == "2.147.2" + assert record["built_with_model_version"] == "2.132.6" + assert VAT_LIMIT in record["limitations"] + assert set(DEFAULTED_INPUTS) <= set(record["defaulted_inputs"]) + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_regional_acceptance_never_widens_national_coverage(tmp_path): + national = _microcosm_uk_national_payload() + national["build"]["built_with_model_package"]["version"] = "2.120.0" + national["compatible_model_packages"][0]["specifier"] = "==2.120.0" + with pytest.raises(CertificationError, match="new data build|compatibility claim"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=input_evidence(), + ) From b90eada3aecbedbd1a8c0f98a00d0d0f653c3f64 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:46:35 -0400 Subject: [PATCH 16/20] Bootstrap regional audits before target-model certification --- scripts/audit_regional_input_surface.py | 2 ++ tests/test_measure_regional_inputs.py | 30 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py index 635a610da..cbe5cd1a2 100644 --- a/scripts/audit_regional_input_surface.py +++ b/scripts/audit_regional_input_surface.py @@ -13,6 +13,8 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] +# The target model must be auditable before the current bundle certifies it. +os.environ.setdefault("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") sys.path.insert(0, str(REPO_ROOT / "src")) from policyengine.provenance.regional_input_surface import ( # noqa: E402 diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index 8cf4006af..4bff297b6 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -343,3 +343,33 @@ def measure(**kwargs): assert captured["artifact_paths"] == {"dense": path} assert captured["limitations"] == [LIMITATION] assert json.loads(output.read_text()) == result.model_dump(mode="json") + + +def test_audit_cli_bootstraps_before_target_model_is_certified(tmp_path): + """The target must be auditable before it matches the current bundle.""" + import subprocess + + script = Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + code = """ +import importlib.util +import os +import runpy +import sys +os.environ.pop('POLICYENGINE_SKIP_COUNTRY_IMPORTS', None) +original = importlib.util.find_spec +def refuse_country_autoload(name, *args, **kwargs): + if name in {'policyengine_uk', 'policyengine_us'}: + raise RuntimeError('audit triggered automatic country import') + return original(name, *args, **kwargs) +importlib.util.find_spec = refuse_country_autoload +sys.argv = [sys.argv[1], '--help'] +runpy.run_path(sys.argv[0], run_name='__main__') +""" + result = subprocess.run( + [sys.executable, "-c", code, str(script)], + capture_output=True, + text=True, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + assert "--model-version" in result.stdout From aa59b0d704852eab63087d8146f7da667d2178e0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 09:49:56 -0400 Subject: [PATCH 17/20] Record Budget review fix validation and certification handoff --- .job/HANDOFF.md | 61 +++++++++++++++++++++---------------------------- .job/PR_BODY.md | 19 +++++++++++++++ 2 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 .job/PR_BODY.md diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md index d0029679a..e14bc0201 100644 --- a/.job/HANDOFF.md +++ b/.job/HANDOFF.md @@ -1,44 +1,35 @@ -# PR #704 review-fix handoff +# PR #699 Budget review fix -- Draft PR: https://github.com/PolicyEngine/policyengine.py/pull/704 -- Branch: `budget-local-data-manifest`; confirmed reviewed remote head `fd4e099ed3ff768041f3a537e6e38527c07b0bbf`. -- Validated implementation head SHA: `0d7bf1a641d61b8951f47ab12d8d4867f57df140`. -- The following commit updates only this handoff. Its exact final head is recorded in `.job/HEAD` and `.job/local-data-manifest.bundle`. +- Confirmed origin heads: #699 `afee349d2c14de1b121b0b7e4f03b695d03f418c`; #704 `da2ae8fb8e97145093e35366686df99c1e38449e`. +- Merged #704 into `accept-regional-model-mismatch` with merge commit `c1a65198`. No history rewritten. Release order remains main → #704 → #699 → certification. +- Implementation validated through `b90eada3`; the final report commit is recorded in `.job/HEAD`. -## Fixes +Changes: -- Scan every TRO definition of the manifest artifact ID, including direct graph nodes, nested definitions and artifact references; require one distinct valid SHA-256 assertion. Preserve canonical inline/location checks. -- Validate both local and fetched manifest artifacts after revision rewriting. The certified/default artifact must be present and anchored; every overlapping managed artifact must agree with bundled repo/path/revision, repository type and pinned SHA-256. Compare identity fields separately to prevent ambiguous URI concatenation. Primary manifests may omit separately pinned regional artifacts and overlays. -- Share the existing release-relative diagnostics/evidence normalization with certification. Exact-pair model/core/build and fingerprint acceptance logic remains unchanged; unset fetch/cache/offline behavior is preserved for consistent bundles. -- Replace function attributes with a typed callable cache facade. -- Make the documented export conditional on successful download/verification/atomic replacement; remove release coordination from permanent public docs. -- PR body records the invariants and both downstream actions. No other repository was edited. +- Require typed audited evidence bound to the exact regional cut URI, original manifest digest, every regional H5 identity/hash, and exact target model. Recompute the surface comparison and refuse every dropped column. Record every defaulted input by inspecting all stored periods and the country loader's aliases. +- Port Microcosm #1348's comparison, strengthening its older-model proxy to measure actual byte-verified H5 columns. New producer: `scripts/audit_regional_input_surface.py`. The CLI works before the current bundle certifies its target model. Microcosm handoff callers must generate and forward the new evidence JSON. +- Preserve the regional manifest's raw-byte digest. Keep limitations, including the pre-2.135.7 VAT fix/dashboard limit, through Pydantic validation and bundle hashing. +- Expose the full regional record in receipts and TRACE (`@json` preserves JSON-LD semantics). Area receipts stay uncertified and identify their dense build model. National certification never widens. Explicit reversal removes the record, digest, area routes and dense artifact; the national certificate stays identical. +- Regenerate the receipt schema. Dashboard follow-up: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json`. -## Validation +Validation: -**3,322 unique cases passed; five skipped.** +- Tests first demonstrated missing-evidence acceptance, stale evidence/dropped-column failures, missing receipt/TRACE records, lost JSON-LD semantics, and premature audit CLI model initialization. +- **1,248 selected cases passed; two skipped**, across the UK run and successful reruns of three US metadata cases in the environment with the bundled US pin installed. The UK run initially reported 1,245 passes plus those three missing-package failures; all three reruns passed. One unrelated US offline-import integration case was explicitly deselected. +- Covered all changed provenance/certification modules, the existing roundtrip, required reversal cases, and #704's three local-manifest modules. Logs: `.job/logs/uk-final.txt`, `.job/logs/us-metadata-rerun.txt`. +- Expanded 53-module validation reported 3,008 passes and five skips with no failures before stalling for over nine minutes; it was interrupted and is not claimed complete. A subsequent wider pinned-environment attempt was also interrupted during slow package inspection. Initial setup problems (missing packages, inherited Git variables and an unpinned model) were diagnosed; final test subprocesses clear Git overrides. No interrupted test processes remain. +- `make format`, `make lint`, receipt schema generation/equality, `bundle.py check`, and diff checks pass. +- Mypy on all changed source modules: 24 diagnostics, identical to the merged baseline ignoring moved line numbers. Both new source modules have zero errors. Comparison: `.job/logs/mypy-comparison.txt`. +- Every pytest launch checked load1 ≤110 and free disk ≥55,000,000,000 bytes. At most two workers per invocation. Final launches: load1 75.71 / 97.69; available disk 103.7 GB decimal. -- Initial local/TRO-pin run: 88 passed, including two 80-example Hypothesis properties (byte mutation and duplicate placement). -- Broad 56-module provenance/certification/disclosure run: 3,256 passed, five skipped, one failure from the old missing-default error-message assertion. It now refuses earlier in the shared validator; the corrected complete TRO regeneration module passed all 40 cases on rerun. -- Release/bootstrap-focused run: 189 passed, including 59 cases outside the broad selection and two new default-anchor cases. -- Final affected manifest/certification run: 280 passed, including all 82 local-manifest cases, all 12 TRO-pin cases and four new repo/path-boundary cases. No failures remain in pytest's failure cache. -- `make format`, `make lint`, focused `mypy --follow-imports=silent src/policyengine/provenance/manifest.py`, `git diff --check`, and generated receipt schema checks passed. Documentation shell block passes `bash -n`. -- Every pytest invocation checked load1 <=110 and available disk >=55,000,000,000 bytes; no pytest workers. Observed load1 9.04–19.02; available disk 144.3–146.9 GB decimal. Tests used Python 3.13.9. -- Detailed logs and selected modules remain under `.job/`. No private manifests or microdata were committed. +Actual Budget pair: -## Downstream actions +- National `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2), dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (2.132.6). +- The exact model-inventory comparison has no dropped names and 16 added pure inputs, including both `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is in certification guidance. +- The private dense H5 was unavailable (no HF credential; HTTP 401). This inventory is not a byte audit, and this job does not certify the actual pair. The certification follow-up must run the new audit against the real H5 and record its complete default list. -- Dashboard: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` when upgrading. -- Scorecard: `pipeline/uk_replay_variants.py:730` retains the setting in offline isolation, while `:750` requires exact runtime bundle equality. Clear `POLICYENGINE_*_DATA_MANIFEST_FILE` during isolation. -- Leave the PR draft; do not merge. No versions, country pins, data or certified releases changed. +Delivery: -## Workspace Git metadata and delivery - -The protected workspace `.git` points into the caller's repository. All mutable Git metadata is instead inside this workspace: - -```bash -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git status -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -4 --oneline -``` - -The original pointer and caller checkout were not changed. `.job/local-data-manifest.bundle` is portable relative to main `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. The single final push is attempted after committing this handoff; its outcome is recorded in `.job/PUSH_RESULT.txt`. `.job/HEAD` contains the final branch SHA. No history was rewritten and nothing was pushed to main or master. +- All writes stayed in the assigned workspace. The protected `.git` points into the caller's checkout; mutable Git metadata is isolated in `.job/repo.git`. Use `source .job/env` before Git commands here. The caller checkout was not written. +- Portable commits: `.job/budget-fix-699.bundle`; final head: `.job/HEAD`. The single push attempt's output is saved in `.job/PUSH_RESULT.txt`. No main/master push; leave #699 draft. +- GitHub blocked the PR body update because the connector requires approval and approval policy is `never`. The replacement body, including the invariants, is saved in `.job/PR_BODY.md`; no PR metadata change was made. diff --git a/.job/PR_BODY.md b/.job/PR_BODY.md new file mode 100644 index 000000000..1b3c5fa33 --- /dev/null +++ b/.job/PR_BODY.md @@ -0,0 +1,19 @@ +Require audited evidence for a UK regional release served on an accepted model mismatch. + +This branch stacks #704 (`da2ae8fb`) into #699 (`afee349d2`) with a merge commit. The intended release order remains main → #704 → #699 → certification. + +Invariants: + +- National certification never widens. The national dataset name/hash, model coverage and core checks remain separate from the regional exception. +- An accepted area mismatch requires a reviewer reason and audited evidence bound to the exact dense cut URI, original manifest bytes, artifact identity/SHA-256, and exact target model package/version. Certification recomputes the comparison and refuses every dropped column. There is no dropped-column override. +- Every defaulted input is recorded across every stored period, accounting for the country loader's input aliases. Free-text limitations and the dense cut's pre-fix VAT limitation survive Pydantic validation and are hashed with the bundle. +- TRACE and execution receipts expose the full regional identity, acceptance, reason, input audit and limitations. TRACE preserves the record as a JSON-LD JSON literal; area receipts remain uncertified and identify their actual build model. +- Re-certifying with `--drop-regional-release` removes the exception, regional manifest digest, area routes and dense artifact without changing the national certificate. + +The comparison is ported from Microcosm #1348's `tools/certify_uk_cut_in_policyengine_py.py` and strengthened to inspect byte-verified H5 entity-table columns by period. `scripts/audit_regional_input_surface.py` produces the required JSON; `certify-data --regional-input-surface-evidence` consumes it with the accepted reason. The original regional manifest's raw-byte digest is retained. + +Budget route: national `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2) and dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (built on 2.132.6). The exact model-inventory comparison yields 16 added pure inputs, including `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is documented. Actual dense-byte evidence must be generated by certification with authenticated access to the H5. This implementation job has no HF credential and does not certify that pair. + +Required downstream refresh: update the dashboard's vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` from this repository's regenerated schema. + +Validation: 1,248 selected cases passed and two skipped across the UK run and three successful US metadata reruns. One unrelated US offline-import integration case was explicitly deselected. Formatting, lint, generated-schema equality and bundle checks pass. Mypy retains the same 24 baseline diagnostics, with zero errors in the new modules. Expanded validation was interrupted after 3,008 reported passes and five skips; it is not claimed complete. Details are recorded in `.job/HANDOFF.md`. Regression tests demonstrated failures before implementation. Leave this PR draft. From 878392cf5a42d7b4916ae7b311c84aa30d383d38 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:19:21 -0400 Subject: [PATCH 18/20] Reject foreign regional models and preserve empty mismatch reasons --- changelog.d/699.fixed.md | 2 + docs/engineering/skills/data-certification.md | 9 ++++ scripts/bundle.py | 2 +- src/policyengine/provenance/certification.py | 27 +++++++--- .../provenance/regional_input_surface.py | 6 +++ tests/test_certify_data_release.py | 53 +++++++++++++++++++ tests/test_regional_input_evidence.py | 30 +++++++++++ 7 files changed, 120 insertions(+), 9 deletions(-) diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md index 55eb27701..55fc07874 100644 --- a/changelog.d/699.fixed.md +++ b/changelog.d/699.fixed.md @@ -1 +1,3 @@ Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. + +Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them. Document that the UK loader silently skips unknown columns. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 91c2dc2ec..e4805591e 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -112,6 +112,15 @@ Microcosm national release with one of: column**. It records every defaulted input and the free-text limitations in the validated `regional_certification`; the bundle hash binds that record. + The policyengine-uk 2.147.2 dataset loaders silently skip columns absent + from the model's variables: both the + [generic dataset loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127) + and the + [UK multi-year loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196) + use `continue` without warning. A removed or renamed input can therefore + disappear silently at runtime; the audit must catch that column loss before + certification. + The national certificate never widens. Area status remains `uncertified`: the certificate still covers the national dataset only (#685). TRACE and execution receipts expose the regional identity, build model, acceptance, diff --git a/scripts/bundle.py b/scripts/bundle.py index 125635cd6..9dd4040a4 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -48,7 +48,7 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-release-kind", args.regional_release_kind]) if args.drop_regional_release: argv.append("--drop-regional-release") - if args.accept_regional_model_mismatch: + if args.accept_regional_model_mismatch is not None: argv.extend( ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] ) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index e773ba900..5ddec8e47 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -425,6 +425,23 @@ def model_compatibility_basis( ) +def _regional_build_model_version( + regional_manifest: DataReleaseManifest, model_package: str +) -> Optional[str]: + """Return the recorded regional build version; refuse a different package.""" + built = ( + regional_manifest.build.built_with_model_package + if regional_manifest.build is not None + else None + ) + if built is not None and built.name != model_package: + raise CertificationError( + f"Regional release was built with {built.name}, not {model_package}; " + "regional releases must use the target model package." + ) + return built.version if built is not None else None + + def accepted_regional_mismatch( regional_manifest: DataReleaseManifest, model_package: str, @@ -449,6 +466,7 @@ def accepted_regional_mismatch( raise CertificationError( "--accept-regional-model-mismatch needs a reason a reviewer can read." ) + built_with = _regional_build_model_version(regional_manifest, model_package) if evidence is None: raise CertificationError( "An accepted regional model mismatch requires audited input-surface " @@ -542,14 +560,6 @@ def accepted_regional_mismatch( "Dropped regional input columns are refused: " + ", ".join(comparison["dropped_columns"]) ) - built = ( - regional_manifest.build.built_with_model_package - if (regional_manifest.build is not None) - else None - ) - built_with = ( - built.version if built is not None and built.name == model_package else None - ) limitations = list(audit.limitations) try: predates_vat_fix = built_with is not None and Version(built_with) < Version( @@ -1399,6 +1409,7 @@ def certify( regional_manifest, regional_manifest_sha256, regional_uri_parts = ( fetch_release_manifest(regional_manifest_uri, token=token) ) + _regional_build_model_version(regional_manifest, model_package) # The regional release runs on the same model and core, so it must # cover the pinned versions by its own build or claims. A qualified # pair names the primary release only, so it does not extend here. diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py index d156d64c0..460c53337 100644 --- a/src/policyengine/provenance/regional_input_surface.py +++ b/src/policyengine/provenance/regional_input_surface.py @@ -5,6 +5,12 @@ artifact and period. Country-model input moves are read from that installed model's constructor, so an alias cannot make a supplied input look defaulted. No household values are included in the resulting audit. + +policyengine-uk 2.147.2 silently skips unknown columns in both dataset loaders; +the audit must detect that loss before certification. See the ``continue`` +branches in ``build_from_dataset`` and ``build_from_multi_year_dataset``: +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127 +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196 """ from __future__ import annotations diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index b00893fca..2b9e00975 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1575,6 +1575,59 @@ def fake_main(argv): == "regional-audit.json" ) + def test__given_bundle_wrapper_empty_reason__then_certifier_refuses_it( + self, monkeypatch, tmp_path + ): + bundle_script = _load_bundle_script(monkeypatch) + spec = importlib.util.spec_from_file_location( + "certify_data_release", + Path(__file__).resolve().parents[1] / "scripts" / "certify_data_release.py", + ) + certifier = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(certifier) + monkeypatch.setitem(sys.modules, "certify_data_release", certifier) + bundle_path = _tmp_bundle(tmp_path) + original_bundle = bundle_path.read_bytes() + captured = {} + + def certify_temporary_bundle(**kwargs): + captured.update(kwargs) + kwargs["bundle_path"] = bundle_path + return certify_data_release(**kwargs) + + monkeypatch.setattr(certifier, "certify_data_release", certify_temporary_bundle) + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + ), + pytest.raises(CertificationError, match="needs a reason"), + ): + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--model-version", + UK_MODEL_VERSION, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + "", + "--no-generate", + "--no-changelog", + "--skip-artifact-check", + ] + ) + assert captured["regional_model_mismatch_reason"] == "" + assert bundle_path.read_bytes() == original_bundle + def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path ): diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index 5efdd99a0..71f08401f 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -92,6 +92,36 @@ def test_accepted_mismatch_without_audited_evidence_refuses(tmp_path): ) +@pytest.mark.parametrize("model_package", ["policyengine-us", "another-model"]) +@pytest.mark.parametrize("accept_mismatch", [False, True]) +def test_regional_release_built_with_another_model_package_refuses( + tmp_path, model_package, accept_mismatch +): + dense = older_dense() + dense["build"]["built_with_model_package"]["name"] = model_package + if not accept_mismatch: + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ] + bundle_path = _tmp_bundle(tmp_path) + before = bundle_path.read_bytes() + + with pytest.raises( + CertificationError, + match=f"built with {model_package}, not policyengine-uk", + ): + if accept_mismatch: + certify_with_evidence(bundle_path, input_evidence(dense), dense) + else: + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + ) + + assert bundle_path.read_bytes() == before + + @pytest.mark.parametrize( ("field", "wrong"), [ From c125eae17b5d861ee63c0e540385857e0e2e1d2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:19:54 -0400 Subject: [PATCH 19/20] Record owner-findings fixes and reservation-limited validation --- .job/HANDOFF.md | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md index e14bc0201..77b4a5c57 100644 --- a/.job/HANDOFF.md +++ b/.job/HANDOFF.md @@ -1,35 +1,23 @@ -# PR #699 Budget review fix +# PR #699 owner findings — Budget Monday re-cut -- Confirmed origin heads: #699 `afee349d2c14de1b121b0b7e4f03b695d03f418c`; #704 `da2ae8fb8e97145093e35366686df99c1e38449e`. -- Merged #704 into `accept-regional-model-mismatch` with merge commit `c1a65198`. No history rewritten. Release order remains main → #704 → #699 → certification. -- Implementation validated through `b90eada3`; the final report commit is recorded in `.job/HEAD`. +- Confirmed GitHub head before editing: `aa59b0d704852eab63087d8146f7da667d2178e0`, branch `accept-regional-model-mismatch`, canonical repository, draft PR. +- Implementation head SHA: `878392cf5a42d7b4916ae7b311c84aa30d383d38`. This handoff is a separate report-only commit; its final delivery SHA is saved in `.job/HEAD`. Changes: -- Require typed audited evidence bound to the exact regional cut URI, original manifest digest, every regional H5 identity/hash, and exact target model. Recompute the surface comparison and refuse every dropped column. Record every defaulted input by inspecting all stored periods and the country loader's aliases. -- Port Microcosm #1348's comparison, strengthening its older-model proxy to measure actual byte-verified H5 columns. New producer: `scripts/audit_regional_input_surface.py`. The CLI works before the current bundle certifies its target model. Microcosm handoff callers must generate and forward the new evidence JSON. -- Preserve the regional manifest's raw-byte digest. Keep limitations, including the pre-2.135.7 VAT fix/dashboard limit, through Pydantic validation and bundle hashing. -- Expose the full regional record in receipts and TRACE (`@json` preserves JSON-LD semantics). Area receipts stay uncertified and identify their dense build model. National certification never widens. Explicit reversal removes the record, digest, area routes and dense artifact; the national certificate stays identical. -- Regenerate the receipt schema. Dashboard follow-up: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json`. +1. Refuse any explicitly different regional `built_with_model_package.name` before checking publisher claims or accepting a version mismatch. Missing build metadata retains its previous behavior. Four regression cases cover `policyengine-us` and another package, with audited mismatch acceptance or a UK compatibility claim; each checks that refusal leaves the bundle unchanged. +2. Forward `--accept-regional-model-mismatch` whenever the value `is not None`. One regression follows the real wrapper and certifier CLI into certification, confirms the empty string is forwarded and refused, and checks that the bundle is unchanged. +3. Document silent unknown-column skipping in both the certification skill and input-audit module docstring. Inspected policyengine-uk 2.147.2, immutable commit `fe0b99a36bd309a722b19937345e34290cb6dd73`: [`build_from_dataset`, lines 1116–1117](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127) and [`build_from_multi_year_dataset`, lines 1194–1195](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196) use `continue` without warning. The old core-warning claim was already absent from the approved head; the documentation now states the actual UK behavior explicitly. Validation: -- Tests first demonstrated missing-evidence acceptance, stale evidence/dropped-column failures, missing receipt/TRACE records, lost JSON-LD semantics, and premature audit CLI model initialization. -- **1,248 selected cases passed; two skipped**, across the UK run and successful reruns of three US metadata cases in the environment with the bundled US pin installed. The UK run initially reported 1,245 passes plus those three missing-package failures; all three reruns passed. One unrelated US offline-import integration case was explicitly deselected. -- Covered all changed provenance/certification modules, the existing roundtrip, required reversal cases, and #704's three local-manifest modules. Logs: `.job/logs/uk-final.txt`, `.job/logs/us-metadata-rerun.txt`. -- Expanded 53-module validation reported 3,008 passes and five skips with no failures before stalling for over nine minutes; it was interrupted and is not claimed complete. A subsequent wider pinned-environment attempt was also interrupted during slow package inspection. Initial setup problems (missing packages, inherited Git variables and an unpinned model) were diagnosed; final test subprocesses clear Git overrides. No interrupted test processes remain. -- `make format`, `make lint`, receipt schema generation/equality, `bundle.py check`, and diff checks pass. -- Mypy on all changed source modules: 24 diagnostics, identical to the merged baseline ignoring moved line numbers. Both new source modules have zero errors. Comparison: `.job/logs/mypy-comparison.txt`. -- Every pytest launch checked load1 ≤110 and free disk ≥55,000,000,000 bytes. At most two workers per invocation. Final launches: load1 75.71 / 97.69; available disk 103.7 GB decimal. - -Actual Budget pair: - -- National `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2), dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (2.132.6). -- The exact model-inventory comparison has no dropped names and 16 added pure inputs, including both `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is in certification guidance. -- The private dense H5 was unavailable (no HF credential; HTTP 401). This inventory is not a byte audit, and this job does not certify the actual pair. The certification follow-up must run the new audit against the real H5 and record its complete default list. +- `python3 ~/reviews/fleet-ops/tools/reservation.py allowed uk-hub` refused: `held back by route-a-release-rerun-d1811-2026-10-23 (reservation.py status)`. +- **0 pytest cases executed; 0 passed, 0 failed. Five new regression cases await CI.** No collection, full suite, country-model run, or audit was launched. Rely on CI as requested. Focused follow-up, once permitted: `pytest tests/test_certify_data_release.py tests/test_regional_input_evidence.py -n 2`. +- Ruff lint and formatting checks passed for all five changed Python files: the bundle wrapper, certification, input audit, and both test modules. `git diff --check` passed. +- Independent source/diff review found no blockers. No data release was certified. Delivery: -- All writes stayed in the assigned workspace. The protected `.git` points into the caller's checkout; mutable Git metadata is isolated in `.job/repo.git`. Use `source .job/env` before Git commands here. The caller checkout was not written. -- Portable commits: `.job/budget-fix-699.bundle`; final head: `.job/HEAD`. The single push attempt's output is saved in `.job/PUSH_RESULT.txt`. No main/master push; leave #699 draft. -- GitHub blocked the PR body update because the connector requires approval and approval policy is `never`. The replacement body, including the invariants, is saved in `.job/PR_BODY.md`; no PR metadata change was made. +- All writes stay in this assigned workspace; mutable Git metadata is isolated in `.job/repo.git`. Run `source .job/env` before Git commands here. The protected `.git` still points to the caller's read-only metadata. +- Two added commits: the fixes above and this report. No history rewrite. The final SHA is in `.job/HEAD`; portable new commits are in `.job/budget-fix-699-r3.bundle`. +- One normal push to `origin HEAD:accept-regional-model-mismatch` is attempted after the report commit. Its outcome is saved in `.job/PUSH_RESULT.txt` and reported to the caller. No push to main/master. Leave #699 draft. From 463c3a953ccaeb110e3e5a7f7c494ed46597d52d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:44:49 -0400 Subject: [PATCH 20/20] Remove the build job's hand-off note from the PR The note was a Subfleet job artifact, not part of the change. The hub keeps a copy outside the repository. Co-Authored-By: Claude Opus 5.5 --- .job/HANDOFF.md | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .job/HANDOFF.md diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md deleted file mode 100644 index d0029679a..000000000 --- a/.job/HANDOFF.md +++ /dev/null @@ -1,44 +0,0 @@ -# PR #704 review-fix handoff - -- Draft PR: https://github.com/PolicyEngine/policyengine.py/pull/704 -- Branch: `budget-local-data-manifest`; confirmed reviewed remote head `fd4e099ed3ff768041f3a537e6e38527c07b0bbf`. -- Validated implementation head SHA: `0d7bf1a641d61b8951f47ab12d8d4867f57df140`. -- The following commit updates only this handoff. Its exact final head is recorded in `.job/HEAD` and `.job/local-data-manifest.bundle`. - -## Fixes - -- Scan every TRO definition of the manifest artifact ID, including direct graph nodes, nested definitions and artifact references; require one distinct valid SHA-256 assertion. Preserve canonical inline/location checks. -- Validate both local and fetched manifest artifacts after revision rewriting. The certified/default artifact must be present and anchored; every overlapping managed artifact must agree with bundled repo/path/revision, repository type and pinned SHA-256. Compare identity fields separately to prevent ambiguous URI concatenation. Primary manifests may omit separately pinned regional artifacts and overlays. -- Share the existing release-relative diagnostics/evidence normalization with certification. Exact-pair model/core/build and fingerprint acceptance logic remains unchanged; unset fetch/cache/offline behavior is preserved for consistent bundles. -- Replace function attributes with a typed callable cache facade. -- Make the documented export conditional on successful download/verification/atomic replacement; remove release coordination from permanent public docs. -- PR body records the invariants and both downstream actions. No other repository was edited. - -## Validation - -**3,322 unique cases passed; five skipped.** - -- Initial local/TRO-pin run: 88 passed, including two 80-example Hypothesis properties (byte mutation and duplicate placement). -- Broad 56-module provenance/certification/disclosure run: 3,256 passed, five skipped, one failure from the old missing-default error-message assertion. It now refuses earlier in the shared validator; the corrected complete TRO regeneration module passed all 40 cases on rerun. -- Release/bootstrap-focused run: 189 passed, including 59 cases outside the broad selection and two new default-anchor cases. -- Final affected manifest/certification run: 280 passed, including all 82 local-manifest cases, all 12 TRO-pin cases and four new repo/path-boundary cases. No failures remain in pytest's failure cache. -- `make format`, `make lint`, focused `mypy --follow-imports=silent src/policyengine/provenance/manifest.py`, `git diff --check`, and generated receipt schema checks passed. Documentation shell block passes `bash -n`. -- Every pytest invocation checked load1 <=110 and available disk >=55,000,000,000 bytes; no pytest workers. Observed load1 9.04–19.02; available disk 144.3–146.9 GB decimal. Tests used Python 3.13.9. -- Detailed logs and selected modules remain under `.job/`. No private manifests or microdata were committed. - -## Downstream actions - -- Dashboard: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` when upgrading. -- Scorecard: `pipeline/uk_replay_variants.py:730` retains the setting in offline isolation, while `:750` requires exact runtime bundle equality. Clear `POLICYENGINE_*_DATA_MANIFEST_FILE` during isolation. -- Leave the PR draft; do not merge. No versions, country pins, data or certified releases changed. - -## Workspace Git metadata and delivery - -The protected workspace `.git` points into the caller's repository. All mutable Git metadata is instead inside this workspace: - -```bash -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git status -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -4 --oneline -``` - -The original pointer and caller checkout were not changed. `.job/local-data-manifest.bundle` is portable relative to main `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. The single final push is attempted after committing this handoff; its outcome is recorded in `.job/PUSH_RESULT.txt`. `.job/HEAD` contains the final branch SHA. No history was rewritten and nothing was pushed to main or master.