diff --git a/changelog.d/1294-execution-overlay-file.added.md b/changelog.d/1294-execution-overlay-file.added.md new file mode 100644 index 0000000000..3837e6dc96 --- /dev/null +++ b/changelog.d/1294-execution-overlay-file.added.md @@ -0,0 +1 @@ +Add `microcosm.graph.execution_overlay.execution_overlay_bytes` and `load_execution_overlay`, which write and read the execution overlay (`microcosm.graph.execution-evidence.v1`) as its own canonical JSON file. Projecting a schema with the overlay read back from that file gives byte-identical Orrery JSON, and the reader needs only the standard library. It refuses non-regular files, files over the schema's 32 MiB serialized limit, duplicate keys, non-finite numbers and other protocols. diff --git a/changelog.d/1294-orrery-projection-without-store.changed.md b/changelog.d/1294-orrery-projection-without-store.changed.md new file mode 100644 index 0000000000..4d206b8f19 --- /dev/null +++ b/changelog.d/1294-orrery-projection-without-store.changed.md @@ -0,0 +1 @@ +Load `microcosm.graph`'s runtime exports on first access, and take the execution overlay's protocol name and the SHA-256 helper from modules that need only the standard library. `microcosm.graph.orrery` now imports and projects a saved schema and execution overlay in a process without numpy, pandas, microcosm-frame or the content store; the package's declared dependencies are unchanged. diff --git a/changelog.d/1294-uk-execution-overlay-output.added.md b/changelog.d/1294-uk-execution-overlay-output.added.md new file mode 100644 index 0000000000..60cfe95773 --- /dev/null +++ b/changelog.d/1294-uk-execution-overlay-output.added.md @@ -0,0 +1 @@ +UK finalize now writes the execution overlay it projects as `graph.execution.json`, beside `graph.orrery.json`, and registers it as a build output: in the dataset manifest, in a refused build's evidence manifest and in a failed build's bundle. Projecting `graph.schema.json` with it reproduces `graph.orrery.json` byte for byte. It stays out of the Orrery publication set (`orrery-publication/` and `orrery.upload.json`), and every existing file keeps its bytes, apart from the new entry in the manifest that registers it. diff --git a/docs/orrery-adapter.md b/docs/orrery-adapter.md index b095093260..469ab3e36d 100644 --- a/docs/orrery-adapter.md +++ b/docs/orrery-adapter.md @@ -24,6 +24,14 @@ Microcosm executes. The direct API performs steps 2 and 3 in memory. A saved schema lets another process repeat step 3 without importing the original graph construction code. +That process needs none of the build runtime either. `microcosm.graph.orrery` +and the modules it imports use only the standard library, and `microcosm.graph` +loads its runtime exports (the executor, content store, kernels, codecs and +evidence capture, which need numpy, pandas and microcosm-frame) on first +access. A test runs the projection, with an execution overlay, and the command +in an interpreter that refuses to import numpy, pandas, microcosm-frame or the +store. The package's declared dependencies are unchanged. + A kernel is the implementation of one graph operation, with a declared input, output, numerical and execution contract. @@ -243,6 +251,39 @@ npx orrery --input build/graph.orrery.json --output build/graph.orrery.html Native evidence links and saved-schema relative references are rebased to the command's output directory; keep the referenced files available with the HTML. +`collect_execution_evidence` returns the execution overlay: the +`microcosm.graph.execution-evidence.v1` object that the projection embeds at +`metadata.execution`. `microcosm.graph.execution_overlay` saves it as its own +file, so a later projection needs only the schema and that file, not the store: + +```python +from microcosm.graph.execution_overlay import ( + execution_overlay_bytes, + load_execution_overlay, +) + +Path("build/graph.execution.json").write_bytes(execution_overlay_bytes(execution)) + +# Later, in a process without the store, numpy, pandas or microcosm-frame: +schema = json.loads(Path("build/graph.schema.json").read_bytes()) +overlay = load_execution_overlay(Path("build/graph.execution.json")) +Path("build/graph.orrery.json").write_text( + orrery_json_from_schema(schema, execution=overlay), encoding="utf-8" +) +``` + +The file is the overlay's canonical JSON: UTF-8, sorted keys and no +whitespace. It has none of the transport tags for numbers JavaScript cannot +represent; the projection adds those. Reading it back gives an equal object, +so the projection is byte-identical to projecting the in-memory overlay. The +reader takes only a regular file within the schema's 32 MiB serialized limit, +which evidence collection and projection already apply to the overlay. It +refuses duplicate keys, non-finite numbers, values outside the schema's JSON +bounds and any other protocol. The overlay's graph binding and records are +checked when it is projected. Its native-evidence links are relative to the +directory it was collected for (`reference_base`), so project into that +directory, beside the linked files. + Other countries supply the same three inputs: a presentation dictionary (groups, scope, optional labels and references), an artifact-summary registry keyed by artifact type, and driver wiring that calls `save_graph_schema`, @@ -256,6 +297,11 @@ that case, pass the attempt's `graph.schema.json` and `graph-evidence/execution.evidence.json`, and the same store. The spine sidecar identifies its native index under the checkpoint directory; use that directory's `graph.schema.json` and `node-graph` store. Orrery JSON and HTML remain opt-in. +When UK finalize exports `graph.orrery.json` from recorded execution, it also +writes the overlay it projected as `graph.execution.json` and registers it as a +build output, so `graph.schema.json` and `graph.execution.json` reproduce the +export byte for byte without the store. The overlay is not part of the Orrery +publication set (see [UK graph publication](orrery-publication.md)). The final national schema, declaration and inventory all include the appended `uk.full.national.readback` operation. Earlier endpoint runs remain separate diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index b6b0a59a68..e073acd931 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -6,6 +6,14 @@ public-safe file inventory in the attempt directory, attempt publication before Hugging Face artifact staging, and register graph/evidence files and an `orrery.publication.json` receipt snapshot in the artifact manifest. +When execution was recorded, they also write the execution overlay that +`graph.orrery.json` embeds as `graph.execution.json` and register it in the +same manifest. A failed build's bundle carries it too. It is not copied to the +preserved `orrery-publication/` directory or listed in `orrery.upload.json`: +the emitter's file roles and the runs service's contract do not admit it yet. +Its content is already inside `graph.orrery.json`, so registering it discloses +nothing new. + Microcosm's metadata adapter starts one local emitter service with two independent workers: live telemetry and completed graph publication. The `microcosm-emitter` package owns sockets, authentication, persistence, retries and resource sampling. diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py index 3cd1be5c4b..d3666f8537 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py @@ -26,11 +26,13 @@ from microcosm.graph import ContentStore from microcosm.graph.canonical import canonical_json from microcosm.graph.evidence import collect_execution_evidence, load_run_evidence +from microcosm.graph.execution_overlay import execution_overlay_bytes from microcosm.graph.orrery import orrery_json_from_schema RECEIPT_NAME = "orrery.publication.json" INVENTORY_NAME = "orrery.upload.json" EVIDENCE_MANIFEST_NAME = "orrery.evidence-manifest.json" +EXECUTION_OVERLAY_NAME = "graph.execution.json" FAILURE_MARKER_NAME = "orrery.failure.json" FAILURE_BUNDLE_DIRECTORY = "orrery-failure" # How long a build waits for a first upload result after the durable enqueue. @@ -141,6 +143,18 @@ def finalize_graph( roles[run[field]["path"]] = role payload = orrery_json_from_schema(schema, execution=execution).encode() materialize_bytes(payload, output / "graph.orrery.json") + # The overlay is the projection's only input besides the schema, + # so a later projection can start from these two files. It is a + # registered build output but not part of the Orrery publication: + # the emitter's file roles and the runs service's contract do not + # admit it yet. + unpublished: list[str] = [] + if execution is not None: + materialize_bytes( + execution_overlay_bytes(execution), + output / EXECUTION_OVERLAY_NAME, + ) + unpublished.append(EXECUTION_OVERLAY_NAME) for url in _urls(json.loads(payload)): name = url.split("#", 1)[0] if re.fullmatch(r"upstream-[a-f0-9]{64}\.json", name): @@ -149,7 +163,7 @@ def finalize_graph( roles[name] = "upstream" # Keep exact sources outside the build's temporary directory and # outside HF's immutable receipt snapshot, including when offline. - names = list(roles) + names = [*roles, *unpublished] preserved = Path(args.attempt_evidence) / "orrery-publication" for name in roles: materialize_bytes((output / name).read_bytes(), preserved / name) @@ -169,7 +183,7 @@ def finalize_graph( raise RuntimeError("Local graph inventory service is unavailable.") materialize_bytes(canonical_json(inventory), preserved / INVENTORY_NAME) materialize_bytes(canonical_json(inventory), output / INVENTORY_NAME) - names = [*roles, INVENTORY_NAME] + names = [*roles, *unpublished, INVENTORY_NAME] receipt = { "version": 1, "publication_id": inventory["publication_id"], diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py index d35c21ea5e..393e4040eb 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py @@ -1,19 +1,36 @@ """UK exports and staging order, independent of the runs application's code.""" +import hashlib import json from argparse import Namespace +from pathlib import Path import pytest +import test_support.microcosm_build.uk_full_build_cli as full_build_support from microcosm.build.uk_runtime import ( full_build_cli, orrery_publication, rowwise_staging, ) from microcosm.build.uk_runtime.orrery_contract import save_uk_graph_schema -from microcosm.graph import Graph, Node, Owned, SourceRef, StructuralDelta +from microcosm.graph import ( + ContentStore, + Graph, + Node, + Owned, + SourceRef, + StructuralDelta, + collect_execution_evidence, + load_run_evidence, +) +from microcosm.graph.execution_overlay import load_execution_overlay +from microcosm.graph.orrery import orrery_json_from_schema from test_support.microcosm_build.telemetry import fake_publication_inventory +#: The execution overlay's file name, which the collector's inputs will name. +OVERLAY_NAME = "graph.execution.json" + def fixture(tmp_path, **options): out = tmp_path / "temporary" @@ -323,3 +340,110 @@ def publish_graph(self, directory, inventory, *, wait_seconds): assert (args.out / "graph.orrery.json").is_file() assert (args.out / "orrery.upload.json").is_file() assert (args.out / "orrery.evidence-manifest.json").is_file() + + +def synthetic_build_request(tmp_path, monkeypatch, failure=None): + """A synthetic dense build request whose run records execution.""" + pytest.importorskip("tables") + full_build_support.patch_certification(monkeypatch) + args = full_build_support.arguments(tmp_path) + return args, full_build_support.prepared(tmp_path, failure) + + +def synthetic_full_build(tmp_path, monkeypatch, failure=None): + """Run a synthetic dense build; return its arguments and exit status.""" + args, build = synthetic_build_request(tmp_path, monkeypatch, failure) + return args, full_build_support.cli.execute_full_build(build, args) + + +def test_execution_overlay_reprojects_to_the_exported_orrery_artifact( + tmp_path, monkeypatch +): + args, status = synthetic_full_build(tmp_path, monkeypatch) + assert status == 0 + schema = json.loads((args.out / "graph.schema.json").read_bytes()) + store = ContentStore(args.out / ".graph-store", create=False) + collected = collect_execution_evidence( + schema, + runs=load_run_evidence(args.out / "execution.evidence.json", store=store), + store=store, + ) + + overlay = load_execution_overlay(args.out / OVERLAY_NAME) + + assert overlay == collected + assert ( + orrery_json_from_schema(schema, execution=overlay).encode() + == (args.out / "graph.orrery.json").read_bytes() + ) + + +def test_execution_overlay_is_registered_but_not_published(tmp_path, monkeypatch): + args, status = synthetic_full_build(tmp_path, monkeypatch) + assert status == 0 + overlay_path = args.out / OVERLAY_NAME + manifest = json.loads( + (args.out / full_build_support.cli.MANIFEST_FILENAME).read_text() + ) + registered = manifest["outputs"][f"orrery/{OVERLAY_NAME}"] + assert Path(registered["path"]) == overlay_path + assert registered["sha256"] == hashlib.sha256(overlay_path.read_bytes()).hexdigest() + assert registered["bytes"] == overlay_path.stat().st_size + preserved = args.attempt_evidence / "orrery-publication" + inventory = json.loads((preserved / "orrery.upload.json").read_bytes()) + assert (preserved / "graph.orrery.json").is_file() + assert not (preserved / OVERLAY_NAME).exists() + assert OVERLAY_NAME not in {entry["name"] for entry in inventory["files"]} + + +def test_refused_build_registers_its_execution_overlay_in_its_evidence_record( + tmp_path, monkeypatch +): + args, status = synthetic_full_build( + tmp_path, monkeypatch, "uk_local_geography_ladder_post_calibration" + ) + assert status == 1 + evidence = json.loads( + (args.out / orrery_publication.EVIDENCE_MANIFEST_NAME).read_bytes() + ) + overlay_path = args.out / OVERLAY_NAME + assert evidence["orrery_publication"]["recorded_execution"] is True + assert evidence["outputs"][OVERLAY_NAME]["sha256"] == ( + hashlib.sha256(overlay_path.read_bytes()).hexdigest() + ) + + +def test_failed_build_keeps_its_execution_overlay_in_the_failure_bundle( + tmp_path, monkeypatch +): + args, build = synthetic_build_request(tmp_path, monkeypatch) + + def refuse_manifest(*arguments, **options): + raise RuntimeError("manifest projection failed") + + monkeypatch.setattr( + full_build_support.cli, "rowwise_candidate_manifest_from_graph", refuse_manifest + ) + with pytest.raises(RuntimeError, match="manifest projection failed"): + full_build_support.cli.execute_full_build(build, args) + bundle = args.attempt_evidence / orrery_publication.FAILURE_BUNDLE_DIRECTORY + evidence = json.loads( + (bundle / orrery_publication.EVIDENCE_MANIFEST_NAME).read_bytes() + ) + assert evidence["orrery_publication"]["recorded_execution"] is True + assert Path(evidence["outputs"][OVERLAY_NAME]["path"]) == bundle / OVERLAY_NAME + assert load_execution_overlay(bundle / OVERLAY_NAME)["phases"] + assert not (args.attempt_evidence / "orrery-publication" / OVERLAY_NAME).exists() + + +def test_build_without_recorded_execution_writes_no_execution_overlay( + tmp_path, monkeypatch +): + args = fixture(tmp_path) + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", None) + manifest = {"outputs": {}} + result = orrery_publication.finalize_graph(args, manifest=manifest) + assert result.receipt["recorded_execution"] is False + assert not (args.out / OVERLAY_NAME).exists() + assert OVERLAY_NAME not in result.files + assert f"orrery/{OVERLAY_NAME}" not in manifest["outputs"] diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index 28dde48d34..a3caf57a6b 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -2,11 +2,18 @@ Public API. ``decl`` and ``kernel`` are the frozen interfaces; the runtime modules implement execution, storage, provenance, and inspection against them. + +Declarations, the compiler schema and the Orrery projection need only the +standard library, so they import eagerly. Exports from the runtime modules, +which need numpy, pandas and microcosm-frame, load on first access: a process +that only projects saved schemas and execution overlays imports +``microcosm.graph.orrery`` without those dependencies. """ from __future__ import annotations from collections.abc import Mapping +from importlib import import_module from importlib import metadata as _metadata from .decl import ( @@ -42,25 +49,8 @@ StoreMissError, StoreUnavailableError, ) -from .kernel import ( - ArtifactValue, - Capabilities, - Determinism, - Kernel, - KernelBase, - KernelContext, - KernelRegistry, - KernelResult, - KernelRole, - Numeric, - NumericScope, - SeedSource, - Tolerance, - source_hash, -) -from .keys import platform_fingerprint -from .randomness import keyed_uniform from .schema import graph_schema +from .serialize import graph_from_json, graph_to_json from .weight_update import WEIGHT_UPDATE_AXIS_SCHEMA, weight_update_receipt __all__ = [ @@ -139,8 +129,76 @@ "weight_update_receipt", "orrery_document", "orrery_json", + "ArtifactSummaryContext", + "RecordedRun", + "checkpoint_references", + "collect_execution_evidence", + "load_run_evidence", + "publish_run_evidence", + "record_run_binding", + "save_graph_schema", + "save_run_evidence", ] +#: The runtime modules and the exports each one supplies on first access. +_RUNTIME_MODULE_EXPORTS: dict[str, tuple[str, ...]] = { + "codecs": ( + "SOURCE_CODECS", + "SourceBytesCodec", + "SourceCodec", + "SourceCodecRegistry", + "load_source", + "load_source_bytes", + ), + "evidence": ( + "ArtifactSummaryContext", + "RecordedRun", + "checkpoint_references", + "collect_execution_evidence", + "load_run_evidence", + "publish_run_evidence", + "record_run_binding", + "save_graph_schema", + "save_run_evidence", + ), + "executor": ("NodeRejected", "run_graph"), + "explain": ("explain_html",), + "kernel": ( + "ArtifactValue", + "Capabilities", + "Determinism", + "Kernel", + "KernelBase", + "KernelContext", + "KernelRegistry", + "KernelResult", + "KernelRole", + "Numeric", + "NumericScope", + "SeedSource", + "Tolerance", + "source_hash", + ), + "keys": ("platform_fingerprint",), + "manifest": ("Decision", "NodeReceipt", "PopulationView", "RunManifest"), + "population": ("MassRecord", "Population", "PopulationError"), + "randomness": ("keyed_uniform",), + "store": ( + "ContentStore", + "ResumePolicy", + "StoreCorrupt", + "StoreError", + "StoreMiss", + "StoreUnavailable", + ), + "view": ("describe",), +} +_RUNTIME_EXPORTS = { + export: module_name + for module_name, exports in _RUNTIME_MODULE_EXPORTS.items() + for export in exports +} + _FRAME_SERIES = "0.1" @@ -156,9 +214,25 @@ def _check_frame_version() -> None: ) +# The installed series is checked before any runtime module can be imported. _check_frame_version() +def __getattr__(name: str) -> object: + """Import a runtime export's defining module the first time it is used.""" + + module_name = _RUNTIME_EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(f"{__name__}.{module_name}"), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted({*globals(), *_RUNTIME_EXPORTS}) + + def orrery_document( graph: Graph | CompiledGraph, *, @@ -185,50 +259,3 @@ def orrery_json( from .orrery import orrery_json as export return export(graph, title=title, extensions=extensions, execution=execution) - - -from .codecs import ( # noqa: E402 - check dependency series before runtime import - SOURCE_CODECS, - SourceBytesCodec, - SourceCodec, - SourceCodecRegistry, - load_source, - load_source_bytes, -) -from .evidence import ( # noqa: E402 - ArtifactSummaryContext, - RecordedRun, - checkpoint_references, - collect_execution_evidence, - load_run_evidence, - publish_run_evidence, - record_run_binding, - save_graph_schema, - save_run_evidence, -) -from .executor import NodeRejected, run_graph # noqa: E402 -from .explain import explain_html # noqa: E402 -from .manifest import Decision, NodeReceipt, PopulationView, RunManifest # noqa: E402 -from .population import MassRecord, Population, PopulationError # noqa: E402 -from .serialize import graph_from_json, graph_to_json # noqa: E402 -from .store import ( # noqa: E402 - ContentStore, - ResumePolicy, - StoreCorrupt, - StoreError, - StoreMiss, - StoreUnavailable, -) -from .view import describe # noqa: E402 - -__all__ += [ - "ArtifactSummaryContext", - "RecordedRun", - "checkpoint_references", - "collect_execution_evidence", - "load_run_evidence", - "publish_run_evidence", - "record_run_binding", - "save_graph_schema", - "save_run_evidence", -] diff --git a/packages/microcosm-graph/src/microcosm/graph/canonical.py b/packages/microcosm-graph/src/microcosm/graph/canonical.py index 50e6dd65c8..a9e80411af 100644 --- a/packages/microcosm-graph/src/microcosm/graph/canonical.py +++ b/packages/microcosm-graph/src/microcosm/graph/canonical.py @@ -12,7 +12,7 @@ from .decl import DESCRIPTIVE_FIELDS -__all__ = ["canonical_json", "normative", "sha256_domain"] +__all__ = ["canonical_json", "normative", "sha256", "sha256_domain"] def _json_value(value: object) -> object: @@ -58,6 +58,12 @@ def canonical_json(obj: object) -> bytes: ).encode("utf-8") +def sha256(payload: bytes) -> str: + """Return the lowercase hexadecimal SHA-256 digest of ``payload``.""" + + return hashlib.sha256(payload).hexdigest() + + def sha256_domain(domain: str, payload: bytes) -> str: """Hash ``payload`` after a UTF-8 domain and a NUL separator.""" diff --git a/packages/microcosm-graph/src/microcosm/graph/evidence.py b/packages/microcosm-graph/src/microcosm/graph/evidence.py index 8b887a05b7..690cef4730 100644 --- a/packages/microcosm-graph/src/microcosm/graph/evidence.py +++ b/packages/microcosm-graph/src/microcosm/graph/evidence.py @@ -7,7 +7,6 @@ from __future__ import annotations -import hashlib import json import logging import math @@ -20,7 +19,7 @@ from urllib.parse import urlsplit from .availability import execution_state, unavailable_artifacts -from .canonical import canonical_json, normative +from .canonical import canonical_json, normative, sha256 from .decl import ( GATE_OUTCOMES, CompiledGraph, @@ -29,6 +28,7 @@ StructuralDelta, compile_graph, ) +from .execution_overlay import EXECUTION_PROTOCOL from .kernel import Capabilities, KernelRole from .keys import ( artifact_key, @@ -52,7 +52,6 @@ BINDING_PROTOCOL = "microcosm.graph.run-binding.v1" EVIDENCE_INPUT_PROTOCOL = "microcosm.graph.execution-input.v1" -EXECUTION_PROTOCOL = "microcosm.graph.execution-evidence.v1" # Native evidence files are exact copies of run manifests and graphs, which a # real build already holds in memory; they are bounded by bytes and nesting # depth only. The per-item complexity charge of the compiler-schema reader is @@ -69,10 +68,6 @@ _LOG = logging.getLogger(__name__) -def sha256(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - def _plain(value: object) -> object: return _plain_json(json.loads(canonical_json(value))) diff --git a/packages/microcosm-graph/src/microcosm/graph/execution_overlay.py b/packages/microcosm-graph/src/microcosm/graph/execution_overlay.py new file mode 100644 index 0000000000..124cebf91e --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/execution_overlay.py @@ -0,0 +1,80 @@ +"""The execution overlay as its own file, shared by evidence capture and projection. + +:func:`microcosm.graph.evidence.collect_execution_evidence` verifies recorded +runs against the content store and returns the overlay; +:mod:`microcosm.graph.orrery` projects it. The file holds that same object as +canonical JSON, so a process that only projects can read it back. This module +needs only the standard library, so that process needs neither the store nor +numpy, pandas or microcosm-frame. +""" + +from __future__ import annotations + +import json +import os +import stat +from pathlib import Path + +from .canonical import canonical_json +from .presentation import require +from .schema import _MAX_BYTES as _MAX_OVERLAY_BYTES +from .schema import _plain_json + +__all__ = ["EXECUTION_PROTOCOL", "execution_overlay_bytes", "load_execution_overlay"] + +EXECUTION_PROTOCOL = "microcosm.graph.execution-evidence.v1" + + +def _overlay_object(value: object) -> dict: + require( + type(value) is dict and value.get("protocol") == EXECUTION_PROTOCOL, + f"execution overlay must be a {EXECUTION_PROTOCOL} object", + ) + return value # type: ignore[return-value] + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + require(key not in result, "duplicate JSON key in execution overlay") + result[key] = value + return result + + +def _reject_constant(_value: str) -> None: + raise ValueError("Graph evidence: execution overlay requires finite JSON numbers.") + + +def execution_overlay_bytes(overlay: object) -> bytes: + """Serialize an overlay from ``collect_execution_evidence`` as canonical JSON. + + :func:`load_execution_overlay` reads the bytes back as an equal object, so + projecting it gives the same Orrery JSON as projecting ``overlay`` itself. + """ + + return canonical_json(_overlay_object(_plain_json(overlay))) + + +def load_execution_overlay(path: Path) -> dict: + """Read an overlay file written from :func:`execution_overlay_bytes`. + + Only a UTF-8 regular file within the schema's serialized size limit, which + evidence collection and projection both apply to the overlay, is read. + Duplicate keys, non-finite numbers, values outside the schema's JSON bounds + and any other protocol are refused. The overlay's graph binding and its + records are checked when it is projected. + """ + + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(descriptor, "rb") as stream: + info = os.fstat(stream.fileno()) + require(stat.S_ISREG(info.st_mode), "execution overlay must be a regular file") + require(info.st_size <= _MAX_OVERLAY_BYTES, "execution overlay byte limit") + payload = stream.read(_MAX_OVERLAY_BYTES + 1) + require(len(payload) <= _MAX_OVERLAY_BYTES, "execution overlay byte limit") + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_object, + parse_constant=_reject_constant, + ) + return _overlay_object(_plain_json(value)) diff --git a/packages/microcosm-graph/src/microcosm/graph/orrery_evidence.py b/packages/microcosm-graph/src/microcosm/graph/orrery_evidence.py index 79670dbafa..dee1b6597e 100644 --- a/packages/microcosm-graph/src/microcosm/graph/orrery_evidence.py +++ b/packages/microcosm-graph/src/microcosm/graph/orrery_evidence.py @@ -2,8 +2,8 @@ from __future__ import annotations -from .canonical import canonical_json -from .evidence import EXECUTION_PROTOCOL, sha256 +from .canonical import canonical_json, sha256 +from .execution_overlay import EXECUTION_PROTOCOL from .presentation import ( digest, presentation_id, diff --git a/packages/microcosm-graph/tests/engine_free/shared/test_graph_orrery_projection_without_store.py b/packages/microcosm-graph/tests/engine_free/shared/test_graph_orrery_projection_without_store.py new file mode 100644 index 0000000000..29ae544dae --- /dev/null +++ b/packages/microcosm-graph/tests/engine_free/shared/test_graph_orrery_projection_without_store.py @@ -0,0 +1,280 @@ +"""The Orrery projection runs without the build runtime. + +A collector worker will project a saved compiler schema and execution overlay +file without the content store or the population libraries. These tests run +that projection in a fresh interpreter that refuses to import numpy, pandas, +microcosm-frame or the store, and compare its bytes with the projection a +build makes in-process. +""" + +import importlib.util +import json +import os +import subprocess +import sys + +import pytest + +import microcosm.graph as graph_api +from microcosm.graph import ( + collect_execution_evidence, + execution_overlay, + graph_schema, + record_run_binding, +) +from microcosm.graph.canonical import canonical_json +from microcosm.graph.orrery import main, orrery_json_from_schema +from microcosm.graph.presentation import PRESENTATION_EXTENSION, PRESENTATION_PROTOCOL +from test_support.paths import paths_for + +_PATHS = paths_for("microcosm-graph") +if "_toy" not in sys.modules: + _SPEC = importlib.util.spec_from_file_location("_toy", _PATHS.tests / "_toy.py") + _TOY = importlib.util.module_from_spec(_SPEC) + sys.modules["_toy"] = _TOY + _SPEC.loader.exec_module(_TOY) +toy = sys.modules["_toy"] + +#: Modules a projection process must be able to do without. +RUNTIME_MODULES = ("numpy", "pandas", "microcosm.frame", "microcosm.graph.store") + +#: Installs the refusal before anything else imports, then runs the body. +_REFUSE_RUNTIME_IMPORTS = r""" +import importlib.abc, json, sys + +refused = tuple(sys.argv[1].split(",")) + + +def is_refused(name): + return any(name == prefix or name.startswith(prefix + ".") for prefix in refused) + + +class RefuseRuntimeImports(importlib.abc.MetaPathFinder): + def find_spec(self, name, path=None, target=None): + if is_refused(name): + raise ImportError(f"the Orrery projection imported {name}") + return None + + +sys.meta_path.insert(0, RefuseRuntimeImports()) +""" + +_RESTRICTED_PROJECTION = ( + _REFUSE_RUNTIME_IMPORTS + + r""" +from pathlib import Path + +from microcosm.graph.execution_overlay import load_execution_overlay +from microcosm.graph.orrery import orrery_json_from_schema + +schema_path, overlay_path, output_path = (Path(argument) for argument in sys.argv[2:5]) +schema = json.loads(schema_path.read_bytes()) +overlay = load_execution_overlay(overlay_path) +output_path.write_text( + orrery_json_from_schema(schema, execution=overlay), encoding="utf-8" +) +print(json.dumps(sorted(name for name in sys.modules if is_refused(name)))) +""" +) + +_RESTRICTED_COMMAND = ( + _REFUSE_RUNTIME_IMPORTS + + r""" +import runpy + +sys.argv = ["microcosm.graph.orrery", *sys.argv[2:]] +runpy.run_module("microcosm.graph.orrery", run_name="__main__") +""" +) + + +def _run_restricted(script: str, *arguments: str, cwd) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", script, ",".join(RUNTIME_MODULES), *arguments], + capture_output=True, + text=True, + timeout=300, + check=False, + cwd=cwd, + ) + + +@pytest.fixture +def recorded(tmp_path): + """A computing phase and a replay of the toy graph, with their schema.""" + computed = toy.run_toy(toy.full_graph(), tmp_path) + replayed = toy.run_toy( + computed.compiled.graph, + tmp_path, + sources=computed.sources, + registry=computed.registry, + store=computed.store, + ) + schema = graph_schema( + computed.compiled, + extensions={ + PRESENTATION_EXTENSION: { + "protocol": PRESENTATION_PROTOCOL, + "scope": {"id": "fixture", "label": "Synthetic country"}, + "groups": [ + {"id": "sources", "label": "Sources", "sources": ["survey"]}, + { + "id": "work", + "label": "Processing", + "operations": list(computed.compiled.order), + }, + ], + } + }, + ) + runs = [ + record_run_binding( + computed.compiled, computed.manifest, attempt_id="attempt", phase="first" + ), + record_run_binding( + replayed.compiled, replayed.manifest, attempt_id="attempt", phase="replay" + ), + ] + overlay = collect_execution_evidence(schema, runs=runs, store=computed.store) + return schema, overlay + + +def test_orrery_projection_runs_where_numpy_pandas_frame_and_store_cannot_import( + recorded, tmp_path +): + schema, overlay = recorded + schema_path = tmp_path / "graph.schema.json" + overlay_path = tmp_path / "graph.execution.json" + output_path = tmp_path / "graph.orrery.json" + schema_path.write_bytes(canonical_json(schema)) + overlay_path.write_bytes(execution_overlay.execution_overlay_bytes(overlay)) + + completed = _run_restricted( + _RESTRICTED_PROJECTION, + str(schema_path), + str(overlay_path), + str(output_path), + cwd=tmp_path, + ) + + assert completed.returncode == 0, completed.stderr[-4000:] + assert json.loads(completed.stdout) == [] + assert output_path.read_bytes() == ( + orrery_json_from_schema(schema, execution=overlay).encode("utf-8") + ) + + +def test_orrery_command_runs_where_numpy_pandas_frame_and_store_cannot_import( + recorded, tmp_path +): + schema, _overlay = recorded + schema_path = tmp_path / "graph.schema.json" + schema_path.write_bytes(canonical_json(schema)) + restricted_output = tmp_path / "restricted" / "graph.orrery.json" + in_process_output = tmp_path / "in-process" / "graph.orrery.json" + + completed = _run_restricted( + _RESTRICTED_COMMAND, + "--schema", + str(schema_path), + "--output", + str(restricted_output), + cwd=tmp_path, + ) + + assert completed.returncode == 0, completed.stderr[-4000:] + assert main(["--schema", str(schema_path), "--output", str(in_process_output)]) == 0 + assert restricted_output.read_bytes() == in_process_output.read_bytes() + + +def test_package_runtime_exports_still_resolve_from_the_package(): + from microcosm.graph import executor, store + + for name in graph_api.__all__: + assert getattr(graph_api, name) is not None, name + assert set(graph_api.__all__) <= set(dir(graph_api)) + assert graph_api.ContentStore is store.ContentStore + assert graph_api.run_graph is executor.run_graph + with pytest.raises(AttributeError, match="not_an_export"): + graph_api.not_an_export # noqa: B018 + + +def test_execution_overlay_file_round_trips_to_byte_identical_orrery_json( + recorded, tmp_path +): + schema, overlay = recorded + overlay_path = tmp_path / "graph.execution.json" + overlay_path.write_bytes(execution_overlay.execution_overlay_bytes(overlay)) + + loaded = execution_overlay.load_execution_overlay(overlay_path) + + assert overlay_path.read_bytes() == canonical_json(overlay) + assert loaded["protocol"] == "microcosm.graph.execution-evidence.v1" + assert loaded == overlay + assert orrery_json_from_schema(schema, execution=loaded) == ( + orrery_json_from_schema(schema, execution=overlay) + ) + + +_OVERLAY_PROTOCOL_FIELD = b'"protocol":"microcosm.graph.execution-evidence.v1"' + + +@pytest.mark.parametrize( + "payload,message", + [ + (b'{"protocol":"microcosm.graph.execution-input.v1"}', "evidence.v1 object"), + (b'["microcosm.graph.execution-evidence.v1"]', "evidence.v1 object"), + ( + b"{" + _OVERLAY_PROTOCOL_FIELD + b',"history":"a","history":"b"}', + "duplicate", + ), + (b"{" + _OVERLAY_PROTOCOL_FIELD + b',"history":NaN}', "finite JSON numbers"), + (b"{" + _OVERLAY_PROTOCOL_FIELD + b',"history":1e400}', "finite numbers"), + (b"{" + _OVERLAY_PROTOCOL_FIELD.replace(b"evidence", b"\xff") + b"}", "utf-8"), + ], + ids=[ + "another protocol", + "not an object", + "duplicate key", + "NaN literal", + "overflowing number", + "not UTF-8", + ], +) +def test_execution_overlay_reader_refuses_malformed_files(tmp_path, payload, message): + overlay_path = tmp_path / "graph.execution.json" + overlay_path.write_bytes(payload) + with pytest.raises(ValueError, match=message): + execution_overlay.load_execution_overlay(overlay_path) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="requires POSIX FIFO") +def test_execution_overlay_reader_refuses_a_fifo_without_waiting_for_a_writer( + tmp_path, +): + overlay_path = tmp_path / "graph.execution.json" + os.mkfifo(overlay_path) + with pytest.raises(ValueError, match="regular file"): + execution_overlay.load_execution_overlay(overlay_path) + + +def test_execution_overlay_reader_refuses_a_file_over_the_byte_limit( + recorded, tmp_path, monkeypatch +): + _schema, overlay = recorded + overlay_path = tmp_path / "graph.execution.json" + payload = execution_overlay.execution_overlay_bytes(overlay) + overlay_path.write_bytes(payload) + monkeypatch.setattr(execution_overlay, "_MAX_OVERLAY_BYTES", len(payload) - 1) + with pytest.raises(ValueError, match="byte limit"): + execution_overlay.load_execution_overlay(overlay_path) + + +def test_execution_overlay_writer_refuses_a_value_without_the_overlay_protocol( + recorded, +): + _schema, overlay = recorded + another_protocol = {**overlay, "protocol": "microcosm.graph.execution-input.v1"} + for value in (another_protocol, [overlay]): + with pytest.raises(ValueError, match="evidence.v1 object"): + execution_overlay.execution_overlay_bytes(value)