diff --git a/CHANGELOG.md b/CHANGELOG.md index eddbfec7e..6e9abf26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ ### Changed +- The metric contract and the quality validators move from `inferrsi.model` to + Benchmark (Refs #498, slice 1). There is no forwarding module or alias, and the + import gate fails any import of an old path, even when no file is behind it. + - `inferrsi.model.protocols.metrics` is now `inferrsi.benchmark.metrics`. + `inferrsi.model.validators` (`flare`, `image`) is now + `inferrsi.benchmark.validators`. + - The Flare adapter command is now + `python -m inferrsi.benchmark.validators.flare`, with the same arguments, + outputs and exit codes. The old `python -m inferrsi.model.validators.flare` + fails with `ModuleNotFoundError`. + - New `inferrsi.benchmark.artifacts` holds Benchmark's own `sha256`, + `write_json` and `atomic_write_json`. Their code equals the copies in + `inferrsi.infra.artifacts` and Harness, and + `tests/test_artifact_helper_copies.py` keeps the copies equal. + `inferrsi.infra.artifacts` stays for its other users. + - A new Flare receipt keeps its schema and keys. Its `adapter` digests are those + of `benchmark/validators/flare.py` and `benchmark/artifacts.py`. Existing + receipts are not rewritten. + - The MoE task's `benchmark.py` imports the Benchmark modules. Each MoE + `evaluate` call prepares its own copy of the task files and launches that copy + in the same call, so a new run binds the changed `benchmark.py` bytes and no + directory prepared before this change is launched again. + - Pickled objects that name an old module path no longer load. The Knowledge + records that cite `src/inferrsi/model/protocols/metrics.py` at commit + `b3ef95a` keep that pinned path. - The five runtime resource loaders are closed and the combined tree is isolated (Refs #412, R3). `first_wave_report()` now has no violation and no blocker, and `new_core_closure_isolated` and `isolation_complete` are both true. diff --git a/docs/architecture.md b/docs/architecture.md index cba3b66f5..a08f2d6a4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,7 +92,7 @@ interfaces; existing CLI commands remain supported. | --- | --- | --- | | Entry and knowledge | CLI arguments, immutable RunSpec, cited knowledge/context export | Model execution or automatic source editing | | Harness | AB/BA order, deadlines, source checks, scenario composition, evidence, performance adjudication and reporting | Device-specific conditions, request schemas or kernel dispatch | -| Model | Request payloads, metric schemas, output decoding and explicit quality checks | Scheduling, worker lifetime or hardware selection | +| Model | Request payloads and output decoding | Scheduling, worker lifetime, hardware selection, metric contracts or quality checks | | Infra | Repository identity, device discovery/qualification, process ownership, HTTP transport, storage and artifacts | Model constants, campaign gates or performance verdicts | Dependencies flow from entrypoints to Harness, from scenarios to Model/Infra, @@ -163,6 +163,11 @@ src/inferrsi/ registry.py Generic task resource discovery tasks/ Task-specific definitions, workloads and assessment legacy_run_v0.py Old run gates: metric contract, summaries and verdict + metrics.py Finite positive dotted-key metric contract + artifacts.py Benchmark-owned file digests and JSON writes + validators/ + flare.py Offline Flare delivery-evidence adapter and pair check + image.py Explicit PNG dimensions/mode, hashes, RGB/alpha quality gates sampling.py Warmup seeds, first-response retention, sample/artifact groups skill_evaluation.py Skill trial check validity, verdict and condition statistics skill_eval_diagnostic.py Synthetic KV admission diagnosis task and grader @@ -258,10 +263,7 @@ src/inferrsi/ external.py External command trials; metric validation injected model/ protocols/ - metrics.py Finite positive dotted-key metric contract image.py OpenAI-style image payload and PNG response protocol - validators/ - image.py Explicit PNG dimensions/mode, hashes, RGB/alpha quality gates infra/ processes.py Supervised Linux serving workers and their group cleanup targets/git.py Repository identity and source snapshots @@ -301,7 +303,7 @@ path to its replacement and lists what still keeps the isolation gate false. | Add | Main extension locations | Required qualification | | --- | --- | --- | -| Model or scenario | `model/protocols`, `model/validators`, `harness/scenarios` | Frozen request and metric contract; model-specific quality coverage | +| Model or scenario | `model/protocols`, `benchmark/metrics.py`, `benchmark/validators`, `harness/scenarios` | Frozen request and metric contract; model-specific quality coverage | | Repository | `infra/targets`, a composed scenario, `data/repo_packs` | Stable source identity; command/result contract; environment isolation | | Hardware | `infra/platforms` | Device selection, process exclusivity and independent backend/model evidence; a backend is registered in `infra/platforms/__init__.py` and its probe command is defined in the implementing module under `infra/platforms/`, not in package data | @@ -326,7 +328,7 @@ benchmark format, reuse the standard executor and only replace metric interpreta from functools import partial from inferrsi.apps.legacy.run import execute_trial, run_experiment -from inferrsi.model.protocols.metrics import read_metric +from inferrsi.benchmark.metrics import read_metric def read_engine_metric(document, key): @@ -346,7 +348,7 @@ cleanup. A custom `execute(command, cwd, trial_dir, deadline, metric_key)` must preserve that trial record contract. Prefer composing `execute_trial` over reimplementing process lifetime. Metric readers convert a result document into a float; `benchmark.legacy_run_v0.trial_metric` then applies the one finite, positive -metric contract (`model.protocols.metrics.validate_metric`) to whatever the reader +metric contract (`benchmark.metrics.validate_metric`) to whatever the reader returns, so a reader that yields `0.0`, a `bool` or a non-finite value records a metric error rather than entering the evaluation. The Harness executor `harness.scenarios.external.execute_trial` holds no metric rule: a caller that diff --git a/docs/contributing.md b/docs/contributing.md index 003057fd4..a9efdff8b 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -35,7 +35,7 @@ policy does not renormalize existing blobs or authorize edits to frozen files. Nightly checks cover Linux and Windows checkouts with `core.autocrlf` set to `false`, `true` and `input`. -Use `harness/scenarios` and `model/protocols` / `model/validators` for new models, +Use `harness/scenarios`, `model/protocols` and `benchmark/validators` for new models, `infra/targets` with `data/repo_packs` for repositories, and `infra/platforms` for hardware. A backend is registered in `infra/platforms/__init__.py`; its probe command is defined in the implementing module under `infra/platforms/`. That diff --git a/docs/flare-qualification.md b/docs/flare-qualification.md index e8fce79ad..2b3217866 100644 --- a/docs/flare-qualification.md +++ b/docs/flare-qualification.md @@ -2,7 +2,7 @@ The adapter reads existing Flare qualification artifacts and writes a separate receipt. It never launches Flare, downloads a model or executes imported code. -Run it with `python -m inferrsi.model.validators.flare`. The existing +Run it with `python -m inferrsi.benchmark.validators.flare`. The existing `external_benchmark` scenario, `validation_command` and evaluator own execution and verdicts. @@ -92,11 +92,11 @@ synthetic status, package bytes, corpus bytes and prepared capacity. This keeps an incomplete or incomparable pair out of the evaluator's performance inputs. ```text -python -m inferrsi.model.validators.flare measure \ +python -m inferrsi.benchmark.validators.flare measure \ --packet /packet.json --peer /packet.json \ --output /baseline.json -python -m inferrsi.model.validators.flare validate-pair \ +python -m inferrsi.benchmark.validators.flare validate-pair \ --baseline /packet.json --candidate /packet.json \ --output /pair.json ``` @@ -104,7 +104,7 @@ python -m inferrsi.model.validators.flare validate-pair \ For `RunSpec`, use these argv sequences: ```python -cli = (python, "-m", "inferrsi.model.validators.flare") +cli = (python, "-m", "inferrsi.benchmark.validators.flare") baseline_command = cli + ( "measure", "--packet", baseline, "--peer", candidate, "--output", "{result}", ) @@ -145,7 +145,7 @@ From a development checkout with InferRSI installed: ```text python tests/fixtures/flare_packet.py -python -m inferrsi.model.validators.flare measure \ +python -m inferrsi.benchmark.validators.flare measure \ --packet /packet.json --peer /packet.json \ --output /receipt.json pytest tests/test_flare_adapter.py diff --git a/docs/module-layout.md b/docs/module-layout.md index 3550cf7a5..870792851 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -145,10 +145,10 @@ Several flat modules remain shared seams until their slice starts. | `apps/mindie/` (`project.py`, `context.py`, `mcp.py`, `read_worker.py`) | The local MindIE adapter: project grant, stdio MCP, context export and the read worker | M1 (#452) moved the four root MindIE files here; no Harness, Knowledge or Infra module names MindIE, and the initializer imports nothing | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `harness/runner.py`, `harness/scenarios/external.py`, `harness/atomic_json.py` | Old run execution facts and Harness-owned JSON writes | Ports injected; no Model edge, and no Infra edge except the local process executor | -| `benchmark/legacy_run_v0.py`, `benchmark/sampling.py` | Old run gates and metric validation; image/video sample collection | Imports only `model/protocols/metrics`; owns its rule inputs and verdict labels | +| `benchmark/legacy_run_v0.py`, `benchmark/sampling.py` | Old run gates and metric validation; image/video sample collection | Imports only `benchmark/metrics.py`; owns its rule inputs and verdict labels | | `apps/legacy/` | Old run composition, old `evaluate` entry and HTML report | Supplies the Git source snapshot from `infra/targets`; remains in Apps | | `apps/inspection.py` | Prepared-environment inspection bound to this host | Binds the `infra/inspection.py` facts API to `infra.commands.probe_command` and `infra.targets.repository_identity`; replaces root `inspection.py`, which R1 deleted with the three old-run root forwarders | -| `model/validators/` | Benchmark-leaning checks | Not moved | +| `benchmark/metrics.py`, `benchmark/validators/`, `benchmark/artifacts.py` | The metric contract, the Flare and image quality checks, and Benchmark's own file digests and JSON writes | #498 slice 1 moved the first two from `model/protocols/metrics.py` and `model/validators/`, with no forwarder or alias; the gate fails any import of the old paths | | `infra/` | Seams used by the current tree, including the `infra/inspection.py` facts API | Not a fourth core; reaches no Knowledge, Harness, Benchmark or Apps module | | `apps/views/summary_chart.py` | Stdlib SVG gain chart and report figure spec | A2 (#408) moved root `summary_chart.py` here; the renderer, its specification checks and its `main` are unchanged, and `tools/campaign_summary_chart.py` calls it | | `contracts/` (`runspec.py`, `json_v1.py`, `claim.py`, `evidence.py`) | Technical-only contracts: RunSpec, canonical JSON, verdict vocabulary, evidence segment and path rules | Exact module set; the initializer re-exports only `RunSpec`; not a fourth core | diff --git a/docs/operations.md b/docs/operations.md index 9832c1479..5d473c213 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -57,7 +57,7 @@ Effect classes used below: | Assemble an optimization trace | CLI `campaign assemble-trace`; [offline dashboard](dashboard.md#capture-the-optimization-trace) | Local evidence-writing | Copies an explicitly reviewed context into a new bounded campaign-root snapshot; caller owns redaction and narrative accuracy | | Import a MindIE read | CLI `campaign import-mindie`; [Codex workflow](codex-plugin.md#mindie-knowledge) | Local evidence-writing | Retains caller-supplied query/explain bytes and hashes, exports delivered graded bodies and a trace context fragment; no activation, model call, transcript capture or automatic caller review | | Associate runs with a local campaign | CLI `campaign init` / `campaign record` / `campaign decide`; [offline dashboard](dashboard.md) | Local evidence-writing | Legacy manual records reference existing artifacts and never start an agent; record/decide cannot mutate a managed optimization campaign | -| Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | +| Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.benchmark.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | | Export a cloud-generated parser pair | [Generation-only protocol](skill-eval-generation-only.md), `python -m tools.skill_eval_cloud_export` | Local evidence-writing; no execution or publication | Validates frozen inputs and retained sources; writes one bounded inert JSON bundle, never executes parser source | diff --git a/src/inferrsi/benchmark/artifacts.py b/src/inferrsi/benchmark/artifacts.py new file mode 100644 index 000000000..6fb3a116c --- /dev/null +++ b/src/inferrsi/benchmark/artifacts.py @@ -0,0 +1,51 @@ +# Benchmark-owned file digests and JSON evidence writes. The function bodies equal +# those in the other cores' copies; tests/test_artifact_helper_copies.py keeps them +# equal. Benchmark imports no other inferrsi module here. + +import hashlib +import json +import os +import time +import uuid +from pathlib import Path + +# Windows refuses os.replace while another handle holds the destination open. +# A bounded retry covers the observed contention without hiding a real failure. +REPLACE_ATTEMPTS = 8 +REPLACE_DELAY_SECONDS = 0.05 + + +def sha256(path): + hasher = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(block) + return hasher.hexdigest() + + +def write_json(path, value): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + + +def atomic_write_json(path: Path, value: object) -> None: + path = Path(path) + # A per-writer temporary name keeps two writers from sharing one file. + temporary = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + try: + for attempt in range(REPLACE_ATTEMPTS): + try: + temporary.replace(path) + return + except PermissionError: + if attempt + 1 == REPLACE_ATTEMPTS: + raise + time.sleep(REPLACE_DELAY_SECONDS) + finally: + if temporary.exists(): + temporary.unlink() diff --git a/src/inferrsi/benchmark/legacy_run_v0.py b/src/inferrsi/benchmark/legacy_run_v0.py index 9f87f53d2..9727311d3 100644 --- a/src/inferrsi/benchmark/legacy_run_v0.py +++ b/src/inferrsi/benchmark/legacy_run_v0.py @@ -4,8 +4,8 @@ from collections.abc import Callable from dataclasses import dataclass -from ..model.protocols.metrics import read_metric as read_metric -from ..model.protocols.metrics import validate_metric +from .metrics import read_metric as read_metric +from .metrics import validate_metric # The v0 protocol's own labels. Benchmark shares no verdict type with another module. PASS, FAIL, INCONCLUSIVE = "PASS", "FAIL", "INCONCLUSIVE" diff --git a/src/inferrsi/model/protocols/metrics.py b/src/inferrsi/benchmark/metrics.py similarity index 100% rename from src/inferrsi/model/protocols/metrics.py rename to src/inferrsi/benchmark/metrics.py diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py index e735be59c..9030a60b2 100644 --- a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py @@ -16,8 +16,8 @@ import uuid from pathlib import Path -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model.protocols.metrics import read_metric +from inferrsi.benchmark.artifacts import atomic_write_json +from inferrsi.benchmark.metrics import read_metric # Worker cleanup for this MoE invocation, including detached torchrun sessions. # torchrun workers may create their own sessions, outside the executor's process diff --git a/src/inferrsi/model/validators/__init__.py b/src/inferrsi/benchmark/validators/__init__.py similarity index 100% rename from src/inferrsi/model/validators/__init__.py rename to src/inferrsi/benchmark/validators/__init__.py diff --git a/src/inferrsi/model/validators/flare.py b/src/inferrsi/benchmark/validators/flare.py similarity index 99% rename from src/inferrsi/model/validators/flare.py rename to src/inferrsi/benchmark/validators/flare.py index 5eb58da29..4d43b5903 100644 --- a/src/inferrsi/model/validators/flare.py +++ b/src/inferrsi/benchmark/validators/flare.py @@ -6,7 +6,7 @@ from decimal import Decimal, InvalidOperation from pathlib import Path -from ...infra import artifacts +from .. import artifacts SOURCE = { "repository": "https://github.com/NVIDIA/flare-local-inference", diff --git a/src/inferrsi/model/validators/image.py b/src/inferrsi/benchmark/validators/image.py similarity index 98% rename from src/inferrsi/model/validators/image.py rename to src/inferrsi/benchmark/validators/image.py index 87d1674d3..2b8a427a4 100644 --- a/src/inferrsi/model/validators/image.py +++ b/src/inferrsi/benchmark/validators/image.py @@ -7,7 +7,7 @@ import math from dataclasses import dataclass -from ...infra.artifacts import sha256 +from ..artifacts import sha256 @dataclass(frozen=True) diff --git a/src/inferrsi/model/README.md b/src/inferrsi/model/README.md index 29702e88c..fd1080a7b 100644 --- a/src/inferrsi/model/README.md +++ b/src/inferrsi/model/README.md @@ -1,8 +1,9 @@ # Model -`protocols/` defines request and metric contracts. `validators/` owns output -quality checks with explicit campaign-supplied gates. The image validator's -optional dependencies are available through `inferrsi[image]`. +`protocols/` defines request contracts. The metric contract and the output +quality checks moved to `inferrsi.benchmark.metrics` and +`inferrsi.benchmark.validators` (#498 slice 1). The image validator's optional +dependencies are available through `inferrsi[image]`. Model/revision pins and numeric acceptance gates stay in campaign plans. Frozen historical snapshots remain in `tests/fixtures/legacy/`; current examples and diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index 990e6d9ff..fe959a664 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -89,6 +89,11 @@ } ) RETIRED_ROOT_MODULES = RETIRED_BY_R1 | RETIRED_BY_A1 | RETIRED_BY_A2 | RETIRED_BY_M1 +# #498 slice 1 moved these packages and modules out of ``inferrsi.model``, with no +# forwarder or alias: ``inferrsi.benchmark.metrics`` and ``inferrsi.benchmark.validators``. +# Each name covers everything below it. A file under one fails, and so does any import +# that names one, even when its target is absent. +RETIRED_MODULES = frozenset({"inferrsi.model.protocols.metrics", "inferrsi.model.validators"}) # #412 R1, permanent: the one importer of the neutral data leaf outside # ``inferrsi.knowledge``, with the only name it may bind (D020: Infra reads one pack # description and reaches no Knowledge or Harness). This is not a manifest @@ -153,6 +158,22 @@ def _from_base(node: ast.ImportFrom, package: str) -> str: return ".".join(keep + ([node.module] if node.module else [])) +def _retired_imports(tree: ast.Module, package: str) -> set[str]: + """Every name under ``RETIRED_MODULES`` that an import statement in ``tree`` spells.""" + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + base = _from_base(node, package) + names = [base, *(f"{base}.{alias.name}" for alias in node.names)] + else: + continue + found.update( + name for name in names if any(_matches(name, old) for old in RETIRED_MODULES)) + return found + + def _is_entry_delegation(node: ast.stmt, package: str) -> bool: """The module entry's one admitted statement, in its exact plain shape.""" module, name = MODULE_ENTRY_DELEGATION @@ -729,6 +750,16 @@ def first_wave_report( for name in modules if name in RETIRED_ROOT_MODULES and name not in sources ] + violations += [ + f"{name} is a retired module" + for name in modules + if any(_matches(name, old) for old in RETIRED_MODULES) + ] + violations += [ + f"{name} imports retired module {target}" + for name in modules + for target in sorted(_retired_imports(graph.trees[name], graph.packages[name])) + ] # Outside Knowledge the leaf has its one permanent importer and any staged pair # that names it. Every other importer fails. declared_importers = dict(LEAF_IMPORTERS) diff --git a/tests/check_installed.py b/tests/check_installed.py index 666416730..7e4c57482 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -776,3 +776,26 @@ def _inventory(root, prefix): check=False, cwd=launch_cwd, ) assert launch.returncode == 0, launch.stderr +# #498 slice 1: the metric contract and the quality validators ship only under +# inferrsi.benchmark. The old inferrsi.model paths are not in the wheel and have no alias. +for retired in ("inferrsi.model.protocols.metrics", "inferrsi.model.validators"): + assert find_spec(retired) is None, retired + try: + import_module(retired) + except ModuleNotFoundError: + pass + else: + raise AssertionError(f"retired module {retired} must not import") +for canonical in ("inferrsi.benchmark.metrics", "inferrsi.benchmark.artifacts", + "inferrsi.benchmark.validators.flare", "inferrsi.benchmark.validators.image"): + assert find_spec(canonical) is not None, canonical +# The flare CLI runs from the installed wheel only under its new path. +with tempfile.TemporaryDirectory(prefix="inferrsi-flare-cli-") as cli_cwd: + for module, code in (("inferrsi.benchmark.validators.flare", 0), + ("inferrsi.model.validators.flare", 1)): + flare_cli = subprocess.run( + [sys.executable, "-I", "-B", "-m", module, "--help"], capture_output=True, + text=True, check=False, cwd=cli_cwd, + ) + assert flare_cli.returncode == code, (module, flare_cli.returncode, flare_cli.stderr) + assert "No module named 'inferrsi.model.validators'" in flare_cli.stderr, flare_cli.stderr diff --git a/tests/fixtures/flare_packet.py b/tests/fixtures/flare_packet.py index bcc261c96..7a3811926 100644 --- a/tests/fixtures/flare_packet.py +++ b/tests/fixtures/flare_packet.py @@ -4,7 +4,7 @@ import struct from pathlib import Path -from inferrsi.model.validators.flare import METRIC, SCHEMA, SOURCE, UPSTREAM_CLOCK +from inferrsi.benchmark.validators.flare import METRIC, SCHEMA, SOURCE, UPSTREAM_CLOCK def encoded(value): diff --git a/tests/test_artifact_helper_copies.py b/tests/test_artifact_helper_copies.py new file mode 100644 index 000000000..321d9d260 --- /dev/null +++ b/tests/test_artifact_helper_copies.py @@ -0,0 +1,75 @@ +"""Every core's copy of the shared file helpers stays the same code with the same bytes. + +#412 R2 gave Harness its own copies and #498 slice 1 gave Benchmark its own, so no core +imports another for them. This test fails when one copy drifts from the others. +""" + +import importlib +import inspect +import json + +import pytest + +# Each helper name and the modules that hold a copy. #498 slice 5 drops ``infra`` here +# when it deletes the package; a later slice adds the Apps copy. +COPIES = { + "sha256": ("inferrsi.infra.artifacts", "inferrsi.harness.artifact_primitives", + "inferrsi.benchmark.artifacts"), + "write_json": ("inferrsi.infra.artifacts", "inferrsi.benchmark.artifacts"), + "atomic_write_json": ("inferrsi.infra.artifacts", "inferrsi.harness.atomic_json", + "inferrsi.benchmark.artifacts"), +} +CONSTANTS = ("REPLACE_ATTEMPTS", "REPLACE_DELAY_SECONDS") +VALUE = {"b": [1.5, None, True, 1e-07, 1e16], "a": "非 ASCII", "nested": {"z": 0.1 + 0.2}} + + +@pytest.mark.parametrize("name", sorted(COPIES)) +def test_every_copy_has_the_same_source(name): + sources = {module: inspect.getsource(getattr(importlib.import_module(module), name)) + for module in COPIES[name]} + assert len(set(sources.values())) == 1, sorted(sources) + + +def test_the_atomic_writers_share_their_retry_constants(): + for constant in CONSTANTS: + values = {getattr(importlib.import_module(module), constant) + for module in COPIES["atomic_write_json"]} + assert len(values) == 1, constant + + +def test_the_benchmark_copy_imports_no_other_inferrsi_module(): + import ast + + from inferrsi.benchmark import artifacts + + tree = ast.parse(inspect.getsource(artifacts)) + assert not [node for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + and (node.level or (node.module or "").split(".")[0] == "inferrsi")] + assert not [node for node in ast.walk(tree) if isinstance(node, ast.Import) + and any(alias.name.split(".")[0] == "inferrsi" for alias in node.names)] + + +@pytest.mark.parametrize("name", ["write_json", "atomic_write_json"]) +def test_every_writer_writes_the_same_bytes(tmp_path, name): + written = set() + for index, module in enumerate(COPIES[name]): + path = tmp_path / f"{index}.json" + getattr(importlib.import_module(module), name)(path, VALUE) + written.add(path.read_bytes()) + assert len(written) == 1 + assert json.loads(written.pop()) == VALUE + for module in COPIES[name]: + with pytest.raises(ValueError): + getattr(importlib.import_module(module), name)(tmp_path / "nan.json", + {"value": float("nan")}) + + +def test_every_digest_copy_reads_the_same_bytes(tmp_path): + blob = tmp_path / "blob.bin" + blob.write_bytes(bytes(range(256)) * 9000) # crosses the 1 MiB read block + digests = {getattr(importlib.import_module(module), "sha256")(blob) + for module in COPIES["sha256"]} + assert len(digests) == 1 + for module in COPIES["sha256"]: + with pytest.raises(FileNotFoundError): + importlib.import_module(module).sha256(tmp_path / "missing.bin") diff --git a/tests/test_diffusion_reference_psnr.py b/tests/test_diffusion_reference_psnr.py index a400a6174..a3bfc1950 100644 --- a/tests/test_diffusion_reference_psnr.py +++ b/tests/test_diffusion_reference_psnr.py @@ -21,7 +21,7 @@ import math import unittest -from inferrsi.model.validators.image import ImageQualityGate +from inferrsi.benchmark.validators.image import ImageQualityGate HEIGHT = 64 WIDTH = 64 diff --git a/tests/test_flare_adapter.py b/tests/test_flare_adapter.py index b94e49241..8467e1483 100644 --- a/tests/test_flare_adapter.py +++ b/tests/test_flare_adapter.py @@ -1,5 +1,6 @@ import hashlib import json +import os import subprocess import sys from pathlib import Path @@ -8,8 +9,8 @@ from fixtures.flare_packet import build_packet, write_json from inferrsi.apps.legacy.run import run_experiment +from inferrsi.benchmark.validators.flare import adapt, main, validate_pair from inferrsi.contracts import RunSpec -from inferrsi.model.validators.flare import adapt, main, validate_pair @pytest.fixture @@ -95,6 +96,36 @@ def change_summaries(measurements): assert json.loads(output.read_bytes())["identities"]["manifest_sha256"] +def test_new_receipts_bind_the_benchmark_adapter_files(packet, tmp_path): + # #498 slice 1: same schema and keys; the digests are those of the moved files. + from inferrsi.benchmark import artifacts + from inferrsi.benchmark.validators import flare + + output = tmp_path / "receipt.json" + assert main(["measure", "--packet", str(packet), "--peer", str(packet), + "--output", str(output)]) == 0 + receipt = json.loads(output.read_bytes()) + assert receipt["schema"] == "inferrsi-flare-delivery-receipt-v1" + assert receipt["adapter"] == { + "flare.py": hashlib.sha256(Path(flare.__file__).read_bytes()).hexdigest(), + "artifacts.py": hashlib.sha256(Path(artifacts.__file__).read_bytes()).hexdigest(), + } + + +def test_the_old_flare_cli_path_is_gone(packet, tmp_path): + # The command moved to inferrsi.benchmark.validators.flare with no forwarder. + source = Path(__file__).resolve().parents[1] / "src" + result = subprocess.run( + [sys.executable, "-m", "inferrsi.model.validators.flare", "measure", "--packet", + str(packet), "--peer", str(packet), "--output", str(tmp_path / "receipt.json")], + capture_output=True, text=True, check=False, cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(source)}, + ) + assert result.returncode == 1 + assert "No module named 'inferrsi.model.validators'" in result.stderr + assert not (tmp_path / "receipt.json").exists() + + def test_identical_unaugmented_observations_remain_supported(packet): def remove_derived(measurements): for observation in measurements["observations"]: @@ -276,7 +307,7 @@ def test_real_runner_uses_existing_validation_command_and_rejection_is_inconclus ): subprocess.run(command, cwd=repo, check=True, capture_output=True) candidate = build_packet(tmp_path / "candidate", ttft_ns=80_000_000) - cli = (sys.executable, "-m", "inferrsi.model.validators.flare") + cli = (sys.executable, "-m", "inferrsi.benchmark.validators.flare") spec = RunSpec( repo=str(repo), repetitions=1, diff --git a/tests/test_image_modules.py b/tests/test_image_modules.py index 5f565b787..63cf1763d 100644 --- a/tests/test_image_modules.py +++ b/tests/test_image_modules.py @@ -8,9 +8,9 @@ import pytest from inferrsi.benchmark.sampling import collect_samples +from inferrsi.benchmark.validators.image import ImageQualityGate, compare_images from inferrsi.infra.artifacts import sha256 from inferrsi.model.protocols.image import ImageProtocol -from inferrsi.model.validators.image import ImageQualityGate, compare_images def test_image_protocol_and_sample_collection_use_real_http(tmp_path): diff --git a/tests/test_legacy_run_split.py b/tests/test_legacy_run_split.py index 2d73f811a..313ff4a77 100644 --- a/tests/test_legacy_run_split.py +++ b/tests/test_legacy_run_split.py @@ -16,13 +16,12 @@ from inferrsi.apps.legacy import evaluation as legacy_evaluation from inferrsi.apps.legacy import reporting as legacy_reporting from inferrsi.apps.legacy import run as legacy_run -from inferrsi.benchmark import legacy_run_v0, sampling +from inferrsi.benchmark import legacy_run_v0, metrics, sampling from inferrsi.contracts import RunSpec, claim from inferrsi.harness import atomic_json from inferrsi.harness.runner import run_experiment from inferrsi.harness.scenarios.external import execute_trial from inferrsi.infra import artifacts, targets -from inferrsi.model.protocols import metrics from tests._gate_policy_support import ROOT, _scan, _walk GRAPH = _scan(ROOT, frozenset()) @@ -62,9 +61,9 @@ def test_runner_and_external_lost_their_model_artifact_and_target_edges(): def test_benchmark_rules_import_only_the_metric_contract(): + # #498 slice 1 moved the metric contract into Benchmark. assert _outside("inferrsi.benchmark.legacy_run_v0", "inferrsi.") == { - "inferrsi.benchmark", "inferrsi.model", "inferrsi.model.protocols", - "inferrsi.model.protocols.metrics", + "inferrsi.benchmark", "inferrsi.benchmark.metrics", } assert _outside("inferrsi.benchmark.sampling", "inferrsi.") == {"inferrsi.benchmark"} # The v0 labels are Benchmark's own and equal the legacy vocabulary. diff --git a/tests/test_module_boundaries.py b/tests/test_module_boundaries.py index 0b1ab0ea8..45a8b9ab4 100644 --- a/tests/test_module_boundaries.py +++ b/tests/test_module_boundaries.py @@ -11,10 +11,10 @@ import pytest from inferrsi.apps.legacy.run import execute_trial, run_experiment +from inferrsi.benchmark.metrics import read_metric from inferrsi.contracts import RunSpec from inferrsi.infra.platforms import PLATFORM_PROBES, inspect_device from inferrsi.infra.platforms.cuda import qualify_devices -from inferrsi.model.protocols.metrics import read_metric def test_new_scenario_target_and_backend_share_abba_and_verdict(tmp_path): @@ -134,7 +134,7 @@ def test_control_imports_need_no_inference_or_image_dependencies(): sys.executable, "-c", "import sys; import inferrsi.apps.cli; " - "import inferrsi.model.validators.image; " + "import inferrsi.benchmark.validators.image; " "assert not {'torch', 'vllm', 'numpy', 'PIL', 'skimage'}.intersection(sys.modules)", ], check=False, diff --git a/tests/test_moe_benchmark.py b/tests/test_moe_benchmark.py index f43035213..acbb2f320 100644 --- a/tests/test_moe_benchmark.py +++ b/tests/test_moe_benchmark.py @@ -12,8 +12,8 @@ import pytest +from inferrsi.benchmark.metrics import read_metric from inferrsi.harness.scenarios.external import execute_trial -from inferrsi.model.protocols.metrics import read_metric from tests.engineering_task_support import load_task_module moe = load_task_module("moe-forward-npu", "benchmark") diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index d05d352b3..a6b417da8 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -154,6 +154,64 @@ def test_by_path_loading_is_admitted_nowhere_else(tmp_path, relative): assert report.runtime_execution == () +@pytest.mark.parametrize("importer, source, target", [ + # #498 slice 1: the old paths fail even though no file is behind them. + ("benchmark/runner.py", "from inferrsi.model.protocols.metrics import read_metric\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/runner.py", "from inferrsi.model.protocols import metrics\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/runner.py", "import inferrsi.model.validators.flare\n", + "inferrsi.model.validators.flare"), + ("benchmark/runner.py", "from inferrsi.model import validators\n", + "inferrsi.model.validators"), + ("benchmark/runner.py", "from ..model.validators.image import ImageQualityGate\n", + "inferrsi.model.validators.image"), + ("apps/runner.py", "from inferrsi.model.validators.flare import main\n", + "inferrsi.model.validators.flare"), + ("model/protocols/image.py", "from .metrics import read_metric\n", + "inferrsi.model.protocols.metrics"), +]) +def test_retired_model_paths_fail_to_import(tmp_path, importer, source, target): + report = _report(tmp_path, {importer: source}) + assert f"{_module(importer)} imports retired module {target}" in report.violations + assert "violations" in blocker_kinds(report) + + +@pytest.mark.parametrize("importer, source, target", [ + # Review of f8f1553: an initializer resolves relative imports from its own package. + ("benchmark/__init__.py", "from ..model.protocols import metrics\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/sub/__init__.py", "from ...model.validators import flare\n", + "inferrsi.model.validators.flare"), + ("benchmark/sub/__init__.py", "from ...model import validators\n", + "inferrsi.model.validators"), +]) +def test_retired_model_paths_fail_from_a_package_initializer(tmp_path, importer, source, + target): + report = _report(tmp_path, {importer: source}) + package = _module(importer).removesuffix(".__init__") + assert f"{package} imports retired module {target}" in report.violations + + +@pytest.mark.parametrize("relative", [ + "model/protocols/metrics.py", + "model/validators/__init__.py", + "model/validators/flare.py", +]) +def test_retired_model_files_fail_when_they_return(tmp_path, relative): + report = _report(tmp_path, {relative: "VALUE = 1\n"}) + assert f"{_module(relative).removesuffix('.__init__')} is a retired module" in ( + report.violations) + + +def test_the_model_paths_that_remain_are_not_retired(tmp_path): + report = _report(tmp_path, { + "benchmark/runner.py": "from inferrsi.model.protocols import image\n", + "apps/runner.py": "from inferrsi.model.protocols.image import ImageProtocol\n", + }) + assert not [item for item in report.violations if "retired module" in item] + + def _scan_tree(files: dict[str, str]) -> list[str]: """Cross-core violations of a synthetic package root holding only ``files``.""" with tempfile.TemporaryDirectory() as temporary: