From f8f1553f92bf5c3c89e71ca56463e3e9512993d9 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 21:37:38 +0800 Subject: [PATCH 1/2] refactor(benchmark): move the metric contract and validators out of model The finite positive metric contract and the Flare and image quality checks are Benchmark rules, so they move to Benchmark with no forwarder or alias: - model/protocols/metrics.py -> benchmark/metrics.py - model/validators/{__init__,flare,image}.py -> benchmark/validators/ benchmark/artifacts.py holds Benchmark's own sha256, write_json and atomic_write_json, with the same function source as the Infra and Harness copies, so Benchmark imports no Infra module for them. The new tests/test_artifact_helper_copies.py keeps every copy equal in source and written bytes. infra/artifacts.py stays for its other users. The legacy run gates and the MoE task program import the Benchmark modules. The Flare command is now python -m inferrsi.benchmark.validators.flare with the same arguments, outputs and exit codes. A new Flare receipt keeps its schema and keys, and its adapter digests name the moved files. The gate now fails a file under a retired path and any import that names one, even when no file is behind it. The installed-wheel check asserts that the old paths are absent, the new ones ship, and only the new Flare command runs. Docs and CHANGELOG name the new paths. Knowledge records that pin src/inferrsi/model/protocols/metrics.py at an earlier commit keep that path. Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 25 +++++++ docs/architecture.md | 16 ++-- docs/contributing.md | 2 +- docs/flare-qualification.md | 10 +-- docs/module-layout.md | 4 +- docs/operations.md | 2 +- src/inferrsi/benchmark/artifacts.py | 51 +++++++++++++ src/inferrsi/benchmark/legacy_run_v0.py | 4 +- .../{model/protocols => benchmark}/metrics.py | 0 .../tasks/moe-forward-npu/benchmark.py | 4 +- .../validators/__init__.py | 0 .../{model => benchmark}/validators/flare.py | 2 +- .../{model => benchmark}/validators/image.py | 2 +- src/inferrsi/model/README.md | 7 +- tests/_gate_policy_support.py | 31 ++++++++ tests/check_installed.py | 23 ++++++ tests/fixtures/flare_packet.py | 2 +- tests/test_artifact_helper_copies.py | 75 +++++++++++++++++++ tests/test_diffusion_reference_psnr.py | 2 +- tests/test_flare_adapter.py | 35 ++++++++- tests/test_image_modules.py | 2 +- tests/test_legacy_run_split.py | 7 +- tests/test_module_boundaries.py | 4 +- tests/test_moe_benchmark.py | 2 +- tests/test_target_module_import_gate.py | 42 +++++++++++ 25 files changed, 317 insertions(+), 37 deletions(-) create mode 100644 src/inferrsi/benchmark/artifacts.py rename src/inferrsi/{model/protocols => benchmark}/metrics.py (100%) rename src/inferrsi/{model => benchmark}/validators/__init__.py (100%) rename src/inferrsi/{model => benchmark}/validators/flare.py (99%) rename src/inferrsi/{model => benchmark}/validators/image.py (98%) create mode 100644 tests/test_artifact_helper_copies.py diff --git a/CHANGELOG.md b/CHANGELOG.md index eddbfec7e..6e9abf26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ ### Changed +- The metric contract and the quality validators move from `inferrsi.model` to + Benchmark (Refs #498, slice 1). There is no forwarding module or alias, and the + import gate fails any import of an old path, even when no file is behind it. + - `inferrsi.model.protocols.metrics` is now `inferrsi.benchmark.metrics`. + `inferrsi.model.validators` (`flare`, `image`) is now + `inferrsi.benchmark.validators`. + - The Flare adapter command is now + `python -m inferrsi.benchmark.validators.flare`, with the same arguments, + outputs and exit codes. The old `python -m inferrsi.model.validators.flare` + fails with `ModuleNotFoundError`. + - New `inferrsi.benchmark.artifacts` holds Benchmark's own `sha256`, + `write_json` and `atomic_write_json`. Their code equals the copies in + `inferrsi.infra.artifacts` and Harness, and + `tests/test_artifact_helper_copies.py` keeps the copies equal. + `inferrsi.infra.artifacts` stays for its other users. + - A new Flare receipt keeps its schema and keys. Its `adapter` digests are those + of `benchmark/validators/flare.py` and `benchmark/artifacts.py`. Existing + receipts are not rewritten. + - The MoE task's `benchmark.py` imports the Benchmark modules. Each MoE + `evaluate` call prepares its own copy of the task files and launches that copy + in the same call, so a new run binds the changed `benchmark.py` bytes and no + directory prepared before this change is launched again. + - Pickled objects that name an old module path no longer load. The Knowledge + records that cite `src/inferrsi/model/protocols/metrics.py` at commit + `b3ef95a` keep that pinned path. - The five runtime resource loaders are closed and the combined tree is isolated (Refs #412, R3). `first_wave_report()` now has no violation and no blocker, and `new_core_closure_isolated` and `isolation_complete` are both true. diff --git a/docs/architecture.md b/docs/architecture.md index cba3b66f5..a08f2d6a4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,7 +92,7 @@ interfaces; existing CLI commands remain supported. | --- | --- | --- | | Entry and knowledge | CLI arguments, immutable RunSpec, cited knowledge/context export | Model execution or automatic source editing | | Harness | AB/BA order, deadlines, source checks, scenario composition, evidence, performance adjudication and reporting | Device-specific conditions, request schemas or kernel dispatch | -| Model | Request payloads, metric schemas, output decoding and explicit quality checks | Scheduling, worker lifetime or hardware selection | +| Model | Request payloads and output decoding | Scheduling, worker lifetime, hardware selection, metric contracts or quality checks | | Infra | Repository identity, device discovery/qualification, process ownership, HTTP transport, storage and artifacts | Model constants, campaign gates or performance verdicts | Dependencies flow from entrypoints to Harness, from scenarios to Model/Infra, @@ -163,6 +163,11 @@ src/inferrsi/ registry.py Generic task resource discovery tasks/ Task-specific definitions, workloads and assessment legacy_run_v0.py Old run gates: metric contract, summaries and verdict + metrics.py Finite positive dotted-key metric contract + artifacts.py Benchmark-owned file digests and JSON writes + validators/ + flare.py Offline Flare delivery-evidence adapter and pair check + image.py Explicit PNG dimensions/mode, hashes, RGB/alpha quality gates sampling.py Warmup seeds, first-response retention, sample/artifact groups skill_evaluation.py Skill trial check validity, verdict and condition statistics skill_eval_diagnostic.py Synthetic KV admission diagnosis task and grader @@ -258,10 +263,7 @@ src/inferrsi/ external.py External command trials; metric validation injected model/ protocols/ - metrics.py Finite positive dotted-key metric contract image.py OpenAI-style image payload and PNG response protocol - validators/ - image.py Explicit PNG dimensions/mode, hashes, RGB/alpha quality gates infra/ processes.py Supervised Linux serving workers and their group cleanup targets/git.py Repository identity and source snapshots @@ -301,7 +303,7 @@ path to its replacement and lists what still keeps the isolation gate false. | Add | Main extension locations | Required qualification | | --- | --- | --- | -| Model or scenario | `model/protocols`, `model/validators`, `harness/scenarios` | Frozen request and metric contract; model-specific quality coverage | +| Model or scenario | `model/protocols`, `benchmark/metrics.py`, `benchmark/validators`, `harness/scenarios` | Frozen request and metric contract; model-specific quality coverage | | Repository | `infra/targets`, a composed scenario, `data/repo_packs` | Stable source identity; command/result contract; environment isolation | | Hardware | `infra/platforms` | Device selection, process exclusivity and independent backend/model evidence; a backend is registered in `infra/platforms/__init__.py` and its probe command is defined in the implementing module under `infra/platforms/`, not in package data | @@ -326,7 +328,7 @@ benchmark format, reuse the standard executor and only replace metric interpreta from functools import partial from inferrsi.apps.legacy.run import execute_trial, run_experiment -from inferrsi.model.protocols.metrics import read_metric +from inferrsi.benchmark.metrics import read_metric def read_engine_metric(document, key): @@ -346,7 +348,7 @@ cleanup. A custom `execute(command, cwd, trial_dir, deadline, metric_key)` must preserve that trial record contract. Prefer composing `execute_trial` over reimplementing process lifetime. Metric readers convert a result document into a float; `benchmark.legacy_run_v0.trial_metric` then applies the one finite, positive -metric contract (`model.protocols.metrics.validate_metric`) to whatever the reader +metric contract (`benchmark.metrics.validate_metric`) to whatever the reader returns, so a reader that yields `0.0`, a `bool` or a non-finite value records a metric error rather than entering the evaluation. The Harness executor `harness.scenarios.external.execute_trial` holds no metric rule: a caller that diff --git a/docs/contributing.md b/docs/contributing.md index 003057fd4..a9efdff8b 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -35,7 +35,7 @@ policy does not renormalize existing blobs or authorize edits to frozen files. Nightly checks cover Linux and Windows checkouts with `core.autocrlf` set to `false`, `true` and `input`. -Use `harness/scenarios` and `model/protocols` / `model/validators` for new models, +Use `harness/scenarios`, `model/protocols` and `benchmark/validators` for new models, `infra/targets` with `data/repo_packs` for repositories, and `infra/platforms` for hardware. A backend is registered in `infra/platforms/__init__.py`; its probe command is defined in the implementing module under `infra/platforms/`. That diff --git a/docs/flare-qualification.md b/docs/flare-qualification.md index e8fce79ad..2b3217866 100644 --- a/docs/flare-qualification.md +++ b/docs/flare-qualification.md @@ -2,7 +2,7 @@ The adapter reads existing Flare qualification artifacts and writes a separate receipt. It never launches Flare, downloads a model or executes imported code. -Run it with `python -m inferrsi.model.validators.flare`. The existing +Run it with `python -m inferrsi.benchmark.validators.flare`. The existing `external_benchmark` scenario, `validation_command` and evaluator own execution and verdicts. @@ -92,11 +92,11 @@ synthetic status, package bytes, corpus bytes and prepared capacity. This keeps an incomplete or incomparable pair out of the evaluator's performance inputs. ```text -python -m inferrsi.model.validators.flare measure \ +python -m inferrsi.benchmark.validators.flare measure \ --packet /packet.json --peer /packet.json \ --output /baseline.json -python -m inferrsi.model.validators.flare validate-pair \ +python -m inferrsi.benchmark.validators.flare validate-pair \ --baseline /packet.json --candidate /packet.json \ --output /pair.json ``` @@ -104,7 +104,7 @@ python -m inferrsi.model.validators.flare validate-pair \ For `RunSpec`, use these argv sequences: ```python -cli = (python, "-m", "inferrsi.model.validators.flare") +cli = (python, "-m", "inferrsi.benchmark.validators.flare") baseline_command = cli + ( "measure", "--packet", baseline, "--peer", candidate, "--output", "{result}", ) @@ -145,7 +145,7 @@ From a development checkout with InferRSI installed: ```text python tests/fixtures/flare_packet.py -python -m inferrsi.model.validators.flare measure \ +python -m inferrsi.benchmark.validators.flare measure \ --packet /packet.json --peer /packet.json \ --output /receipt.json pytest tests/test_flare_adapter.py diff --git a/docs/module-layout.md b/docs/module-layout.md index 3550cf7a5..870792851 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -145,10 +145,10 @@ Several flat modules remain shared seams until their slice starts. | `apps/mindie/` (`project.py`, `context.py`, `mcp.py`, `read_worker.py`) | The local MindIE adapter: project grant, stdio MCP, context export and the read worker | M1 (#452) moved the four root MindIE files here; no Harness, Knowledge or Infra module names MindIE, and the initializer imports nothing | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `harness/runner.py`, `harness/scenarios/external.py`, `harness/atomic_json.py` | Old run execution facts and Harness-owned JSON writes | Ports injected; no Model edge, and no Infra edge except the local process executor | -| `benchmark/legacy_run_v0.py`, `benchmark/sampling.py` | Old run gates and metric validation; image/video sample collection | Imports only `model/protocols/metrics`; owns its rule inputs and verdict labels | +| `benchmark/legacy_run_v0.py`, `benchmark/sampling.py` | Old run gates and metric validation; image/video sample collection | Imports only `benchmark/metrics.py`; owns its rule inputs and verdict labels | | `apps/legacy/` | Old run composition, old `evaluate` entry and HTML report | Supplies the Git source snapshot from `infra/targets`; remains in Apps | | `apps/inspection.py` | Prepared-environment inspection bound to this host | Binds the `infra/inspection.py` facts API to `infra.commands.probe_command` and `infra.targets.repository_identity`; replaces root `inspection.py`, which R1 deleted with the three old-run root forwarders | -| `model/validators/` | Benchmark-leaning checks | Not moved | +| `benchmark/metrics.py`, `benchmark/validators/`, `benchmark/artifacts.py` | The metric contract, the Flare and image quality checks, and Benchmark's own file digests and JSON writes | #498 slice 1 moved the first two from `model/protocols/metrics.py` and `model/validators/`, with no forwarder or alias; the gate fails any import of the old paths | | `infra/` | Seams used by the current tree, including the `infra/inspection.py` facts API | Not a fourth core; reaches no Knowledge, Harness, Benchmark or Apps module | | `apps/views/summary_chart.py` | Stdlib SVG gain chart and report figure spec | A2 (#408) moved root `summary_chart.py` here; the renderer, its specification checks and its `main` are unchanged, and `tools/campaign_summary_chart.py` calls it | | `contracts/` (`runspec.py`, `json_v1.py`, `claim.py`, `evidence.py`) | Technical-only contracts: RunSpec, canonical JSON, verdict vocabulary, evidence segment and path rules | Exact module set; the initializer re-exports only `RunSpec`; not a fourth core | diff --git a/docs/operations.md b/docs/operations.md index 9832c1479..5d473c213 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -57,7 +57,7 @@ Effect classes used below: | Assemble an optimization trace | CLI `campaign assemble-trace`; [offline dashboard](dashboard.md#capture-the-optimization-trace) | Local evidence-writing | Copies an explicitly reviewed context into a new bounded campaign-root snapshot; caller owns redaction and narrative accuracy | | Import a MindIE read | CLI `campaign import-mindie`; [Codex workflow](codex-plugin.md#mindie-knowledge) | Local evidence-writing | Retains caller-supplied query/explain bytes and hashes, exports delivered graded bodies and a trace context fragment; no activation, model call, transcript capture or automatic caller review | | Associate runs with a local campaign | CLI `campaign init` / `campaign record` / `campaign decide`; [offline dashboard](dashboard.md) | Local evidence-writing | Legacy manual records reference existing artifacts and never start an agent; record/decide cannot mutate a managed optimization campaign | -| Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | +| Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.benchmark.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | | Export a cloud-generated parser pair | [Generation-only protocol](skill-eval-generation-only.md), `python -m tools.skill_eval_cloud_export` | Local evidence-writing; no execution or publication | Validates frozen inputs and retained sources; writes one bounded inert JSON bundle, never executes parser source | diff --git a/src/inferrsi/benchmark/artifacts.py b/src/inferrsi/benchmark/artifacts.py new file mode 100644 index 000000000..6fb3a116c --- /dev/null +++ b/src/inferrsi/benchmark/artifacts.py @@ -0,0 +1,51 @@ +# Benchmark-owned file digests and JSON evidence writes. The function bodies equal +# those in the other cores' copies; tests/test_artifact_helper_copies.py keeps them +# equal. Benchmark imports no other inferrsi module here. + +import hashlib +import json +import os +import time +import uuid +from pathlib import Path + +# Windows refuses os.replace while another handle holds the destination open. +# A bounded retry covers the observed contention without hiding a real failure. +REPLACE_ATTEMPTS = 8 +REPLACE_DELAY_SECONDS = 0.05 + + +def sha256(path): + hasher = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(block) + return hasher.hexdigest() + + +def write_json(path, value): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + + +def atomic_write_json(path: Path, value: object) -> None: + path = Path(path) + # A per-writer temporary name keeps two writers from sharing one file. + temporary = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + try: + for attempt in range(REPLACE_ATTEMPTS): + try: + temporary.replace(path) + return + except PermissionError: + if attempt + 1 == REPLACE_ATTEMPTS: + raise + time.sleep(REPLACE_DELAY_SECONDS) + finally: + if temporary.exists(): + temporary.unlink() diff --git a/src/inferrsi/benchmark/legacy_run_v0.py b/src/inferrsi/benchmark/legacy_run_v0.py index 9f87f53d2..9727311d3 100644 --- a/src/inferrsi/benchmark/legacy_run_v0.py +++ b/src/inferrsi/benchmark/legacy_run_v0.py @@ -4,8 +4,8 @@ from collections.abc import Callable from dataclasses import dataclass -from ..model.protocols.metrics import read_metric as read_metric -from ..model.protocols.metrics import validate_metric +from .metrics import read_metric as read_metric +from .metrics import validate_metric # The v0 protocol's own labels. Benchmark shares no verdict type with another module. PASS, FAIL, INCONCLUSIVE = "PASS", "FAIL", "INCONCLUSIVE" diff --git a/src/inferrsi/model/protocols/metrics.py b/src/inferrsi/benchmark/metrics.py similarity index 100% rename from src/inferrsi/model/protocols/metrics.py rename to src/inferrsi/benchmark/metrics.py diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py index e735be59c..9030a60b2 100644 --- a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py @@ -16,8 +16,8 @@ import uuid from pathlib import Path -from inferrsi.infra.artifacts import atomic_write_json -from inferrsi.model.protocols.metrics import read_metric +from inferrsi.benchmark.artifacts import atomic_write_json +from inferrsi.benchmark.metrics import read_metric # Worker cleanup for this MoE invocation, including detached torchrun sessions. # torchrun workers may create their own sessions, outside the executor's process diff --git a/src/inferrsi/model/validators/__init__.py b/src/inferrsi/benchmark/validators/__init__.py similarity index 100% rename from src/inferrsi/model/validators/__init__.py rename to src/inferrsi/benchmark/validators/__init__.py diff --git a/src/inferrsi/model/validators/flare.py b/src/inferrsi/benchmark/validators/flare.py similarity index 99% rename from src/inferrsi/model/validators/flare.py rename to src/inferrsi/benchmark/validators/flare.py index 5eb58da29..4d43b5903 100644 --- a/src/inferrsi/model/validators/flare.py +++ b/src/inferrsi/benchmark/validators/flare.py @@ -6,7 +6,7 @@ from decimal import Decimal, InvalidOperation from pathlib import Path -from ...infra import artifacts +from .. import artifacts SOURCE = { "repository": "https://github.com/NVIDIA/flare-local-inference", diff --git a/src/inferrsi/model/validators/image.py b/src/inferrsi/benchmark/validators/image.py similarity index 98% rename from src/inferrsi/model/validators/image.py rename to src/inferrsi/benchmark/validators/image.py index 87d1674d3..2b8a427a4 100644 --- a/src/inferrsi/model/validators/image.py +++ b/src/inferrsi/benchmark/validators/image.py @@ -7,7 +7,7 @@ import math from dataclasses import dataclass -from ...infra.artifacts import sha256 +from ..artifacts import sha256 @dataclass(frozen=True) diff --git a/src/inferrsi/model/README.md b/src/inferrsi/model/README.md index 29702e88c..fd1080a7b 100644 --- a/src/inferrsi/model/README.md +++ b/src/inferrsi/model/README.md @@ -1,8 +1,9 @@ # Model -`protocols/` defines request and metric contracts. `validators/` owns output -quality checks with explicit campaign-supplied gates. The image validator's -optional dependencies are available through `inferrsi[image]`. +`protocols/` defines request contracts. The metric contract and the output +quality checks moved to `inferrsi.benchmark.metrics` and +`inferrsi.benchmark.validators` (#498 slice 1). The image validator's optional +dependencies are available through `inferrsi[image]`. Model/revision pins and numeric acceptance gates stay in campaign plans. Frozen historical snapshots remain in `tests/fixtures/legacy/`; current examples and diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index 990e6d9ff..fe959a664 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -89,6 +89,11 @@ } ) RETIRED_ROOT_MODULES = RETIRED_BY_R1 | RETIRED_BY_A1 | RETIRED_BY_A2 | RETIRED_BY_M1 +# #498 slice 1 moved these packages and modules out of ``inferrsi.model``, with no +# forwarder or alias: ``inferrsi.benchmark.metrics`` and ``inferrsi.benchmark.validators``. +# Each name covers everything below it. A file under one fails, and so does any import +# that names one, even when its target is absent. +RETIRED_MODULES = frozenset({"inferrsi.model.protocols.metrics", "inferrsi.model.validators"}) # #412 R1, permanent: the one importer of the neutral data leaf outside # ``inferrsi.knowledge``, with the only name it may bind (D020: Infra reads one pack # description and reaches no Knowledge or Harness). This is not a manifest @@ -153,6 +158,22 @@ def _from_base(node: ast.ImportFrom, package: str) -> str: return ".".join(keep + ([node.module] if node.module else [])) +def _retired_imports(tree: ast.Module, package: str) -> set[str]: + """Every name under ``RETIRED_MODULES`` that an import statement in ``tree`` spells.""" + found: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + base = _from_base(node, package) + names = [base, *(f"{base}.{alias.name}" for alias in node.names)] + else: + continue + found.update( + name for name in names if any(_matches(name, old) for old in RETIRED_MODULES)) + return found + + def _is_entry_delegation(node: ast.stmt, package: str) -> bool: """The module entry's one admitted statement, in its exact plain shape.""" module, name = MODULE_ENTRY_DELEGATION @@ -729,6 +750,16 @@ def first_wave_report( for name in modules if name in RETIRED_ROOT_MODULES and name not in sources ] + violations += [ + f"{name} is a retired module" + for name in modules + if any(_matches(name, old) for old in RETIRED_MODULES) + ] + violations += [ + f"{name} imports retired module {target}" + for name in modules + for target in sorted(_retired_imports(graph.trees[name], graph.packages[name])) + ] # Outside Knowledge the leaf has its one permanent importer and any staged pair # that names it. Every other importer fails. declared_importers = dict(LEAF_IMPORTERS) diff --git a/tests/check_installed.py b/tests/check_installed.py index 666416730..7e4c57482 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -776,3 +776,26 @@ def _inventory(root, prefix): check=False, cwd=launch_cwd, ) assert launch.returncode == 0, launch.stderr +# #498 slice 1: the metric contract and the quality validators ship only under +# inferrsi.benchmark. The old inferrsi.model paths are not in the wheel and have no alias. +for retired in ("inferrsi.model.protocols.metrics", "inferrsi.model.validators"): + assert find_spec(retired) is None, retired + try: + import_module(retired) + except ModuleNotFoundError: + pass + else: + raise AssertionError(f"retired module {retired} must not import") +for canonical in ("inferrsi.benchmark.metrics", "inferrsi.benchmark.artifacts", + "inferrsi.benchmark.validators.flare", "inferrsi.benchmark.validators.image"): + assert find_spec(canonical) is not None, canonical +# The flare CLI runs from the installed wheel only under its new path. +with tempfile.TemporaryDirectory(prefix="inferrsi-flare-cli-") as cli_cwd: + for module, code in (("inferrsi.benchmark.validators.flare", 0), + ("inferrsi.model.validators.flare", 1)): + flare_cli = subprocess.run( + [sys.executable, "-I", "-B", "-m", module, "--help"], capture_output=True, + text=True, check=False, cwd=cli_cwd, + ) + assert flare_cli.returncode == code, (module, flare_cli.returncode, flare_cli.stderr) + assert "No module named 'inferrsi.model.validators'" in flare_cli.stderr, flare_cli.stderr diff --git a/tests/fixtures/flare_packet.py b/tests/fixtures/flare_packet.py index bcc261c96..7a3811926 100644 --- a/tests/fixtures/flare_packet.py +++ b/tests/fixtures/flare_packet.py @@ -4,7 +4,7 @@ import struct from pathlib import Path -from inferrsi.model.validators.flare import METRIC, SCHEMA, SOURCE, UPSTREAM_CLOCK +from inferrsi.benchmark.validators.flare import METRIC, SCHEMA, SOURCE, UPSTREAM_CLOCK def encoded(value): diff --git a/tests/test_artifact_helper_copies.py b/tests/test_artifact_helper_copies.py new file mode 100644 index 000000000..321d9d260 --- /dev/null +++ b/tests/test_artifact_helper_copies.py @@ -0,0 +1,75 @@ +"""Every core's copy of the shared file helpers stays the same code with the same bytes. + +#412 R2 gave Harness its own copies and #498 slice 1 gave Benchmark its own, so no core +imports another for them. This test fails when one copy drifts from the others. +""" + +import importlib +import inspect +import json + +import pytest + +# Each helper name and the modules that hold a copy. #498 slice 5 drops ``infra`` here +# when it deletes the package; a later slice adds the Apps copy. +COPIES = { + "sha256": ("inferrsi.infra.artifacts", "inferrsi.harness.artifact_primitives", + "inferrsi.benchmark.artifacts"), + "write_json": ("inferrsi.infra.artifacts", "inferrsi.benchmark.artifacts"), + "atomic_write_json": ("inferrsi.infra.artifacts", "inferrsi.harness.atomic_json", + "inferrsi.benchmark.artifacts"), +} +CONSTANTS = ("REPLACE_ATTEMPTS", "REPLACE_DELAY_SECONDS") +VALUE = {"b": [1.5, None, True, 1e-07, 1e16], "a": "非 ASCII", "nested": {"z": 0.1 + 0.2}} + + +@pytest.mark.parametrize("name", sorted(COPIES)) +def test_every_copy_has_the_same_source(name): + sources = {module: inspect.getsource(getattr(importlib.import_module(module), name)) + for module in COPIES[name]} + assert len(set(sources.values())) == 1, sorted(sources) + + +def test_the_atomic_writers_share_their_retry_constants(): + for constant in CONSTANTS: + values = {getattr(importlib.import_module(module), constant) + for module in COPIES["atomic_write_json"]} + assert len(values) == 1, constant + + +def test_the_benchmark_copy_imports_no_other_inferrsi_module(): + import ast + + from inferrsi.benchmark import artifacts + + tree = ast.parse(inspect.getsource(artifacts)) + assert not [node for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + and (node.level or (node.module or "").split(".")[0] == "inferrsi")] + assert not [node for node in ast.walk(tree) if isinstance(node, ast.Import) + and any(alias.name.split(".")[0] == "inferrsi" for alias in node.names)] + + +@pytest.mark.parametrize("name", ["write_json", "atomic_write_json"]) +def test_every_writer_writes_the_same_bytes(tmp_path, name): + written = set() + for index, module in enumerate(COPIES[name]): + path = tmp_path / f"{index}.json" + getattr(importlib.import_module(module), name)(path, VALUE) + written.add(path.read_bytes()) + assert len(written) == 1 + assert json.loads(written.pop()) == VALUE + for module in COPIES[name]: + with pytest.raises(ValueError): + getattr(importlib.import_module(module), name)(tmp_path / "nan.json", + {"value": float("nan")}) + + +def test_every_digest_copy_reads_the_same_bytes(tmp_path): + blob = tmp_path / "blob.bin" + blob.write_bytes(bytes(range(256)) * 9000) # crosses the 1 MiB read block + digests = {getattr(importlib.import_module(module), "sha256")(blob) + for module in COPIES["sha256"]} + assert len(digests) == 1 + for module in COPIES["sha256"]: + with pytest.raises(FileNotFoundError): + importlib.import_module(module).sha256(tmp_path / "missing.bin") diff --git a/tests/test_diffusion_reference_psnr.py b/tests/test_diffusion_reference_psnr.py index a400a6174..a3bfc1950 100644 --- a/tests/test_diffusion_reference_psnr.py +++ b/tests/test_diffusion_reference_psnr.py @@ -21,7 +21,7 @@ import math import unittest -from inferrsi.model.validators.image import ImageQualityGate +from inferrsi.benchmark.validators.image import ImageQualityGate HEIGHT = 64 WIDTH = 64 diff --git a/tests/test_flare_adapter.py b/tests/test_flare_adapter.py index b94e49241..8467e1483 100644 --- a/tests/test_flare_adapter.py +++ b/tests/test_flare_adapter.py @@ -1,5 +1,6 @@ import hashlib import json +import os import subprocess import sys from pathlib import Path @@ -8,8 +9,8 @@ from fixtures.flare_packet import build_packet, write_json from inferrsi.apps.legacy.run import run_experiment +from inferrsi.benchmark.validators.flare import adapt, main, validate_pair from inferrsi.contracts import RunSpec -from inferrsi.model.validators.flare import adapt, main, validate_pair @pytest.fixture @@ -95,6 +96,36 @@ def change_summaries(measurements): assert json.loads(output.read_bytes())["identities"]["manifest_sha256"] +def test_new_receipts_bind_the_benchmark_adapter_files(packet, tmp_path): + # #498 slice 1: same schema and keys; the digests are those of the moved files. + from inferrsi.benchmark import artifacts + from inferrsi.benchmark.validators import flare + + output = tmp_path / "receipt.json" + assert main(["measure", "--packet", str(packet), "--peer", str(packet), + "--output", str(output)]) == 0 + receipt = json.loads(output.read_bytes()) + assert receipt["schema"] == "inferrsi-flare-delivery-receipt-v1" + assert receipt["adapter"] == { + "flare.py": hashlib.sha256(Path(flare.__file__).read_bytes()).hexdigest(), + "artifacts.py": hashlib.sha256(Path(artifacts.__file__).read_bytes()).hexdigest(), + } + + +def test_the_old_flare_cli_path_is_gone(packet, tmp_path): + # The command moved to inferrsi.benchmark.validators.flare with no forwarder. + source = Path(__file__).resolve().parents[1] / "src" + result = subprocess.run( + [sys.executable, "-m", "inferrsi.model.validators.flare", "measure", "--packet", + str(packet), "--peer", str(packet), "--output", str(tmp_path / "receipt.json")], + capture_output=True, text=True, check=False, cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(source)}, + ) + assert result.returncode == 1 + assert "No module named 'inferrsi.model.validators'" in result.stderr + assert not (tmp_path / "receipt.json").exists() + + def test_identical_unaugmented_observations_remain_supported(packet): def remove_derived(measurements): for observation in measurements["observations"]: @@ -276,7 +307,7 @@ def test_real_runner_uses_existing_validation_command_and_rejection_is_inconclus ): subprocess.run(command, cwd=repo, check=True, capture_output=True) candidate = build_packet(tmp_path / "candidate", ttft_ns=80_000_000) - cli = (sys.executable, "-m", "inferrsi.model.validators.flare") + cli = (sys.executable, "-m", "inferrsi.benchmark.validators.flare") spec = RunSpec( repo=str(repo), repetitions=1, diff --git a/tests/test_image_modules.py b/tests/test_image_modules.py index 5f565b787..63cf1763d 100644 --- a/tests/test_image_modules.py +++ b/tests/test_image_modules.py @@ -8,9 +8,9 @@ import pytest from inferrsi.benchmark.sampling import collect_samples +from inferrsi.benchmark.validators.image import ImageQualityGate, compare_images from inferrsi.infra.artifacts import sha256 from inferrsi.model.protocols.image import ImageProtocol -from inferrsi.model.validators.image import ImageQualityGate, compare_images def test_image_protocol_and_sample_collection_use_real_http(tmp_path): diff --git a/tests/test_legacy_run_split.py b/tests/test_legacy_run_split.py index 2d73f811a..313ff4a77 100644 --- a/tests/test_legacy_run_split.py +++ b/tests/test_legacy_run_split.py @@ -16,13 +16,12 @@ from inferrsi.apps.legacy import evaluation as legacy_evaluation from inferrsi.apps.legacy import reporting as legacy_reporting from inferrsi.apps.legacy import run as legacy_run -from inferrsi.benchmark import legacy_run_v0, sampling +from inferrsi.benchmark import legacy_run_v0, metrics, sampling from inferrsi.contracts import RunSpec, claim from inferrsi.harness import atomic_json from inferrsi.harness.runner import run_experiment from inferrsi.harness.scenarios.external import execute_trial from inferrsi.infra import artifacts, targets -from inferrsi.model.protocols import metrics from tests._gate_policy_support import ROOT, _scan, _walk GRAPH = _scan(ROOT, frozenset()) @@ -62,9 +61,9 @@ def test_runner_and_external_lost_their_model_artifact_and_target_edges(): def test_benchmark_rules_import_only_the_metric_contract(): + # #498 slice 1 moved the metric contract into Benchmark. assert _outside("inferrsi.benchmark.legacy_run_v0", "inferrsi.") == { - "inferrsi.benchmark", "inferrsi.model", "inferrsi.model.protocols", - "inferrsi.model.protocols.metrics", + "inferrsi.benchmark", "inferrsi.benchmark.metrics", } assert _outside("inferrsi.benchmark.sampling", "inferrsi.") == {"inferrsi.benchmark"} # The v0 labels are Benchmark's own and equal the legacy vocabulary. diff --git a/tests/test_module_boundaries.py b/tests/test_module_boundaries.py index 0b1ab0ea8..45a8b9ab4 100644 --- a/tests/test_module_boundaries.py +++ b/tests/test_module_boundaries.py @@ -11,10 +11,10 @@ import pytest from inferrsi.apps.legacy.run import execute_trial, run_experiment +from inferrsi.benchmark.metrics import read_metric from inferrsi.contracts import RunSpec from inferrsi.infra.platforms import PLATFORM_PROBES, inspect_device from inferrsi.infra.platforms.cuda import qualify_devices -from inferrsi.model.protocols.metrics import read_metric def test_new_scenario_target_and_backend_share_abba_and_verdict(tmp_path): @@ -134,7 +134,7 @@ def test_control_imports_need_no_inference_or_image_dependencies(): sys.executable, "-c", "import sys; import inferrsi.apps.cli; " - "import inferrsi.model.validators.image; " + "import inferrsi.benchmark.validators.image; " "assert not {'torch', 'vllm', 'numpy', 'PIL', 'skimage'}.intersection(sys.modules)", ], check=False, diff --git a/tests/test_moe_benchmark.py b/tests/test_moe_benchmark.py index f43035213..acbb2f320 100644 --- a/tests/test_moe_benchmark.py +++ b/tests/test_moe_benchmark.py @@ -12,8 +12,8 @@ import pytest +from inferrsi.benchmark.metrics import read_metric from inferrsi.harness.scenarios.external import execute_trial -from inferrsi.model.protocols.metrics import read_metric from tests.engineering_task_support import load_task_module moe = load_task_module("moe-forward-npu", "benchmark") diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index d05d352b3..7bc2aa890 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -154,6 +154,48 @@ def test_by_path_loading_is_admitted_nowhere_else(tmp_path, relative): assert report.runtime_execution == () +@pytest.mark.parametrize("importer, source, target", [ + # #498 slice 1: the old paths fail even though no file is behind them. + ("benchmark/runner.py", "from inferrsi.model.protocols.metrics import read_metric\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/runner.py", "from inferrsi.model.protocols import metrics\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/runner.py", "import inferrsi.model.validators.flare\n", + "inferrsi.model.validators.flare"), + ("benchmark/runner.py", "from inferrsi.model import validators\n", + "inferrsi.model.validators"), + ("benchmark/runner.py", "from ..model.validators.image import ImageQualityGate\n", + "inferrsi.model.validators.image"), + ("apps/runner.py", "from inferrsi.model.validators.flare import main\n", + "inferrsi.model.validators.flare"), + ("model/protocols/image.py", "from .metrics import read_metric\n", + "inferrsi.model.protocols.metrics"), +]) +def test_retired_model_paths_fail_to_import(tmp_path, importer, source, target): + report = _report(tmp_path, {importer: source}) + assert f"{_module(importer)} imports retired module {target}" in report.violations + assert "violations" in blocker_kinds(report) + + +@pytest.mark.parametrize("relative", [ + "model/protocols/metrics.py", + "model/validators/__init__.py", + "model/validators/flare.py", +]) +def test_retired_model_files_fail_when_they_return(tmp_path, relative): + report = _report(tmp_path, {relative: "VALUE = 1\n"}) + assert f"{_module(relative).removesuffix('.__init__')} is a retired module" in ( + report.violations) + + +def test_the_model_paths_that_remain_are_not_retired(tmp_path): + report = _report(tmp_path, { + "benchmark/runner.py": "from inferrsi.model.protocols import image\n", + "apps/runner.py": "from inferrsi.model.protocols.image import ImageProtocol\n", + }) + assert not [item for item in report.violations if "retired module" in item] + + def _scan_tree(files: dict[str, str]) -> list[str]: """Cross-core violations of a synthetic package root holding only ``files``.""" with tempfile.TemporaryDirectory() as temporary: From b508bd32c39c0dbf373f26b8a210c57fd49e8b92 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 23:24:10 +0800 Subject: [PATCH 2/2] test(gate): reject retired model paths from a package initializer Review of f8f1553: the retired-path controls used ordinary modules only. A package initializer resolves a relative import from its own package, so three new controls import the retired metric contract and validators relatively from benchmark/__init__.py and benchmark/sub/__init__.py, and expect the violation under the package name. Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- tests/test_target_module_import_gate.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index 7bc2aa890..a6b417da8 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -177,6 +177,22 @@ def test_retired_model_paths_fail_to_import(tmp_path, importer, source, target): assert "violations" in blocker_kinds(report) +@pytest.mark.parametrize("importer, source, target", [ + # Review of f8f1553: an initializer resolves relative imports from its own package. + ("benchmark/__init__.py", "from ..model.protocols import metrics\n", + "inferrsi.model.protocols.metrics"), + ("benchmark/sub/__init__.py", "from ...model.validators import flare\n", + "inferrsi.model.validators.flare"), + ("benchmark/sub/__init__.py", "from ...model import validators\n", + "inferrsi.model.validators"), +]) +def test_retired_model_paths_fail_from_a_package_initializer(tmp_path, importer, source, + target): + report = _report(tmp_path, {importer: source}) + package = _module(importer).removesuffix(".__init__") + assert f"{package} imports retired module {target}" in report.violations + + @pytest.mark.parametrize("relative", [ "model/protocols/metrics.py", "model/validators/__init__.py",