From 5ee69128d228b921aa2446f3f0211057b22d56fa Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 02:56:02 +0800 Subject: [PATCH 1/7] refactor(harness,benchmark): close the five runtime resource loaders (R3) Close the last first-wave blocker. first_wave_report() now has no violation and no blocker, and new_core_closure_isolated and isolation_complete are both true. - Harness no longer runs task protocol code in its own process. Each validate_arguments / arguments / assess call runs the new harness/protocol_child.py as `python -I -B` under a 120 s deadline, inside its own process group or Windows Job, and answers with one versioned strict JSON envelope on a private copy of stdout. The child imports only the standard library and compiles only the protocol bytes that match the frozen SHA-256. Failures stay distinct (request, identity, protocol_exception, result, malformed_response, invalid_response, exit_status, timeout, launch, cleanup, output_limit); a protocol SystemExit keeps its exit code, so argparse usage errors still exit 2. task_protocol.protocol() is replaced by run(). - moe-forward-npu folds processes.py into benchmark.py, and qwen3-tts-npu folds evaluation.py and its CLI into benchmark.py, so no bundle loads a sibling module by path. - The MoE grader and Qwen21 evaluator keep their in-process candidate loaders under a permanent gate rule (EVALUATOR_LOADERS): one exact call each, only in a task bundle program, with negative controls for every other placement. first_wave_report().runtime_execution lists them, so static isolation is not read as runtime isolation. - The optimization session's runtime identity adds harness/protocol_child.py. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 25 + docs/contracts/current-design.md | 2 +- docs/engineering-benchmarks.md | 8 +- docs/module-layout.md | 75 +-- docs/tts-benchmark.md | 5 +- .../tasks/moe-forward-npu/benchmark.py | 83 ++- .../tasks/moe-forward-npu/processes.py | 80 --- .../tasks/qwen3-tts-npu/benchmark.py | 509 +++++++++++++++++- .../tasks/qwen3-tts-npu/evaluation.py | 462 ---------------- src/inferrsi/harness/optimization_session.py | 15 +- src/inferrsi/harness/protocol_child.py | 164 ++++++ src/inferrsi/harness/task_protocol.py | 157 +++++- tests/_gate_policy_support.py | 52 +- tests/test_engineering_benchmark_plugins.py | 3 +- tests/test_evaluator_loaders.py | 58 ++ tests/test_knowledge_compatibility_shells.py | 13 +- tests/test_legacy_run_forwarder_gate.py | 8 +- tests/test_moe_benchmark.py | 3 +- tests/test_optimization_session.py | 2 + tests/test_target_module_import_gate.py | 127 +++-- tests/test_task_protocol_child.py | 274 ++++++++++ tests/test_tts_benchmark.py | 4 +- tests/test_tts_standalone.py | 4 +- 23 files changed, 1428 insertions(+), 705 deletions(-) delete mode 100644 src/inferrsi/benchmark/tasks/moe-forward-npu/processes.py delete mode 100644 src/inferrsi/benchmark/tasks/qwen3-tts-npu/evaluation.py create mode 100644 src/inferrsi/harness/protocol_child.py create mode 100644 tests/test_evaluator_loaders.py create mode 100644 tests/test_task_protocol_child.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a44e7d8d1..c293fdf0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ ### Changed +- The five runtime resource loaders are closed and the combined tree is isolated + (Refs #412, R3). `first_wave_report()` now has no violation and no blocker, and + `new_core_closure_isolated` and `isolation_complete` are both true. + - The Harness task protocol no longer runs in the Harness process. Each + `validate_arguments`, `arguments` and `assess` call runs the new + `harness/protocol_child.py` as `python -I -B` under a 120 second deadline, in + its own process group or Windows Job, and returns one strict JSON envelope. + The child compiles only protocol bytes that match the frozen SHA-256, and a + protocol `SystemExit` keeps its exit code (argparse usage errors still exit 2). + `task_protocol.protocol()` is removed; callers use `task_protocol.run()`. A + task `optimization.py` must take and return JSON values. + - `moe-forward-npu`: `processes.py` is folded into `benchmark.py` + (`MARKER`, `cleanup` and `_members` unchanged) and leaves `TASK_FILES`. + - `qwen3-tts-npu`: `evaluation.py` and the CLI are one `benchmark.py`; the + prepared bundle no longer has `evaluation.py`. A bundle prepared before this + change fails its frozen digests and must be prepared again. Retained results + are not rewritten. + - The MoE grader and Qwen21 evaluator keep their in-process candidate loaders + under a permanent gate rule (`EVALUATOR_LOADERS`): one exact call each, only in + a task bundle program. `first_wave_report().runtime_execution` lists them, + because static isolation does not cover what the loaded code does at run time. + - The optimization session's runtime identity adds `harness/protocol_child.py`, + and `harness/task_protocol.py` changed, so a session started before this + change does not resume after it. + - Six Harness modules stop importing Infra (Refs #412, R2). `agent_sessions`, `candidates`, `optimization`, `optimization_session`, `project` and `task_protocol` take `digest`, `sha256` and `file_lock` from the new diff --git a/docs/contracts/current-design.md b/docs/contracts/current-design.md index a9e2eccfe..4664cf0b1 100644 --- a/docs/contracts/current-design.md +++ b/docs/contracts/current-design.md @@ -166,7 +166,7 @@ Current status after R1 (#412): - The seven root leaves are deleted, so `compatibility_pairs` in [`../legacy-knowledge-manifest.json`](../legacy-knowledge-manifest.json) is empty. The nine pair records stay in its `revision_history`, unchanged, as the record of this decision. - The former ninth pair, `infra.inspection` → `inferrsi.resources.repository_packs`, is a live import that no root deletion removes. It is now a permanent exact rule outside the allowlist: that one module binds the one name `repository_pack` with one plain module-level import. The leaf's bytes, SHA-256, allowed imports, shape and import-free parent initializer stay locked, and no `resources.` or `contracts.` prefix is allowed. - Root `arguments.py` → the same leaf, for `REPO_TYPES`, was a separate temporary entry with its own blocker. A1 (#408) removed both with the root file, and no Apps importer of the leaf replaced them. -- `overall isolation_complete` is still false. A1 (#408) cleared that temporary entry; twelve `harness` → `inferrsi.infra` hits in six Harness modules (R2) and five declared runtime resource loaders (R3) remain. The package root holds eight tracked files; the final two-file root waits for A2 (#408) and M1 (#452). +- `overall isolation_complete` is true since R3 (#412). A1 (#408) cleared that temporary entry, R2 cleared the twelve `harness` → `inferrsi.infra` hits in six Harness modules and R3 cleared the five runtime resource loaders (a child-process protocol, two folded sibling modules and a permanent two-evaluator rule). The package root holds the two tracked files `__init__.py` and `__main__.py`. - The permanent rule and that manifest revision require formal architecture review. The implementer does not self-approve them, and passing tests are not that review. [`../module-layout.md`](../module-layout.md) has the exact rules and their negative controls. diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index cc58bf4ea..fcd8a41ea 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -47,10 +47,10 @@ inferrsi benchmarks run qwen21-denoise-npu --help inferrsi benchmarks run qwen3-tts-npu --help ``` -MoE owns `benchmark.py`, `grader.py` and its torchrun cleanup helper. The application -adapter injects the generic Harness executor into its entry. Python callers -provide the `execute_trial` argument explicitly. TTS owns `benchmark.py` and -`evaluation.py`, alongside its inference and service implementation. The image +MoE owns `benchmark.py` (which holds its torchrun cleanup helper) and `grader.py`. The +application adapter injects the generic Harness executor into its entry. Python callers +provide the `execute_trial` argument explicitly. TTS owns one `benchmark.py` with its +CLI and evaluation, alongside its inference and service implementation. The image and TTS entries also run directly by file path. ## Directory ownership diff --git a/docs/module-layout.md b/docs/module-layout.md index bfd686330..6c2568437 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -175,29 +175,46 @@ directories have no `__init__.py`. They import only the standard library and external engines, so the Knowledge closure rule holds for them unchanged. K02 did not change the gate's scan scope or any closure rule. A dynamic import (`__import__`, `importlib.import_module`, or `module_from_spec`) inside a checked closure is an undeclared edge and fails -closed, except for the exact resource execution declarations below. +closed, except for the two permanent evaluator declarations below. Other dynamic forms (`getattr`, `exec`, `eval`, `runpy`, `pkgutil`) are out of scope for this package. `tools/` and `tests/` are separate registered consumers; this gate does not scan them. -The task integration declares these existing resource execution boundaries in -`tests/_gate_policy_support.py`, independently of the Knowledge compatibility -pairs. Each permits exactly one `importlib.util.module_from_spec` call, with the -listed local spec variable, in the named function or module scope. Other dynamic -calls, additional calls, adjacent modules and static cross-core imports remain -checked. These declarations do not make arbitrary candidate code isolated. +#412 R3 retired the five temporary resource-loader declarations and their +`runtime resource loaders` blocker. Three of the five were removed: + +- `harness.task_protocol` no longer loads task code. Each protocol operation + (`validate_arguments`, `arguments`, `assess`) runs `harness/protocol_child.py` + as `python -I -B` in its own process group or Windows Job under a deadline, and + answers with one strict JSON envelope on a private copy of stdout. The child + imports only the standard library and compiles the exact protocol bytes it has + checked against the frozen SHA-256. This is an interpreter and process + boundary, not a filesystem or network sandbox. +- `moe-forward-npu` folded its sibling `processes.py` into `benchmark.py`, and + `qwen3-tts-npu` folded its sibling `evaluation.py` into `benchmark.py`. The + bundles no longer load a sibling module by path, and a bundle prepared before + R3 is refused by its frozen digests and must be prepared again. + +The two evaluators that must call submitted code in their own process to time it +stay, under a permanent rule in `tests/_gate_policy_support.py` +(`EVALUATOR_LOADERS`) that is independent of the Knowledge compatibility pairs. +The rule permits exactly one `importlib.util.module_from_spec` call, with the +listed local spec variable, in the named function of a task bundle program (a file +in a directory without `__init__.py` under `benchmark/tasks/`). A second call, a +call elsewhere, the same shape in any other module, a declared module that becomes +importable or loses its call, other dynamic calls and static cross-core imports +fail. | Relative module | Scope | Local spec | | --- | --- | --- | -| `harness.task_protocol` | `protocol` | `spec` | -| `benchmark.tasks.moe-forward-npu.benchmark` | module | `_process_spec` | | `benchmark.tasks.moe-forward-npu.grader` | `load` | `spec` | | `benchmark.tasks.qwen21-denoise-npu.benchmark` | `load` | `spec` | -| `benchmark.tasks.qwen3-tts-npu.benchmark` | module | `_evaluation_spec` | -The report adds a `runtime resource loaders` blocker when these loaders are -present and does not claim either complete isolation or new-core closure -isolation. R1 changed none of these five declarations; closing them is R3 (#412). +These two loaders run frozen run files in their own child interpreter. The +callers verify the file identities (`identity.json` in both tasks), and each load +returns a fresh module that is not registered in `sys.modules`. They are not a +sandbox for candidate code. `first_wave_report().runtime_execution` lists them, so +the static `isolation_complete` result never reads as runtime isolation. R1 (#412) deleted the three root forwarders of the old run chain and removed their `LEGACY_RUN_FORWARDERS` registration from `tests/_gate_policy_support.py`. @@ -215,23 +232,18 @@ With the seven Knowledge shells and root `inspection`, their names are now No `legacy run forwarders` blocker exists any more. [Retired root modules](#retired-root-modules) gives each replacement. -A clean result means no new cross-core import of those names. It does not mean -the modules are isolated. Isolation stays false while any one of these holds: -`knowledge/`, `benchmark/`, or `apps/` is absent or has no scannable -module; the compatibility allowlist -is non-empty; -`harness/**` still imports Model or Infra, -directly or through another module; a declared runtime resource loader is -present; or the scan reports a violation. The three packages may appear in stages. -A clean graph for the new cores is reported separately as "new core business -closure isolated" and never sets overall `isolation_complete` to true. - -On the tree after R2 (#412), `first_wave_report()` has no violation and one -blocker, so `isolation_complete` is false and R2 does not claim otherwise: - -| Blocker | Cause | Closed by | -| --- | --- | --- | -| `runtime resource loaders: 5` | The five declared loaders in the table above | R3 (#412) | +A clean result means no new cross-core import of those names. Isolation stays false +(`isolation_complete`, a static import-closure result) while any one of +these holds: `knowledge/`, `benchmark/`, or `apps/` is absent or has no scannable +module; the compatibility allowlist is non-empty; `harness/**` still imports Model +or Infra, directly or through another module; or the scan reports a violation. A +clean graph for the new cores is reported separately as "new core business closure +isolated". Neither flag says anything about what frozen task or candidate code does +once it runs: that is `runtime_execution`. + +On the tree after R3 (#412), `first_wave_report()` has no violation and no +blocker, `new_core_closure_isolated` and `isolation_complete` are both true, and +`runtime_execution` lists the two permanent evaluator loaders. R1 cleared the `allowlist` and `legacy run forwarders` blockers, and A1 cleared the `temporary leaf importers` blocker that R1 left for root `arguments.py`. R2 @@ -241,8 +253,7 @@ cleared `harness reaches model/infra` (12 hits): `agent_sessions`, `candidates`, and `file_lock` from `harness/artifact_primitives.py` and `atomic_write_json` from `harness/atomic_json.py`, whose function bodies equal those in `infra/artifacts.py`. `infra/artifacts.py` stays for its other users. No Harness module reaches -`inferrsi.infra` or `inferrsi.model`. The loaders also keep -`new_core_closure_isolated` false. +`inferrsi.infra` or `inferrsi.model`. R3 cleared `runtime resource loaders: 5`. `knowledge-model-policy` stays the only required status. Until a separate process decision makes this import gate required, every migration PR attaches diff --git a/docs/tts-benchmark.md b/docs/tts-benchmark.md index e4ea19302..ae00b7e2b 100644 --- a/docs/tts-benchmark.md +++ b/docs/tts-benchmark.md @@ -15,8 +15,7 @@ SHA256;启动器阻止导入 vLLM、vLLM-Omni、SGLang 和历史独立引擎 | 文件 | 做什么 | 优化时的作用 | | --- | --- | --- | -| `benchmark.py` | 解析 prepare/serve/collect/assess 参数 | 题包自身的可执行入口 | -| `evaluation.py` | 冻结判据、采集音频和计时、离线判定 | 评测规则和结果复核 | +| `benchmark.py` | 解析 prepare/serve/collect/assess 参数;冻结判据、采集音频和计时、离线判定 | 题包自身的可执行入口,也是评测规则和结果复核 | | `baseline.py` | 校验模型资产、启用引擎导入隔离、启动服务 | 固定参考入口,记录代码与权重身份 | | `inference.py` | 文本和指令编码 → Talker/code predictor → codec 解码 → PCM | 推理调度、缓存、算子、同步和解码优化的入口 | | `service.py` | HTTP/SSE、请求校验、有界队列、单模型线程 | 批处理、排队和增量音频交付的入口 | @@ -39,7 +38,7 @@ SHA256;启动器阻止导入 vLLM、vLLM-Omni、SGLang 和历史独立引擎 → 独立 ASR/音色评分 → assess 核对 → 反馈下一轮 → 独立 final ``` -这里的 prepare、serve、collect、assess 由题包内 benchmark.py 和 evaluation.py 实现。RSI 的实验调用方 +这里的 prepare、serve、collect、assess 由题包内 benchmark.py 实现(#412 R3 把原来的 evaluation.py 并入它;此前准备好的 prepared 目录因冻结摘要不符会被拒绝,需要重新 prepare)。RSI 的实验调用方 负责触发它们;描述任务的 Markdown 不会自动启动评测。task、运行入口和判据对应 #362 的 Benchmark 职责,当前沿 scenario 入口提供,后续公共契约按 S1 对接。 diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py index 3864957f8..e735be59c 100644 --- a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py @@ -6,10 +6,10 @@ import argparse import hashlib -import importlib.util import json import math import os +import signal import statistics import sys import time @@ -19,16 +19,83 @@ from inferrsi.infra.artifacts import atomic_write_json from inferrsi.model.protocols.metrics import read_metric -_process_spec = importlib.util.spec_from_file_location( - "moe_task_processes", Path(__file__).resolve().with_name("processes.py") -) -_processes = importlib.util.module_from_spec(_process_spec) -_process_spec.loader.exec_module(_processes) -MARKER, cleanup = _processes.MARKER, _processes.cleanup +# Worker cleanup for this MoE invocation, including detached torchrun sessions. +# torchrun workers may create their own sessions, outside the executor's process +# group. Only the launcher receives a fresh marker; it cannot reach the invoking +# shell or allocator. This is lifecycle supervision for reviewed code, not a sandbox. +# The marker must be generated in this driver after it starts, as evaluate() does. +MARKER = "INFERRSI_MOE_RUN_ID" + + +def _members(token: str, proc: Path = Path("/proc"), *, errors=None) -> list[dict]: + marker = (MARKER + "=" + token).encode() + found = [] + # A fresh marker can only be inherited by processes started after this + # driver. Older same-user services may have unreadable environments; they + # cannot own this invocation. Keep same-tick processes in scope because + # Linux start times have tick granularity, not a strict ordering within it. + driver_fields = (proc / "self" / "stat").read_text().rsplit(")", 1)[1].split() + driver_start = int(driver_fields[19]) + # A missing procfs must raise, never be interpreted as successful cleanup. + for entry in proc.iterdir(): + if not entry.name.isdigit() or int(entry.name) == os.getpid(): + continue + try: + if entry.stat().st_uid != os.getuid(): + continue + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if fields[0] == "Z": + continue + start_ticks = int(fields[19]) + if start_ticks < driver_start: + continue + if marker in (entry / "environ").read_bytes().split(b"\0"): + found.append({"pid": int(entry.name), "start_ticks": start_ticks}) + except (FileNotFoundError, ProcessLookupError): + continue + except (OSError, ValueError, IndexError) as exc: + if errors is None: + raise + # Preserve confirmed workers even when another process is unreadable. + # The caller must keep the resulting cleanup verdict inconclusive. + errors.add(str(exc)) + return found + + +def cleanup(token: str, *, grace_seconds: float = 5.0) -> dict: + """Clean a marker minted by this driver; preserve PID-reuse protection.""" + killed = set() + errors = set() + deadline = time.monotonic() + grace_seconds + try: + while True: + members = _members(token, errors=errors) + if not members: + return {"verified": not errors, "signalled_pids": sorted(killed), + "remaining": [], **({"error": "; ".join(sorted(errors))} if errors else {})} + for item in members: + try: + fields = Path(f"/proc/{item['pid']}/stat").read_text().rsplit(")", 1)[1].split() + if int(fields[19]) == item["start_ticks"]: + os.kill(item["pid"], signal.SIGKILL) + killed.add(item["pid"]) + except (FileNotFoundError, ProcessLookupError): + pass + except (OSError, ValueError, IndexError) as exc: + errors.add(str(exc)) + if time.monotonic() >= deadline: + remaining = _members(token, errors=errors) + return {"verified": False, "signalled_pids": sorted(killed), + "remaining": remaining, + **({"error": "; ".join(sorted(errors))} if errors else {})} + time.sleep(0.05) + except (OSError, ValueError, IndexError) as exc: + return {"verified": False, "signalled_pids": sorted(killed), "error": str(exc)} + TASK_FILES = ( "task.json", "TASK.md", "baseline.py", "oracle.py", "workload.py", "LICENSE", - "benchmark.py", "grader.py", "processes.py", + "benchmark.py", "grader.py", ) diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/processes.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/processes.py deleted file mode 100644 index aa7629046..000000000 --- a/src/inferrsi/benchmark/tasks/moe-forward-npu/processes.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Clean this MoE invocation's torchrun workers, including detached sessions. - -torchrun workers may create their own sessions, outside the executor's process -group. Only the launcher receives a fresh marker; it cannot reach the invoking -shell or allocator. This is lifecycle supervision for reviewed code, not a sandbox. -The marker must be generated in this driver after it starts, as evaluate() does. -""" - -import os -import signal -import time -from pathlib import Path - -MARKER = "INFERRSI_MOE_RUN_ID" - - -def _members(token: str, proc: Path = Path("/proc"), *, errors=None) -> list[dict]: - marker = (MARKER + "=" + token).encode() - found = [] - # A fresh marker can only be inherited by processes started after this - # driver. Older same-user services may have unreadable environments; they - # cannot own this invocation. Keep same-tick processes in scope because - # Linux start times have tick granularity, not a strict ordering within it. - driver_fields = (proc / "self" / "stat").read_text().rsplit(")", 1)[1].split() - driver_start = int(driver_fields[19]) - # A missing procfs must raise, never be interpreted as successful cleanup. - for entry in proc.iterdir(): - if not entry.name.isdigit() or int(entry.name) == os.getpid(): - continue - try: - if entry.stat().st_uid != os.getuid(): - continue - fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() - if fields[0] == "Z": - continue - start_ticks = int(fields[19]) - if start_ticks < driver_start: - continue - if marker in (entry / "environ").read_bytes().split(b"\0"): - found.append({"pid": int(entry.name), "start_ticks": start_ticks}) - except (FileNotFoundError, ProcessLookupError): - continue - except (OSError, ValueError, IndexError) as exc: - if errors is None: - raise - # Preserve confirmed workers even when another process is unreadable. - # The caller must keep the resulting cleanup verdict inconclusive. - errors.add(str(exc)) - return found - - -def cleanup(token: str, *, grace_seconds: float = 5.0) -> dict: - """Clean a marker minted by this driver; preserve PID-reuse protection.""" - killed = set() - errors = set() - deadline = time.monotonic() + grace_seconds - try: - while True: - members = _members(token, errors=errors) - if not members: - return {"verified": not errors, "signalled_pids": sorted(killed), - "remaining": [], **({"error": "; ".join(sorted(errors))} if errors else {})} - for item in members: - try: - fields = Path(f"/proc/{item['pid']}/stat").read_text().rsplit(")", 1)[1].split() - if int(fields[19]) == item["start_ticks"]: - os.kill(item["pid"], signal.SIGKILL) - killed.add(item["pid"]) - except (FileNotFoundError, ProcessLookupError): - pass - except (OSError, ValueError, IndexError) as exc: - errors.add(str(exc)) - if time.monotonic() >= deadline: - remaining = _members(token, errors=errors) - return {"verified": False, "signalled_pids": sorted(killed), - "remaining": remaining, - **({"error": "; ".join(sorted(errors))} if errors else {})} - time.sleep(0.05) - except (OSError, ValueError, IndexError) as exc: - return {"verified": False, "signalled_pids": sorted(killed), "error": str(exc)} diff --git a/src/inferrsi/benchmark/tasks/qwen3-tts-npu/benchmark.py b/src/inferrsi/benchmark/tasks/qwen3-tts-npu/benchmark.py index 89151e0e3..d6cce42fd 100644 --- a/src/inferrsi/benchmark/tasks/qwen3-tts-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/qwen3-tts-npu/benchmark.py @@ -1,17 +1,469 @@ -"""CLI assembly for the Qwen3-TTS streaming service task.""" +"""Qwen3-TTS streaming service task: CLI, streaming observations and offline assessment. + +The evaluation and its CLI share this one file, so no sibling module is loaded by path. +""" import argparse -import importlib.util +import array +import base64 +import concurrent.futures +import hashlib +import http.client +import io import json +import math import os +import statistics import sys +import time +import urllib.request +import wave from pathlib import Path -_evaluation_spec = importlib.util.spec_from_file_location( - "tts_task_evaluation", Path(__file__).resolve().with_name("evaluation.py") +TASK_FILES = ( + "task.json", + "workload.jsonl", + "baseline.py", + "inference.py", + "service.py", + "model-assets.json", + "TASK.md", + "sources.json", + "LICENSE", + "benchmark.py", ) -tts = importlib.util.module_from_spec(_evaluation_spec) -_evaluation_spec.loader.exec_module(tts) + + +def task_resources(): + return Path(__file__).resolve().parent + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def read(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def write(path, value): + Path(path).write_text( + json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + + +def positive(value): + return type(value) in (int, float) and math.isfinite(value) and value > 0 + + +def prepare(output, *, concurrencies, warmups, gates, scorer_id): + """Freeze task, original texts and caller-selected gates before candidate search.""" + if ( + not concurrencies + or any(type(c) is not int or c < 1 for c in concurrencies) + or len(set(concurrencies)) != len(concurrencies) + ): + raise ValueError("Concurrencies must be distinct positive integers") + if type(warmups) is not int or warmups < 1: + raise ValueError("At least one explicit warmup is required") + if not scorer_id.strip(): + raise ValueError("An immutable audio scorer identity is required") + required = { + "min_improvement_pct", + "max_error_rate_increase", + "max_speaker_drop", + "max_duration_change_pct", + "max_ttfa_regression_pct", + } + if set(gates) != required or any( + type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in gates.values() + ): + raise ValueError("All five finite nonnegative campaign gates are required") + if gates["max_duration_change_pct"] >= 100: + raise ValueError("The duration gate must reject empty or nearly empty responses") + workload_bytes = task_resources().joinpath("workload.jsonl").read_bytes() + request_count = len([line for line in workload_bytes.splitlines() if line.strip()]) + if max(concurrencies) > request_count: + raise ValueError("Concurrency exceeds the number of requests in the workload") + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + for name in TASK_FILES: + (output / name).write_bytes(task_resources().joinpath(name).read_bytes()) + policy = { + "schema_version": 1, + "task_sha256": sha((output / "task.json").read_bytes()), + "workload_sha256": sha((output / "workload.jsonl").read_bytes()), + "concurrencies": concurrencies, + "warmups": warmups, + "quality_scorer_id": scorer_id, + "gates": gates, + "grader_sha256": sha(Path(__file__).read_bytes()), + "baseline_launcher_sha256": sha((output / "baseline.py").read_bytes()), + "runtime_sha256": {name: sha((output / name).read_bytes()) for name in TASK_FILES}, + } + write(output / "policy.json", policy) + return policy + + +def load_prepared(root): + root = Path(root) + policy = read(root / "policy.json") + if policy["grader_sha256"] != sha(Path(__file__).read_bytes()) or policy[ + "baseline_launcher_sha256" + ] != sha((root / "baseline.py").read_bytes()): + raise ValueError("Prepared grader or baseline launcher changed") + for name, key in (("task.json", "task_sha256"), ("workload.jsonl", "workload_sha256")): + if sha((root / name).read_bytes()) != policy[key]: + raise ValueError("Prepared evaluation bytes changed") + for name in TASK_FILES: + if policy["runtime_sha256"][name] != sha((root / name).read_bytes()): + raise ValueError("Prepared task runtime changed: " + name) + workload = [ + json.loads(line) + for line in (root / "workload.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + return policy, workload, sha((root / "policy.json").read_bytes()) + + +def _remaining(deadline): + remaining = deadline - time.perf_counter() + if remaining <= 0: + raise TimeoutError("Request deadline exceeded") + return remaining + + +class _DeadlineReader(io.RawIOBase): + """Bound each socket read, including partial headers, SSE lines and HTTP chunks.""" + + def __init__(self, raw, sock, deadline): + self.raw, self.sock, self.deadline = raw, sock, deadline + + def readable(self): + return True + + def readinto(self, buffer): + self.sock.settimeout(_remaining(self.deadline)) + count = self.raw.readinto(buffer) + _remaining(self.deadline) + return count + + def close(self): + try: + self.raw.close() + finally: + super().close() + + +class _DeadlineHandler(urllib.request.HTTPHandler, urllib.request.HTTPSHandler): + """Keep urllib routing and TLS verification; wrap its response before parsing.""" + + def __init__(self, deadline): + urllib.request.HTTPSHandler.__init__(self) + self.deadline = deadline + + def do_open(self, connection_type, request, **kwargs): + def response(sock, **options): + result = http.client.HTTPResponse(sock, **options) + result.fp = io.BufferedReader( + _DeadlineReader(result.fp.detach(), sock, self.deadline) + ) + return result + + def connection(host, **options): + result = connection_type(host, **options) + result.response_class = response + return result + + request.timeout = _remaining(self.deadline) + return super().do_open(connection, request, **kwargs) + + +def _request(endpoint, payload, output, request_id, timeout): + """One attempt, including failed partial audio. SSE chunks may split PCM samples.""" + started = time.perf_counter() + deadline = started + timeout + pcm = bytearray() + record = { + "id": request_id, + "payload_sha256": sha(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()), + "success": False, + "ttfa_seconds": None, + "terminal": None, + "chunks": [], + } + try: + request = urllib.request.Request( + endpoint, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + opener = urllib.request.build_opener(_DeadlineHandler(deadline)) + with opener.open(request, timeout=_remaining(deadline)) as response: + if response.status != 200 or "text/event-stream" not in response.headers.get( + "Content-Type", "" + ): + raise ValueError("A successful SSE response is required") + for raw in response: + _remaining(deadline) + if not raw.startswith(b"data:"): + continue + event = json.loads(raw[5:]) + kind = event.get("type") + if record["terminal"] is not None: + raise ValueError("Data after terminal event") + if kind == "speech.audio.delta": + if event.get("sample_rate", 24000) != 24000 or event.get("channels", 1) != 1: + raise ValueError("Expected 24 kHz mono PCM16") + block = base64.b64decode(event["audio"], validate=True) + pcm.extend(block) + elapsed = time.perf_counter() - started + if len(pcm) >= 2 and record["ttfa_seconds"] is None: + record["ttfa_seconds"] = elapsed + record["chunks"].append({"seconds": elapsed, "bytes": len(block)}) + elif kind == "speech.audio.done": + if event.get("finish_reason") in {"length", "abort", "error", "cancelled"}: + raise ValueError("Unsuccessful generation termination") + record["terminal"] = event + else: + raise ValueError("Unexpected SSE event or server error") + if len(pcm) > 24000 * 2 * 600: + raise ValueError("Response exceeds ten minutes of PCM") + _remaining(deadline) + if record["terminal"] is None or not pcm or len(pcm) % 2: + raise ValueError("Missing terminal, empty PCM or incomplete sample") + samples = array.array("h", pcm) + if sys.byteorder != "little": + samples.byteswap() + if min(samples) == max(samples): + raise ValueError("Constant or silent audio") + record["success"] = True + except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException) as error: + # Preserve the error class without storing private endpoint/HTTP error bodies. + record["error"] = type(error).__name__ + record.update( + elapsed_seconds=time.perf_counter() - started, + pcm_sha256=sha(pcm), + pcm_bytes=len(pcm), + audio_seconds=len(pcm) / 48000, + ) + with wave.open(str(output / (request_id + ".wav")), "wb") as audio: + audio.setparams((1, 2, 24000, 0, "NONE", "not compressed")) + audio.writeframes(pcm) + write(output / (request_id + ".json"), record) + return record + + +def summarize(records, wall_seconds): + if not records or not positive(wall_seconds) or not all(r["success"] for r in records): + return None + return { + "rtf": statistics.mean(r["elapsed_seconds"] / r["audio_seconds"] for r in records), + "ttfa_median_ms": statistics.median(r["ttfa_seconds"] for r in records) * 1000, + "audio_seconds_per_second": sum(r["audio_seconds"] for r in records) / wall_seconds, + } + + +def collect(prepared, output, *, endpoint, model, timeout): + """Observe a caller-managed service. Its lifecycle is owned by the outer Harness.""" + if not positive(timeout): + raise ValueError("Socket/request timeout must be positive and finite") + policy, workload, policy_hash = load_prepared(prepared) + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + records = { + "schema_version": 1, + "policy_sha256": policy_hash, + "model": model, + "cohorts": [], + "status": "running", + "quality": "NOT_EVALUATED", + } + write(output / "result.json", records) + payloads = [ + {**row, "model": model, "stream": True, "stream_format": "sse", "response_format": "pcm"} + for row in workload + ] + for concurrency in policy["concurrencies"]: + cohort = {"concurrency": concurrency, "warmups": [], "requests": []} + records["cohorts"].append(cohort) + for index in range(policy["warmups"]): + record = _request( + endpoint, + payloads[index % len(payloads)], + output, + f"c{concurrency}-w{index}", + timeout, + ) + cohort["warmups"].append(record) + if not record["success"]: + records["status"] = "request_failure" + write(output / "result.json", records) + return records + started = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: + futures = [ + pool.submit( + _request, endpoint, payload, output, f"c{concurrency}-r{index}", timeout + ) + for index, payload in enumerate(payloads) + ] + cohort["requests"] = [future.result() for future in futures] + cohort["wall_seconds"] = time.perf_counter() - started + cohort["metrics"] = summarize(cohort["requests"], cohort["wall_seconds"]) + write(output / "result.json", records) + if cohort["metrics"] is None: + records["status"] = "request_failure" + write(output / "result.json", records) + return records + records["status"] = "collected" + write(output / "result.json", records) + return records + + +def _verify_trial(directory, policy, workload, policy_hash): + document = read(Path(directory) / "result.json") + if document["policy_sha256"] != policy_hash or document["status"] != "collected": + raise ValueError("Trial is incomplete or belongs to another prepared evaluation") + if [c["concurrency"] for c in document["cohorts"]] != policy["concurrencies"]: + raise ValueError("Missing, reordered or duplicated concurrency cohort") + retained = {} + for cohort in document["cohorts"]: + concurrency = cohort["concurrency"] + for phase, count, marker in ( + ("warmups", policy["warmups"], "w"), + ("requests", len(workload), "r"), + ): + rows = cohort[phase] + if [r["id"] for r in rows] != [f"c{concurrency}-{marker}{i}" for i in range(count)]: + raise ValueError("Missing, duplicated or reordered request") + for index, row in enumerate(rows): + payload = { + **workload[index % len(workload)], + "model": document["model"], + "stream": True, + "stream_format": "sse", + "response_format": "pcm", + } + expected_hash = sha( + json.dumps(payload, sort_keys=True, ensure_ascii=False).encode() + ) + if not row["success"] or row["payload_sha256"] != expected_hash: + raise ValueError("Failed request or changed payload") + with wave.open(str(Path(directory) / (row["id"] + ".wav")), "rb") as audio: + if (audio.getnchannels(), audio.getsampwidth(), audio.getframerate()) != ( + 1, + 2, + 24000, + ): + raise ValueError("Changed audio format") + pcm = audio.readframes(audio.getnframes()) + if ( + not pcm + or sha(pcm) != row["pcm_sha256"] + or len(pcm) != row["pcm_bytes"] + or len(pcm) / 48000 != row["audio_seconds"] + or not positive(row["elapsed_seconds"]) + or not positive(row["ttfa_seconds"]) + or row["ttfa_seconds"] > row["elapsed_seconds"] + or not isinstance(row["terminal"], dict) + or row["terminal"].get("type") != "speech.audio.done" + or row["terminal"].get("finish_reason") + in {"length", "abort", "error", "cancelled"} + ): + raise ValueError("Missing or inconsistent retained audio/timing evidence") + samples = array.array("h", pcm) + if sys.byteorder != "little": + samples.byteswap() + if min(samples) == max(samples) or sum(c["bytes"] for c in row["chunks"]) != len( + pcm + ): + raise ValueError("Constant audio or inconsistent chunk evidence") + retained[row["id"]] = row + if ( + not positive(cohort["wall_seconds"]) + or summarize(cohort["requests"], cohort["wall_seconds"]) != cohort["metrics"] + ): + raise ValueError("Metrics do not match retained requests") + return document, retained + + +def assess(prepared, baseline, candidate, quality): + """Read retained bytes only. Supplied ASR/speaker scores have a frozen scorer ID.""" + result = { + "scope": "retained streaming audio and supplied independent quality scores", + "verdict": "INCONCLUSIVE", + "reasons": [], + "speedup": None, + "policy_sha256": None, + "cases": [], + } + try: + policy, workload, policy_hash = load_prepared(prepared) + result["policy_sha256"] = policy_hash + before, base_rows = _verify_trial(baseline, policy, workload, policy_hash) + after, cand_rows = _verify_trial(candidate, policy, workload, policy_hash) + if before["model"] != after["model"]: + raise ValueError("Served model identities differ") + scores = read(quality) + if scores["scorer_id"] != policy["quality_scorer_id"]: + raise ValueError("Quality scorer identity changed") + by_side = {} + for side, rows in (("baseline", base_rows), ("candidate", cand_rows)): + values = scores[side] + if len(values) != len(rows) or {s["id"] for s in values} != set(rows): + raise ValueError("Audio scoring coverage is incomplete") + by_side[side] = {s["id"]: s for s in values} + for score in values: + if score["pcm_sha256"] != rows[score["id"]]["pcm_sha256"]: + raise ValueError("Audio scores refer to different PCM bytes") + if ( + type(score["error_rate"]) not in (float, int) + or not math.isfinite(score["error_rate"]) + or score["error_rate"] < 0 + or type(score["speaker_similarity"]) not in (float, int) + or not math.isfinite(score["speaker_similarity"]) + or not -1 <= score["speaker_similarity"] <= 1 + ): + raise ValueError("Invalid audio score") + failed = [] + gates = policy["gates"] + for request_id, base in base_rows.items(): + cand = cand_rows[request_id] + bs, cs = by_side["baseline"][request_id], by_side["candidate"][request_id] + if ( + cs["error_rate"] > bs["error_rate"] + gates["max_error_rate_increase"] + or cs["speaker_similarity"] < bs["speaker_similarity"] - gates["max_speaker_drop"] + or abs(cand["audio_seconds"] / base["audio_seconds"] - 1) * 100 + > gates["max_duration_change_pct"] + ): + failed.append("Audio quality/duration gate failed for " + request_id) + for bc, cc in zip(before["cohorts"], after["cohorts"]): + bm, cm = bc["metrics"], cc["metrics"] + speedup = bm["rtf"] / cm["rtf"] + result["cases"].append( + { + "concurrency": bc["concurrency"], + "baseline": bm, + "candidate": cm, + "observed_speedup": speedup, + } + ) + gain = (1 - cm["rtf"] / bm["rtf"]) * 100 + if gain <= 0 or gain < gates["min_improvement_pct"]: + failed.append("RTF gain gate failed at concurrency " + str(bc["concurrency"])) + if cm["ttfa_median_ms"] > bm["ttfa_median_ms"] * ( + 1 + gates["max_ttfa_regression_pct"] / 100 + ): + failed.append("First-audio gate failed at concurrency " + str(bc["concurrency"])) + result.update(verdict="FAIL" if failed else "PASS", reasons=failed) + if not failed: + result["speedup"] = math.exp( + statistics.mean(math.log(c["observed_speedup"]) for c in result["cases"]) + ) + except (OSError, ValueError, KeyError, TypeError, wave.Error, ZeroDivisionError) as error: + result["reasons"].append(type(error).__name__ + ": incomplete or inconsistent evidence") + return result def main(argv=None): @@ -22,11 +474,12 @@ def main(argv=None): serve.add_argument("--model-path", type=Path, required=True) serve.add_argument("--port", type=int, required=True) serve.add_argument("--identity-output", type=Path, required=True) - prepare = commands.add_parser("prepare", help="Freeze task, texts and explicit campaign gates") - prepare.add_argument("--output-dir", type=Path, required=True) - prepare.add_argument("--concurrency", type=int, nargs="+", required=True) - prepare.add_argument("--warmups", type=int, required=True) - prepare.add_argument("--quality-scorer-id", required=True) + prepare_command = commands.add_parser( + "prepare", help="Freeze task, texts and explicit campaign gates") + prepare_command.add_argument("--output-dir", type=Path, required=True) + prepare_command.add_argument("--concurrency", type=int, nargs="+", required=True) + prepare_command.add_argument("--warmups", type=int, required=True) + prepare_command.add_argument("--quality-scorer-id", required=True) for name in ( "min-improvement-pct", "max-error-rate-increase", @@ -34,21 +487,23 @@ def main(argv=None): "max-duration-change-pct", "max-ttfa-regression-pct", ): - prepare.add_argument("--" + name, type=float, required=True) - collect = commands.add_parser("collect", help="Collect a caller-managed service's SSE audio") - collect.add_argument("--prepared", type=Path, required=True) - collect.add_argument("--output-dir", type=Path, required=True) - collect.add_argument("--endpoint", required=True) - collect.add_argument("--model", required=True) - collect.add_argument("--timeout-seconds", type=float, required=True) - assess = commands.add_parser("assess", help="Assess retained audio and external quality scores") - assess.add_argument("--prepared", type=Path, required=True) - assess.add_argument("--baseline", type=Path, required=True) - assess.add_argument("--candidate", type=Path, required=True) - assess.add_argument("--quality", type=Path, required=True) + prepare_command.add_argument("--" + name, type=float, required=True) + collect_command = commands.add_parser( + "collect", help="Collect a caller-managed service's SSE audio") + collect_command.add_argument("--prepared", type=Path, required=True) + collect_command.add_argument("--output-dir", type=Path, required=True) + collect_command.add_argument("--endpoint", required=True) + collect_command.add_argument("--model", required=True) + collect_command.add_argument("--timeout-seconds", type=float, required=True) + assess_command = commands.add_parser( + "assess", help="Assess retained audio and external quality scores") + assess_command.add_argument("--prepared", type=Path, required=True) + assess_command.add_argument("--baseline", type=Path, required=True) + assess_command.add_argument("--candidate", type=Path, required=True) + assess_command.add_argument("--quality", type=Path, required=True) args = parser.parse_args(argv) if args.action == "serve": - tts.load_prepared(args.prepared) + load_prepared(args.prepared) command = [ sys.executable, str((args.prepared / "baseline.py").resolve()), @@ -71,7 +526,7 @@ def main(argv=None): "max_ttfa_regression_pct", ) } - result = tts.prepare( + result = prepare( args.output_dir, concurrencies=args.concurrency, warmups=args.warmups, @@ -79,7 +534,7 @@ def main(argv=None): scorer_id=args.quality_scorer_id, ) elif args.action == "collect": - result = tts.collect( + result = collect( args.prepared, args.output_dir, endpoint=args.endpoint, @@ -87,7 +542,7 @@ def main(argv=None): timeout=args.timeout_seconds, ) else: - result = tts.assess(args.prepared, args.baseline, args.candidate, args.quality) + result = assess(args.prepared, args.baseline, args.candidate, args.quality) print(json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False)) if args.action == "assess": return {"PASS": 0, "FAIL": 1, "INCONCLUSIVE": 2}[result["verdict"]] diff --git a/src/inferrsi/benchmark/tasks/qwen3-tts-npu/evaluation.py b/src/inferrsi/benchmark/tasks/qwen3-tts-npu/evaluation.py deleted file mode 100644 index 9f60549cc..000000000 --- a/src/inferrsi/benchmark/tasks/qwen3-tts-npu/evaluation.py +++ /dev/null @@ -1,462 +0,0 @@ -"""Streaming TTS observations and offline assessment of retained audio evidence.""" - -import array -import base64 -import concurrent.futures -import hashlib -import http.client -import io -import json -import math -import statistics -import sys -import time -import urllib.request -import wave -from pathlib import Path - -TASK_FILES = ( - "task.json", - "workload.jsonl", - "baseline.py", - "inference.py", - "service.py", - "model-assets.json", - "TASK.md", - "sources.json", - "LICENSE", - "benchmark.py", - "evaluation.py", -) - - -def task_resources(): - return Path(__file__).resolve().parent - - -def sha(data): - return hashlib.sha256(data).hexdigest() - - -def read(path): - return json.loads(Path(path).read_text(encoding="utf-8")) - - -def write(path, value): - Path(path).write_text( - json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" - ) - - -def positive(value): - return type(value) in (int, float) and math.isfinite(value) and value > 0 - - -def prepare(output, *, concurrencies, warmups, gates, scorer_id): - """Freeze task, original texts and caller-selected gates before candidate search.""" - if ( - not concurrencies - or any(type(c) is not int or c < 1 for c in concurrencies) - or len(set(concurrencies)) != len(concurrencies) - ): - raise ValueError("Concurrencies must be distinct positive integers") - if type(warmups) is not int or warmups < 1: - raise ValueError("At least one explicit warmup is required") - if not scorer_id.strip(): - raise ValueError("An immutable audio scorer identity is required") - required = { - "min_improvement_pct", - "max_error_rate_increase", - "max_speaker_drop", - "max_duration_change_pct", - "max_ttfa_regression_pct", - } - if set(gates) != required or any( - type(v) not in (int, float) or not math.isfinite(v) or v < 0 for v in gates.values() - ): - raise ValueError("All five finite nonnegative campaign gates are required") - if gates["max_duration_change_pct"] >= 100: - raise ValueError("The duration gate must reject empty or nearly empty responses") - workload_bytes = task_resources().joinpath("workload.jsonl").read_bytes() - request_count = len([line for line in workload_bytes.splitlines() if line.strip()]) - if max(concurrencies) > request_count: - raise ValueError("Concurrency exceeds the number of requests in the workload") - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - for name in TASK_FILES: - (output / name).write_bytes(task_resources().joinpath(name).read_bytes()) - policy = { - "schema_version": 1, - "task_sha256": sha((output / "task.json").read_bytes()), - "workload_sha256": sha((output / "workload.jsonl").read_bytes()), - "concurrencies": concurrencies, - "warmups": warmups, - "quality_scorer_id": scorer_id, - "gates": gates, - "grader_sha256": sha(Path(__file__).read_bytes()), - "baseline_launcher_sha256": sha((output / "baseline.py").read_bytes()), - "runtime_sha256": {name: sha((output / name).read_bytes()) for name in TASK_FILES}, - } - write(output / "policy.json", policy) - return policy - - -def load_prepared(root): - root = Path(root) - policy = read(root / "policy.json") - if policy["grader_sha256"] != sha(Path(__file__).read_bytes()) or policy[ - "baseline_launcher_sha256" - ] != sha((root / "baseline.py").read_bytes()): - raise ValueError("Prepared grader or baseline launcher changed") - for name, key in (("task.json", "task_sha256"), ("workload.jsonl", "workload_sha256")): - if sha((root / name).read_bytes()) != policy[key]: - raise ValueError("Prepared evaluation bytes changed") - for name in TASK_FILES: - if policy["runtime_sha256"][name] != sha((root / name).read_bytes()): - raise ValueError("Prepared task runtime changed: " + name) - workload = [ - json.loads(line) - for line in (root / "workload.jsonl").read_text(encoding="utf-8").splitlines() - if line.strip() - ] - return policy, workload, sha((root / "policy.json").read_bytes()) - - -def _remaining(deadline): - remaining = deadline - time.perf_counter() - if remaining <= 0: - raise TimeoutError("Request deadline exceeded") - return remaining - - -class _DeadlineReader(io.RawIOBase): - """Bound each socket read, including partial headers, SSE lines and HTTP chunks.""" - - def __init__(self, raw, sock, deadline): - self.raw, self.sock, self.deadline = raw, sock, deadline - - def readable(self): - return True - - def readinto(self, buffer): - self.sock.settimeout(_remaining(self.deadline)) - count = self.raw.readinto(buffer) - _remaining(self.deadline) - return count - - def close(self): - try: - self.raw.close() - finally: - super().close() - - -class _DeadlineHandler(urllib.request.HTTPHandler, urllib.request.HTTPSHandler): - """Keep urllib routing and TLS verification; wrap its response before parsing.""" - - def __init__(self, deadline): - urllib.request.HTTPSHandler.__init__(self) - self.deadline = deadline - - def do_open(self, connection_type, request, **kwargs): - def response(sock, **options): - result = http.client.HTTPResponse(sock, **options) - result.fp = io.BufferedReader( - _DeadlineReader(result.fp.detach(), sock, self.deadline) - ) - return result - - def connection(host, **options): - result = connection_type(host, **options) - result.response_class = response - return result - - request.timeout = _remaining(self.deadline) - return super().do_open(connection, request, **kwargs) - - -def _request(endpoint, payload, output, request_id, timeout): - """One attempt, including failed partial audio. SSE chunks may split PCM samples.""" - started = time.perf_counter() - deadline = started + timeout - pcm = bytearray() - record = { - "id": request_id, - "payload_sha256": sha(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()), - "success": False, - "ttfa_seconds": None, - "terminal": None, - "chunks": [], - } - try: - request = urllib.request.Request( - endpoint, - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json"}, - ) - opener = urllib.request.build_opener(_DeadlineHandler(deadline)) - with opener.open(request, timeout=_remaining(deadline)) as response: - if response.status != 200 or "text/event-stream" not in response.headers.get( - "Content-Type", "" - ): - raise ValueError("A successful SSE response is required") - for raw in response: - _remaining(deadline) - if not raw.startswith(b"data:"): - continue - event = json.loads(raw[5:]) - kind = event.get("type") - if record["terminal"] is not None: - raise ValueError("Data after terminal event") - if kind == "speech.audio.delta": - if event.get("sample_rate", 24000) != 24000 or event.get("channels", 1) != 1: - raise ValueError("Expected 24 kHz mono PCM16") - block = base64.b64decode(event["audio"], validate=True) - pcm.extend(block) - elapsed = time.perf_counter() - started - if len(pcm) >= 2 and record["ttfa_seconds"] is None: - record["ttfa_seconds"] = elapsed - record["chunks"].append({"seconds": elapsed, "bytes": len(block)}) - elif kind == "speech.audio.done": - if event.get("finish_reason") in {"length", "abort", "error", "cancelled"}: - raise ValueError("Unsuccessful generation termination") - record["terminal"] = event - else: - raise ValueError("Unexpected SSE event or server error") - if len(pcm) > 24000 * 2 * 600: - raise ValueError("Response exceeds ten minutes of PCM") - _remaining(deadline) - if record["terminal"] is None or not pcm or len(pcm) % 2: - raise ValueError("Missing terminal, empty PCM or incomplete sample") - samples = array.array("h", pcm) - if sys.byteorder != "little": - samples.byteswap() - if min(samples) == max(samples): - raise ValueError("Constant or silent audio") - record["success"] = True - except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException) as error: - # Preserve the error class without storing private endpoint/HTTP error bodies. - record["error"] = type(error).__name__ - record.update( - elapsed_seconds=time.perf_counter() - started, - pcm_sha256=sha(pcm), - pcm_bytes=len(pcm), - audio_seconds=len(pcm) / 48000, - ) - with wave.open(str(output / (request_id + ".wav")), "wb") as audio: - audio.setparams((1, 2, 24000, 0, "NONE", "not compressed")) - audio.writeframes(pcm) - write(output / (request_id + ".json"), record) - return record - - -def summarize(records, wall_seconds): - if not records or not positive(wall_seconds) or not all(r["success"] for r in records): - return None - return { - "rtf": statistics.mean(r["elapsed_seconds"] / r["audio_seconds"] for r in records), - "ttfa_median_ms": statistics.median(r["ttfa_seconds"] for r in records) * 1000, - "audio_seconds_per_second": sum(r["audio_seconds"] for r in records) / wall_seconds, - } - - -def collect(prepared, output, *, endpoint, model, timeout): - """Observe a caller-managed service. Its lifecycle is owned by the outer Harness.""" - if not positive(timeout): - raise ValueError("Socket/request timeout must be positive and finite") - policy, workload, policy_hash = load_prepared(prepared) - output = Path(output) - output.mkdir(parents=True, exist_ok=False) - records = { - "schema_version": 1, - "policy_sha256": policy_hash, - "model": model, - "cohorts": [], - "status": "running", - "quality": "NOT_EVALUATED", - } - write(output / "result.json", records) - payloads = [ - {**row, "model": model, "stream": True, "stream_format": "sse", "response_format": "pcm"} - for row in workload - ] - for concurrency in policy["concurrencies"]: - cohort = {"concurrency": concurrency, "warmups": [], "requests": []} - records["cohorts"].append(cohort) - for index in range(policy["warmups"]): - record = _request( - endpoint, - payloads[index % len(payloads)], - output, - f"c{concurrency}-w{index}", - timeout, - ) - cohort["warmups"].append(record) - if not record["success"]: - records["status"] = "request_failure" - write(output / "result.json", records) - return records - started = time.perf_counter() - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - futures = [ - pool.submit( - _request, endpoint, payload, output, f"c{concurrency}-r{index}", timeout - ) - for index, payload in enumerate(payloads) - ] - cohort["requests"] = [future.result() for future in futures] - cohort["wall_seconds"] = time.perf_counter() - started - cohort["metrics"] = summarize(cohort["requests"], cohort["wall_seconds"]) - write(output / "result.json", records) - if cohort["metrics"] is None: - records["status"] = "request_failure" - write(output / "result.json", records) - return records - records["status"] = "collected" - write(output / "result.json", records) - return records - - -def _verify_trial(directory, policy, workload, policy_hash): - document = read(Path(directory) / "result.json") - if document["policy_sha256"] != policy_hash or document["status"] != "collected": - raise ValueError("Trial is incomplete or belongs to another prepared evaluation") - if [c["concurrency"] for c in document["cohorts"]] != policy["concurrencies"]: - raise ValueError("Missing, reordered or duplicated concurrency cohort") - retained = {} - for cohort in document["cohorts"]: - concurrency = cohort["concurrency"] - for phase, count, marker in ( - ("warmups", policy["warmups"], "w"), - ("requests", len(workload), "r"), - ): - rows = cohort[phase] - if [r["id"] for r in rows] != [f"c{concurrency}-{marker}{i}" for i in range(count)]: - raise ValueError("Missing, duplicated or reordered request") - for index, row in enumerate(rows): - payload = { - **workload[index % len(workload)], - "model": document["model"], - "stream": True, - "stream_format": "sse", - "response_format": "pcm", - } - expected_hash = sha( - json.dumps(payload, sort_keys=True, ensure_ascii=False).encode() - ) - if not row["success"] or row["payload_sha256"] != expected_hash: - raise ValueError("Failed request or changed payload") - with wave.open(str(Path(directory) / (row["id"] + ".wav")), "rb") as audio: - if (audio.getnchannels(), audio.getsampwidth(), audio.getframerate()) != ( - 1, - 2, - 24000, - ): - raise ValueError("Changed audio format") - pcm = audio.readframes(audio.getnframes()) - if ( - not pcm - or sha(pcm) != row["pcm_sha256"] - or len(pcm) != row["pcm_bytes"] - or len(pcm) / 48000 != row["audio_seconds"] - or not positive(row["elapsed_seconds"]) - or not positive(row["ttfa_seconds"]) - or row["ttfa_seconds"] > row["elapsed_seconds"] - or not isinstance(row["terminal"], dict) - or row["terminal"].get("type") != "speech.audio.done" - or row["terminal"].get("finish_reason") - in {"length", "abort", "error", "cancelled"} - ): - raise ValueError("Missing or inconsistent retained audio/timing evidence") - samples = array.array("h", pcm) - if sys.byteorder != "little": - samples.byteswap() - if min(samples) == max(samples) or sum(c["bytes"] for c in row["chunks"]) != len( - pcm - ): - raise ValueError("Constant audio or inconsistent chunk evidence") - retained[row["id"]] = row - if ( - not positive(cohort["wall_seconds"]) - or summarize(cohort["requests"], cohort["wall_seconds"]) != cohort["metrics"] - ): - raise ValueError("Metrics do not match retained requests") - return document, retained - - -def assess(prepared, baseline, candidate, quality): - """Read retained bytes only. Supplied ASR/speaker scores have a frozen scorer ID.""" - result = { - "scope": "retained streaming audio and supplied independent quality scores", - "verdict": "INCONCLUSIVE", - "reasons": [], - "speedup": None, - "policy_sha256": None, - "cases": [], - } - try: - policy, workload, policy_hash = load_prepared(prepared) - result["policy_sha256"] = policy_hash - before, base_rows = _verify_trial(baseline, policy, workload, policy_hash) - after, cand_rows = _verify_trial(candidate, policy, workload, policy_hash) - if before["model"] != after["model"]: - raise ValueError("Served model identities differ") - scores = read(quality) - if scores["scorer_id"] != policy["quality_scorer_id"]: - raise ValueError("Quality scorer identity changed") - by_side = {} - for side, rows in (("baseline", base_rows), ("candidate", cand_rows)): - values = scores[side] - if len(values) != len(rows) or {s["id"] for s in values} != set(rows): - raise ValueError("Audio scoring coverage is incomplete") - by_side[side] = {s["id"]: s for s in values} - for score in values: - if score["pcm_sha256"] != rows[score["id"]]["pcm_sha256"]: - raise ValueError("Audio scores refer to different PCM bytes") - if ( - type(score["error_rate"]) not in (float, int) - or not math.isfinite(score["error_rate"]) - or score["error_rate"] < 0 - or type(score["speaker_similarity"]) not in (float, int) - or not math.isfinite(score["speaker_similarity"]) - or not -1 <= score["speaker_similarity"] <= 1 - ): - raise ValueError("Invalid audio score") - failed = [] - gates = policy["gates"] - for request_id, base in base_rows.items(): - cand = cand_rows[request_id] - bs, cs = by_side["baseline"][request_id], by_side["candidate"][request_id] - if ( - cs["error_rate"] > bs["error_rate"] + gates["max_error_rate_increase"] - or cs["speaker_similarity"] < bs["speaker_similarity"] - gates["max_speaker_drop"] - or abs(cand["audio_seconds"] / base["audio_seconds"] - 1) * 100 - > gates["max_duration_change_pct"] - ): - failed.append("Audio quality/duration gate failed for " + request_id) - for bc, cc in zip(before["cohorts"], after["cohorts"]): - bm, cm = bc["metrics"], cc["metrics"] - speedup = bm["rtf"] / cm["rtf"] - result["cases"].append( - { - "concurrency": bc["concurrency"], - "baseline": bm, - "candidate": cm, - "observed_speedup": speedup, - } - ) - gain = (1 - cm["rtf"] / bm["rtf"]) * 100 - if gain <= 0 or gain < gates["min_improvement_pct"]: - failed.append("RTF gain gate failed at concurrency " + str(bc["concurrency"])) - if cm["ttfa_median_ms"] > bm["ttfa_median_ms"] * ( - 1 + gates["max_ttfa_regression_pct"] / 100 - ): - failed.append("First-audio gate failed at concurrency " + str(bc["concurrency"])) - result.update(verdict="FAIL" if failed else "PASS", reasons=failed) - if not failed: - result["speedup"] = math.exp( - statistics.mean(math.log(c["observed_speedup"]) for c in result["cases"]) - ) - except (OSError, ValueError, KeyError, TypeError, wave.Error, ZeroDivisionError) as error: - result["reasons"].append(type(error).__name__ + ": incomplete or inconsistent evidence") - return result diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index b123dc794..b4e8d2775 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -28,13 +28,15 @@ def runtime_environment() -> dict: def _session_execute(root, directory, plan, deadline): from inferrsi.harness.scenarios.external import execute_trial - from inferrsi.harness.task_protocol import protocol + from inferrsi.harness.task_protocol import run as run_protocol if plan["task_source"] == "project": return _project_execute(root, directory, plan, deadline) task = root / "frozen" - arguments = protocol(task).arguments(directory / "candidate.py", directory / "measurement", - plan["evaluation_settings"]) + arguments = run_protocol(task, plan["frozen_files"]["optimization.py"], "arguments", + candidate=directory / "candidate.py", + output=directory / "measurement", + settings=plan["evaluation_settings"]) command = (*plan["evaluation_prefix"], plan["python"], str(Path(__file__).resolve()), str(task / "benchmark.py"), *arguments) timeout = min(plan["evaluation_timeout_seconds"], max(0, deadline - time.time())) @@ -45,13 +47,15 @@ def _session_execute(root, directory, plan, deadline): def _session_assess(root, directory, plan, execution): - from inferrsi.harness.task_protocol import protocol + from inferrsi.harness.task_protocol import run as run_protocol if plan["task_source"] == "project": from inferrsi.harness.project import assess return assess(directory, plan["evaluation_settings"], execution) - return protocol(root / "frozen").assess(directory, plan["evaluation_settings"], execution) + return run_protocol(root / "frozen", plan["frozen_files"]["optimization.py"], "assess", + directory=directory, settings=plan["evaluation_settings"], + execution=execution) def _project_execute(root, directory, plan, deadline): @@ -115,6 +119,7 @@ def _runtime_identity(): "infra/processes.py", "infra/artifacts.py", "harness/scenarios/external.py", "harness/atomic_json.py", "harness/artifact_primitives.py", "harness/usage.py", "harness/agent_sessions.py", "harness/task_protocol.py", + "harness/protocol_child.py", "harness/candidates.py", "harness/project.py", "harness/records/knowledge_capture.py", "apps/campaign.py", "knowledge/context_projection.py") diff --git a/src/inferrsi/harness/protocol_child.py b/src/inferrsi/harness/protocol_child.py new file mode 100644 index 000000000..7fc4b902b --- /dev/null +++ b/src/inferrsi/harness/protocol_child.py @@ -0,0 +1,164 @@ +# Child entry for one operation of a frozen task protocol (#412 R3). +# +# Harness starts this file as ``python -I -B protocol_child.py REQUEST`` from +# ``task_protocol.run``; nothing executes protocol code in the Harness process. The file +# imports only the standard library, so the child never loads inferrsi. It reads the +# protocol bytes once, checks them against the request's SHA-256, compiles exactly those +# bytes and calls one protocol function. Descriptor 1 carries only the response envelope: +# everything the protocol prints, from Python, native code or its own children, goes to +# stderr. This is an interpreter and process boundary, not a filesystem or network sandbox. +# ``task_protocol`` imports this module only for the shared envelope contract below. + +import hashlib +import json +import math +import os +import sys +import types +from pathlib import Path + +SCHEMA_VERSION = 1 +# Operation name -> its request argument names, in call order. +OPERATIONS = { + "validate_arguments": ("arguments",), + "arguments": ("candidate", "output", "settings"), + "assess": ("directory", "settings", "execution"), +} +REQUEST_KEYS = frozenset({"schema_version", "operation", "protocol", "protocol_sha256", + "arguments"}) +# Failure kinds the child itself reports in an error envelope. +CHILD_ERROR_KINDS = frozenset({"request", "identity", "protocol_exception", "system_exit", + "result"}) + + +def _unique_pairs(pairs): + keys = [key for key, _ in pairs] + if len(set(keys)) != len(keys): + raise ValueError("duplicate JSON object key") + return dict(pairs) + + +def _non_finite(name): + raise ValueError(f"non-finite JSON number {name}") + + +def _finite_float(text): + value = float(text) + if not math.isfinite(value): + raise ValueError(f"non-finite JSON number {text}") + return value + + +def loads_strict(data: bytes): + """Parse UTF-8 JSON that has no duplicate key and no NaN or infinite number.""" + return json.loads(data.decode("utf-8"), object_pairs_hook=_unique_pairs, + parse_constant=_non_finite, parse_float=_finite_float) + + +def dumps_strict(value) -> bytes: + """Encode JSON in ASCII; NaN, infinity and non-JSON values raise ValueError or TypeError.""" + return json.dumps(value, allow_nan=False, separators=(",", ":")).encode("ascii") + + +def check_request(request) -> None: + """Raise ValueError unless ``request`` is a complete version-1 request.""" + if not isinstance(request, dict) or set(request) != REQUEST_KEYS: + raise ValueError("request must hold exactly " + ", ".join(sorted(REQUEST_KEYS))) + if type(request["schema_version"]) is not int or request["schema_version"] != SCHEMA_VERSION: + raise ValueError(f"request schema_version must be {SCHEMA_VERSION}") + operation = request["operation"] + if operation not in OPERATIONS: + raise ValueError(f"unknown protocol operation {operation!r}") + if not isinstance(request["protocol"], str) or not Path(request["protocol"]).is_absolute(): + raise ValueError("protocol must be an absolute path") + digest = request["protocol_sha256"] + if not isinstance(digest, str) or len(digest) != 64 or set(digest) - set("0123456789abcdef"): + raise ValueError("protocol_sha256 must be 64 lowercase hex digits") + arguments = request["arguments"] + if not isinstance(arguments, dict) or set(arguments) != set(OPERATIONS[operation]): + raise ValueError(f"{operation} takes exactly " + ", ".join(OPERATIONS[operation])) + if operation == "validate_arguments": + values = arguments["arguments"] + if not isinstance(values, list) or not all(isinstance(item, str) for item in values): + raise ValueError("validate_arguments takes a list of strings") + return + paths = ("candidate", "output") if operation == "arguments" else ("directory",) + for name in paths: + if not isinstance(arguments[name], str) or not Path(arguments[name]).is_absolute(): + raise ValueError(f"{name} must be an absolute path") + objects = ("settings",) if operation == "arguments" else ("settings", "execution") + for name in objects: + if not isinstance(arguments[name], dict): + raise ValueError(f"{name} must be an object") + + +def _error(operation, kind, error_type, message, **extra): + return {"schema_version": SCHEMA_VERSION, "operation": operation, + "error": {"kind": kind, "type": error_type, "message": message, **extra}} + + +def _call(module, operation, arguments): + if operation == "validate_arguments": + return module.validate_arguments(list(arguments["arguments"])) + if operation == "arguments": + return module.arguments(Path(arguments["candidate"]), Path(arguments["output"]), + arguments["settings"]) + return module.assess(Path(arguments["directory"]), arguments["settings"], + arguments["execution"]) + + +def _respond(argv) -> dict: + if len(argv) != 1: + return _error(None, "request", "UsageError", "expected exactly one request path") + try: + request = loads_strict(Path(argv[0]).read_bytes()) + check_request(request) + except (OSError, UnicodeDecodeError, ValueError) as error: + return _error(None, "request", type(error).__name__, str(error)) + operation = request["operation"] + path = Path(request["protocol"]) + try: + source = path.read_bytes() + except OSError as error: + return _error(operation, "identity", type(error).__name__, str(error)) + if hashlib.sha256(source).hexdigest() != request["protocol_sha256"]: + return _error(operation, "identity", "ProtocolIdentityError", + "protocol bytes differ from the frozen digest") + # argparse in a protocol names itself after argv[0]. + sys.argv = [str(path)] + module = types.ModuleType("task_optimization_protocol") + module.__file__ = str(path) + try: + # The digest-checked bytes themselves are compiled; no bytecode cache is involved. + exec(compile(source, str(path), "exec"), module.__dict__) + result = _call(module, operation, request["arguments"]) + except SystemExit as stop: + code = stop.code + if code is not None and type(code) is not int: + print(code, file=sys.stderr) + code = 1 + return _error(operation, "system_exit", "SystemExit", "protocol exited", + exit_code=0 if code is None else code) + except Exception as error: + return _error(operation, "protocol_exception", type(error).__name__, str(error)) + try: + dumps_strict(result) + except (TypeError, ValueError) as error: + return _error(operation, "result", type(error).__name__, str(error)) + return {"schema_version": SCHEMA_VERSION, "operation": operation, "result": result} + + +def main(argv) -> int: + # Keep a private copy of descriptor 1 for the envelope, then point descriptor 1 at + # stderr so protocol output and inherited descriptors cannot reach the envelope. + envelope = os.fdopen(os.dup(1), "wb") + os.dup2(2, 1) + response = _respond(argv) + sys.stdout.flush() + with envelope: + envelope.write(dumps_strict(response)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index b62feaf1b..14d948e88 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -1,34 +1,155 @@ -"""Freeze supplied task resources and load their task-owned evaluation protocol. +"""Freeze supplied task resources and run their task-owned evaluation protocol. Task discovery belongs to the caller. This module accepts a resource directory; -it neither imports Benchmark nor contains model-specific acceptance rules. +it neither imports Benchmark nor contains model-specific acceptance rules. Protocol +code never runs in this process: each operation runs ``protocol_child.py`` in a +separate isolated interpreter under a deadline, inside its own process group or +Windows Job, and returns one strict JSON envelope (#412 R3). """ -import importlib.util +import os import shutil +import sys +import tempfile +import time from pathlib import Path +from . import local_process, protocol_child from .artifact_primitives import sha256 +# One protocol operation must finish within this time; the child and every process in +# its group or Job are then stopped and the operation fails with kind "timeout". +PROTOCOL_TIMEOUT_SECONDS = 120.0 +# Harness reads at most this many bytes of the envelope on stdout ... +STDOUT_LIMIT_BYTES = 16 * 1024 * 1024 +# ... and quotes at most this tail of stderr in a failure message. +STDERR_TAIL_BYTES = 4096 +# The child entry that every operation runs as a separate script. +CHILD_ENTRY = Path(protocol_child.__file__).resolve() +_RESULT_TYPES = {"validate_arguments": dict, "arguments": list, "assess": dict} + + +class TaskProtocolError(ValueError): + """A protocol operation gave no valid result. ``kind`` names the failure class.""" + + def __init__(self, kind: str, message: str): + super().__init__(f"{kind}: {message}") + self.kind = kind + + +def _tail(path: Path) -> str: + with path.open("rb") as stream: + stream.seek(max(0, path.stat().st_size - STDERR_TAIL_BYTES)) + return stream.read().decode("utf-8", "replace") -def protocol(root: Path): - path = root / "optimization.py" - if not path.is_file(): - raise ValueError("This task has no iterative evaluation protocol") - spec = importlib.util.spec_from_file_location("task_optimization_protocol", path) - module = importlib.util.module_from_spec(spec) - # Always read the frozen source; do not create or consume a mutable bytecode cache. - exec(compile(path.read_bytes(), str(path), "exec"), module.__dict__) - return module + +def _response_problem(response, operation: str) -> str | None: + if not isinstance(response, dict): + return "envelope is not an object" + version = response.get("schema_version") + if type(version) is not int or version != protocol_child.SCHEMA_VERSION: + return "envelope schema_version mismatch" + keys = set(response) + if keys == {"schema_version", "operation", "result"}: + if response["operation"] != operation: + return "envelope answers another operation" + result = response["result"] + if not isinstance(result, _RESULT_TYPES[operation]): + return f"{operation} result must be a {_RESULT_TYPES[operation].__name__}" + if operation == "arguments" and not all(isinstance(item, str) for item in result): + return "arguments result must be a list of strings" + return None + if keys != {"schema_version", "operation", "error"}: + return "envelope must hold either result or error" + error = response["error"] + if not isinstance(error, dict) or error.get("kind") not in protocol_child.CHILD_ERROR_KINDS: + return "envelope error has no known kind" + expected = {"kind", "type", "message"} | ( + {"exit_code"} if error["kind"] == "system_exit" else set()) + if (set(error) != expected or not isinstance(error["type"], str) + or not isinstance(error["message"], str) + or (error["kind"] == "system_exit" and type(error["exit_code"]) is not int)): + return "envelope error object is malformed" + if response["operation"] != (None if error["kind"] == "request" else operation): + return "envelope answers another operation" + return None + + +def run(root: Path, protocol_sha256: str, operation: str, **arguments): + """Run one operation of ``root/optimization.py`` whose bytes have ``protocol_sha256``. + + Path arguments must be absolute. A protocol ``SystemExit`` is raised again with its + exit code after the child's stderr tail is copied to this process's stderr; every + other failure raises ``TaskProtocolError``. Descendants that the protocol leaves + running are stopped with the child. + """ + path = Path(root).resolve() / "optimization.py" + if sha256(path) != protocol_sha256: + raise TaskProtocolError("identity", "protocol bytes differ from the frozen digest") + request = {"schema_version": protocol_child.SCHEMA_VERSION, "operation": operation, + "protocol": str(path), "protocol_sha256": protocol_sha256, + "arguments": {name: str(value) if isinstance(value, Path) else value + for name, value in arguments.items()}} + try: + protocol_child.check_request(request) + data = protocol_child.dumps_strict(request) + except (TypeError, ValueError) as error: + raise TaskProtocolError("request", str(error)) from error + with tempfile.TemporaryDirectory(prefix="inferrsi-protocol-") as temporary: + work = Path(temporary) + (work / "request.json").write_bytes(data) + argv = [sys.executable, "-I", "-B", str(CHILD_ENTRY), str(work / "request.json")] + # The scratch directory is the working directory, so a stray relative write + # cannot land among the frozen resources. + record = local_process._supervise( + argv, work, dict(os.environ), work / "stdout", work / "stderr", + time.monotonic() + PROTOCOL_TIMEOUT_SECONDS) + stderr = _tail(work / "stderr") if (work / "stderr").is_file() else "" + if record["status"] == "cancelled" and record["cancellation_source"] == ( + "keyboard_interrupt"): + raise KeyboardInterrupt + if record["status"] == "timeout": + raise TaskProtocolError( + "timeout", f"{operation} exceeded {PROTOCOL_TIMEOUT_SECONDS:g} s; {stderr}") + if record["status"] not in ("completed", "failed"): + raise TaskProtocolError("launch", f"{record['status']}: {record['error']}") + if record["returncode"] != 0: + raise TaskProtocolError( + "exit_status", f"child exited with {record['returncode']}; {stderr}") + if record["domain_empty"] is not True: + raise TaskProtocolError("cleanup", "the protocol left a live process behind") + if (work / "stdout").stat().st_size > STDOUT_LIMIT_BYTES: + raise TaskProtocolError("output_limit", f"envelope exceeds {STDOUT_LIMIT_BYTES} bytes") + raw = (work / "stdout").read_bytes() + try: + response = protocol_child.loads_strict(raw) + except (UnicodeDecodeError, ValueError, RecursionError) as error: + raise TaskProtocolError("malformed_response", str(error)) from error + problem = _response_problem(response, operation) + if problem is not None: + raise TaskProtocolError("invalid_response", problem) + if "result" in response: + return response["result"] + error = response["error"] + if error["kind"] == "system_exit": + # argparse explains a usage error on stderr; keep that message visible. + sys.stderr.write(stderr) + raise SystemExit(error["exit_code"]) + raise TaskProtocolError(error["kind"], f"{error['type']}: {error['message']}") def prepare(source: Path, destination: Path, arguments: list[str]) -> dict: - settings = protocol(source).validate_arguments(arguments) + path = source / "optimization.py" + if not path.is_file(): + raise ValueError("This task has no iterative evaluation protocol") + digest = sha256(path) + settings = run(source, digest, "validate_arguments", arguments=arguments) if any(path.is_symlink() for path in source.rglob("*")): raise ValueError("Task resources must be regular files") shutil.copytree(source, destination, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) - return { - "evaluation_settings": settings, - "frozen_files": {path.relative_to(destination).as_posix(): sha256(path) - for path in destination.rglob("*") if path.is_file()}, - } + frozen = {path.relative_to(destination).as_posix(): sha256(path) + for path in destination.rglob("*") if path.is_file()} + # The validated bytes are the frozen bytes that every later operation must match. + if frozen["optimization.py"] != digest: + raise ValueError("The task protocol changed while its resources were frozen") + return {"evaluation_settings": settings, "frozen_files": frozen} diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index 1eb2f021d..1caf92a5d 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -34,16 +34,21 @@ # file is covered, in the root or under a package. MODULE_ENTRY = "__main__.py" MODULE_ENTRY_DELEGATION = ("inferrsi.apps.cli", "main") -# Reviewed execution boundaries: owner function (or module scope) and the exact -# local spec variable. They execute task resources/candidate code, not a hidden -# core compatibility import. Their presence keeps isolation incomplete. -RESOURCE_LOADERS = { - "inferrsi.harness.task_protocol": ("protocol", "spec"), - "inferrsi.benchmark.tasks.moe-forward-npu.benchmark": ("", "_process_spec"), +# #412 R3, permanent: the two task evaluators that must call submitted code in their own +# process to time it. Each entry is a task bundle program (a file in a directory without +# ``__init__.py`` under ``benchmark/tasks/``), the owner function and the exact local +# spec variable of its one admitted ``importlib.util.module_from_spec`` call. A second +# call, a call elsewhere in the file, the same shape in any other module, a declared +# module that becomes importable or loses its call, and every other dynamic import fail. +# These loaders execute frozen run files; they are listed in ``runtime_execution``, +# which ``isolation_complete`` (a static import-closure result) does not cover. R3 +# retired the temporary ``RESOURCE_LOADERS`` table and its blocker: the Harness task +# protocol runs in a child interpreter and the two sibling loads were merged away. +EVALUATOR_LOADERS = { "inferrsi.benchmark.tasks.moe-forward-npu.grader": ("load", "spec"), "inferrsi.benchmark.tasks.qwen21-denoise-npu.benchmark": ("load", "spec"), - "inferrsi.benchmark.tasks.qwen3-tts-npu.benchmark": ("", "_evaluation_spec"), } +TASK_BUNDLES = "inferrsi.benchmark.tasks" # #412 R1 deleted these eleven root modules: the seven Knowledge shells, the three # old-run forwarders and the root inspection binding. None is registered any more. # A file that returns under one of these names is a violation by itself, and every @@ -357,8 +362,9 @@ def _scan(root: Path, pairs: frozenset[tuple[str, str]]) -> _Graph: return _Graph(trees, packages, edges) -def _resource_loader(tree: ast.Module, module: str) -> ast.Attribute | None: - declaration = RESOURCE_LOADERS.get(module) +def _evaluator_loader(tree: ast.Module, module: str) -> ast.Attribute | None: + """The one declared evaluator call in ``module``; None unless exactly one exists.""" + declaration = EVALUATOR_LOADERS.get(module) if declaration is None: return None matches = [] @@ -381,7 +387,7 @@ def visit(node, owner=""): def _has_dynamic_import(tree: ast.Module, module: str = "") -> bool: - declared = _resource_loader(tree, module) + declared = _evaluator_loader(tree, module) for node in ast.walk(tree): if node is declared: continue @@ -683,7 +689,10 @@ class IsolationReport: """Staged first-wave result (A01-4). ``new_core_closure_isolated`` covers only the new-core closures present in - the tree. It never stands in for ``isolation_complete``. + the tree. It never stands in for ``isolation_complete``. Both are static + import-closure results. ``runtime_execution`` lists the declared sites that + execute frozen task or candidate files in-process at run time; neither flag + says anything about what that code does once it runs. """ violations: tuple[str, ...] @@ -691,6 +700,7 @@ class IsolationReport: harness_model_infra: tuple[str, ...] blockers: tuple[str, ...] new_core_closure_isolated: bool + runtime_execution: tuple[str, ...] = () @property def isolation_complete(self) -> bool: @@ -755,6 +765,20 @@ def first_wave_report( for parent in sorted(leaf_parents): violations += _initializer_errors(parent, graph.trees[parent], graph.packages[parent]) violations += _contracts_errors(graph) + # #412 R3: a declared evaluator must stay a task bundle program and keep exactly its + # one call. A stale or widened declaration fails instead of passing silently. + runtime_execution = [] + for name in sorted(set(EVALUATOR_LOADERS) & set(modules)): + bundle = name.rpartition(".")[0] + if bundle.rpartition(".")[0] != TASK_BUNDLES or bundle in graph.trees: + violations.append(f"{name} declares an evaluator loader outside a task bundle program") + if _evaluator_loader(graph.trees[name], name) is None: + violations.append(f"{name} does not hold exactly its one declared evaluator loader") + else: + function, spec = EVALUATOR_LOADERS[name] + runtime_execution.append( + f"{name}.{function}: importlib.util.module_from_spec({spec}) executes frozen " + "run files in-process") # Inside the contracts package a Knowledge module may name only the encoder. The # initializer and its RunSpec module load with that import and are never named. violations += [ @@ -833,12 +857,8 @@ def h_entry_forbidden(node: str) -> bool: blockers.append(f"harness reaches model/infra: {legacy[0]} ({len(legacy)} hits)") if violations: blockers.append(f"violations: {len(violations)}") - resource_loaders = [name for name in modules - if _resource_loader(graph.trees[name], name) is not None] - if resource_loaders: - blockers.append(f"runtime resource loaders: {len(resource_loaders)}") return IsolationReport( tuple(violations), tuple(legacy), tuple(blockers), - bool(starts) and not new_core and not resource_loaders, + bool(starts) and not new_core, tuple(runtime_execution), ) diff --git a/tests/test_engineering_benchmark_plugins.py b/tests/test_engineering_benchmark_plugins.py index e4abc8e92..b135515fc 100644 --- a/tests/test_engineering_benchmark_plugins.py +++ b/tests/test_engineering_benchmark_plugins.py @@ -30,7 +30,8 @@ def test_isolated_tts_entry_prepares_its_bundled_workload(distribution, tmp_path ) assert result.returncode == 0, result.stderr assert len((prepared / "workload.jsonl").read_text(encoding="utf-8").splitlines()) == 12 - assert (prepared / "benchmark.py").is_file() and (prepared / "evaluation.py").is_file() + # #412 R3: the evaluation lives in benchmark.py; no sibling evaluation.py is frozen. + assert (prepared / "benchmark.py").is_file() and not (prepared / "evaluation.py").exists() assert json.loads((prepared / "policy.json").read_text())["quality_scorer_id"] == ( "synthetic-software-control" ) diff --git a/tests/test_evaluator_loaders.py b/tests/test_evaluator_loaders.py new file mode 100644 index 000000000..e5b81b444 --- /dev/null +++ b/tests/test_evaluator_loaders.py @@ -0,0 +1,58 @@ +"""#412 R3: the two permanent evaluator loaders give fresh, unregistered modules. + +Frozen-file identity is checked by their callers (MoE ``_adjudicate`` and the Qwen21 +``main``/protocol ``assess``); these tests pin the loaders' own contract. +""" + +import sys + +import pytest + +from tests.engineering_task_support import load_task_module + +grader = load_task_module("moe-forward-npu", "grader") +qwen = load_task_module("qwen21-denoise-npu", "benchmark") + + +def source(tmp_path, body, name="candidate.py"): + path = tmp_path / name + path.write_text(body, encoding="utf-8") + return path + + +def test_moe_grader_load_returns_a_fresh_compute_each_time(tmp_path): + path = source(tmp_path, "STATE = []\ndef compute(x, params, *, config, group):\n" + " STATE.append(x)\n return STATE\n") + first = grader.load(path, "evaluated_candidate") + second = grader.load(path, "evaluated_candidate") + assert first is not second and first.__globals__ is not second.__globals__ + assert first(1, {}, config={}, group=None) == [1] + assert second(2, {}, config={}, group=None) == [2] + assert "evaluated_candidate" not in sys.modules + + +@pytest.mark.parametrize("body", ["", "compute = 3\n"]) +def test_moe_grader_load_requires_a_callable_compute(tmp_path, body): + with pytest.raises(ValueError, match="Candidate must define compute"): + grader.load(source(tmp_path, body), "evaluated_candidate") + + +def test_qwen21_load_returns_a_fresh_unregistered_module_each_time(tmp_path): + path = source(tmp_path, "STATE = []\n", "workload.py") + first = qwen.load(path, "task_workload") + second = qwen.load(path, "task_workload") + first.STATE.append(1) + assert first is not second and second.STATE == [] + assert "task_workload" not in sys.modules + + +@pytest.mark.parametrize("body, message", [ + ("", "Candidate must define callable compute"), + ("compute = 3\n", "Candidate must define callable compute"), + ("def compute(x):\n pass\n", "compute does not accept the task arguments"), + ("def compute(:\n", "invalid syntax"), +]) +def test_qwen21_candidate_entry_requires_the_task_compute(tmp_path, body, message): + with pytest.raises(qwen.CandidateContractError, match=message): + qwen.candidate_function(source(tmp_path, body)) + assert "submitted_candidate" not in sys.modules diff --git a/tests/test_knowledge_compatibility_shells.py b/tests/test_knowledge_compatibility_shells.py index f3bd04d3d..c6482a8a2 100644 --- a/tests/test_knowledge_compatibility_shells.py +++ b/tests/test_knowledge_compatibility_shells.py @@ -280,14 +280,13 @@ def test_package_init_and_infra_inspection_load_no_knowledge_body(): ) -def test_knowledge_closure_is_clean_while_overall_stays_false(): +def test_knowledge_closure_is_clean_and_the_final_tree_is_isolated(): # K01-5: the Knowledge package exists and its closure is clean. R1 cleared the - # allowlist and the old-run forwarders, A1 the parser entry and R2 the twelve - # harness -> infra hits from six modules. The five runtime resource loaders (R3) - # remain, and they keep new_core_closure_isolated false. + # allowlist and the old-run forwarders, A1 the parser entry, R2 the twelve + # harness -> infra hits from six modules and R3 the five runtime resource loaders. report = first_wave_report() assert report.violations == () - assert report.blockers == ("runtime resource loaders: 5",) + assert report.blockers == () assert report.harness_model_infra == () - assert not report.new_core_closure_isolated - assert not report.isolation_complete + assert report.new_core_closure_isolated + assert report.isolation_complete diff --git a/tests/test_legacy_run_forwarder_gate.py b/tests/test_legacy_run_forwarder_gate.py index d9f6cd121..ad5e2e662 100644 --- a/tests/test_legacy_run_forwarder_gate.py +++ b/tests/test_legacy_run_forwarder_gate.py @@ -89,11 +89,11 @@ def test_forwarder_misuse_is_a_violation(changes, expected): assert not report.isolation_complete -def test_real_tree_holds_no_forwarder_and_stays_incomplete(): +def test_real_tree_holds_no_forwarder_and_is_isolated(): report = first_wave_report() assert report.violations == () for name in RETIRED_FORWARDERS: assert not (support.ROOT / f"{name}.py").exists(), name - # What is left belongs to R3: R2 closed the harness -> infra hits. - assert support.blocker_kinds(report) == {"runtime resource loaders"} - assert not report.isolation_complete + # R2 closed the harness -> infra hits and R3 the runtime resource loaders. + assert report.blockers == () + assert report.isolation_complete diff --git a/tests/test_moe_benchmark.py b/tests/test_moe_benchmark.py index ff32469aa..f43035213 100644 --- a/tests/test_moe_benchmark.py +++ b/tests/test_moe_benchmark.py @@ -17,7 +17,8 @@ from tests.engineering_task_support import load_task_module moe = load_task_module("moe-forward-npu", "benchmark") -_moe_processes = moe._processes +# #412 R3: worker cleanup moved from the sibling processes.py into benchmark.py. +_moe_processes = moe def _write(path, value): diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index 9e9c37f2c..2768e6756 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -354,6 +354,8 @@ def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executo assert "harness/local_process.py" in identity # #412 R2: the Harness-owned primitives replace infra.artifacts in six modules. assert "harness/artifact_primitives.py" in identity + # #412 R3: the protocol child entry runs every task protocol operation. + assert "harness/protocol_child.py" in identity assert "infra/executors/local.py" not in identity for name in identity: assert (root / name).is_file(), name diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index 561a42aaa..fe737fce9 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -25,35 +25,92 @@ load_manifest, ) +# #412 R3: the one admitted evaluator loader shape, as in the MoE grader. +EVALUATOR = "benchmark/tasks/moe-forward-npu/grader.py" +EVALUATOR_MODULE = "inferrsi.benchmark.tasks.moe-forward-npu.grader" +LOADER = ( + "import importlib.util\n" + "def load(path, name):\n" + " spec = importlib.util.spec_from_file_location(name, path)\n" + " module = importlib.util.module_from_spec(spec)\n" + " spec.loader.exec_module(module)\n" + " return module\n" +) +MISSING = f"{EVALUATOR_MODULE} does not hold exactly its one declared evaluator loader" + + +def _report(root, files): + for relative, source in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + return first_wave_report(root) + + +def test_declared_evaluator_loader_is_admitted_listed_and_never_a_blocker(tmp_path): + report = _report(tmp_path, {EVALUATOR: LOADER}) + assert report.violations == () + assert "runtime resource loaders" not in blocker_kinds(report) + assert report.runtime_execution == ( + f"{EVALUATOR_MODULE}.load: importlib.util.module_from_spec(spec) executes frozen " + "run files in-process", + ) + @pytest.mark.parametrize("extra", [ "\nimport importlib\nimportlib.import_module('inferrsi.knowledge')\n", + "\n__import__('json')\n", "\nimportlib.util.module_from_spec(spec)\n", - "\ndef another():\n return importlib.util.module_from_spec(spec)\n", + "\ndef another(spec):\n return importlib.util.module_from_spec(spec)\n", ]) -def test_declared_resource_loader_does_not_allow_an_extra_dynamic_edge(tmp_path, extra): - task = tmp_path / "harness/task_protocol.py" - task.parent.mkdir() - source = ("import importlib.util\ndef protocol():\n" - " return importlib.util.module_from_spec(spec)\n") - task.write_text(source, encoding="utf-8") - report = first_wave_report(tmp_path) - assert not report.violations - assert "runtime resource loaders" in blocker_kinds(report) +def test_declared_evaluator_loader_does_not_allow_an_extra_dynamic_edge(tmp_path, extra): + report = _report(tmp_path, {EVALUATOR: LOADER + extra}) + assert any("dynamic import" in item for item in report.violations) assert not report.isolation_complete - task.write_text(source + extra, encoding="utf-8") - assert any("dynamic import" in item for item in first_wave_report(tmp_path).violations) -def test_resource_loader_declaration_does_not_cover_an_adjacent_module(tmp_path): - task = tmp_path / "harness/other_protocol.py" - task.parent.mkdir() - task.write_text( - "import importlib.util\ndef protocol():\n" - " return importlib.util.module_from_spec(spec)\n", - encoding="utf-8", - ) - assert any("dynamic import" in item for item in first_wave_report(tmp_path).violations) +@pytest.mark.parametrize("source", [ + # A second call in the declared function voids the declaration for both calls. + LOADER.replace(" return module\n", + " importlib.util.module_from_spec(spec)\n return module\n"), + # The call under another spec variable or in another function is not the declared one. + LOADER.replace("spec =", "other =").replace("(spec)", "(other)") + .replace("spec.loader", "other.loader"), + LOADER.replace("def load(", "def load_module("), +]) +def test_declared_evaluator_loader_shape_is_exact(tmp_path, source): + report = _report(tmp_path, {EVALUATOR: source}) + assert MISSING in report.violations + assert any("dynamic import" in item for item in report.violations) + assert report.runtime_execution == () + + +def test_declared_evaluator_without_its_call_is_a_stale_declaration(tmp_path): + report = _report(tmp_path, {EVALUATOR: "def load(path, name):\n return None\n"}) + assert MISSING in report.violations + assert report.runtime_execution == () + + +def test_declared_evaluator_inside_an_importable_package_fails(tmp_path): + report = _report(tmp_path, {EVALUATOR: LOADER, + "benchmark/tasks/moe-forward-npu/__init__.py": ""}) + assert (f"{EVALUATOR_MODULE} declares an evaluator loader outside a task bundle program" + in report.violations) + + +@pytest.mark.parametrize("relative", [ + "benchmark/tasks/moe-forward-npu/workload.py", + "benchmark/tasks/qwen3-tts-npu/benchmark.py", + "benchmark/registry.py", + # The Harness protocol loader that R3 retired, and its neighbours. + "harness/task_protocol.py", + "harness/other_protocol.py", + "apps/benchmark_operations.py", +]) +def test_the_evaluator_loader_shape_is_admitted_nowhere_else(tmp_path, relative): + report = _report(tmp_path, {relative: LOADER}) + assert any("dynamic import" in item for item in report.violations) + assert report.runtime_execution == () def _scan_tree(files: dict[str, str]) -> list[str]: @@ -135,14 +192,11 @@ def test_clean_scan_is_not_isolation(self) -> None: ["__init__.py", "repository_packs.py"], ) self.assertTrue((ROOT / "apps" / "evidence_layout.py").is_file()) - # A01-4 staged model: packages may appear one at a time, but each - # remaining trigger keeps overall false. + # #412 R3: every staged trigger is gone, so the final tree is fully isolated in + # the static import-closure sense. report = first_wave_report() - self.assertFalse(report.isolation_complete) for name in NEW_CORES: - absent = not (ROOT / name).is_dir() - self.assertEqual(f"absent: {name}/" in report.blockers, absent) - self.assertNotIn(f"unscannable: {name}/", report.blockers) + self.assertTrue((ROOT / name).is_dir()) kinds = blocker_kinds(report) # #412 R1: the allowlist is empty and no old-run forwarder is registered. self.assertNotIn("allowlist", kinds) @@ -153,12 +207,21 @@ def test_clean_scan_is_not_isolation(self) -> None: self.assertFalse(hasattr(report, "residuals")) self.assertNotIn("residual", kinds) self.assertEqual(report.violations, ()) - # #412 R2 closed the six Harness modules: no Harness module reaches the - # package-level ``inferrsi.infra``, ``inferrsi.infra.artifacts`` or - # ``inferrsi.model``. What keeps isolation false is the five loaders that R3 closes. - self.assertEqual(report.blockers, ("runtime resource loaders: 5",)) + # R2 closed the Harness -> Infra/Model edges; R3 retired the five temporary + # resource loaders and their blocker. + self.assertFalse(hasattr(support, "RESOURCE_LOADERS")) self.assertEqual(report.harness_model_infra, ()) - self.assertFalse(report.new_core_closure_isolated) + self.assertEqual(report.blockers, ()) + self.assertTrue(report.new_core_closure_isolated) + self.assertTrue(report.isolation_complete) + # Static closure is not runtime isolation: the two declared evaluators still + # execute frozen run files in-process, and the report says so. + self.assertEqual(report.runtime_execution, ( + "inferrsi.benchmark.tasks.moe-forward-npu.grader.load: " + "importlib.util.module_from_spec(spec) executes frozen run files in-process", + "inferrsi.benchmark.tasks.qwen21-denoise-npu.benchmark.load: " + "importlib.util.module_from_spec(spec) executes frozen run files in-process", + )) def test_split_skill_eval_modules_reach_no_model_or_infra(self) -> None: # S01: the split Harness modules own their writes and digests. The diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py new file mode 100644 index 000000000..4b1c6e773 --- /dev/null +++ b/tests/test_task_protocol_child.py @@ -0,0 +1,274 @@ +"""#412 R3: Harness runs a frozen task protocol only in a child interpreter. + +Each operation is one ``python -I -B protocol_child.py`` process under a deadline. The +tests drive real child processes; the envelope cases substitute the child script itself +to inject faults the real entry never writes. +""" + +import ast +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import protocol_child, task_protocol +from inferrsi.harness.task_protocol import TaskProtocolError, run + +SETTINGS = {"pairs": 2, "samples": 1, "warmups": 1} +QWEN_ARGUMENTS = ["--pairs", "2", "--samples", "1", "--warmups", "1"] + + +def write_protocol(root, body): + root.mkdir(parents=True, exist_ok=True) + path = root / "optimization.py" + path.write_text(body, encoding="utf-8") + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def failure(root, digest, operation="validate_arguments", **arguments): + with pytest.raises(TaskProtocolError) as error: + run(root, digest, operation, **(arguments or {"arguments": []})) + return error.value + + +def qwen(): + root = Path(str(task_resources("qwen21-denoise-npu"))) + return root, hashlib.sha256((root / "optimization.py").read_bytes()).hexdigest() + + +def test_qwen21_protocol_answers_all_three_operations(tmp_path): + root, digest = qwen() + assert run(root, digest, "validate_arguments", arguments=QWEN_ARGUMENTS) == SETTINGS + candidate, output = tmp_path / "candidate.py", tmp_path / "measurement" + assert run(root, digest, "arguments", candidate=candidate, output=output, + settings=SETTINGS) == [ + "--candidate", str(candidate), "--output-dir", str(output), "--device", "npu", + "--pairs", "2", "--samples", "1", "--warmups", "1"] + assessment = run(root, digest, "assess", directory=tmp_path, settings=SETTINGS, + execution={"status": "completed", "returncode": 0}) + assert assessment == {"status": "infrastructure_failure", "score": None, + "verdict": "INCONCLUSIVE", "reason": "missing_task_result"} + + +def test_argparse_usage_error_keeps_exit_status_two_and_its_message(capsys): + root, digest = qwen() + with pytest.raises(SystemExit) as stop: + run(root, digest, "validate_arguments", arguments=["--pairs", "0x"]) + assert stop.value.code == 2 + assert "invalid int value" in capsys.readouterr().err + + +def test_protocol_exception_keeps_its_type_and_message(tmp_path): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n" + " raise TypeError('bad settings')\n") + error = failure(tmp_path, digest) + assert error.kind == "protocol_exception" + assert str(error) == "protocol_exception: TypeError: bad settings" + + +@pytest.mark.parametrize("value, message", [ + ("float('nan')", "Out of range float values are not JSON compliant"), + ("object()", "is not JSON serializable"), +]) +def test_unserializable_results_are_rejected_in_the_child(tmp_path, value, message): + digest = write_protocol(tmp_path, f"def validate_arguments(a):\n return {{'x': {value}}}\n") + error = failure(tmp_path, digest) + assert error.kind == "result" and message in str(error) + + +@pytest.mark.parametrize("result", ["[1]", "'text'", "None"]) +def test_result_of_the_wrong_type_is_an_invalid_response(tmp_path, result): + digest = write_protocol(tmp_path, f"def validate_arguments(a):\n return {result}\n") + error = failure(tmp_path, digest) + assert error.kind == "invalid_response" + assert "validate_arguments result must be a dict" in str(error) + + +def test_arguments_result_must_be_strings(tmp_path): + digest = write_protocol(tmp_path, "def arguments(c, o, s):\n return ['ok', 3]\n") + error = failure(tmp_path, digest, "arguments", candidate=tmp_path / "c.py", + output=tmp_path / "o", settings={}) + assert error.kind == "invalid_response" + + +def test_protocol_output_goes_to_stderr_and_never_reaches_the_envelope(tmp_path): + digest = write_protocol(tmp_path, ( + "import os, subprocess, sys\n" + "def validate_arguments(a):\n" + " print('python print')\n" + " os.write(1, b'raw descriptor write\\n')\n" + " subprocess.run([sys.executable, '-c', 'print(\"grandchild\")'], check=True)\n" + " return {'kept': True}\n")) + assert run(tmp_path, digest, "validate_arguments", arguments=[]) == {"kept": True} + + +@pytest.mark.parametrize("payload, kind", [ + (b"", "malformed_response"), + (b"{not json", "malformed_response"), + (b'{"a":1,"a":2}', "malformed_response"), + (b'{"x":NaN}', "malformed_response"), + (b'{"x":1e999}', "malformed_response"), + (b'{"schema_version":2,"operation":"validate_arguments","result":{}}', + "invalid_response"), + (b'{"schema_version":1,"operation":"assess","result":{}}', "invalid_response"), + (b'{"schema_version":1,"operation":"validate_arguments","result":{},"error":{}}', + "invalid_response"), + (b'{"schema_version":1,"operation":"validate_arguments",' + b'"error":{"kind":"unknown","type":"X","message":"m"}}', "invalid_response"), +]) +def test_malformed_or_foreign_envelopes_are_distinct_failures(tmp_path, monkeypatch, payload, + kind): + # Fault injection: a real child process writes exactly these envelope bytes. + child = tmp_path / "child.py" + child.write_text(f"import os\nos.write(1, {payload!r})\n", encoding="utf-8") + monkeypatch.setattr(task_protocol, "CHILD_ENTRY", child) + digest = write_protocol(tmp_path / "task", "def validate_arguments(a):\n return {}\n") + assert failure(tmp_path / "task", digest).kind == kind + + +def test_child_exit_without_an_envelope_is_an_exit_status_failure(tmp_path): + digest = write_protocol(tmp_path, ( + "import os, sys\n" + "def validate_arguments(a):\n" + " sys.stderr.write('protocol broke')\n" + " sys.stderr.flush()\n" + " os._exit(3)\n")) + error = failure(tmp_path, digest) + assert error.kind == "exit_status" + assert "exited with 3" in str(error) and "protocol broke" in str(error) + + +def test_launch_failure_is_a_transport_failure(tmp_path, monkeypatch): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + monkeypatch.setattr(task_protocol.sys, "executable", str(tmp_path / "missing-python")) + assert failure(tmp_path, digest).kind == "launch" + + +def test_oversized_envelope_is_rejected(tmp_path, monkeypatch): + monkeypatch.setattr(task_protocol, "STDOUT_LIMIT_BYTES", 64) + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {'x': 'y' * 100}\n") + assert failure(tmp_path, digest).kind == "output_limit" + + +def _late_writer(marker): + # repr() quotes the whole child program, so a Windows path inside it stays valid source. + code = f"import pathlib, time; time.sleep(4); pathlib.Path({str(marker)!r}).write_text('x')" + return f" subprocess.Popen([sys.executable, '-c', {code!r}])\n" + + +def test_deadline_stops_the_child_and_its_descendants(tmp_path, monkeypatch): + marker = tmp_path / "late.txt" + monkeypatch.setattr(task_protocol, "PROTOCOL_TIMEOUT_SECONDS", 3.0) + digest = write_protocol(tmp_path / "task", ( + "import subprocess, sys, time\n" + "def validate_arguments(a):\n" + _late_writer(marker) + " time.sleep(60)\n")) + started = time.monotonic() + assert failure(tmp_path / "task", digest).kind == "timeout" + assert time.monotonic() - started < 30 + time.sleep(5) + assert not marker.exists() + + +def test_descendants_left_running_are_stopped_with_the_child(tmp_path): + marker = tmp_path / "late.txt" + digest = write_protocol(tmp_path / "task", ( + "import subprocess, sys\n" + "def validate_arguments(a):\n" + _late_writer(marker) + " return {'done': True}\n")) + assert run(tmp_path / "task", digest, "validate_arguments", arguments=[]) == {"done": True} + time.sleep(5) + assert not marker.exists() + + +def test_changed_protocol_bytes_never_run(tmp_path): + marker = tmp_path / "ran.txt" + digest = write_protocol(tmp_path / "task", "def validate_arguments(a):\n return {}\n") + (tmp_path / "task/optimization.py").write_text( + f"import pathlib\npathlib.Path({str(marker)!r}).write_text('x')\n" + "def validate_arguments(a):\n return {}\n", encoding="utf-8") + assert failure(tmp_path / "task", digest).kind == "identity" + assert not marker.exists() + + +def test_child_checks_the_digest_of_the_bytes_it_compiles(tmp_path): + # The child re-reads and checks the protocol itself, so bytes swapped after the + # parent's check still never run. + marker = tmp_path / "ran.txt" + good = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + (tmp_path / "optimization.py").write_text( + f"import pathlib\npathlib.Path({str(marker)!r}).write_text('x')\n", encoding="utf-8") + request = tmp_path / "request.json" + request.write_bytes(protocol_child.dumps_strict({ + "schema_version": 1, "operation": "validate_arguments", + "protocol": str(tmp_path / "optimization.py"), "protocol_sha256": good, + "arguments": {"arguments": []}})) + completed = subprocess.run( + [sys.executable, "-I", "-B", str(task_protocol.CHILD_ENTRY), str(request)], + capture_output=True, check=True, timeout=60) + assert json.loads(completed.stdout)["error"]["kind"] == "identity" + assert not marker.exists() + + +@pytest.mark.parametrize("arguments, message", [ + ({"arguments": "not a list"}, "list of strings"), + ({"arguments": [], "extra": 1}, "takes exactly"), +]) +def test_invalid_requests_fail_before_any_child_starts(tmp_path, arguments, message): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + error = failure(tmp_path, digest, **arguments) + assert error.kind == "request" and message in str(error) + + +def test_relative_paths_are_rejected(tmp_path): + digest = write_protocol(tmp_path, "def arguments(c, o, s):\n return []\n") + error = failure(tmp_path, digest, "arguments", candidate=Path("candidate.py"), + output=tmp_path, settings={}) + assert error.kind == "request" and "absolute" in str(error) + + +def test_child_is_isolated_loads_no_inferrsi_and_writes_no_bytecode(tmp_path, monkeypatch): + # -I drops PYTHONPATH and the working directory from sys.path; -B writes no cache. + shadow = tmp_path / "shadow" + shadow.mkdir() + (shadow / "json.py").write_text("raise RuntimeError('shadowed')\n", encoding="utf-8") + monkeypatch.setenv("PYTHONPATH", str(shadow)) + monkeypatch.setenv("R3_PROBE", "kept") + digest = write_protocol(tmp_path / "task", ( + "import json, os, sys\n" + "def validate_arguments(a):\n" + " return {'isolated': sys.flags.isolated, 'no_bytecode': sys.dont_write_bytecode,\n" + " 'inferrsi': [m for m in sys.modules if m.startswith('inferrsi')],\n" + " 'probe': os.environ.get('R3_PROBE')}\n")) + assert run(tmp_path / "task", digest, "validate_arguments", arguments=[]) == { + "isolated": 1, "no_bytecode": True, "inferrsi": [], "probe": "kept"} + assert not list((tmp_path / "task").rglob("__pycache__")) + + +def test_harness_never_executes_protocol_code_in_process(tmp_path): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + run(tmp_path, digest, "validate_arguments", arguments=[]) + assert "task_optimization_protocol" not in sys.modules + source = Path(task_protocol.__file__).read_text(encoding="utf-8") + assert "module_from_spec" not in source and "exec(" not in source + + +def test_child_entry_imports_only_the_standard_library(): + tree = ast.parse(task_protocol.CHILD_ENTRY.read_text(encoding="utf-8")) + imported = {alias.name.split(".")[0] for node in ast.walk(tree) + if isinstance(node, ast.Import) for alias in node.names} + imported |= {(node.module or "").split(".")[0] for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom)} + assert not [node for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level] + assert imported <= set(sys.stdlib_module_names), imported + + +def test_prepare_freezes_exactly_the_validated_protocol(tmp_path): + source, digest = qwen() + settings = task_protocol.prepare(source, tmp_path / "frozen", QWEN_ARGUMENTS) + assert settings["evaluation_settings"] == SETTINGS + assert settings["frozen_files"]["optimization.py"] == digest diff --git a/tests/test_tts_benchmark.py b/tests/test_tts_benchmark.py index d298078c1..fbdc8a4f8 100644 --- a/tests/test_tts_benchmark.py +++ b/tests/test_tts_benchmark.py @@ -13,8 +13,8 @@ from tests.engineering_task_support import load_task_module -tts = load_task_module("qwen3-tts-npu", "evaluation") -main = load_task_module("qwen3-tts-npu", "benchmark").main +tts = load_task_module("qwen3-tts-npu", "benchmark") +main = tts.main @contextlib.contextmanager diff --git a/tests/test_tts_standalone.py b/tests/test_tts_standalone.py index 61bfeea48..114fed0d1 100644 --- a/tests/test_tts_standalone.py +++ b/tests/test_tts_standalone.py @@ -18,7 +18,7 @@ from tests.engineering_task_support import load_task_module -tts = load_task_module("qwen3-tts-npu", "evaluation") +tts = load_task_module("qwen3-tts-npu", "benchmark") def module(name): @@ -264,7 +264,7 @@ def test_invalid_request_never_reaches_model(payload): def test_serve_cli_execs_verified_bundle(tmp_path, monkeypatch): - main = load_task_module("qwen3-tts-npu", "benchmark").main + main = tts.main prepared = tmp_path / "prepared" tts.prepare( From 70f6b56b7d64df99586987f80ad70c4832a37c0d Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 04:31:56 +0800 Subject: [PATCH 2/7] fix(harness,tests): forward protocol diagnostics; update a missed real-tree gate pin (R3) - full-check failed on both Python versions at one test: tests/test_campaign_record_injection.py still asserted that the real tree is not isolated. R3 makes it isolated, so the test now requires no violation, no blocker and isolation_complete. Its intent, that walking the extra records entries adds no violation, is unchanged. - Review (zengchuang-hw, 5ee6912): the child's stderr reached the caller only for a SystemExit. task_protocol.run now copies the last 4 KiB of the child's stderr, protocol prints included, to the caller's stderr after every operation, then deletes the scratch directory. The constants and docs state that the size limits apply when Harness reads the child's files and that the deadline bounds the run. Tests assert the forwarded text on success, on a protocol exception and for the tail bound. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 4 ++++ docs/module-layout.md | 6 +++-- src/inferrsi/harness/task_protocol.py | 21 ++++++++++------- tests/test_campaign_record_injection.py | 4 +++- tests/test_task_protocol_child.py | 30 ++++++++++++++++++++++++- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c293fdf0b..c892adb09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ its own process group or Windows Job, and returns one strict JSON envelope. The child compiles only protocol bytes that match the frozen SHA-256, and a protocol `SystemExit` keeps its exit code (argparse usage errors still exit 2). + After every operation the last 4 KiB of the child's stderr, protocol prints + included, is copied to the caller's stderr; the scratch directory is then + deleted. The 16 MiB envelope limit and the stderr tail apply when Harness reads + the files, and the deadline bounds the run. `task_protocol.protocol()` is removed; callers use `task_protocol.run()`. A task `optimization.py` must take and return JSON values. - `moe-forward-npu`: `processes.py` is folded into `benchmark.py` diff --git a/docs/module-layout.md b/docs/module-layout.md index 6c2568437..9ebed1194 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -188,8 +188,10 @@ registered consumers; this gate does not scan them. as `python -I -B` in its own process group or Windows Job under a deadline, and answers with one strict JSON envelope on a private copy of stdout. The child imports only the standard library and compiles the exact protocol bytes it has - checked against the frozen SHA-256. This is an interpreter and process - boundary, not a filesystem or network sandbox. + checked against the frozen SHA-256. After every operation the last 4 KiB of the + child's stderr is copied to the caller's stderr; size limits apply when Harness + reads the child's output files, and the deadline bounds the run. This is an + interpreter and process boundary, not a filesystem or network sandbox. - `moe-forward-npu` folded its sibling `processes.py` into `benchmark.py`, and `qwen3-tts-npu` folded its sibling `evaluation.py` into `benchmark.py`. The bundles no longer load a sibling module by path, and a bundle prepared before diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 14d948e88..7128e2026 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -20,9 +20,11 @@ # One protocol operation must finish within this time; the child and every process in # its group or Job are then stopped and the operation fails with kind "timeout". PROTOCOL_TIMEOUT_SECONDS = 120.0 -# Harness reads at most this many bytes of the envelope on stdout ... +# The child writes stdout and stderr to files in a private scratch directory. These two +# limits apply when Harness reads those files, not while the child writes them; the +# deadline above bounds the run. Harness reads at most this many bytes of envelope ... STDOUT_LIMIT_BYTES = 16 * 1024 * 1024 -# ... and quotes at most this tail of stderr in a failure message. +# ... and copies at most this tail of stderr to its own stderr after every operation. STDERR_TAIL_BYTES = 4096 # The child entry that every operation runs as a separate script. CHILD_ENTRY = Path(protocol_child.__file__).resolve() @@ -78,10 +80,10 @@ def _response_problem(response, operation: str) -> str | None: def run(root: Path, protocol_sha256: str, operation: str, **arguments): """Run one operation of ``root/optimization.py`` whose bytes have ``protocol_sha256``. - Path arguments must be absolute. A protocol ``SystemExit`` is raised again with its - exit code after the child's stderr tail is copied to this process's stderr; every - other failure raises ``TaskProtocolError``. Descendants that the protocol leaves - running are stopped with the child. + Path arguments must be absolute. The child's stderr tail is copied to this process's + stderr after every operation. A protocol ``SystemExit`` is raised again with its + exit code; every other failure raises ``TaskProtocolError``. Descendants that the + protocol leaves running are stopped with the child. """ path = Path(root).resolve() / "optimization.py" if sha256(path) != protocol_sha256: @@ -105,6 +107,11 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): argv, work, dict(os.environ), work / "stdout", work / "stderr", time.monotonic() + PROTOCOL_TIMEOUT_SECONDS) stderr = _tail(work / "stderr") if (work / "stderr").is_file() else "" + # Diagnostics contract: after every operation, whatever its result, the last + # STDERR_TAIL_BYTES of the child's stderr (protocol prints included) are copied to + # this process's stderr. The scratch directory with the full logs is then deleted. + sys.stderr.write(stderr) + sys.stderr.flush() if record["status"] == "cancelled" and record["cancellation_source"] == ( "keyboard_interrupt"): raise KeyboardInterrupt @@ -132,8 +139,6 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): return response["result"] error = response["error"] if error["kind"] == "system_exit": - # argparse explains a usage error on stderr; keep that message visible. - sys.stderr.write(stderr) raise SystemExit(error["exit_code"]) raise TaskProtocolError(error["kind"], f"{error['type']}: {error['message']}") diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index eca883bd9..1c94519ba 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -407,7 +407,9 @@ def test_harness_records_closure_holds_as_declared_entries(): # Declared entries are walked on the full graph, without the residual exemption. report = first_wave_report(h_entries=NEW_H_ENTRIES + records) assert report.violations == () - assert not report.isolation_complete + # #412 R3 closed the last blocker, so walking the extra entries adds none. + assert report.blockers == () + assert report.isolation_complete def test_observability_package_is_dissolved(): diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index 4b1c6e773..2a003ef3b 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -96,15 +96,43 @@ def test_arguments_result_must_be_strings(tmp_path): assert error.kind == "invalid_response" -def test_protocol_output_goes_to_stderr_and_never_reaches_the_envelope(tmp_path): +def test_protocol_output_goes_to_stderr_and_never_reaches_the_envelope(tmp_path, capsys): digest = write_protocol(tmp_path, ( "import os, subprocess, sys\n" "def validate_arguments(a):\n" " print('python print')\n" + " sys.stdout.flush()\n" " os.write(1, b'raw descriptor write\\n')\n" " subprocess.run([sys.executable, '-c', 'print(\"grandchild\")'], check=True)\n" + " print('diagnostic line', file=sys.stderr)\n" " return {'kept': True}\n")) assert run(tmp_path, digest, "validate_arguments", arguments=[]) == {"kept": True} + # The diagnostics contract: every byte the protocol wrote reaches the caller's stderr. + forwarded = capsys.readouterr().err + for line in ("python print", "raw descriptor write", "grandchild", "diagnostic line"): + assert line in forwarded + + +def test_diagnostics_are_forwarded_when_the_protocol_raises(tmp_path, capsys): + digest = write_protocol(tmp_path, ( + "import sys\n" + "def validate_arguments(a):\n" + " print('context before failure', file=sys.stderr)\n" + " raise ValueError('rejected')\n")) + error = failure(tmp_path, digest) + assert str(error) == "protocol_exception: ValueError: rejected" + assert "context before failure" in capsys.readouterr().err + + +def test_only_the_stderr_tail_is_forwarded(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(task_protocol, "STDERR_TAIL_BYTES", 16) + digest = write_protocol(tmp_path, ( + "import sys\n" + "def validate_arguments(a):\n" + " sys.stderr.write('x' * 100 + 'END-OF-LOG')\n" + " return {}\n")) + run(tmp_path, digest, "validate_arguments", arguments=[]) + assert capsys.readouterr().err == "xxxxxxEND-OF-LOG" @pytest.mark.parametrize("payload, kind", [ From 36384866a10738ff8506a4f8af42653a335aad2d Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 05:24:38 +0800 Subject: [PATCH 3/7] test(harness): assert stderr forwarding and scratch cleanup on every protocol path (R3) Review follow-up (zengchuang-hw, 70f6b56): the diagnostics contract says the child's stderr tail reaches the caller after every operation, but the tests covered only success, a protocol exception and the tail bound. Add the forwarded-stderr assertion to the timeout, nonzero-exit and malformed-envelope paths, and assert that the scratch directory is gone after a successful and a failed operation. Reword the success test's comment so it does not promise unlimited retention. Test-only. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- tests/test_task_protocol_child.py | 37 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index 2a003ef3b..f24a4676f 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -107,7 +107,7 @@ def test_protocol_output_goes_to_stderr_and_never_reaches_the_envelope(tmp_path, " print('diagnostic line', file=sys.stderr)\n" " return {'kept': True}\n")) assert run(tmp_path, digest, "validate_arguments", arguments=[]) == {"kept": True} - # The diagnostics contract: every byte the protocol wrote reaches the caller's stderr. + # Each of these small writes reaches the caller's stderr within the forwarded tail. forwarded = capsys.readouterr().err for line in ("python print", "raw descriptor write", "grandchild", "diagnostic line"): assert line in forwarded @@ -149,17 +149,19 @@ def test_only_the_stderr_tail_is_forwarded(tmp_path, capsys, monkeypatch): (b'{"schema_version":1,"operation":"validate_arguments",' b'"error":{"kind":"unknown","type":"X","message":"m"}}', "invalid_response"), ]) -def test_malformed_or_foreign_envelopes_are_distinct_failures(tmp_path, monkeypatch, payload, - kind): +def test_malformed_or_foreign_envelopes_are_distinct_failures(tmp_path, monkeypatch, capsys, + payload, kind): # Fault injection: a real child process writes exactly these envelope bytes. child = tmp_path / "child.py" - child.write_text(f"import os\nos.write(1, {payload!r})\n", encoding="utf-8") + child.write_text("import os, sys\nsys.stderr.write('child diagnostics')\n" + f"os.write(1, {payload!r})\n", encoding="utf-8") monkeypatch.setattr(task_protocol, "CHILD_ENTRY", child) digest = write_protocol(tmp_path / "task", "def validate_arguments(a):\n return {}\n") assert failure(tmp_path / "task", digest).kind == kind + assert "child diagnostics" in capsys.readouterr().err -def test_child_exit_without_an_envelope_is_an_exit_status_failure(tmp_path): +def test_child_exit_without_an_envelope_is_an_exit_status_failure(tmp_path, capsys): digest = write_protocol(tmp_path, ( "import os, sys\n" "def validate_arguments(a):\n" @@ -169,6 +171,7 @@ def test_child_exit_without_an_envelope_is_an_exit_status_failure(tmp_path): error = failure(tmp_path, digest) assert error.kind == "exit_status" assert "exited with 3" in str(error) and "protocol broke" in str(error) + assert "protocol broke" in capsys.readouterr().err def test_launch_failure_is_a_transport_failure(tmp_path, monkeypatch): @@ -189,19 +192,39 @@ def _late_writer(marker): return f" subprocess.Popen([sys.executable, '-c', {code!r}])\n" -def test_deadline_stops_the_child_and_its_descendants(tmp_path, monkeypatch): +def test_deadline_stops_the_child_and_its_descendants(tmp_path, monkeypatch, capsys): marker = tmp_path / "late.txt" monkeypatch.setattr(task_protocol, "PROTOCOL_TIMEOUT_SECONDS", 3.0) digest = write_protocol(tmp_path / "task", ( "import subprocess, sys, time\n" - "def validate_arguments(a):\n" + _late_writer(marker) + " time.sleep(60)\n")) + "def validate_arguments(a):\n" + _late_writer(marker) + + " print('waiting past the deadline', file=sys.stderr, flush=True)\n" + " time.sleep(60)\n")) started = time.monotonic() assert failure(tmp_path / "task", digest).kind == "timeout" assert time.monotonic() - started < 30 + assert "waiting past the deadline" in capsys.readouterr().err time.sleep(5) assert not marker.exists() +@pytest.mark.parametrize("body, raises", [ + (" return {}\n", False), + (" raise ValueError('rejected')\n", True), +]) +def test_scratch_directory_is_removed_after_every_operation(tmp_path, monkeypatch, body, + raises): + scratch = tmp_path / "scratch" + scratch.mkdir() + monkeypatch.setattr(task_protocol.tempfile, "tempdir", str(scratch)) + digest = write_protocol(tmp_path / "task", "def validate_arguments(a):\n" + body) + if raises: + assert failure(tmp_path / "task", digest).kind == "protocol_exception" + else: + assert run(tmp_path / "task", digest, "validate_arguments", arguments=[]) == {} + assert list(scratch.iterdir()) == [] + + def test_descendants_left_running_are_stopped_with_the_child(tmp_path): marker = tmp_path / "late.txt" digest = write_protocol(tmp_path / "task", ( From 096f93c74d63eb3870d58457a92b0403014050dd Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 09:21:33 +0800 Subject: [PATCH 4/7] fix(harness): keep a protocol exception's ": " across the child (R3) Review (cursor[bot], 3638486): when a frozen optimization.py raised, the settled attempt recorded "TaskProtocolError: protocol_exception: TypeError: invalid task assessment" instead of "TypeError: invalid task assessment", the reason the in-process call gave before R3 and the one the R3 design promised. The existing test monkeypatched the assessor in process, so it never crossed the child transport. - TaskProtocolError for kind protocol_exception now keeps the protocol's own message as str() and its ": " as `reason`. Every other kind keeps naming the transport failure, so transport, identity and result failures stay distinguishable. - optimization._failure records `error.reason` for a TaskProtocolError and is unchanged for every other error. - A ValueError from validate_arguments at `session start` again prints its own message through the CLI parser error, as before R3. - New regressions drive the real child through `session start` and `round`: an assessor TypeError settles a blocked attempt with "TypeError: invalid task assessment" in the view and the retained observation.json; a validate_arguments ValueError exits 2 with its own message and creates no session. Protocol-child tests pin str(), reason and a transport failure's reason. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 3 ++ src/inferrsi/harness/optimization.py | 7 +++- src/inferrsi/harness/task_protocol.py | 16 ++++++++-- tests/test_optimization_session.py | 46 +++++++++++++++++++++++++++ tests/test_task_protocol_child.py | 15 +++++++-- 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c892adb09..bbf737f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ its own process group or Windows Job, and returns one strict JSON envelope. The child compiles only protocol bytes that match the frozen SHA-256, and a protocol `SystemExit` keeps its exit code (argparse usage errors still exit 2). + An exception the protocol raises keeps its own message and its + `": "` settlement reason, as before; transport, identity and + result failures are reported as `TaskProtocolError` with their kind. After every operation the last 4 KiB of the child's stderr, protocol prints included, is copied to the caller's stderr; the scratch directory is then deleted. The 16 MiB envelope limit and the stderr tail apply when Harness reads diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 09d738dfc..2d3e86b6f 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -10,6 +10,7 @@ from . import candidates from .artifact_primitives import digest, file_lock, sha256 from .atomic_json import atomic_write_json +from .task_protocol import TaskProtocolError from .usage import normalize_usage, usage_totals @@ -210,8 +211,12 @@ def _summary(root, plan, state): def _failure(error): + # A task protocol that raised keeps its own ": " across the child + # transport (#412 R3); any other error is named by its own type. + reason = (error.reason if isinstance(error, TaskProtocolError) + else f"{type(error).__name__}: {error}") return {"status": "infrastructure_failure", "score": None, - "verdict": "INCONCLUSIVE", "reason": f"{type(error).__name__}: {error}"} + "verdict": "INCONCLUSIVE", "reason": reason} def _settle(root, plan, state, pending, execution, assess): diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 7128e2026..139abfdef 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -32,11 +32,19 @@ class TaskProtocolError(ValueError): - """A protocol operation gave no valid result. ``kind`` names the failure class.""" + """A protocol operation gave no valid result. ``kind`` names the failure class. - def __init__(self, kind: str, message: str): - super().__init__(f"{kind}: {message}") + For ``protocol_exception`` the protocol itself raised: ``str()`` is its own message + and ``reason`` its ``": "``, the text the in-process call gave before + #412 R3. For every other kind both name the transport failure, so the two stay + distinguishable. + """ + + def __init__(self, kind: str, message: str, *, error_type: str | None = None): + super().__init__(message if kind == "protocol_exception" else f"{kind}: {message}") self.kind = kind + self.reason = (f"{error_type}: {message}" if kind == "protocol_exception" + else f"{type(self).__name__}: {self}") def _tail(path: Path) -> str: @@ -140,6 +148,8 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): error = response["error"] if error["kind"] == "system_exit": raise SystemExit(error["exit_code"]) + if error["kind"] == "protocol_exception": + raise TaskProtocolError("protocol_exception", error["message"], error_type=error["type"]) raise TaskProtocolError(error["kind"], f"{error['type']}: {error['message']}") diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index 2768e6756..03d5b57f9 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -305,6 +305,52 @@ def broken_assessor(*args): assert (session / "calls.log").read_text().splitlines() == ["2"] +def start_args(task, root): + return ["benchmarks", "session", "start", task.name, "--session", str(root), + "--rounds", "3", "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "synthetic-current-agent", "--evaluation-args", ""] + + +def test_child_assessor_exception_keeps_its_reason_through_a_real_round(task, tmp_path, capsys): + # #412 R3: the frozen protocol's assess runs in the protocol child. Its exception must + # settle the attempt with the same ": " as the in-process call did. + path = task / "optimization.py" + path.write_text(path.read_text(encoding="utf-8").replace( + "def assess(directory, settings, execution):\n", + "def assess(directory, settings, execution):\n" + " raise TypeError('invalid task assessment')\n"), encoding="utf-8") + root = tmp_path / "session" + assert main(start_args(task, root)) == 0 + capsys.readouterr() + candidate = root / "work/candidate.py" + candidate.write_text("FACTOR = 2\n", encoding="utf-8") + assert main(["benchmarks", "session", "round", "--session", str(root), + "--candidate", str(candidate), "--hypothesis", "Child assessor error"]) == 2 + view = json.loads(capsys.readouterr().out) + assert view["state"] == "blocked" and view["pending"] is None + assessment = view["latest_observation"]["assessment"] + assert assessment["reason"] == "TypeError: invalid task assessment" + assert assessment["verdict"] == "INCONCLUSIVE" and assessment["score"] is None + retained = json.loads((root / view["latest_observation"]["path"] / "observation.json") + .read_text()) + assert retained["assessment"] == assessment + assert (root / "calls.log").read_text().splitlines() == ["2"] + + +def test_child_argument_error_keeps_its_cli_message_at_start(task, tmp_path, capsys): + path = task / "optimization.py" + path.write_text(path.read_text(encoding="utf-8").replace( + "def validate_arguments(args):\n", + "def validate_arguments(args):\n raise ValueError('pairs must be positive')\n"), + encoding="utf-8") + with pytest.raises(SystemExit) as stop: + main(start_args(task, tmp_path / "session")) + assert stop.value.code == 2 + error = capsys.readouterr().err + assert "error: pairs must be positive" in error and "protocol_exception" not in error + assert not (tmp_path / "session").exists() + + def test_task_runner_does_not_expose_harness_as_top_level_modules(task, tmp_path): from inferrsi.apps.benchmark_operations import run_task diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index f24a4676f..c3837f87e 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -68,7 +68,18 @@ def test_protocol_exception_keeps_its_type_and_message(tmp_path): " raise TypeError('bad settings')\n") error = failure(tmp_path, digest) assert error.kind == "protocol_exception" - assert str(error) == "protocol_exception: TypeError: bad settings" + # The protocol's own message and ": " survive the child transport. + assert str(error) == "bad settings" + assert error.reason == "TypeError: bad settings" + + +def test_transport_failures_stay_distinguishable_from_protocol_exceptions(tmp_path): + digest = write_protocol(tmp_path, + "def validate_arguments(a):\n return {'x': float('nan')}\n") + error = failure(tmp_path, digest) + assert error.kind == "result" + assert error.reason == f"TaskProtocolError: {error}" + assert str(error).startswith("result: ValueError: ") @pytest.mark.parametrize("value, message", [ @@ -120,7 +131,7 @@ def test_diagnostics_are_forwarded_when_the_protocol_raises(tmp_path, capsys): " print('context before failure', file=sys.stderr)\n" " raise ValueError('rejected')\n")) error = failure(tmp_path, digest) - assert str(error) == "protocol_exception: ValueError: rejected" + assert str(error) == "rejected" and error.reason == "ValueError: rejected" assert "context before failure" in capsys.readouterr().err From bcbfbde6a593ba7d3f1f06d9b30b7f2ebfb096a8 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 10:59:36 +0800 Subject: [PATCH 5/7] fix(harness): keep a protocol's own exception type in the coordinator record (R3) Review (zengchuang-hw, 096f93c): when a frozen optimization.py raised in its `arguments` operation, coordinator-error.json recorded type "TaskProtocolError" instead of the protocol's own type, which the in-process call recorded before R3. The settlement reason on that path was and stays the existing coordinator behaviour, "ValueError: ". - TaskProtocolError gains `type_name`: the protocol's own exception type for kind protocol_exception, "TaskProtocolError" for every other kind. It replaces the `reason` attribute added in 096f93c. - optimization._type_name() feeds both _failure and coordinator-error.json, so all three operations leave the records they left before R3. - New regression drives `session start` and a real `round` with an `arguments` TypeError: coordinator-error.json is {"type": "TypeError", "error": "bad arguments"}, the attempt settles as "ValueError: bad arguments" and the benchmark never starts. - CHANGELOG states the record per operation instead of one broad claim. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 9 ++++++--- src/inferrsi/harness/optimization.py | 14 ++++++++------ src/inferrsi/harness/task_protocol.py | 12 ++++++------ tests/test_optimization_session.py | 28 +++++++++++++++++++++++++++ tests/test_task_protocol_child.py | 6 +++--- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbf737f1c..7eae6e629 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,12 @@ its own process group or Windows Job, and returns one strict JSON envelope. The child compiles only protocol bytes that match the frozen SHA-256, and a protocol `SystemExit` keeps its exit code (argparse usage errors still exit 2). - An exception the protocol raises keeps its own message and its - `": "` settlement reason, as before; transport, identity and - result failures are reported as `TaskProtocolError` with their kind. + An exception the protocol raises is recorded as before R3, under its own type and + message: an `assess` failure settles as `": "`, an `arguments` + failure keeps its type and message in `coordinator-error.json` and settles through + the existing coordinator path, and a `validate_arguments` failure prints its own + message at `session start`. Transport, identity and result failures are recorded + as `TaskProtocolError` with their kind. After every operation the last 4 KiB of the child's stderr, protocol prints included, is copied to the caller's stderr; the scratch directory is then deleted. The 16 MiB envelope limit and the stderr tail apply when Harness reads diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 2d3e86b6f..02c703b30 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -210,13 +210,15 @@ def _summary(root, plan, state): } +def _type_name(error): + # A task protocol that raised is recorded under its own exception type across the + # child transport (#412 R3); any other error under its own Python type. + return error.type_name if isinstance(error, TaskProtocolError) else type(error).__name__ + + def _failure(error): - # A task protocol that raised keeps its own ": " across the child - # transport (#412 R3); any other error is named by its own type. - reason = (error.reason if isinstance(error, TaskProtocolError) - else f"{type(error).__name__}: {error}") return {"status": "infrastructure_failure", "score": None, - "verdict": "INCONCLUSIVE", "reason": reason} + "verdict": "INCONCLUSIVE", "reason": f"{_type_name(error)}: {error}"} def _settle(root, plan, state, pending, execution, assess): @@ -348,7 +350,7 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, # failed. Collect it without launching the same candidate again. path = directory / "execution/execution.json" atomic_write_json(directory / "coordinator-error.json", { - "type": type(error).__name__, "error": str(error)}) + "type": _type_name(error), "error": str(error)}) if path.is_file(): execution = json.loads(path.read_text(encoding="utf-8")) else: diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 139abfdef..340f12181 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -34,17 +34,17 @@ class TaskProtocolError(ValueError): """A protocol operation gave no valid result. ``kind`` names the failure class. - For ``protocol_exception`` the protocol itself raised: ``str()`` is its own message - and ``reason`` its ``": "``, the text the in-process call gave before - #412 R3. For every other kind both name the transport failure, so the two stay - distinguishable. + ``type_name`` is the type a caller records for this failure. For + ``protocol_exception`` the protocol itself raised: ``str()`` is its own message and + ``type_name`` its own exception type, the record the in-process call left before + #412 R3. For every other kind ``type_name`` is ``TaskProtocolError`` and ``str()`` + starts with the kind, so transport, identity and result failures stay distinct. """ def __init__(self, kind: str, message: str, *, error_type: str | None = None): super().__init__(message if kind == "protocol_exception" else f"{kind}: {message}") self.kind = kind - self.reason = (f"{error_type}: {message}" if kind == "protocol_exception" - else f"{type(self).__name__}: {self}") + self.type_name = error_type if kind == "protocol_exception" else type(self).__name__ def _tail(path: Path) -> str: diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index 03d5b57f9..e01c148eb 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -337,6 +337,34 @@ def test_child_assessor_exception_keeps_its_reason_through_a_real_round(task, tm assert (root / "calls.log").read_text().splitlines() == ["2"] +def test_child_arguments_exception_keeps_the_coordinator_record_through_a_real_round( + task, tmp_path, capsys, +): + # #412 R3: a frozen protocol's `arguments` failure leaves the same coordinator record + # as the in-process call did: the protocol's own type and message, and the attempt + # settles through the existing coordinator path. The benchmark never starts. + path = task / "optimization.py" + path.write_text(path.read_text(encoding="utf-8").replace( + "def arguments(candidate, output, settings):\n", + "def arguments(candidate, output, settings):\n" + " raise TypeError('bad arguments')\n"), encoding="utf-8") + root = tmp_path / "session" + assert main(start_args(task, root)) == 0 + capsys.readouterr() + candidate = root / "work/candidate.py" + candidate.write_text("FACTOR = 2\n", encoding="utf-8") + assert main(["benchmarks", "session", "round", "--session", str(root), + "--candidate", str(candidate), "--hypothesis", "Child arguments error"]) == 2 + view = json.loads(capsys.readouterr().out) + directory = root / view["latest_observation"]["path"] + assert json.loads((directory / "coordinator-error.json").read_text()) == { + "type": "TypeError", "error": "bad arguments"} + assessment = view["latest_observation"]["assessment"] + assert assessment["reason"] == "ValueError: bad arguments" + assert assessment["verdict"] == "INCONCLUSIVE" and assessment["score"] is None + assert not (root / "calls.log").exists() + + def test_child_argument_error_keeps_its_cli_message_at_start(task, tmp_path, capsys): path = task / "optimization.py" path.write_text(path.read_text(encoding="utf-8").replace( diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index c3837f87e..0446015c8 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -70,7 +70,7 @@ def test_protocol_exception_keeps_its_type_and_message(tmp_path): assert error.kind == "protocol_exception" # The protocol's own message and ": " survive the child transport. assert str(error) == "bad settings" - assert error.reason == "TypeError: bad settings" + assert error.type_name == "TypeError" def test_transport_failures_stay_distinguishable_from_protocol_exceptions(tmp_path): @@ -78,7 +78,7 @@ def test_transport_failures_stay_distinguishable_from_protocol_exceptions(tmp_pa "def validate_arguments(a):\n return {'x': float('nan')}\n") error = failure(tmp_path, digest) assert error.kind == "result" - assert error.reason == f"TaskProtocolError: {error}" + assert error.type_name == "TaskProtocolError" assert str(error).startswith("result: ValueError: ") @@ -131,7 +131,7 @@ def test_diagnostics_are_forwarded_when_the_protocol_raises(tmp_path, capsys): " print('context before failure', file=sys.stderr)\n" " raise ValueError('rejected')\n")) error = failure(tmp_path, digest) - assert str(error) == "rejected" and error.reason == "ValueError: rejected" + assert str(error) == "rejected" and error.type_name == "ValueError" assert "context before failure" in capsys.readouterr().err From 2d0fa299473c31a88bae94f9c3b9b8c59b99c559 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 11:42:22 +0800 Subject: [PATCH 6/7] refactor(tests): admit by-path loading by bundle location, not by named task (R3) Owner decision (#412): the gate must not name specific tasks. The two named EVALUATOR_LOADERS entries (MoE grader, Qwen21 evaluator) tied it to those tasks, so every new task that times submitted code would need a gate edit. - A task bundle program is a file directly inside benchmark/tasks// where that directory has no __init__.py. No package code imports it; Harness and Apps run it as its own process. - In a bundle program, importlib.util.module_from_spec calls are admitted. Name-based lookups still fail there (__import__, import_module, module_from_spec imported by name, a find_spec call), and so do static imports of other cores. The same call fails everywhere else. - EVALUATOR_LOADERS and its per-function shape check are removed. first_wave_report().runtime_execution is computed: every bundle program that loads by path. - Gate tests cover both sides: any bundle and file name is admitted, name-based lookups and a static Knowledge import fail, a finder class that only defines find_spec stays admitted, and the same call fails in a deeper bundle file, a bundle with __init__.py, other Benchmark modules, Harness, Apps and knowledge/data scripts. - docs/module-layout.md and CHANGELOG describe the rule. No task bundle code changes. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 12 ++- docs/module-layout.md | 47 +++++----- tests/_gate_policy_support.py | 106 ++++++++++------------ tests/test_evaluator_loaders.py | 2 +- tests/test_target_module_import_gate.py | 113 ++++++++++++++---------- 5 files changed, 146 insertions(+), 134 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7eae6e629..e4ca88a44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,10 +31,14 @@ prepared bundle no longer has `evaluation.py`. A bundle prepared before this change fails its frozen digests and must be prepared again. Retained results are not rewritten. - - The MoE grader and Qwen21 evaluator keep their in-process candidate loaders - under a permanent gate rule (`EVALUATOR_LOADERS`): one exact call each, only in - a task bundle program. `first_wave_report().runtime_execution` lists them, - because static isolation does not cover what the loaded code does at run time. + - The MoE grader and Qwen21 evaluator keep loading the frozen candidate by path in + their own process, which timing requires. The gate admits + `importlib.util.module_from_spec` only in task bundle programs, files directly + inside a `benchmark/tasks//` directory without `__init__.py`, and names no + task, so a new bundle needs no gate change. Name-based lookups (`__import__`, + `import_module`, `find_spec`) still fail there. `first_wave_report().runtime_execution` + lists each bundle program that loads by path, because static isolation does not + cover what the loaded code does at run time. - The optimization session's runtime identity adds `harness/protocol_child.py`, and `harness/task_protocol.py` changed, so a session started before this change does not resume after it. diff --git a/docs/module-layout.md b/docs/module-layout.md index 9ebed1194..0c58fccfe 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -175,7 +175,7 @@ directories have no `__init__.py`. They import only the standard library and external engines, so the Knowledge closure rule holds for them unchanged. K02 did not change the gate's scan scope or any closure rule. A dynamic import (`__import__`, `importlib.import_module`, or `module_from_spec`) inside a checked closure is an undeclared edge and fails -closed, except for the two permanent evaluator declarations below. +closed, except for by-path loading in task bundle programs, described below. Other dynamic forms (`getattr`, `exec`, `eval`, `runpy`, `pkgutil`) are out of scope for this package. `tools/` and `tests/` are separate registered consumers; this gate does not scan them. @@ -197,26 +197,29 @@ registered consumers; this gate does not scan them. bundles no longer load a sibling module by path, and a bundle prepared before R3 is refused by its frozen digests and must be prepared again. -The two evaluators that must call submitted code in their own process to time it -stay, under a permanent rule in `tests/_gate_policy_support.py` -(`EVALUATOR_LOADERS`) that is independent of the Knowledge compatibility pairs. -The rule permits exactly one `importlib.util.module_from_spec` call, with the -listed local spec variable, in the named function of a task bundle program (a file -in a directory without `__init__.py` under `benchmark/tasks/`). A second call, a -call elsewhere, the same shape in any other module, a declared module that becomes -importable or loses its call, other dynamic calls and static cross-core imports -fail. - -| Relative module | Scope | Local spec | -| --- | --- | --- | -| `benchmark.tasks.moe-forward-npu.grader` | `load` | `spec` | -| `benchmark.tasks.qwen21-denoise-npu.benchmark` | `load` | `spec` | - -These two loaders run frozen run files in their own child interpreter. The -callers verify the file identities (`identity.json` in both tasks), and each load -returns a fresh module that is not registered in `sys.modules`. They are not a -sandbox for candidate code. `first_wave_report().runtime_execution` lists them, so -the static `isolation_complete` result never reads as runtime isolation. +An evaluator that times submitted code must call it in its own process, so it loads +the frozen run file by path. The gate decides this by file location, not by task: + +- A **task bundle program** is a file directly inside `benchmark/tasks//` + where that directory has no `__init__.py`. No package code imports it, and the + hyphenated directory cannot be imported as a package; Harness and Apps run it as + its own process. +- In a bundle program, `importlib.util.module_from_spec` calls are admitted. Every + name-based lookup still fails there: `__import__`, `importlib.import_module`, a + `module_from_spec` imported by name and a call to `find_spec`. Static imports of + Knowledge, Harness or Apps fail as in any Benchmark module. +- Everywhere else, including a deeper file inside a bundle, a bundle directory with an + `__init__.py`, `knowledge/data/**` scripts, Harness and Apps, the same call is a + dynamic import and fails. + +No table lists tasks or functions, so a new bundle needs no gate change. Today the +MoE grader and the Qwen21 evaluator load this way. Their callers verify the file +identities (`identity.json` in both tasks), and each load returns a fresh module that +is not registered in `sys.modules`. The gate cannot see the path a bundle program +loads, so the rule is a statement about where by-path loading is allowed, not a +sandbox for candidate code. `first_wave_report().runtime_execution` lists every +bundle program that loads by path, so the static `isolation_complete` result never +reads as runtime isolation. R1 (#412) deleted the three root forwarders of the old run chain and removed their `LEGACY_RUN_FORWARDERS` registration from `tests/_gate_policy_support.py`. @@ -245,7 +248,7 @@ once it runs: that is `runtime_execution`. On the tree after R3 (#412), `first_wave_report()` has no violation and no blocker, `new_core_closure_isolated` and `isolation_complete` are both true, and -`runtime_execution` lists the two permanent evaluator loaders. +`runtime_execution` lists the two bundle programs that load files by path. R1 cleared the `allowlist` and `legacy run forwarders` blockers, and A1 cleared the `temporary leaf importers` blocker that R1 left for root `arguments.py`. R2 diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index 1caf92a5d..d3439b921 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -34,21 +34,22 @@ # file is covered, in the root or under a package. MODULE_ENTRY = "__main__.py" MODULE_ENTRY_DELEGATION = ("inferrsi.apps.cli", "main") -# #412 R3, permanent: the two task evaluators that must call submitted code in their own -# process to time it. Each entry is a task bundle program (a file in a directory without -# ``__init__.py`` under ``benchmark/tasks/``), the owner function and the exact local -# spec variable of its one admitted ``importlib.util.module_from_spec`` call. A second -# call, a call elsewhere in the file, the same shape in any other module, a declared -# module that becomes importable or loses its call, and every other dynamic import fail. -# These loaders execute frozen run files; they are listed in ``runtime_execution``, -# which ``isolation_complete`` (a static import-closure result) does not cover. R3 -# retired the temporary ``RESOURCE_LOADERS`` table and its blocker: the Harness task -# protocol runs in a child interpreter and the two sibling loads were merged away. -EVALUATOR_LOADERS = { - "inferrsi.benchmark.tasks.moe-forward-npu.grader": ("load", "spec"), - "inferrsi.benchmark.tasks.qwen21-denoise-npu.benchmark": ("load", "spec"), -} +# #412 R3, permanent: task bundle programs. A bundle program is a file directly inside +# ``benchmark/tasks//`` where that directory has no ``__init__.py``. No package +# code imports such a file, and the directory name cannot be imported as a package; +# Harness and Apps run it as its own process. Its evaluator may load a frozen run file +# (a candidate, baseline or reference) by path with ``importlib.util.module_from_spec`` +# to call and time it in-process; the gate admits that call there and nowhere else, by +# file location and without naming any task. Inside a bundle program every name-based +# lookup still fails: ``__import__``, ``import_module``, a ``module_from_spec`` imported +# by name, and a call to ``find_spec``. Static imports of other cores fail as before. +# ``runtime_execution`` lists each bundle program that loads files by path, which the +# static ``isolation_complete`` result does not cover. R3 retired the temporary +# ``RESOURCE_LOADERS`` table and its blocker: the Harness task protocol runs in a child +# interpreter and the two sibling loads were merged away. TASK_BUNDLES = "inferrsi.benchmark.tasks" +BY_PATH_LOAD = "importlib.util.module_from_spec" +NAME_LOOKUPS = frozenset({"importlib.util.find_spec", "find_spec"}) # #412 R1 deleted these eleven root modules: the seven Knowledge shells, the three # old-run forwarders and the root inspection binding. None is registered any more. # A file that returns under one of these names is a violation by itself, and every @@ -362,35 +363,30 @@ def _scan(root: Path, pairs: frozenset[tuple[str, str]]) -> _Graph: return _Graph(trees, packages, edges) -def _evaluator_loader(tree: ast.Module, module: str) -> ast.Attribute | None: - """The one declared evaluator call in ``module``; None unless exactly one exists.""" - declaration = EVALUATOR_LOADERS.get(module) - if declaration is None: - return None - matches = [] - - def visit(node, owner=""): - if isinstance(node, ast.ClassDef): - owner += ".." + node.name - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - owner = node.name if owner == "" else owner + "." + node.name - if (isinstance(node, ast.Call) and owner == declaration[0] - and ast.unparse(node.func) == "importlib.util.module_from_spec" - and len(node.args) == 1 and not node.keywords - and isinstance(node.args[0], ast.Name) and node.args[0].id == declaration[1]): - matches.append(node.func) - for child in ast.iter_child_nodes(node): - visit(child, owner) - - visit(tree) - return matches[0] if len(matches) == 1 else None - - -def _has_dynamic_import(tree: ast.Module, module: str = "") -> bool: - declared = _evaluator_loader(tree, module) +def _is_bundle_program(module: str, trees: dict[str, Any]) -> bool: + """A file directly inside a task bundle directory that has no ``__init__.py``.""" + bundle = module.rpartition(".")[0] + return bundle.rpartition(".")[0] == TASK_BUNDLES and bundle not in trees + + +def _by_path_loads(tree: ast.Module, module: str, trees: dict[str, Any]) -> list[ast.expr]: + """The admitted ``importlib.util.module_from_spec`` calls of a bundle program.""" + if not _is_bundle_program(module, trees): + return [] + return [node.func for node in ast.walk(tree) + if isinstance(node, ast.Call) and ast.unparse(node.func) == BY_PATH_LOAD] + + +def _has_dynamic_import( + tree: ast.Module, module: str = "", trees: dict[str, Any] | None = None +) -> bool: + bundle = trees is not None and _is_bundle_program(module, trees) + admitted = {id(node) for node in _by_path_loads(tree, module, trees)} if bundle else set() for node in ast.walk(tree): - if node is declared: + if id(node) in admitted: continue + if bundle and isinstance(node, ast.Call) and ast.unparse(node.func) in NAME_LOOKUPS: + return True if isinstance(node, ast.Name) and node.id in DYNAMIC_IMPORT_NAMES: return True if isinstance(node, ast.Attribute) and node.attr in DYNAMIC_IMPORT_NAMES: @@ -433,7 +429,7 @@ def _closure_violations( found += [ f"{start} reaches an undeclared dynamic import in {node}" for node in expanded - if node in graph.trees and _has_dynamic_import(graph.trees[node], node) + if node in graph.trees and _has_dynamic_import(graph.trees[node], node, graph.trees) ] return found @@ -690,9 +686,9 @@ class IsolationReport: ``new_core_closure_isolated`` covers only the new-core closures present in the tree. It never stands in for ``isolation_complete``. Both are static - import-closure results. ``runtime_execution`` lists the declared sites that - execute frozen task or candidate files in-process at run time; neither flag - says anything about what that code does once it runs. + import-closure results. ``runtime_execution`` lists the task bundle programs that + load frozen run files by path and execute them in-process at run time; neither + flag says anything about what that code does once it runs. """ violations: tuple[str, ...] @@ -765,20 +761,12 @@ def first_wave_report( for parent in sorted(leaf_parents): violations += _initializer_errors(parent, graph.trees[parent], graph.packages[parent]) violations += _contracts_errors(graph) - # #412 R3: a declared evaluator must stay a task bundle program and keep exactly its - # one call. A stale or widened declaration fails instead of passing silently. - runtime_execution = [] - for name in sorted(set(EVALUATOR_LOADERS) & set(modules)): - bundle = name.rpartition(".")[0] - if bundle.rpartition(".")[0] != TASK_BUNDLES or bundle in graph.trees: - violations.append(f"{name} declares an evaluator loader outside a task bundle program") - if _evaluator_loader(graph.trees[name], name) is None: - violations.append(f"{name} does not hold exactly its one declared evaluator loader") - else: - function, spec = EVALUATOR_LOADERS[name] - runtime_execution.append( - f"{name}.{function}: importlib.util.module_from_spec({spec}) executes frozen " - "run files in-process") + # #412 R3: bundle programs that load frozen run files by path. They run as their own + # processes, so this is runtime execution, which the static closure does not cover. + runtime_execution = [ + f"{name}: {BY_PATH_LOAD} executes files loaded by path in-process" + for name in modules if _by_path_loads(graph.trees[name], name, graph.trees) + ] # Inside the contracts package a Knowledge module may name only the encoder. The # initializer and its RunSpec module load with that import and are never named. violations += [ diff --git a/tests/test_evaluator_loaders.py b/tests/test_evaluator_loaders.py index e5b81b444..1bce6e2ac 100644 --- a/tests/test_evaluator_loaders.py +++ b/tests/test_evaluator_loaders.py @@ -1,4 +1,4 @@ -"""#412 R3: the two permanent evaluator loaders give fresh, unregistered modules. +"""#412 R3: the two in-process evaluator loaders give fresh, unregistered modules. Frozen-file identity is checked by their callers (MoE ``_adjudicate`` and the Qwen21 ``main``/protocol ``assess``); these tests pin the loaders' own contract. diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index fe737fce9..cd54daf1f 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -25,9 +25,7 @@ load_manifest, ) -# #412 R3: the one admitted evaluator loader shape, as in the MoE grader. -EVALUATOR = "benchmark/tasks/moe-forward-npu/grader.py" -EVALUATOR_MODULE = "inferrsi.benchmark.tasks.moe-forward-npu.grader" +# #412 R3: a by-path load, as in the MoE grader and the Qwen21 evaluator. LOADER = ( "import importlib.util\n" "def load(path, name):\n" @@ -36,7 +34,6 @@ " spec.loader.exec_module(module)\n" " return module\n" ) -MISSING = f"{EVALUATOR_MODULE} does not hold exactly its one declared evaluator loader" def _report(root, files): @@ -47,67 +44,87 @@ def _report(root, files): return first_wave_report(root) -def test_declared_evaluator_loader_is_admitted_listed_and_never_a_blocker(tmp_path): - report = _report(tmp_path, {EVALUATOR: LOADER}) - assert report.violations == () - assert "runtime resource loaders" not in blocker_kinds(report) - assert report.runtime_execution == ( - f"{EVALUATOR_MODULE}.load: importlib.util.module_from_spec(spec) executes frozen " - "run files in-process", - ) +def _module(relative): + return "inferrsi." + relative.removesuffix(".py").replace("/", ".") + + +def test_retired_loader_tables_do_not_return(): + # R3 admits by file location, so no table names a task, a function or a module. + for name in ("RESOURCE_LOADERS", "EVALUATOR_LOADERS"): + assert not hasattr(support, name), name + + +@pytest.mark.parametrize("relative", [ + "benchmark/tasks/moe-forward-npu/grader.py", + # Any bundle and any file name: nothing in the gate names a task. + "benchmark/tasks/a-new-task/evaluate.py", +]) +def test_bundle_program_may_load_files_by_path_and_is_listed(tmp_path, relative): + module_scope = ("import importlib.util\n" + "spec = importlib.util.spec_from_file_location('c', 'c.py')\n" + "module = importlib.util.module_from_spec(spec)\n") + two_calls = LOADER + "\nload('a.py', 'a')\nother = importlib.util.module_from_spec(spec)\n" + for index, source in enumerate((LOADER, two_calls, module_scope)): + report = _report(tmp_path / str(index), {relative: source}) + # The synthetic tree lacks the other cores, so only their "absent" blockers remain. + assert report.violations == () + assert blocker_kinds(report) == {"absent"} + assert report.runtime_execution == ( + f"{_module(relative)}: importlib.util.module_from_spec executes files loaded " + "by path in-process",) @pytest.mark.parametrize("extra", [ "\nimport importlib\nimportlib.import_module('inferrsi.knowledge')\n", "\n__import__('json')\n", - "\nimportlib.util.module_from_spec(spec)\n", - "\ndef another(spec):\n return importlib.util.module_from_spec(spec)\n", + "\nfrom importlib.util import module_from_spec\n", + "\nimportlib.util.find_spec('inferrsi.knowledge')\n", + "\nfrom importlib.util import find_spec\nfind_spec('inferrsi.knowledge')\n", ]) -def test_declared_evaluator_loader_does_not_allow_an_extra_dynamic_edge(tmp_path, extra): - report = _report(tmp_path, {EVALUATOR: LOADER + extra}) +def test_bundle_program_name_based_lookups_still_fail(tmp_path, extra): + report = _report(tmp_path, {"benchmark/tasks/moe-forward-npu/grader.py": LOADER + extra}) assert any("dynamic import" in item for item in report.violations) assert not report.isolation_complete -@pytest.mark.parametrize("source", [ - # A second call in the declared function voids the declaration for both calls. - LOADER.replace(" return module\n", - " importlib.util.module_from_spec(spec)\n return module\n"), - # The call under another spec variable or in another function is not the declared one. - LOADER.replace("spec =", "other =").replace("(spec)", "(other)") - .replace("spec.loader", "other.loader"), - LOADER.replace("def load(", "def load_module("), -]) -def test_declared_evaluator_loader_shape_is_exact(tmp_path, source): - report = _report(tmp_path, {EVALUATOR: source}) - assert MISSING in report.violations - assert any("dynamic import" in item for item in report.violations) - assert report.runtime_execution == () +def test_bundle_program_static_cross_core_import_still_fails(tmp_path): + report = _report(tmp_path, { + "knowledge/catalog.py": "VALUE = 1\n", + "benchmark/tasks/moe-forward-npu/grader.py": LOADER + "from inferrsi.knowledge " + "import catalog\n"}) + assert any("imports inferrsi.knowledge" in item for item in report.violations) -def test_declared_evaluator_without_its_call_is_a_stale_declaration(tmp_path): - report = _report(tmp_path, {EVALUATOR: "def load(path, name):\n return None\n"}) - assert MISSING in report.violations - assert report.runtime_execution == () +def test_a_finder_class_in_a_bundle_program_is_not_a_lookup(tmp_path): + # The Qwen21 and TTS import guards define a find_spec method; defining it is not a call. + guard = ("import importlib.util\nclass Guard:\n" + " def find_spec(self, fullname, path=None, target=None):\n" + " return importlib.util.spec_from_loader(fullname, self)\n") + report = _report(tmp_path, {"benchmark/tasks/qwen3-tts-npu/baseline.py": guard}) + assert report.violations == () and report.runtime_execution == () -def test_declared_evaluator_inside_an_importable_package_fails(tmp_path): - report = _report(tmp_path, {EVALUATOR: LOADER, - "benchmark/tasks/moe-forward-npu/__init__.py": ""}) - assert (f"{EVALUATOR_MODULE} declares an evaluator loader outside a task bundle program" - in report.violations) +@pytest.mark.parametrize("files", [ + # The bundle directory becomes an importable package. + {"benchmark/tasks/moe-forward-npu/__init__.py": ""}, + # tasks itself does not make a bundle program out of a deeper file. + {"benchmark/tasks/moe-forward-npu/sub/grader.py": LOADER}, +]) +def test_only_files_directly_in_a_non_package_bundle_are_bundle_programs(tmp_path, files): + report = _report(tmp_path, {"benchmark/tasks/moe-forward-npu/grader.py": LOADER, **files}) + assert any("dynamic import" in item for item in report.violations) @pytest.mark.parametrize("relative", [ - "benchmark/tasks/moe-forward-npu/workload.py", - "benchmark/tasks/qwen3-tts-npu/benchmark.py", "benchmark/registry.py", + "benchmark/tasks.py", # The Harness protocol loader that R3 retired, and its neighbours. "harness/task_protocol.py", "harness/other_protocol.py", "apps/benchmark_operations.py", + "knowledge/data/skills/scripts/loader.py", ]) -def test_the_evaluator_loader_shape_is_admitted_nowhere_else(tmp_path, relative): +def test_by_path_loading_is_admitted_nowhere_else(tmp_path, relative): report = _report(tmp_path, {relative: LOADER}) assert any("dynamic import" in item for item in report.violations) assert report.runtime_execution == () @@ -214,13 +231,13 @@ def test_clean_scan_is_not_isolation(self) -> None: self.assertEqual(report.blockers, ()) self.assertTrue(report.new_core_closure_isolated) self.assertTrue(report.isolation_complete) - # Static closure is not runtime isolation: the two declared evaluators still - # execute frozen run files in-process, and the report says so. + # Static closure is not runtime isolation: the two evaluators that time submitted + # code still load frozen run files by path in-process, and the report says so. self.assertEqual(report.runtime_execution, ( - "inferrsi.benchmark.tasks.moe-forward-npu.grader.load: " - "importlib.util.module_from_spec(spec) executes frozen run files in-process", - "inferrsi.benchmark.tasks.qwen21-denoise-npu.benchmark.load: " - "importlib.util.module_from_spec(spec) executes frozen run files in-process", + "inferrsi.benchmark.tasks.moe-forward-npu.grader: " + "importlib.util.module_from_spec executes files loaded by path in-process", + "inferrsi.benchmark.tasks.qwen21-denoise-npu.benchmark: " + "importlib.util.module_from_spec executes files loaded by path in-process", )) def test_split_skill_eval_modules_reach_no_model_or_infra(self) -> None: From 2e2de0462296bc1188a37c058f33c9ac076f7a1d Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Wed, 18 Nov 2026 12:19:09 +0800 Subject: [PATCH 7/7] fix(tests): reject find_spec aliases and package imports of bundle programs (R3) Review (zengchuang-hw, 2d0fa29), two static-policy gaps in the bundle program rule: 1. find_spec was rejected only as a call spelled `find_spec(...)` or `importlib.util.find_spec(...)`. An alias (`from importlib.util import find_spec as resolve`, `import importlib.util as util; util.find_spec(...)`) passed. In a bundle program find_spec is now handled like the other dynamic-import names: any name, attribute or import of it fails, so aliases fail too. A finder class that only defines a find_spec method stays admitted. 2. "No __init__.py" did not make a bundle program unimportable: Python can import it through a namespace package. The gate now rejects any import of a bundle program from package code, Benchmark, Apps or another bundle, so an admitted by-path load cannot join another module's closure. The real tree has no such import. The source comment, docs/module-layout.md and CHANGELOG no longer claim the directory cannot be imported, and call the rule a location-based static policy, not a process or sandbox guarantee. New negative fixtures cover both alias forms, PathFinder.find_spec, module_from_spec reached through an alias, and absolute, relative, Apps and cross-bundle imports of an identifier-named bundle program. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO --- CHANGELOG.md | 5 ++- docs/module-layout.md | 17 +++++---- tests/_gate_policy_support.py | 49 +++++++++++++++---------- tests/test_target_module_import_gate.py | 24 ++++++++++++ 4 files changed, 65 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ca88a44..eddbfec7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,8 +35,9 @@ their own process, which timing requires. The gate admits `importlib.util.module_from_spec` only in task bundle programs, files directly inside a `benchmark/tasks//` directory without `__init__.py`, and names no - task, so a new bundle needs no gate change. Name-based lookups (`__import__`, - `import_module`, `find_spec`) still fail there. `first_wave_report().runtime_execution` + task, so a new bundle needs no gate change. Package code may not import a bundle + program. Name-based lookups (`__import__`, `import_module`, `find_spec` in any + spelling) still fail there. `first_wave_report().runtime_execution` lists each bundle program that loads by path, because static isolation does not cover what the loaded code does at run time. - The optimization session's runtime identity adds `harness/protocol_child.py`, diff --git a/docs/module-layout.md b/docs/module-layout.md index 0c58fccfe..ac6acdb11 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -201,13 +201,14 @@ An evaluator that times submitted code must call it in its own process, so it lo the frozen run file by path. The gate decides this by file location, not by task: - A **task bundle program** is a file directly inside `benchmark/tasks//` - where that directory has no `__init__.py`. No package code imports it, and the - hyphenated directory cannot be imported as a package; Harness and Apps run it as - its own process. + where that directory has no `__init__.py`. Python could still import such a file + through a namespace package, so the gate rejects every import of a bundle program + from package code; Harness and Apps run it as its own process. - In a bundle program, `importlib.util.module_from_spec` calls are admitted. Every - name-based lookup still fails there: `__import__`, `importlib.import_module`, a - `module_from_spec` imported by name and a call to `find_spec`. Static imports of - Knowledge, Harness or Apps fail as in any Benchmark module. + name-based lookup still fails there: `__import__`, `importlib.import_module`, + `module_from_spec` reached by another spelling, and `find_spec` in any spelling, + including an alias. Static imports of Knowledge, Harness or Apps fail as in any + Benchmark module. - Everywhere else, including a deeper file inside a bundle, a bundle directory with an `__init__.py`, `knowledge/data/**` scripts, Harness and Apps, the same call is a dynamic import and fails. @@ -216,8 +217,8 @@ No table lists tasks or functions, so a new bundle needs no gate change. Today t MoE grader and the Qwen21 evaluator load this way. Their callers verify the file identities (`identity.json` in both tasks), and each load returns a fresh module that is not registered in `sys.modules`. The gate cannot see the path a bundle program -loads, so the rule is a statement about where by-path loading is allowed, not a -sandbox for candidate code. `first_wave_report().runtime_execution` lists every +loads, so the rule is a location-based static policy about where by-path loading is +allowed, not a process or sandbox guarantee for candidate code. `first_wave_report().runtime_execution` lists every bundle program that loads by path, so the static `isolation_complete` result never reads as runtime isolation. diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index d3439b921..2dffbdbd4 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -35,21 +35,23 @@ MODULE_ENTRY = "__main__.py" MODULE_ENTRY_DELEGATION = ("inferrsi.apps.cli", "main") # #412 R3, permanent: task bundle programs. A bundle program is a file directly inside -# ``benchmark/tasks//`` where that directory has no ``__init__.py``. No package -# code imports such a file, and the directory name cannot be imported as a package; -# Harness and Apps run it as its own process. Its evaluator may load a frozen run file -# (a candidate, baseline or reference) by path with ``importlib.util.module_from_spec`` -# to call and time it in-process; the gate admits that call there and nowhere else, by -# file location and without naming any task. Inside a bundle program every name-based -# lookup still fails: ``__import__``, ``import_module``, a ``module_from_spec`` imported -# by name, and a call to ``find_spec``. Static imports of other cores fail as before. -# ``runtime_execution`` lists each bundle program that loads files by path, which the -# static ``isolation_complete`` result does not cover. R3 retired the temporary -# ``RESOURCE_LOADERS`` table and its blocker: the Harness task protocol runs in a child -# interpreter and the two sibling loads were merged away. +# ``benchmark/tasks//`` where that directory has no ``__init__.py``. Python could +# still import such a file as part of a namespace package, so the gate itself rejects +# every import of a bundle program from package code; Harness and Apps run it as its own +# process. Its evaluator may load a frozen run file (a candidate, baseline or reference) +# by path with ``importlib.util.module_from_spec`` to call and time it in-process; the +# gate admits that call there and nowhere else, by file location and without naming any +# task. Inside a bundle program every name-based lookup still fails: ``__import__``, +# ``import_module``, a ``module_from_spec`` reached by another spelling, and ``find_spec`` +# in any spelling. Static imports of other cores fail as before. ``runtime_execution`` +# lists each bundle program that loads files by path, which the static +# ``isolation_complete`` result does not cover. This is a location-based static policy, +# not a process or sandbox guarantee. R3 retired the temporary ``RESOURCE_LOADERS`` table +# and its blocker: the Harness task protocol runs in a child interpreter and the two +# sibling loads were merged away. TASK_BUNDLES = "inferrsi.benchmark.tasks" BY_PATH_LOAD = "importlib.util.module_from_spec" -NAME_LOOKUPS = frozenset({"importlib.util.find_spec", "find_spec"}) +BUNDLE_LOOKUP_NAMES = frozenset({"find_spec"}) # #412 R1 deleted these eleven root modules: the seven Knowledge shells, the three # old-run forwarders and the root inspection binding. None is registered any more. # A file that returns under one of these names is a violation by itself, and every @@ -382,18 +384,17 @@ def _has_dynamic_import( ) -> bool: bundle = trees is not None and _is_bundle_program(module, trees) admitted = {id(node) for node in _by_path_loads(tree, module, trees)} if bundle else set() + # A bundle program also may not look a module up by name with find_spec, in any + # spelling: a call, an attribute, an alias or an import of the name. + names = DYNAMIC_IMPORT_NAMES | BUNDLE_LOOKUP_NAMES if bundle else DYNAMIC_IMPORT_NAMES for node in ast.walk(tree): if id(node) in admitted: continue - if bundle and isinstance(node, ast.Call) and ast.unparse(node.func) in NAME_LOOKUPS: + if isinstance(node, ast.Name) and node.id in names: return True - if isinstance(node, ast.Name) and node.id in DYNAMIC_IMPORT_NAMES: + if isinstance(node, ast.Attribute) and node.attr in names: return True - if isinstance(node, ast.Attribute) and node.attr in DYNAMIC_IMPORT_NAMES: - return True - if isinstance(node, ast.ImportFrom) and any( - alias.name in DYNAMIC_IMPORT_NAMES for alias in node.names - ): + if isinstance(node, ast.ImportFrom) and any(alias.name in names for alias in node.names): return True return False @@ -767,6 +768,14 @@ def first_wave_report( f"{name}: {BY_PATH_LOAD} executes files loaded by path in-process" for name in modules if _by_path_loads(graph.trees[name], name, graph.trees) ] + # Package code never imports a bundle program, whatever the bundle's name, so an + # admitted by-path load cannot join another module's closure. + violations += [ + f"{name} imports task bundle program {target}" + for name in modules + for target in sorted(graph.edges[name]) + if target != name and _is_bundle_program(target, graph.trees) + ] # Inside the contracts package a Knowledge module may name only the encoder. The # initializer and its RunSpec module load with that import and are never named. violations += [ diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index cd54daf1f..d05d352b3 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -78,8 +78,14 @@ def test_bundle_program_may_load_files_by_path_and_is_listed(tmp_path, relative) "\nimport importlib\nimportlib.import_module('inferrsi.knowledge')\n", "\n__import__('json')\n", "\nfrom importlib.util import module_from_spec\n", + "\nimport importlib.util as util\nutil.module_from_spec(spec)\n", "\nimportlib.util.find_spec('inferrsi.knowledge')\n", "\nfrom importlib.util import find_spec\nfind_spec('inferrsi.knowledge')\n", + # Review of 2d0fa29: aliases of find_spec are the same name-based lookup. + "\nfrom importlib.util import find_spec as resolve\n" + "spec = resolve('inferrsi.knowledge')\nmodule = importlib.util.module_from_spec(spec)\n", + "\nimport importlib.util as util\nutil.find_spec('inferrsi.knowledge')\n", + "\nimport importlib.machinery\nimportlib.machinery.PathFinder.find_spec('inferrsi')\n", ]) def test_bundle_program_name_based_lookups_still_fail(tmp_path, extra): report = _report(tmp_path, {"benchmark/tasks/moe-forward-npu/grader.py": LOADER + extra}) @@ -87,6 +93,24 @@ def test_bundle_program_name_based_lookups_still_fail(tmp_path, extra): assert not report.isolation_complete +@pytest.mark.parametrize("importer, source", [ + # Review of 2d0fa29: without __init__.py a bundle is still a namespace package, so + # an identifier-named bundle program is importable. The gate rejects that import. + ("benchmark/runner.py", "from inferrsi.benchmark.tasks.new_task import evaluate\n"), + ("benchmark/runner.py", "import inferrsi.benchmark.tasks.new_task.evaluate\n"), + ("benchmark/runner.py", "from .tasks.new_task import evaluate\n"), + ("apps/runner.py", "from inferrsi.benchmark.tasks.new_task import evaluate\n"), + ("benchmark/tasks/other_task/evaluate.py", + "from inferrsi.benchmark.tasks.new_task import evaluate\n"), +]) +def test_package_code_may_not_import_a_bundle_program(tmp_path, importer, source): + target = "inferrsi.benchmark.tasks.new_task.evaluate" + report = _report(tmp_path, {"benchmark/tasks/new_task/evaluate.py": LOADER, + importer: source}) + assert f"{_module(importer)} imports task bundle program {target}" in report.violations + assert not report.isolation_complete + + def test_bundle_program_static_cross_core_import_still_fails(tmp_path): report = _report(tmp_path, { "knowledge/catalog.py": "VALUE = 1\n",