From 9772e690801552402b71d84f9e08368814a1e5ff Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 18 Nov 2026 22:34:39 +0800 Subject: [PATCH 1/4] feat: benchmark RSI knowledge with paired optimization runs Signed-off-by: dengyunyang <584797741@qq.com> Agent-assisted-by: Codex (implementation and self-review) Agent-assisted-by: Codex comparison_rules_report (implementation) Agent-assisted-by: Codex moe_protocol (implementation) --- AGENTS.md | 6 + docs/benchmark-comparison.md | 159 +++++ docs/moe-forward-benchmark.md | 61 +- docs/operations.md | 1 + docs/optimization-benchmarks.md | 8 +- src/inferrsi/apps/benchmark_compare.py | 558 ++++++++++++++++++ src/inferrsi/apps/cli/__init__.py | 5 + src/inferrsi/apps/cli/arguments.py | 39 ++ src/inferrsi/apps/delivery/projection.py | 3 + src/inferrsi/apps/views/comparison_report.py | 363 ++++++++++++ src/inferrsi/benchmark/comparison.py | 231 ++++++++ .../tasks/moe-forward-npu/benchmark.py | 176 +----- .../tasks/moe-forward-npu/optimization.py | 290 +++++++++ src/inferrsi/harness/optimization.py | 15 +- src/inferrsi/harness/optimization_session.py | 2 + tests/test_benchmark_compare.py | 286 +++++++++ tests/test_comparison_report.py | 146 +++++ tests/test_comparison_rules.py | 173 ++++++ tests/test_moe_optimization_protocol.py | 270 +++++++++ tests/test_target_module_import_gate.py | 2 + 20 files changed, 2605 insertions(+), 189 deletions(-) create mode 100644 docs/benchmark-comparison.md create mode 100644 src/inferrsi/apps/benchmark_compare.py create mode 100644 src/inferrsi/apps/views/comparison_report.py create mode 100644 src/inferrsi/benchmark/comparison.py create mode 100644 src/inferrsi/benchmark/tasks/moe-forward-npu/optimization.py create mode 100644 tests/test_benchmark_compare.py create mode 100644 tests/test_comparison_report.py create mode 100644 tests/test_comparison_rules.py create mode 100644 tests/test_moe_optimization_protocol.py diff --git a/AGENTS.md b/AGENTS.md index 1e4aca37..54ee960c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,6 +102,12 @@ Portable operator guidance and candidate procedures live in [src/inferrsi/knowle ## Optimization requests +For requests to use benchmarks to verify RSI knowledge efficiency, start with +`inferrsi benchmarks compare --help` and follow its returned `next_action`. +The default knowledge comparison runs Qwen denoising and MoE forward, each +on/off in independent sessions. Configuration, native handoff, trace and report +details are in [benchmark comparison](docs/benchmark-comparison.md). + Ordinary project optimization stays in the current Agent conversation. Follow the [shared optimization workflow](docs/optimization-workflow-unification.md): prepare explicit workload, quality/measurement scripts, metric, gates, declared diff --git a/docs/benchmark-comparison.md b/docs/benchmark-comparison.md new file mode 100644 index 00000000..0d02b425 --- /dev/null +++ b/docs/benchmark-comparison.md @@ -0,0 +1,159 @@ +# Compare RSI optimization efficiency + +Use the self-describing Apps entry when asked to use benchmarks to determine +whether RSI knowledge improves optimization efficiency. No extra skill is needed. + +```bash +inferrsi benchmarks compare --help +inferrsi benchmarks compare +``` + +Knowledge is the default focus. Its default task set is +`qwen21-denoise-npu` and `moe-forward-npu`, each with knowledge **on/off**. +One repeat means four independent Agent optimizations, not four measurements +of a prewritten candidate. Repeated `--task` replaces the default task set. +Each execution Agent chooses changes from feedback, completes the exact round +budget, and independently retests its selected code with `session final`. + +## Resolve and freeze configuration + +CLI values override `.inferrsi/benchmarks.json` in the current workspace, then +built-ins. `--config` selects another non-secret JSON file. The resolved values +and their sources are frozen in `experiment.json`; each job references its +digest. Resume with the returned `--output`, never by preparing over that path. + +Built-in budgets are eight rounds, one repeat, 60 campaign minutes, 1,200 seconds +per evaluation/final reserve, and 90 minutes per Agent job. These are bounded +run settings, not universal performance gates. Select a prepared execution +environment before dispatch. An unconfigured entry returns +`configuration_required` with an actionable request; it does not fabricate an +NPU host. Read task requirements with `benchmarks show`. A private profile can be: + +```json +{ + "schema_version": 1, + "environment_id": "prepared-ascend-pool", + "evaluation_prefix": "npu run --gpus {devices} --timeout 18m --wait 1m --", + "execution_instructions": "Use the existing authorized target. Return each complete campaign tree to its declared local workspace.", + "rounds": 8, + "repeats": 1, + "policy": { + "metrics": ["speedup", "wall_seconds"], + "tolerances": {"speedup": 0.0, "wall_seconds": 0.0, "tokens": 0.0}, + "minimum_pairs": 1 + } +} +``` + +`{devices}` comes from each task: Qwen needs one NPU, MoE needs two. Default task +arguments are Qwen `--pairs 2 --samples 3 --warmups 1` and MoE +`--pairs 2 --timeout-seconds 900`. Override one with +`--evaluation-args 'TASK=ARGUMENTS'`. Keep allocator wait/run/cleanup within the +outer evaluation timeout. Optional `model` and `effort` are requests, not proof +of actual host settings. Credentials never belong in this configuration. + +## Drive the host handoff + +The command manages experiment records. The initiating Agent calls its actual +host tools for fresh-session creation and observation, following `next_action`. +This is the same native handoff as [Agent sessions](native-agent-sessions.md). +The CLI neither starts a hidden Agent CLI nor invents a conversation identity. + +1. Prepare: `benchmarks compare --config --output `. +2. Read `next_action`; call `compare --action dispatch --output + --trial `. Only `may_create: true` permits one native creation call. +3. Send the exact returned prompt with the declared workspace, and supported + requested model/effort. Bind the confirmed native ID using `--action bind`, + `--host`, `--session-id`, optional `--host-id`, and `--receipt `. +4. Observe that same session. Retain actual host observations with + `--action record --observation `. Uncertain creation stays + `dispatching`; inspecting the original call takes precedence over relaunch. +5. After termination and artifact return, call `--action collect --trial `. + Keep failed/incomplete results. By default, trials run in declared order, + with on/off order alternating between repeats. With sufficient devices, + pass `--parallel` on dispatch for each trial you want to start while others + are active. Keep each allocation separate and note concurrent execution + when interpreting campaign wall time. +6. Generate a progress or final comparison with `--action report`. Read + `--action status` to resume. Reporting never executes missing rounds/finals. + +All non-prepare actions require `--output`. Dispatch/bind/record/collect also +require `--trial`. Jobs are created just before dispatch so waiting for earlier +trials does not consume their host deadlines. Native deadline cancellation +still belongs to the initiating Agent and its supported host tools. + +Native observation JSON uses the existing `agents record` format, plus: + +```json +{ + "host": "", + "host_id": null, + "session_id": "", + "state": "completed", + "native_receipt": {"replace_with": "actual native tool response"}, + "benchmark_observation": { + "model": "", + "effort": "", + "environment_id": "prepared-ascend-pool", + "environment_fingerprint": "", + "knowledge_policy_violation": false + } +} +``` + +These remain caller-exported observations, not authenticated host attestations. +Retain their underlying receipts. Include actual `usage_scope: session_total` +and `usage` counters when available; cached input is a subset of input. Final +host usage is never added to campaign round subtotals. Missing usage stays null. +Do not replace missing final counters with an earlier partial observation. + +## Isolation and evidence + +Every trial has a separate workspace and frozen runtime. Shared code and task +bytes match; on receives the pinned package corpus, off omits `knowledge/data` +and disables managed session retrieval. Both prompts exclude other skill +libraries, external knowledge tools, web research and prior trial solutions. +Public task/CLI instructions and ordinary target APIs remain available to both. +This is runtime-corpus isolation plus an instruction boundary; it is **not an OS +filesystem/network sandbox**. Report observed violations and bound conclusions +to this managed-knowledge treatment. Do not claim all latent or external +knowledge was removed. Remote trials return the exact full campaign tree; +arbitrary summaries cannot substitute for retained evidence. + +Collection verifies the frozen task/protocol, budget, knowledge condition, +runtime identity, raw evidence and independent final candidate binding. Paired +actual model/effort/environment must match. A stale collection cannot conceal +later evidence loss: report generation reads and verifies the campaign again. + +Each round can supply `--strategy ` and repeated +`--knowledge-ref ` in addition to `--hypothesis`. +Harness accepts citations only to contexts already delivered in that campaign. +The trace labels these as Agent declarations; delivery does not prove use. +Unrecorded citations remain unknown, rather than treating every previous search +as knowledge used by every round. + +## Read the report + +Apps writes a new JSON/Markdown/offline-HTML report snapshot under `reports/`. +Start with the conclusion and per-task comparison, then inspect complete traces +(strategy, hypothesis, knowledge citations, round/best speedup, correctness and +selection), independent finals and evidence gaps. JSON digest links the two +readable documents to the same snapshot. Original campaign evidence is retained. + +Benchmark rules compare paired on/off benefit factors: on/off speedup, off/on +wall time or tokens. The default efficiency basis is final optimization speedup +and actual campaign wall time; token usage is displayed separately. Add tokens +to policy.metrics only with comparable whole-session observations. Tolerances +are frozen relative neutral bands, not statistical significance tests. + +Conclusions distinguish observed improvement, no gain, regression, tradeoff and +insufficient evidence. Missing/invalid declared pairs stay in denominators and +make the overall comparison insufficient. Statistics are per-task medians of +paired log ratios; unrelated task latencies are never pooled. Results are +descriptive within the declared tasks, budgets and repeats, never a general +knowledge-efficacy or model/service E2E claim. Set campaign-specific minimum +repeats and neutral tolerances before search when stronger evidence is needed. + +Apps owns configuration, composition and presentation; Benchmark owns task and +comparison rules; Harness owns optimization execution and retained facts; +Knowledge owns the delivered material. No new core-to-core imports are added. diff --git a/docs/moe-forward-benchmark.md b/docs/moe-forward-benchmark.md index ccd15170..8827670b 100644 --- a/docs/moe-forward-benchmark.md +++ b/docs/moe-forward-benchmark.md @@ -88,37 +88,56 @@ CPU 输入/oracle 准备、计时前 barrier、验收、形状预热不在 forwa ## 在 RSI 优化循环里如何使用 -```text -用户目标与题目约束 → Agent 改候选 → 本入口验算并测量 → Agent 看证据再修改 - ↓ - 调用方保留每轮结果、选最佳版本、独立复测 +MoE 题包提供 `optimization.py`,可以使用与 Qwen 相同的 Harness 完整迭代流程。 +它复用单候选入口的同一份双 rank 原始证据规则,不根据 rank 0 汇总或 Agent 自报分数选优。 + +执行 Agent 在已经准备好的环境中启动: + +```bash +inferrsi benchmarks session start moe-forward-npu \ + --session ./outputs/moe-session \ + --rounds --max-minutes \ + --evaluation-timeout-seconds 1500 --final-reserve-seconds 1500 \ + --condition --knowledge \ + --evaluation-prefix 'npu run --gpus 2 --timeout 20m --wait 1m --' \ + --evaluation-args '--pairs 2 --timeout-seconds 900' ``` -调用一次评测一个候选。`inferrsi benchmarks list` 和 -`inferrsi benchmarks show moe-forward-npu` 可以读取题包信息,执行使用本页的 MoE 入口。 -调用方负责启动 Agent、推进轮次、统计模型 tokens 和选择搜索策略, -并将每次 `result.json` 作为下一轮反馈。 -知识库 ON/OFF 或不同 Agent 的差异由调用方控制,测量协议保持一致。 -profiling 可作为 Agent 诊断工具,评测入口不要求先 profiling。 +轮数与总预算由本次实验声明;总预算需要容纳 final 预留。这里的采样和超时是可覆盖的 +命令示例。每轮和 final 使用同一份冻结的配对数与完整 case 集合,评测器内部超时、 +分配器运行/等待超时和外层评测超时应逐层留出启动与清理余量。 + +Agent 阅读返回的 `frozen/TASK.md`,修改 `work/candidate.py`;每次修改后提交假设: + +```bash +inferrsi benchmarks session round --session ./outputs/moe-session \ + --candidate ./outputs/moe-session/work/candidate.py \ + --hypothesis '<本轮优化策略与理由>' + +# 完成声明的轮次后,独立复测已留存的最佳源码 +inferrsi benchmarks session final --session ./outputs/moe-session +``` -可给新 Agent 的简短任务指令: +知识库开启时使用 `benchmarks session knowledge` 检索并留存实际交付内容;关闭时拒绝 +受管检索。外部来源和直接文件读取的隔离由对照实验的执行环境与 Agent 访问约定负责。 +Profiling 由 Agent 按瓶颈选择,不是每轮必经步骤。 -> 阅读 `src/inferrsi/benchmark/tasks/moe-forward-npu/TASK.md`, -> 优化 `./implementation.py` 的完整双 NPU MoE forward。遵守输入只读、专家所有权、 -> 正确性和计时边界。每次候选使用新的输出目录,按已约定的配对数、超时和实验预算, -> 通过 `inferrsi benchmarks run moe-forward-npu` 取得结果;保留失败,不以自报数字替代结果。 -> 最终交付最佳候选、逐轮证据和独立复测记录。 +Harness 留存每轮源码、假设、实际评测、历史最佳选择和独立 final。迭代 assessor 核对 +候选、全部题目/协议文件、grader、launcher 与采样请求的哈希,重算两个 rank 的完整 +调用与计时样本,供标准结果交付和优化 trace 使用。正常结束的候选错误保留为失败轮; +内部或外部执行中断、缺少 rank 证据、清理未确认以及基线失败均阻断后续搜索。 +恢复通过 `session recover` 回收原有尝试,不重新执行同一候选。 -例如要做 8 轮再复测,由调用方为 R1…R8 和 final 分配不同目录并保存候选快照。 -轮数是该次实验的预算,不能把它当成通用默认值。优化曲线按每轮有效分数及历史最佳分数 -绘制;失败轮保留空分数并消耗预算。累计 tokens 需从对应 Agent/session 的原始 usage -记录汇总;本入口的 `elapsed_seconds` 是评测墙钟,不是 Agent 搜索墙钟或模型费用。 +启动新会话与回收使用通用 [Agent 交接](native-agent-sessions.md); +[共享迭代流程](optimization-benchmarks.md) 说明预算、用量导入和标准交付。 +单独的 `benchmarks run moe-forward-npu` 仍只评测一个候选。完整过程的 token 需从 +真实宿主 usage 导出;本题 `elapsed_seconds` 仅是评测墙钟,不是 Agent 搜索消耗。 ## 验证范围与限制 历史冻结题目、baseline、oracle、采样逻辑已有双 NPU 实验;本次发布保留了这些数学和 计时语义,加入便携包装、源码冻结、双 rank 证据复核和独立 session 清理。 -**新增的便携入口尚未在 NPU 上重新跑通**,CPU 测试和 wheel 资源检查只验证接线、 +**新增的迭代协议尚未在 NPU 上重新跑通**,CPU 测试和 wheel 资源检查只验证接线、 失败处理及证据完整性,不能当作新入口的 NPU 验收或性能结论。 当前权重更新用例验证原位更新后的结果,但两次访问之间还运行其他参数用例; diff --git a/docs/operations.md b/docs/operations.md index 9832c147..58a905a9 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -27,6 +27,7 @@ Effect classes used below: | Intent | Entry | Effect class | Boundary | | --- | --- | --- | --- | +| Verify whether RSI knowledge improves optimization efficiency | CLI `benchmarks compare`; [comparison workflow](benchmark-comparison.md) | Local experiment records / explicit native handoff / report-generating | Apps freezes two-task on/off plans, returns host actions, verifies retained finals and reports Benchmark comparisons; no hidden Agent launch | | Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution | | Optimize an ordinary project in the current Agent | CLI `campaign start/knowledge/round/status/final/recover` | Experiment-executing / local records | Harness freezes explicit caller-provided checks, manages multi-file candidates, executes and assesses retained results. This path does not depend on Benchmark or infer acceptance from the goal. See [shared workflow](optimization-workflow-unification.md) | | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index 18c85ad6..d92f8cbb 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -23,10 +23,14 @@ only observes and collects its results. Benchmark supplies task resources and task-specific rules. Harness freezes those inputs, runs the checks, assesses retained outputs and manages the lifecycle; it does not invent project acceptance. -The first task with this session protocol is `qwen21-denoise-npu`. MoE and TTS -currently expose their single-candidate task entries. New session tasks provide +The iterative tasks are `qwen21-denoise-npu` and `moe-forward-npu`; TTS +currently exposes its single-candidate task entry. New session tasks provide their own `optimization.py` alongside the task definition and evaluator. +For paired knowledge on/off experiments over both tasks, use the Apps +[`benchmarks compare` workflow](benchmark-comparison.md). It reuses this session +lifecycle and collects complete optimization traces and independent finals. + ## Start the complete workflow In the execution conversation, run in the prepared NPU environment. The example declares eight optimization diff --git a/src/inferrsi/apps/benchmark_compare.py b/src/inferrsi/apps/benchmark_compare.py new file mode 100644 index 00000000..84e60e40 --- /dev/null +++ b/src/inferrsi/apps/benchmark_compare.py @@ -0,0 +1,558 @@ +"""Compose paired optimization experiments with native host handoffs. + +The Agent calls the host tools returned by this application. Python never +creates a conversation, invents host observations, or edits a candidate. +""" + +import json +import math +import os +import shlex +import shutil +import uuid +from copy import deepcopy +from datetime import datetime, timezone +from importlib.resources import files +from pathlib import Path +from tempfile import TemporaryDirectory + +from inferrsi.benchmark.comparison import DEFAULT_POLICY, DEFAULT_TASKS, summarize, validate_policy +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import agent_sessions, optimization +from inferrsi.harness.artifact_primitives import digest, file_lock, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import _runtime_identity +from inferrsi.harness.task_protocol import run as protocol_operation + +from .delivery import project +from .delivery.schema import decode +from .storage.archive import _publish_directory +from .views.comparison_report import write_comparison_report + +DEFAULT_CONFIG = { + "schema_version": 1, "tasks": list(DEFAULT_TASKS), "repeats": 1, "rounds": 8, + "max_minutes": 60, "evaluation_timeout_seconds": 1200, "final_reserve_seconds": 1200, + "job_max_minutes": 90, "model": None, "effort": None, + "environment_id": None, "evaluation_prefix": "", "execution_instructions": "", + "python": "python", "policy": DEFAULT_POLICY, + "task_arguments": { + "qwen21-denoise-npu": "--pairs 2 --samples 3 --warmups 1", + "moe-forward-npu": "--pairs 2 --timeout-seconds 900", + }, +} +CONFIG_NAME = Path(".inferrsi/benchmarks.json") +OBSERVED_FIELDS = ("model", "effort", "environment_id", "environment_fingerprint") + + +def _io_path(path): + """Use Windows extended paths at I/O boundaries without persisting that spelling.""" + path = Path(path).absolute() + value = str(path) + if os.name == "nt" and not value.startswith("\\\\?\\"): + value = "\\\\?\\UNC\\" + value[2:] if value.startswith("\\\\") else "\\\\?\\" + value + return Path(value) + + +def _object(path): + path = _io_path(path) + if path.stat().st_size > 2 * 1024 * 1024: + raise ValueError("Comparison JSON document exceeds 2 MiB") + value = decode(path.read_bytes()) + if not isinstance(value, dict): + raise ValueError("Comparison JSON must contain an object") + return value + + +def resolve_config(args): + """CLI > project JSON > built-ins; retain field-level provenance.""" + config = deepcopy(DEFAULT_CONFIG) + sources = {key: "default" for key in config} + path = args.config or (CONFIG_NAME if CONFIG_NAME.is_file() else None) + if path is not None: + supplied = _object(path) + if supplied.keys() - config.keys(): + raise ValueError("Unknown comparison configuration fields") + for key, value in supplied.items(): + if key in ("policy", "task_arguments"): + if not isinstance(value, dict): + raise ValueError(f"{key} must be an object") + if key == "policy" and isinstance(value.get("tolerances"), dict): + value = {**value, "tolerances": { + **config[key]["tolerances"], **value["tolerances"]}} + config[key].update(value) + else: + config[key] = value + sources[key] = "config" + for key in config: + value = getattr(args, "task" if key == "tasks" else key, None) + if value is not None: + config[key] = value + sources[key] = "cli" + for override in args.evaluation_args or []: + task, sep, arguments = override.partition("=") + if not sep or task not in config["tasks"]: + raise ValueError("--evaluation-args requires selected TASK=ARGUMENTS") + config["task_arguments"][task] = arguments + sources["task_arguments"] = "cli" + _validate_config(config) + return config, sources + + +def _validate_config(config): + if type(config["schema_version"]) is not int or config["schema_version"] != 1: + raise ValueError("Unsupported comparison configuration version") + for key in ("rounds", "repeats"): + if type(config[key]) is not int or config[key] < 1: + raise ValueError(f"{key} must be a positive integer") + for key in ("max_minutes", "evaluation_timeout_seconds", "final_reserve_seconds", + "job_max_minutes"): + if (type(config[key]) not in (int, float) or not math.isfinite(config[key]) + or config[key] <= 0): + raise ValueError(f"{key} must be finite and positive") + if not config["evaluation_timeout_seconds"] <= config["final_reserve_seconds"] < ( + config["max_minutes"] * 60 + ): + raise ValueError("Reserve at least one evaluation timeout within the campaign budget") + if config["job_max_minutes"] <= config["max_minutes"]: + raise ValueError("Agent job budget must exceed campaign budget for handoff and collection") + tasks = config["tasks"] + if (not isinstance(tasks, list) or not tasks or any(not isinstance(t, str) for t in tasks) + or len(set(tasks)) != len(tasks)): + raise ValueError("tasks must be a nonempty list of unique registered task names") + for task in tasks: + root = Path(str(task_resources(task))) + if not (root / "optimization.py").is_file(): + raise ValueError(f"Task has no iterative protocol: {task}") + if not isinstance(config["task_arguments"].get(task), str): + raise ValueError(f"Missing task arguments: {task}") + for key in ("model", "effort", "environment_id"): + if config[key] is not None and (not isinstance(config[key], str) + or not config[key].strip()): + raise ValueError(f"{key} must be nonempty or null") + for key in ("evaluation_prefix", "execution_instructions", "python"): + if not isinstance(config[key], str) or len(config[key]) > 16384: + raise ValueError(f"{key} must be a bounded string") + if not config["python"].strip(): + raise ValueError("python must name the prepared target interpreter") + # Only the task-owned device count can be interpolated, never a shell command. + shlex.split(config["evaluation_prefix"].replace("{devices}", "1")) + if "{" in config["evaluation_prefix"].replace("{devices}", ""): + raise ValueError("Only {devices} is supported in evaluation_prefix") + config["policy"] = validate_policy(config["policy"]) + + +def _manifest(root): + root = _io_path(root) + paths = sorted(root.rglob("*")) + if root.is_symlink() or any(path.is_symlink() for path in paths): + raise ValueError("Comparison resources cannot contain symbolic links") + return {path.relative_to(root).as_posix(): sha256(path) + for path in paths if path.is_file() + and "__pycache__" not in path.parts and path.suffix != ".pyc"} + + +def _copy_runtime(source, destination, condition): + source, destination = _io_path(source), _io_path(destination) + def ignored(directory, names): + result = [name for name in names if name == "__pycache__" or name.endswith(".pyc")] + if Path(directory) == source / "knowledge" and condition == "off": + result.append("data") + return result + shutil.copytree(source, destination, ignore=ignored) + + +def _prefix(config, resources): + manifest = _object(resources / "task.json") + devices = manifest.get("devices", manifest.get("world_size", 2)) + if type(devices) is not int or devices < 1: + raise ValueError("Task must declare its positive device count") + return config["evaluation_prefix"].replace("{devices}", str(devices)) + + +def prepare(args): + config, sources = resolve_config(args) + if not config["environment_id"]: + return {"state": "configuration_required", "resolved_config": config, + "parameter_sources": sources, + "next_action": { + "operation": "select_prepared_execution_environment", + "instructions": "Select an existing authorized environment satisfying the " + "task manifests. Supply --environment-id, --evaluation-prefix and optional " + "--execution-instructions or project .inferrsi/benchmarks.json. Inspect task " + "requirements with benchmarks show. Credentials stay outside configuration.", + }} + root = (args.output or Path("outputs") / ( + "knowledge-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + "-" + uuid.uuid4().hex[:8] + )).resolve() + if root.exists(): + raise ValueError("Comparison output must be new; use --action status to resume") + runtime = Path(str(files("inferrsi"))).resolve() + if root.is_relative_to(runtime) or any(p.is_symlink() for p in runtime.rglob("*")): + raise ValueError("Runtime must contain regular files and output must be outside it") + task_settings = {} + for name in config["tasks"]: + resource = Path(str(task_resources(name))) + settings = protocol_operation(resource, sha256(resource / "optimization.py"), + "validate_arguments", + arguments=shlex.split(config["task_arguments"][name])) + task_settings[name] = {"settings": settings, "files": _manifest(resource), + "evaluation_prefix": _prefix(config, resource)} + root.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=".compare-", dir=_io_path(root.parent)) as temporary: + staging = Path(temporary) / "experiment" + staging.mkdir() + _prepare_at(config, sources, runtime, task_settings, staging) + _publish_directory(staging, _io_path(root)) + return status(root) + + +def _prepare_at(config, sources, runtime, task_settings, root): + runtime_files = _manifest(runtime) + knowledge_files = {key: value for key, value in runtime_files.items() + if key.startswith("knowledge/data/")} + trials = [] + for task in config["tasks"]: + for repeat in range(1, config["repeats"] + 1): + for condition in (("on", "off") if repeat % 2 else ("off", "on")): + trial_id = f"{task}-r{repeat:03d}-{condition}" + workspace = root / "trials" / f"{len(trials) + 1:04d}" + workspace.mkdir(parents=True) + _copy_runtime(runtime, workspace / "runtime/inferrsi", condition) + trials.append({"id": trial_id, "task": task, "repeat": repeat, + "condition": condition, + "workspace": workspace.relative_to(root).as_posix()}) + plan = {"schema_version": 1, "focus": "knowledge", "config": config, + "parameter_sources": sources, "task_settings": task_settings, + "runtime_files": {key: value for key, value in runtime_files.items() + if key not in knowledge_files}, + "session_runtime_identity": _runtime_identity(), + "knowledge_files": knowledge_files, + "trials": trials, + "knowledge_boundary": "isolated runtime corpus and session gate; native host " + "filesystem/network access follows identical task instructions, not an OS sandbox"} + atomic_write_json(root / "experiment.json", plan) + atomic_write_json(root / "state.json", {"schema_version": 1, "plan_digest": digest(plan), + "collections": {}}) + + +def _load(root): + plan, state = _object(root / "experiment.json"), _object(root / "state.json") + if plan.get("schema_version") != 1 or state.get("plan_digest") != digest(plan): + raise ValueError("Frozen comparison plan changed") + for trial in plan["trials"]: + runtime = root / trial["workspace"] / "runtime/inferrsi" + expected = dict(plan["runtime_files"]) + if trial["condition"] == "on": + expected.update(plan["knowledge_files"]) + if _manifest(runtime) != expected: + raise ValueError(f"Frozen comparison runtime changed: {trial['id']}") + return plan, state + + +def _job(root, trial): + return root / "jobs" / trial["id"] + + +def _select(plan, trial_id): + found = [trial for trial in plan["trials"] if trial["id"] == trial_id] + if len(found) != 1: + raise ValueError("Select a declared --trial ID from comparison status") + return found[0] + + +def _prompt(root, plan, trial): + config = plan["config"] + workspace = root / trial["workspace"] + runtime = _io_path(workspace / "runtime") + campaign = workspace / "campaign" + start = [config["python"], "-B", "-m", "inferrsi", "benchmarks", "session", "start", + trial["task"], "--session", str(campaign), "--rounds", str(config["rounds"]), + "--max-minutes", str(config["max_minutes"]), + "--evaluation-timeout-seconds", str(config["evaluation_timeout_seconds"]), + "--final-reserve-seconds", str(config["final_reserve_seconds"]), + "--condition", trial["id"], "--knowledge", trial["condition"], + "--evaluation-prefix", plan["task_settings"][trial["task"]]["evaluation_prefix"], + "--evaluation-args", config["task_arguments"][trial["task"]], + "--python", config["python"]] + return ( + f"Run complete RSI optimization: {trial['task']}, condition {trial['condition']}.\n" + f"Work only in {workspace}. Set PYTHONPATH to {runtime}; disable bytecode. " + "Do not import another installed InferRSI or parent checkout. Use this fresh conversation; " + "do not inspect other trials, prior solutions, the initiating conversation or reports. " + "Do not delegate or create another conversation.\n" + f"Prepared execution environment: {config['environment_id']}.\n" + f"{config['execution_instructions']}\n" + "Preflight before search; retain Python, device model, framework/runtime versions " + "and allocator profile in environment-observation.json. Preserve shared dependencies. " + "Record actual model/effort from the host observation when available; " + "requested settings alone do not establish actual settings.\n" + "The permitted optimization knowledge source is this runtime's session knowledge operation " + "when enabled. Other skill libraries, stores, external knowledge tools, web research " + "and prior solutions are excluded in both. Public task/command operating instructions " + "and target library APIs remain available to both. The off runtime omits the corpus. " + "Report any observed policy violation; do not claim an OS sandbox.\n" + f"Start argv (a JSON array, execute without a shell): {json.dumps(start)}\n" + "For remote execution, copy this exact runtime to the prepared target. Use a private " + "campaign directory there; return its entire unchanged tree to workspace/campaign above. " + "Preserve bytes and relative paths. Exclude other trials and runtimes.\n" + "Read frozen TASK.md. Analyze feedback and choose a justified optimization each round. " + "Submit candidate, --strategy and --hypothesis; cite delivered --knowledge-ref paths when " + "used and retain failures. When enabled, retrieve/read relevant material with session " + "knowledge. Complete frozen rounds and independently retest the selected candidate " + "using session final. Never replace final with a search score.\n" + "Return campaign, selected code, final delivery and environment observation. Report " + "blocked/failed execution honestly. The initiator collects host receipts and actual usage." + ) + + +def status(root): + return _status(root, *_load(root)) + + +def _status(root, plan, state): + rows, active = [], None + for trial in plan["trials"]: + job = _job(root, trial) + view = agent_sessions.status(job) if job.exists() else None + collection = state["collections"].get(trial["id"]) + row = {**trial, "job": str(job), "campaign": str(root / trial["workspace"] / "campaign"), + "state": view["state"] if view else "pending", "collection": collection, + "host": view} + rows.append(row) + if active is None and collection is None: + active = row + action = {"operation": "report", "instructions": "All declared trials were collected; " + "generate the comparison report, including failures."} + published = state.get("report") + if (active is None and published + and published["collection_digest"] == digest(state["collections"])): + action = {"operation": "completed", "reports": published["paths"], + "instructions": "Read the generated comparison and report its scoped conclusion. " + "Use --action report to reverify evidence or regenerate a snapshot."} + if active is not None: + if active["host"] is None: + action = {"operation": "dispatch", "trial": active["id"], + "instructions": "Reserve one handoff; call the host's fresh-session tool " + "with its exact prompt and workspace. Bind the returned native ID."} + elif active["state"] in agent_sessions.TERMINAL: + action = {"operation": "collect", "trial": active["id"], + "instructions": "Retain actual host metadata and campaign tree; collect. " + "Host completion is separate from independent final acceptance."} + else: + action = {"operation": active["host"]["next_action"], "trial": active["id"], + "native_session": active["host"]["native_session"], + "instructions": "Observe or resolve this original session before dispatching " + "another. Export observations with agents record or compare --action record.", + "observation_fields": { + "host": "bound host", "host_id": "bound host ID or null", + "session_id": "bound session ID", "state": "actual native state", + "native_receipt": "original host tool response object", + "benchmark_observation": { + "model": "actually observed model", "effort": "observed effort", + "environment_id": plan["config"]["environment_id"], + "environment_fingerprint": "digest of observed non-private runtime " + "versions, device model and allocator profile", + "knowledge_policy_violation": "false only when no violation observed", + }, + "usage": "optional actual cumulative input/output/cached_input tokens", + "usage_scope": "session_total when usage is present", + }} + return {"output": str(root), "focus": plan["focus"], "config": plan["config"], + "parameter_sources": plan["parameter_sources"], "trials": rows, + "state": "collected" if active is None else "running", "next_action": action} + + +def dispatch(root, trial_id, *, parallel=False): + plan, state = _load(root) + trial = _select(plan, trial_id) + job = _job(root, trial) + if not job.exists(): + if not parallel: + current = _status(root, plan, state)["next_action"] + if current.get("trial") != trial_id or current["operation"] != "dispatch": + raise ValueError("Dispatch the next declared trial or pass --parallel") + job.parent.mkdir(exist_ok=True) + agent_sessions.prepare( + job, prompt=_prompt(root, plan, trial), workspace=root / trial["workspace"], + title=trial_id, max_seconds=plan["config"]["job_max_minutes"] * 60, + model=plan["config"]["model"], effort=plan["config"]["effort"], + task_reference={"comparison": digest(plan), "trial": trial_id}, + frozen_inputs=(root / "experiment.json",), + ) + return agent_sessions.dispatch(job) + + +def _trial_result(root, plan, trial, host): + result = {"id": trial["id"], "task": trial["task"], "repeat": trial["repeat"], + "condition": trial["condition"], "status": "blocked", "eligible": False, + "gaps": [], "metrics": {"speedup": None, "wall_seconds": None, "tokens": None}, + "trace": [], "final": None, "usage": {}, "knowledge_deliveries": [], + "observed_settings": None} + campaign = root / trial["workspace"] / "campaign" + if host is None or host["state"] not in agent_sessions.TERMINAL: + result["status"] = "pending" if host is None else "running" + result["gaps"].append("Native execution has not terminated") + return result + result["status"] = "failed" if host["state"] != "completed" else "blocked" + receipt = host["last_observation"] + if receipt: + payload = _object(_job(root, trial) / receipt["path"]) + observed = payload.get("benchmark_observation") + if isinstance(observed, dict): + result["observed_settings"] = observed + observed = result["observed_settings"] or {} + if any(not isinstance(observed.get(key), str) or not observed[key].strip() + for key in OBSERVED_FIELDS): + result["gaps"].append("Actual host model, effort or environment observation is missing") + if observed.get("environment_id") != plan["config"]["environment_id"]: + result["gaps"].append("Observed execution environment differs from the frozen profile") + for key in ("model", "effort"): + if plan["config"][key] and observed.get(key) != plan["config"][key]: + result["gaps"].append(f"Observed {key} differs from the requested setting") + if observed.get("knowledge_policy_violation") is not False: + result["gaps"].append("Knowledge-policy observation is missing or reports a violation") + if host["usage"] is not None and host["state"] == "completed": + result["usage"]["host_session_tokens"] = {"value": host["usage"], + "coverage": "complete", "source": "caller_exported_terminal_native_receipt"} + result["metrics"]["tokens"] = host["usage"]["input_tokens"] + host["usage"]["output_tokens"] + if not campaign.is_dir(): + result["gaps"].append("Campaign evidence was not returned") + return result + try: + snapshot, delivery, _ = project(campaign, export_id="comparison-read") + campaign_plan, control, _ = optimization.read_records(campaign) + settings = plan["task_settings"][trial["task"]] + wanted = {"task": trial["task"], "condition": trial["id"], + "knowledge_allowed": trial["condition"] == "on", + "round_limit": plan["config"]["rounds"], "round_mode": "exact", + "max_seconds": plan["config"]["max_minutes"] * 60, + "evaluation_timeout_seconds": plan["config"]["evaluation_timeout_seconds"], + "final_reserve_seconds": plan["config"]["final_reserve_seconds"], + "evaluation_settings": settings["settings"], + "evaluation_prefix": shlex.split(settings["evaluation_prefix"]), + "frozen_files": settings["files"]} + if any(campaign_plan.get(key) != value for key, value in wanted.items()): + result["gaps"].append("Campaign task, condition, protocol or budget differs from plan") + if campaign_plan.get("runtime_identity") != plan["session_runtime_identity"]: + result["gaps"].append("Campaign runtime identity differs from the frozen runtime") + result["knowledge_deliveries"] = control["contexts"] + if trial["condition"] == "off" and control["contexts"]: + result["gaps"].append("Knowledge was delivered in the off condition") + result["usage"].update(delivery["usage"]) + if delivery["acceptance"]["delivery_accepted"] and control["final"]: + final_assessment = control["final"]["assessment"] + result["metrics"]["speedup"] = ( + 1.0 if control["best"]["kind"] == "baseline" + else (final_assessment.get("speedup") + if final_assessment.get("speedup") is not None + else final_assessment.get("score")) + ) + result["metrics"]["wall_seconds"] = delivery["usage"]["wall_seconds"]["value"] + result["final"] = {"assessment": control["final"]["assessment"] + if control["final"] else None, + "speedup": result["metrics"]["speedup"], + "binding": (delivery["final"] or {}).get("binding"), + "selected_candidate": control["best"], + "delivery_accepted": delivery["acceptance"]["delivery_accepted"]} + for index, record in enumerate(control["rounds"], 1): + assessment = record["assessment"] + result["trace"].append({ + "round": index, "strategy": record.get("strategy") or record["hypothesis"], + "hypothesis": record["hypothesis"], + "knowledge_refs": record.get("knowledge_refs"), + "available_contexts": record["contexts"], + "knowledge_attribution": "agent_declared; delivery is not proof of use", + "speedup": assessment.get("speedup", assessment.get("score")), + "best_speedup": record["best_after"]["score"], "quality": assessment["verdict"], + "adopted": record["best_after"].get("candidate_id") == record["candidate_id"], + }) + if (host["state"] != "completed" or not delivery["acceptance"]["delivery_accepted"] + or delivery["completion"]["status"] != "completed"): + result["gaps"].append("A completed, verified independent final is required") + if not snapshot.stable(): + result["gaps"].append("Campaign changed while collecting") + result["status"] = "completed" if control["state"] == "completed" else "blocked" + except (OSError, ValueError, TypeError, KeyError) as error: + result["gaps"].append(f"Campaign evidence could not be verified: {type(error).__name__}") + result["eligible"] = not result["gaps"] + return result + + +def collect(root, trial_id): + plan, state = _load(root) + trial = _select(plan, trial_id) + job = _job(root, trial) + host = agent_sessions.status(job) if job.exists() else None + if host is None or host["state"] not in agent_sessions.TERMINAL: + raise ValueError("Collect only after recording a terminal native host observation") + result = _trial_result(root, plan, trial, host) + directory = root / "collections" + directory.mkdir(exist_ok=True) + path = directory / (trial_id + "-" + uuid.uuid4().hex + ".json") + atomic_write_json(path, result) + state["collections"][trial_id] = {"path": path.relative_to(root).as_posix(), + "sha256": sha256(path)} + atomic_write_json(root / "state.json", state) + return {"trial": result, "next": _status(root, plan, state)["next_action"]} + + +def report(root): + plan, state = _load(root) + trials = [] + for trial in plan["trials"]: + reference = state["collections"].get(trial["id"]) + if reference and sha256(root / reference["path"]) != reference["sha256"]: + raise ValueError("Retained comparison collection changed") + job = _job(root, trial) + host = agent_sessions.status(job) if job.exists() else None + # Reverify current evidence; a stale collection cannot hide later evidence loss. + trials.append(_trial_result(root, plan, trial, host)) + for task in plan["config"]["tasks"]: + for repeat in range(1, plan["config"]["repeats"] + 1): + pair = [t for t in trials if t["task"] == task and t["repeat"] == repeat] + if (all(t["eligible"] for t in pair) and len(pair) == 2 + and any(pair[0]["observed_settings"].get(key) + != pair[1]["observed_settings"].get(key) for key in OBSERVED_FIELDS)): + for trial in pair: + trial["eligible"] = False + trial["gaps"].append("Paired actual Agent/environment settings differ") + result = {"schema_version": 1, "focus": "knowledge", "tasks": plan["config"]["tasks"], + "repeats": plan["config"]["repeats"], "policy": plan["config"]["policy"], + "knowledge_boundary": plan["knowledge_boundary"], "plan_digest": digest(plan), + "trials": trials, "assessment": summarize(trials, plan["config"]["tasks"], + plan["config"]["repeats"], plan["config"]["policy"])} + destination = root / "reports" / uuid.uuid4().hex + paths = write_comparison_report(result, destination) + state["report"] = {"collection_digest": digest(state["collections"]), "paths": paths} + atomic_write_json(root / "state.json", state) + return {"assessment": result["assessment"], "reports": paths, + "next_action": _status(root, plan, state)["next_action"]} + + +def compare_command(args): + if args.action == "prepare": + return prepare(args) + if args.output is None: + raise ValueError("Existing comparison operations require --output") + root = args.output.resolve() + with file_lock(root / ".compare.lock"): + if args.action == "status": + return status(root) + if args.action == "report": + return report(root) + if args.action == "dispatch": + return dispatch(root, args.trial, parallel=args.parallel) + if args.action == "collect": + return collect(root, args.trial) + plan, _ = _load(root) + trial = _select(plan, args.trial) + job = _job(root, trial) + if args.action == "bind": + if not args.host or not args.session_id or args.receipt is None: + raise ValueError("bind requires --host, --session-id and --receipt") + return agent_sessions.bind(job, host=args.host, session_id=args.session_id, + host_id=args.host_id, receipt=args.receipt) + if args.observation is None: + raise ValueError("record requires --observation") + return agent_sessions.record(job, observation=args.observation) diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a..6692ce44 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -350,6 +350,11 @@ def main(argv: list[str] | None = None) -> int: _print(result) return 0 if args.command == "benchmarks": + if args.benchmark_action == "compare": + from ..benchmark_compare import compare_command + + _print(compare_command(args)) + return 0 if args.benchmark_action == "session": from ...harness.optimization_session import session_command from ..delivery import after_final diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 0b0c5da1..b7f8a859 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -89,6 +89,9 @@ def _optimization_actions(stages, *, project=False): if name == "round": stage.add_argument("--candidate", type=Path, required=True) stage.add_argument("--hypothesis", required=True) + stage.add_argument("--strategy", help="Agent-declared optimization strategy") + stage.add_argument("--knowledge-ref", action="append", + help="Delivered context path cited by this round; repeatable") if name == "final": stage.add_argument("--reason", help="Why the caller stopped an up-to-budget search") if name in ("round", "final"): @@ -182,6 +185,42 @@ def parser() -> argparse.ArgumentParser: benchmarks = commands.add_parser("benchmarks", help="Discover packaged engineering tasks") actions = benchmarks.add_subparsers(dest="benchmark_action", required=True) actions.add_parser("list") + compare = actions.add_parser( + "compare", help="Compare complete RSI optimization sessions (default: knowledge on/off)", + description=("Apps coordinates paired fresh-Agent optimizations and reports. Knowledge " + "defaults to qwen21-denoise-npu and moe-forward-npu, each on/off. " + "CLI values override .inferrsi/benchmarks.json, then built-ins. " + "Python records handoffs; the initiating Agent calls native host tools " + "using returned next_action/handoff data. Complete all declared rounds " + "and independent finals, record actual host observations, collect every " + "attempt, then report. No dedicated skill is required."), + ) + compare.add_argument("--focus", choices=("knowledge",), default="knowledge") + compare.add_argument("--action", choices=("prepare", "status", "dispatch", "bind", "record", + "collect", "report"), default="prepare", + help="Prepare a new experiment; use its --output for later operations") + compare.add_argument("--config", type=Path, help="Non-secret project configuration JSON") + compare.add_argument("--output", type=Path, + help="New comparison directory, or existing to resume") + compare.add_argument("--task", action="append", help="Override default task set; repeatable") + for name in ("rounds", "repeats"): + compare.add_argument("--" + name, type=int) + for name in ("max-minutes", "evaluation-timeout-seconds", "final-reserve-seconds", + "job-max-minutes"): + compare.add_argument("--" + name, type=float) + for name in ("model", "effort", "environment-id", "python"): + compare.add_argument("--" + name) + compare.add_argument("--evaluation-prefix", help="Allocator argv; task supplies {devices}") + compare.add_argument("--execution-instructions", help="Prepared target/transfer instructions") + compare.add_argument("--evaluation-args", action="append", help="Override TASK=ARGUMENTS") + compare.add_argument("--trial", help="Declared trial ID from status/next_action") + compare.add_argument("--parallel", action="store_true", + help="Allow dispatch while other declared trials are active") + compare.add_argument("--host") + compare.add_argument("--host-id") + compare.add_argument("--session-id") + compare.add_argument("--receipt", type=Path, help="Original host creation receipt for bind") + compare.add_argument("--observation", type=Path, help="Native host observation JSON") show = actions.add_parser("show") show.add_argument("task", help="Packaged engineering task name") run = actions.add_parser("run", help="Run the selected task's benchmark.py") diff --git a/src/inferrsi/apps/delivery/projection.py b/src/inferrsi/apps/delivery/projection.py index 2774c89b..b9e01c6c 100644 --- a/src/inferrsi/apps/delivery/projection.py +++ b/src/inferrsi/apps/delivery/projection.py @@ -385,6 +385,9 @@ def iterative(snapshot, result): "attempt_id": directory.rsplit("/", 1)[-1], "candidate_fingerprint": record.get("candidate_sha256"), "hypothesis": record.get("hypothesis"), + "strategy": record.get("strategy"), + "knowledge_refs": record.get("knowledge_refs"), + "knowledge_attribution": "agent_declared", "change_summary": record.get("change_summary"), "execution_state": historical.get("execution", {}).get("status"), "assessment_ref": assessment_ref, diff --git a/src/inferrsi/apps/views/comparison_report.py b/src/inferrsi/apps/views/comparison_report.py new file mode 100644 index 00000000..ef7807fc --- /dev/null +++ b/src/inferrsi/apps/views/comparison_report.py @@ -0,0 +1,363 @@ +"""Offline comparison presentation over Apps' normalized, retained result projection.""" + +import hashlib +import json +import math +import os +import tempfile +from html import escape +from pathlib import Path + +_LABELS = { + "efficiency_improved": "Efficiency improved / 效率提高", + "no_gain": "No observed gain / 未观察到收益", + "regression": "Regression / 效率退化", + "tradeoff": "Tradeoff / 存在权衡", + "insufficient_evidence": "Insufficient evidence / 证据不足", +} +_TRACE_HEADERS = ( + "Round", "Strategy", "Hypothesis", "Knowledge references", "Round speedup", + "Best speedup", "Correctness", "Adopted", +) +_METRICS = { + "speedup": "Final speedup", + "wall_seconds": "Campaign time (s)", + "tokens": "Session tokens", +} + + +def _text(value): + if value is None: + return "unknown" + if type(value) is bool: + return "yes" if value else "no" + if type(value) is float: + return f"{value:.6g}" + if type(value) is int: + return f"{value:,}" + return str(value) + + +def _md(value): + value = escape(_text(value), quote=False) + for char in "\\`*_{}[]()#+!|": + value = value.replace(char, "\\" + char) + return value.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "
") + + +def _markdown_table(headers, rows): + return ["| " + " | ".join(_md(value) for value in headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + *["| " + " | ".join(_md(value) for value in row) + " |" for row in rows]] + + +def _html_table(headers, rows): + return ("
" + + "".join(f"" for value in headers) + + "" + + "".join("" + "".join(f"" for value in row) + + "" for row in rows) + "
{escape(_text(value))}
{escape(_text(value))}
") + + +def _task_rows(assessment): + rows = [] + for task in assessment["tasks"]: + for name, label in _METRICS.items(): + metric = task["metrics"][name] + rows.append((task["task"], label, metric["selected"], + _metric_value(name, metric["on_median"]), + _metric_value(name, metric["off_median"]), + _metric_value("speedup", metric["paired_factor"]), metric["pair_count"], + _LABELS[task["conclusion"]["kind"]])) + return rows + + +def _metric_value(name, value): + if value is None: + return None + if name == "speedup": + return f"{value:.3f}×" + if name == "wall_seconds": + return f"{value:,.2f}" + return f"{value:,}" if type(value) is int else f"{value:,.1f}".removesuffix(".0") + + +def _trace_rows(trial): + rows = [] + for row in trial.get("trace", []): + refs = row.get("knowledge_refs") + citations = None if refs is None else "\n".join(refs) or "None" + rows.append((row.get("round"), row.get("strategy"), row.get("hypothesis"), + citations, _metric_value("speedup", row.get("speedup")), + _metric_value("speedup", row.get("best_speedup")), + row.get("quality"), row.get("adopted"))) + return rows + + +def _final_rows(final): + assessment = final.get("assessment") or final + binding = final.get("binding") or {} + candidate = final.get("selected_candidate") or {} + rows = [ + ("Status", assessment.get("status")), + ("Quality verdict", assessment.get("verdict")), + ("Final speedup", _metric_value("speedup", final.get("speedup"))), + ("Selected candidate", candidate.get("candidate_id") or candidate.get("kind")), + ("Candidate binding verified", binding.get("verified")), + ("Delivery accepted", final.get("delivery_accepted")), + ] + if assessment.get("reason"): + rows.append(("Reason", assessment["reason"])) + return rows + + +def _case_rows(final): + rows = [] + for case in (final.get("assessment") or {}).get("case_metrics", []): + samples = case.get("samples_ms") or {} + rows.append((case.get("id"), case.get("baseline_median_ms"), + case.get("candidate_median_ms"), _metric_value("speedup", case.get("speedup")), + *(len(samples[side]) if side in samples else None + for side in ("baseline", "candidate")))) + return rows + + +def _token_rows(trial): + rows = [] + for key, label in (("host_session_tokens", "Agent session"), ("session_tokens", "Campaign"), + ("tokens", "Settled search rounds")): + record = trial.get("usage", {}).get(key, {}) + value = record.get("value") or {} + total = (trial["metrics"].get("tokens") if key == "host_session_tokens" + else value.get("total_tokens")) + rows.append((label, value.get("input_tokens"), value.get("output_tokens"), + value.get("cached_input_tokens"), total, record.get("coverage"))) + return rows + + +def _positive(value): + try: + return type(value) in (int, float) and math.isfinite(value) and value > 0 + except OverflowError: + return False + + +def _curve(trials): + """Keep each actual trial separate; missing points break its recorded curve.""" + if sum(len(trial.get("trace", [])) for trial in trials) > 1000: + return "

Curve omitted for size; all recorded rounds remain in the tables.

" + values = [row["best_speedup"] for trial in trials for row in trial.get("trace", []) + if _positive(row.get("best_speedup"))] + values += [trial["final"]["speedup"] for trial in trials + if isinstance(trial.get("final"), dict) + and _positive(trial["final"].get("speedup"))] + rounds = [row["round"] for trial in trials for row in trial.get("trace", []) + if type(row.get("round")) is int and row["round"] > 0] + if not values or not rounds: + return "

No recorded speedup curve is available.

" + low, high, last = min(values), max(values), max(rounds) + scale = max(high, 1) + lower, upper = low / scale, high / scale + span = upper - lower + if not span: + lower -= 0.05 + span = 0.1 + + def xy(round_number, value): + return (70 + 720 * round_number / (last + 1), + 45 + 210 * (1 - (value / scale - lower) / span)) + + chunks = ['', + '', + f'{high:.4g}x', + f'{low:.4g}x', + 'Recorded rounds; diamond = independent final'] + colors = ("#2563eb", "#16a34a", "#9333ea", "#ea580c") + for index, trial in enumerate(trials): + color, previous = colors[index % len(colors)], None + label = _text(trial.get("id")) + for row in trial.get("trace", []): + position, value = row.get("round"), row.get("best_speedup") + if type(position) is not int or position <= 0 or not _positive(value): + previous = None + continue + x, y = xy(position, value) + if previous and previous[0] < x: + chunks.append(f'') + chunks.append(f'' + f'{escape(label)}: round {position}, ' + f'{value:.6g}x') + previous = x, y + final = trial.get("final") + if isinstance(final, dict) and _positive(final.get("speedup")): + value = final["speedup"] + x, y = xy(last + 1, value) + chunks.append(f'{escape(label)}: independent final, ' + f'{value:.6g}x') + chunks.append("") + return "".join(chunks) + + +def _documents(report, json_digest): + assessment = report["assessment"] + conclusion, counts = assessment["conclusion"], assessment["counts"] + label = _LABELS[conclusion["kind"]] + repeats = report["repeats"] + repeat_label = "repeat" if repeats == 1 else "repeats" + scope = ", ".join(report["tasks"]) + f" · {repeats} {repeat_label} per task" + md = ["# Knowledge comparison", "", "**" + label + "**", "", + _md(conclusion["text"]), "", _md(conclusion["limitation"]), "", _md(scope), ""] + html = ["

Knowledge comparison

", f"

{escape(label)}

", + f"

{escape(conclusion['text'])}

", + f"

{escape(conclusion['limitation'])}

", f"

{escape(scope)}

"] + + def heading(title, level=2): + md.extend(["#" * level + " " + _md(title), ""]) + html.append(f"{escape(title)}") + + def paragraph(value): + md.extend([_md(value), ""]) + html.append(f"

{escape(value)}

") + + def table(headers, rows): + md.extend([*_markdown_table(headers, rows), ""]) + html.append(_html_table(headers, rows)) + + heading("Task comparisons") + table(("Task", "Metric", "Used in verdict", "On median", "Off median", + "Paired benefit factor", "Valid pairs", "Task conclusion"), _task_rows(assessment)) + paragraph("A benefit factor above 1 favors knowledge on. Speedup uses on/off, while time " + "and tokens use off/on. Medians use eligible matched pairs.") + table(("Completed trials", "Eligible trials", "Eligible pairs", "Running", "Pending", + "Failed", "Blocked", "Missing"), + [(f"{counts['statuses']['completed']} / {counts['planned_trials']}", + counts["eligible_trials"], f"{counts['eligible_pairs']} / {counts['planned_pairs']}", + *(counts["statuses"][state] for state in ("running", "pending", "failed", "blocked")), + counts["missing_trials"])]) + + gaps = [(row["task"], gap) for row in assessment["tasks"] for gap in row["gaps"]] + if gaps: + heading("Evidence gaps") + table(("Task", "Gap"), gaps) + + heading("Optimization trace") + html.append(_curve(report["trials"])) + paragraph("Strategies and hypotheses are recorded by the agent. Knowledge references show " + "which material was supplied. Actual model consumption is unverified.") + for index, trial in enumerate(report["trials"], 1): + html.append(f'
') + heading(f"{trial['task']} · {trial['condition'].upper()} · repeat {trial['repeat']}", 3) + status = trial["status"] + eligibility = "eligible for comparison" if trial["eligible"] else "excluded from comparison" + paragraph(f"{status.capitalize()} · {eligibility}") + rows = _trace_rows(trial) + if rows: + table(_TRACE_HEADERS, rows) + else: + paragraph("No rounds recorded.") + heading("Independent final", 4) + final = trial.get("final") + if final and (final.get("assessment") or final.get("verdict")): + fields = _final_rows(final) + table([label for label, _ in fields], [[value for _, value in fields]]) + cases = _case_rows(final) + if cases: + table(("Case", "Baseline median (ms)", "Candidate median (ms)", "Speedup", + "Baseline samples", "Candidate samples"), cases) + else: + paragraph("No independent final recorded.") + if trial.get("gaps"): + table(("Trial evidence gap",), [(gap,) for gap in trial["gaps"]]) + + html.append("
Usage and evidence") + md.extend(["#### Usage and evidence", ""]) + table(("Token scope", "Input", "Output", "Cached input", "Total", "Coverage"), + _token_rows(trial)) + observed = trial.get("observed_settings") or {} + candidate = (final or {}).get("selected_candidate") or {} + evidence = [ + ("Trial ID", trial["id"]), + ("Model", observed.get("model")), + ("Reasoning effort", observed.get("effort")), + ("Environment", observed.get("environment_id")), + ("Candidate path", candidate.get("path")), + ] + if isinstance(trial.get("evidence"), str): + evidence.append(("Evidence reference", trial["evidence"])) + table(("Evidence field", "Value"), evidence) + deliveries = trial.get("knowledge_deliveries") or [] + if deliveries: + table(("Delivered knowledge",), [ + (entry.get("path") if isinstance(entry, dict) else entry,) for entry in deliveries + ]) + html.append("
") + + heading("Comparison settings and source data") + policy = report["policy"] + table(("Metric", "Used in verdict", "Neutral tolerance"), + [(label, name in policy["metrics"], f"{policy['tolerances'][name]:.3g}") + for name, label in _METRICS.items()]) + paragraph(f"Minimum eligible pairs per task: {policy['minimum_pairs']}. " + "Aggregation uses the median of paired log ratios.") + md += ["[Download raw comparison JSON](comparison-result.json)", "", + "JSON SHA-256: `" + json_digest + "`.", ""] + html += ['

Download raw comparison JSON · ' + 'Markdown report

', + f"

JSON SHA-256: {json_digest}

"] + document = ('' + '' + '' + 'Knowledge comparison
' + "".join(html) + '
') + return "\n".join(md), document + + +def write_comparison_report(report: dict, directory: Path) -> dict: + """Write JSON and escaped Markdown/HTML projections of the same derived snapshot. + + Regeneration replaces only these three known derived files. Both human + documents identify the exact JSON digest; no source experiment is modified. + """ + if (not isinstance(report, dict) or type(report.get("schema_version")) is not int + or report["schema_version"] != 1 or report.get("focus") != "knowledge"): + raise ValueError("Expected a version 1 knowledge comparison report") + raw = (json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False) + + "\n").encode("utf-8") + markdown, html = _documents(report, hashlib.sha256(raw).hexdigest()) + directory = Path(directory).absolute() + if any(parent.is_symlink() for parent in (directory, *directory.parents)): + raise ValueError("Comparison report output cannot traverse symbolic links") + directory.mkdir(parents=True, exist_ok=True) + outputs = {"json": ("comparison-result.json", raw), + "markdown": ("report.md", markdown.encode("utf-8")), + "html": ("report.html", html.encode("utf-8"))} + for name, _ in outputs.values(): + destination = directory / name + if destination.is_symlink() or destination.exists() and not destination.is_file(): + raise ValueError("Comparison report destination must be a regular file") + for name, data in outputs.values(): + temporary = None + try: + with tempfile.NamedTemporaryFile( + dir=directory, prefix=".comparison-", delete=False, + ) as out: + temporary = Path(out.name) + out.write(data) + os.replace(temporary, directory / name) + finally: + if temporary is not None and temporary.exists(): + temporary.unlink() + return {key: str(directory / name) for key, (name, _) in outputs.items()} diff --git a/src/inferrsi/benchmark/comparison.py b/src/inferrsi/benchmark/comparison.py new file mode 100644 index 00000000..891496de --- /dev/null +++ b/src/inferrsi/benchmark/comparison.py @@ -0,0 +1,231 @@ +"""Knowledge-comparison rules over normalized retained facts; no execution or I/O.""" + +import math +import statistics + +DEFAULT_TASKS = ("qwen21-denoise-npu", "moe-forward-npu") +DEFAULT_POLICY = { + "metrics": ["speedup", "wall_seconds"], + "tolerances": {"speedup": 0.0, "wall_seconds": 0.0, "tokens": 0.0}, + "minimum_pairs": 1, +} +METRICS = ("speedup", "wall_seconds", "tokens") +STATUSES = ("pending", "running", "completed", "failed", "blocked") + + +def _number(value): + try: + return type(value) in (int, float) and math.isfinite(value) + except OverflowError: + return False + + +def validate_policy(policy: dict) -> dict: + """Return an independent policy, rejecting unknown or ambiguous requirements.""" + if not isinstance(policy, dict) or set(policy) != set(DEFAULT_POLICY): + raise ValueError("Comparison policy requires metrics, tolerances and minimum_pairs") + metrics = policy["metrics"] + if (not isinstance(metrics, list) or not metrics + or any(not isinstance(metric, str) or metric not in METRICS for metric in metrics) + or len(set(metrics)) != len(metrics) or "speedup" not in metrics): + raise ValueError("Select unique comparison metrics including speedup") + tolerances = policy["tolerances"] + if (not isinstance(tolerances, dict) or set(tolerances) != set(METRICS) + or any(not _number(value) or value < 0 for value in tolerances.values())): + raise ValueError("Metric tolerances must be finite nonnegative relative fractions") + minimum = policy["minimum_pairs"] + if type(minimum) is not int or minimum < 1: + raise ValueError("minimum_pairs must be a positive integer") + return {"metrics": list(metrics), "tolerances": dict(tolerances), "minimum_pairs": minimum} + + +def _factor(on, off, metric): + if not _number(on) or not _number(off): + return None + if metric == "tokens" and (type(on) is not int or type(off) is not int): + return None + if metric == "tokens" and on == off == 0: + return 1.0 + if on <= 0 or off <= 0: + return None + try: + value = math.exp(math.log(on) - math.log(off) if metric == "speedup" + else math.log(off) - math.log(on)) + except (ValueError, OverflowError): + return None + return value if math.isfinite(value) and value > 0 else None + + +def _direction(factor, tolerance): + # Reciprocal bounds make swapping on/off symmetric. Tolerance is a relative + # neutral band, not a significance test or an acceptance threshold. + log_factor, band = math.log(factor), math.log1p(tolerance) + return 1 if log_factor > band else -1 if log_factor < -band else 0 + + +def _median(values): + # These are nonnegative observations. statistics.median adds the middle + # two values first, which can overflow for valid large finite measurements. + ordered = sorted(values) + middle = len(ordered) // 2 + return (ordered[middle] if len(ordered) % 2 + else ordered[middle - 1] + (ordered[middle] - ordered[middle - 1]) / 2) + + +def _kind(signs, complete=True): + if not complete: + return "insufficient_evidence" + if 1 in signs and -1 in signs: + return "tradeoff" + if 1 in signs: + return "efficiency_improved" + return "regression" if -1 in signs else "no_gain" + + +def _conclusion(kind, tasks, repeats, metrics): + descriptions = { + "efficiency_improved": ( + "Knowledge on improved the observed selected efficiency metrics without a " + "measured tradeoff in this comparison." + ), + "no_gain": ( + "Knowledge on showed no improvement beyond the frozen neutral tolerances " + "in this comparison." + ), + "regression": ( + "Knowledge on worsened selected efficiency metrics without an offsetting " + "improvement in this comparison." + ), + "tradeoff": "Observed benefits and costs differ between selected metrics or tasks.", + "insufficient_evidence": ( + "The declared comparison is incomplete or lacks comparable evidence; " + "knowledge efficiency cannot be determined." + ), + } + return { + "kind": kind, + "text": descriptions[kind], + "inference": "descriptive_only", + "scope": {"tasks": list(tasks), "repeats": repeats, "metrics": list(metrics), + "aggregation": "median_of_paired_log_ratios"}, + "limitation": ( + "Results describe these tasks, frozen conditions and observed repeats only; " + "they establish no statistical significance or general RSI efficacy." + ), + } + + +def _index(trials, tasks, repeats): + if (not isinstance(tasks, (tuple, list)) or not tasks + or any(not isinstance(task, str) or not task.strip() for task in tasks) + or len(set(tasks)) != len(tasks)): + raise ValueError("Comparison tasks must be unique nonempty names") + if type(repeats) is not int or repeats < 1: + raise ValueError("Comparison repeats must be a positive integer") + if not isinstance(trials, list): + raise ValueError("Comparison trials must be a list") + indexed, identities = {}, set() + for trial in trials: + if not isinstance(trial, dict): + raise ValueError("Comparison trial must be an object") + task, repeat, condition = (trial.get(key) for key in ("task", "repeat", "condition")) + if (task not in tasks or type(repeat) is not int or not 1 <= repeat <= repeats + or condition not in ("on", "off")): + raise ValueError("Trial is outside the declared comparison matrix") + key = task, repeat, condition + identity = trial.get("id") + if (not isinstance(identity, str) or not identity.strip() or identity in identities + or key in indexed): + raise ValueError("Comparison trial IDs and matrix cells must be unique") + if (trial.get("status") not in STATUSES or type(trial.get("eligible")) is not bool + or not isinstance(trial.get("metrics"), dict)): + raise ValueError("Trial requires a known status, eligibility and metric object") + identities.add(identity) + indexed[key] = trial + return indexed + + +def _pair(indexed, task, repeat, policy): + sides = {condition: indexed.get((task, repeat, condition)) for condition in ("on", "off")} + gaps = [] + for condition, trial in sides.items(): + if trial is None: + gaps.append(f"{condition}: trial missing") + elif trial["status"] != "completed" or not trial["eligible"]: + gaps.append(f"{condition}: comparable completed final unavailable") + gaps.extend(f"{condition}: {gap}" for gap in trial.get("gaps", [])) + metrics = {} + for metric in METRICS: + values = {side: (trial["metrics"].get(metric) if trial else None) + for side, trial in sides.items()} + factor = _factor(values["on"], values["off"], metric) + if metric in policy["metrics"] and factor is None: + gaps.append(f"{metric}: positive finite comparable paired values unavailable") + metrics[metric] = {**values, "factor": factor} + return {"task": task, "repeat": repeat, + "on": sides["on"]["id"] if sides["on"] else None, + "off": sides["off"]["id"] if sides["off"] else None, + "eligible": not gaps, "metrics": metrics, "gaps": gaps} + + +def summarize(trials: list[dict], tasks, repeats: int, policy: dict) -> dict: + """Assess every planned pair, retaining failures and missing matrix cells. + + Benefit factors above one favor knowledge on. Speedup is on/off; wall time + and tokens are off/on. Only complete, comparable pairs contribute to paired + aggregates. A missing declared pair cannot disappear from the conclusion. + """ + policy = validate_policy(policy) + indexed = _index(trials, tasks, repeats) + pairs = [_pair(indexed, task, repeat, policy) + for task in tasks for repeat in range(1, repeats + 1)] + task_rows, all_signs = [], [] + for task in tasks: + declared = [pair for pair in pairs if pair["task"] == task] + eligible = [pair for pair in declared if pair["eligible"]] + complete = len(eligible) == repeats and len(eligible) >= policy["minimum_pairs"] + gaps = [f"repeat {pair['repeat']}: {gap}" + for pair in declared for gap in pair["gaps"]] + if len(eligible) < policy["minimum_pairs"]: + gaps.append("The frozen minimum number of eligible pairs was not reached") + metrics, signs = {}, [] + for metric in METRICS: + observed = [pair["metrics"][metric] for pair in eligible + if pair["metrics"][metric]["factor"] is not None] + factor = (math.exp(statistics.median(math.log(row["factor"]) for row in observed)) + if observed else None) + direction = (_direction(factor, policy["tolerances"][metric]) + if factor is not None else None) + metrics[metric] = { + "on_median": _median(row["on"] for row in observed) if observed else None, + "off_median": (_median(row["off"] for row in observed) + if observed else None), + "paired_factor": factor, "direction": direction, "pair_count": len(observed), + "selected": metric in policy["metrics"], + } + if metric in policy["metrics"]: + if direction is None: + complete = False + else: + signs.append(direction) + kind = _kind(signs, complete) + task_rows.append({"task": task, "planned_pairs": repeats, + "eligible_pairs": len(eligible), "metrics": metrics, + "conclusion": _conclusion(kind, [task], repeats, policy["metrics"]), + "gaps": gaps}) + all_signs.extend(signs) + complete = all(row["conclusion"]["kind"] != "insufficient_evidence" for row in task_rows) + kind = _kind(all_signs, complete) + return { + "conclusion": _conclusion(kind, tasks, repeats, policy["metrics"]), + "tasks": task_rows, + "counts": { + "planned_trials": len(tasks) * repeats * 2, "planned_pairs": len(tasks) * repeats, + "observed_trials": len(trials), + "eligible_trials": sum(t["eligible"] and t["status"] == "completed" for t in trials), + "eligible_pairs": sum(pair["eligible"] for pair in pairs), + "statuses": {status: sum(t["status"] == status for t in trials) for status in STATUSES}, + "missing_trials": len(tasks) * repeats * 2 - len(trials), + }, + "pairs": pairs, + } diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py index e735be59..972a1312 100644 --- a/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/moe-forward-npu/benchmark.py @@ -5,12 +5,10 @@ """ import argparse -import hashlib +import importlib.util import json -import math import os import signal -import statistics import sys import time import uuid @@ -93,20 +91,22 @@ def cleanup(token: str, *, grace_seconds: float = 5.0) -> dict: return {"verified": False, "signalled_pids": sorted(killed), "error": str(exc)} -TASK_FILES = ( - "task.json", "TASK.md", "baseline.py", "oracle.py", "workload.py", "LICENSE", - "benchmark.py", "grader.py", -) +# The protocol is a task-owned file, loaded by exact path from this bundle. +# Its pure rules are also used by the isolated Harness protocol interpreter. +_spec = importlib.util.spec_from_file_location("moe_task_protocol", Path(__file__).with_name( + "optimization.py")) +_protocol = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_protocol) +TASK_FILES = _protocol.TASK_FILES +_positive = _protocol._positive +_hashes = _protocol._hashes +_adjudicate = _protocol._adjudicate def task_resources(): return Path(__file__).resolve().parent -def _positive(value): - return type(value) in (int, float) and math.isfinite(value) and value > 0 - - def _preflight(candidate: Path, output: Path, pairs: int, timeout_seconds: float): if type(pairs) is not int or pairs <= 0: raise ValueError("pairs must be a positive integer") @@ -126,10 +126,6 @@ def _preflight(candidate: Path, output: Path, pairs: int, timeout_seconds: float raise ValueError("ASCEND_RT_VISIBLE_DEVICES must contain two distinct allocated devices") -def _hashes(root: Path, names) -> dict: - return {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in names} - - def _prepare(candidate: Path, output: Path, pairs: int) -> dict: output.mkdir(parents=True, exist_ok=False) for directory in ("task", "candidate", "runner", "results"): @@ -157,149 +153,6 @@ def _launch(root: Path, token: str): os.execv(sys.executable, argv) -def _expected_calls(task: dict, pairs: int) -> dict: - expected = {} - measure = task["measurement"] - for index, case in enumerate(task["cases"]): - base = 100000 + index * 10000 - calls = [(base + i, "correctness", "standard", None, None, False) for i in range(2)] - if case["measured"]: - calls += [(base + 10 + i, "warmup", "standard", None, None, False) - for i in range(measure["warmups"])] - calls += [(base + 100 + pair * 100 + sample, phase, variant, pair, sample, timed) - for pair in range(pairs) for sample in range(measure["samples_per_pair"]) - for phase, variant, timed in (("shape_warmup", "shape_warmup", False), - ("measurement", "standard", True))] - for seed, phase, variant, pair, sample, timed in calls: - for label in ("baseline", "candidate"): - expected[(case["id"], seed, label, phase, variant, pair, sample)] = timed - return expected - - -def _score(task: dict, ranks: list[dict], summary: dict, pairs: int) -> dict: - """Recompute from complete raw calls/samples; a rank-0 summary is insufficient.""" - if (len(ranks) != 2 or {r.get("rank") for r in ranks} != {0, 1} - or any(type(r.get("rank")) is not int or r.get("completed") is not True - or r.get("correctness") != "PASS" or r.get("error") for r in ranks)): - raise ValueError("Both ranks must complete correctly without errors") - expected = _expected_calls(task, pairs) - checks = ("shape_dtype_device", "input_parameters_preserved", "finite", "values", "nrmse_pass") - by_rank = [] - for rank in ranks: - observed = {} - for call in rank["calls"]: - key = tuple(call[k] for k in ("case_id", "seed", "label", "phase", "variant", - "pair", "sample")) - if (key in observed or key not in expected - or call["included_in_score"] is not expected[key] - or call["all_ranks_correct"] is not True - or not all(call["checks"].get(k) is True for k in checks) - or not _positive(call["local_elapsed_ms"])): - raise ValueError("Missing, duplicated or invalid forward verification") - observed[key] = call - if set(observed) != set(expected): - raise ValueError("Incomplete correctness/warmup/measurement calls") - samples = {} - for sample in rank["samples"]: - key = tuple(sample[k] for k in ("case_id", "seed", "label", "pair", "sample")) - call_key = key[:3] + ("measurement", "standard") + key[3:] - if (key in samples or call_key not in observed - or sample["local_elapsed_ms"] != observed[call_key]["local_elapsed_ms"] - or sample["input_sha256"] != observed[call_key]["input_sha256"] - or not _positive(sample["slowest_rank_elapsed_ms"])): - raise ValueError("Invalid timed sample") - samples[key] = sample - if len(samples) != sum(expected.values()): - raise ValueError("Incomplete timed samples") - by_rank.append(samples) - if by_rank[0].keys() != by_rank[1].keys(): - raise ValueError("Ranks disagree on timed calls") - for key, first in by_rank[0].items(): - second = by_rank[1][key] - maximum = max(first["local_elapsed_ms"], second["local_elapsed_ms"]) - if (first["slowest_rank_elapsed_ms"] != second["slowest_rank_elapsed_ms"] - or not math.isclose(first["slowest_rank_elapsed_ms"], maximum, rel_tol=1e-6)): - raise ValueError("Timing does not use the maximum of both ranks") - cases = [] - for case in task["cases"]: - if not case["measured"]: - continue - grouped = {label: [s["slowest_rank_elapsed_ms"] for s in by_rank[0].values() - if s["case_id"] == case["id"] and s["label"] == label] - for label in ("baseline", "candidate")} - b, c = (statistics.median(grouped[label]) for label in ("baseline", "candidate")) - cases.append({"id": case["id"], "baseline_median_ms": b, "candidate_median_ms": c, - "speedup": b / c, "samples_ms": grouped}) - speedup = math.exp(statistics.mean(math.log(c["speedup"]) for c in cases)) - score = {"completed": True, "correctness": "PASS", "cases": cases, "speedup": speedup, - "aggregation": "geometric_mean_case_speedups", - "measurement_protocol": - "shape-matched-different-values-warmup-both-labels-before-paired-timing", - "metrics": {"suite_latency_ms": sum(c["candidate_median_ms"] for c in cases)}} - if summary != score or any(r["cases"] != cases or r["speedup"] != speedup for r in ranks): - raise ValueError("Summary disagrees with raw two-rank samples") - return score - - -def _adjudicate(root: Path, execution: dict, guard: dict, identity: dict, pairs: int) -> dict: - result = {"status": "infrastructure_failure", "completed": False, - "correctness": "INCONCLUSIVE", "speedup": None, "metrics": {}, - "round_consumed": False, "process_cleanup": guard, "execution": execution} - ranks = [] - try: - malformed_ranks = [] - for rank in range(2): - path = root / "results" / f"rank-{rank}.json" - if path.exists(): - try: - record = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(record, dict) or record.get("rank") != rank: - raise ValueError("Rank/file identity mismatch") - except (OSError, ValueError): - malformed_ranks.append(rank) - continue - ranks.append(record) - # A peer may leave a truncated record when the job is stopped. - # Preserve observed candidate work before parsing the next rank. - phase, kind, calls = (record.get(k) for k in ("phase", "failure_kind", "calls")) - if ((isinstance(phase, str) and phase.startswith("candidate_")) - or (isinstance(kind, str) and kind.startswith("candidate_")) - or (isinstance(calls, list) and any( - isinstance(c, dict) and c.get("label") == "candidate" for c in calls))): - result["round_consumed"] = True - result["rank_status"] = [{k: r.get(k) for k in ( - "rank", "phase", "completed", "correctness", "failure_kind", "error")} for r in ranks] - failed = any(isinstance(value, str) and value.startswith("candidate_") - for r in ranks for value in (r.get("phase"), r.get("failure_kind"))) - # A later cleanup/integrity failure cannot erase observed candidate work - # and turn a consumed attempt into a free infrastructure retry. - if failed: - result.update(status="candidate_failure", correctness="FAIL", round_consumed=True) - if any(not isinstance(r.get(key, ""), str) - for r in ranks for key in ("phase", "failure_kind")): - raise ValueError("Malformed rank phase or failure_kind") - if malformed_ranks: - raise ValueError("Incomplete or malformed rank records; no usable score") - if not guard.get("verified"): - raise ValueError("Workload process cleanup was not verified") - if _hashes(root, identity) != identity: - raise ValueError("Frozen task, grader, request or candidate changed during execution") - if failed: - raise ValueError("Candidate failed or did not complete; inspect retained rank logs") - if (execution.get("status") != "completed" or execution.get("returncode") != 0 - or execution.get("metric_error") or not _positive(execution.get("metric"))): - raise ValueError("Executor did not complete with a valid result; no usable score") - summary = json.loads((root / "results" / "summary.json").read_text(encoding="utf-8")) - task = json.loads((root / "task" / "task.json").read_text(encoding="utf-8")) - score = _score(task, ranks, summary, pairs) - if execution["metric"] != score["metrics"]["suite_latency_ms"]: - raise ValueError("Execution metric disagrees with raw samples") - result.update(score, status="completed", round_consumed=True) - except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError) as exc: - result["error"] = str(exc) - return result - - def evaluate(candidate: Path, output: Path, *, pairs: int, timeout_seconds: float, execute_trial) -> dict: started = time.monotonic() @@ -312,12 +165,7 @@ def evaluate(candidate: Path, output: Path, *, pairs: int, timeout_seconds: floa # This small launcher is frozen too and receives the marker only after the # executor starts it; the calling shell and npu allocator never inherit it. launcher = output / "runner" / "launch.py" - launcher.write_text( - "from pathlib import Path\nimport runpy, sys\n" - "root = Path(sys.argv[1])\n" - "runpy.run_path(str(root / 'task' / 'benchmark.py'))['_launch'](root, sys.argv[2])\n", - encoding="utf-8", - ) + launcher.write_bytes(_protocol.LAUNCHER_SOURCE.encode("utf-8")) identity.update(_hashes(output, ["runner/launch.py"])) atomic_write_json(output / "identity.json", identity) execution = {"status": "error", "metric": None, "returncode": None} diff --git a/src/inferrsi/benchmark/tasks/moe-forward-npu/optimization.py b/src/inferrsi/benchmark/tasks/moe-forward-npu/optimization.py new file mode 100644 index 00000000..9b08c7b5 --- /dev/null +++ b/src/inferrsi/benchmark/tasks/moe-forward-npu/optimization.py @@ -0,0 +1,290 @@ +"""Frozen full-task protocol and raw-evidence rules for two-rank MoE sessions. + +Only standard-library code runs here. The single-candidate driver and the +iterative assessor share these rules without importing an execution runtime. +""" + +import argparse +import hashlib +import json +import math +import statistics +from pathlib import Path + +TASK_FILES = ( + "task.json", "TASK.md", "baseline.py", "oracle.py", "workload.py", "LICENSE", + "benchmark.py", "grader.py", "optimization.py", +) + +LAUNCHER_SOURCE = ( + "from pathlib import Path\nimport runpy, sys\n" + "root = Path(sys.argv[1])\n" + "runpy.run_path(str(root / 'task' / 'benchmark.py'))['_launch'](root, sys.argv[2])\n" +) + + +def validate_arguments(arguments): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pairs", type=int, required=True) + parser.add_argument("--timeout-seconds", type=float, required=True) + settings = vars(parser.parse_args(arguments)) + if settings["pairs"] < 1 or not _positive(settings["timeout_seconds"]): + raise ValueError("pairs and timeout-seconds must be finite and positive") + return settings + + +def arguments(candidate: Path, output: Path, settings): + return ["--candidate", str(candidate), "--output-dir", str(output), + "--pairs", str(settings["pairs"]), + "--timeout-seconds", str(settings["timeout_seconds"])] + + +def _binding(directory, settings): + output = directory / "measurement" + identity = json.loads((output / "identity.json").read_text(encoding="utf-8")) + required = {"task/" + name for name in TASK_FILES} + required.update({"runner/grade.py", "runner/launch.py", "candidate/implementation.py", + "request.json"}) + if not isinstance(identity, dict) or identity.keys() != required: + raise ValueError("Missing or unexpected evaluated source identities") + expected = _hashes(Path(__file__).parent, TASK_FILES) + for name in TASK_FILES: + if identity["task/" + name] != expected[name]: + raise ValueError("Evaluated task differs from the frozen task") + if identity["runner/grade.py"] != expected["grader.py"]: + raise ValueError("Evaluated grader differs from the frozen grader") + if identity["runner/launch.py"] != hashlib.sha256(LAUNCHER_SOURCE.encode()).hexdigest(): + raise ValueError("Evaluated launcher differs from the frozen launcher") + if identity["candidate/implementation.py"] != hashlib.sha256( + (directory / "candidate.py").read_bytes() + ).hexdigest(): + raise ValueError("Task result is bound to another candidate") + for name, expected_hash in identity.items(): + path = output / name + if (path.is_symlink() or any(parent.is_symlink() for parent in path.parents) + or hashlib.sha256(path.read_bytes()).hexdigest() != expected_hash): + raise ValueError("Evaluated source snapshot changed") + request = json.loads((output / "request.json").read_text(encoding="utf-8")) + if request != {"repetitions": settings["pairs"]}: + raise ValueError("Frozen sampling protocol changed") + return identity + + +def assess(directory: Path, settings, execution): + output = directory / "measurement" + result_path = output / "result.json" + if not result_path.is_file(): + return {"status": "infrastructure_failure", "score": None, + "verdict": "INCONCLUSIVE", "reason": "missing_task_result"} + result = json.loads(result_path.read_text(encoding="utf-8")) + evidence = {"measurement/result.json": hashlib.sha256(result_path.read_bytes()).hexdigest()} + + def incomplete(reason): + return {"status": "infrastructure_failure", "score": None, + "verdict": "INCONCLUSIVE", "reason": reason, "evidence_files": evidence, + "round_consumed": result.get("round_consumed")} + + rejected = result.get("status") == "candidate_failure" + expected_exit = ("failed", 2) if rejected else ("completed", 0) + if (type(execution.get("returncode")) is not int + or (execution.get("status"), execution["returncode"]) != expected_exit): + return incomplete("incomplete_outer_execution") + identity = _binding(directory, settings) + for name, fingerprint in identity.items(): + evidence["measurement/" + name] = fingerprint + evidence["measurement/identity.json"] = hashlib.sha256( + (output / "identity.json").read_bytes()).hexdigest() + if (result.get("pairs") != settings["pairs"] + or result.get("timeout_seconds") != settings["timeout_seconds"]): + raise ValueError("Evaluated sampling or timeout differs from the frozen protocol") + inner_path = output / "execution/execution.json" + if not inner_path.is_file(): + return incomplete("missing_inner_execution") + inner = json.loads(inner_path.read_text(encoding="utf-8")) + evidence["measurement/execution/execution.json"] = hashlib.sha256( + inner_path.read_bytes()).hexdigest() + if inner != result.get("execution"): + raise ValueError("Task result differs from the retained execution receipt") + if (inner.get("status") not in ("completed", "failed") + or type(inner.get("returncode")) is not int or inner["returncode"] < 0): + return incomplete("incomplete_inner_execution") + guard = result.get("process_cleanup", {}) + if guard.get("verified") is not True or guard.get("remaining"): + return incomplete("workload_cleanup_unverified") + # A normal driver exit cannot turn a partial rank record into a resumable + # rejection. Keep raw evidence alongside any rejected score. + for name in ("results/rank-0.json", "results/rank-1.json", "results/summary.json"): + path = output / name + if path.is_file(): + evidence["measurement/" + name] = hashlib.sha256(path.read_bytes()).hexdigest() + recomputed = _adjudicate(output, inner, guard, identity, settings["pairs"]) + if {name: result.get(name) for name in recomputed} != recomputed: + raise ValueError("Task result disagrees with raw two-rank evidence") + if len(recomputed.get("rank_status", [])) != 2: + return incomplete("incomplete_rank_evidence") + if rejected: + if any(isinstance(value, str) and value.startswith("baseline_") + for rank in recomputed["rank_status"] + for value in (rank.get("phase"), rank.get("failure_kind"))): + return incomplete("baseline_failure") + return {"status": "candidate_failure", "score": None, "verdict": "FAIL", + "quality_passed": False if result.get("correctness") == "FAIL" else None, + "reason": recomputed.get("error"), "round_consumed": True, + "evidence_files": evidence} + if recomputed["status"] != "completed": + return incomplete(recomputed.get("error", "incomplete_task")) + return {"status": "completed", "verdict": "PASS", "score": recomputed["speedup"], + "quality_passed": True, "case_metrics": recomputed["cases"], + "round_consumed": True, "evidence_files": evidence} + + +def _positive(value): + return type(value) in (int, float) and math.isfinite(value) and value > 0 + + +def _hashes(root: Path, names) -> dict: + return {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in names} + + +def _expected_calls(task: dict, pairs: int) -> dict: + expected = {} + measure = task["measurement"] + for index, case in enumerate(task["cases"]): + base = 100000 + index * 10000 + calls = [(base + i, "correctness", "standard", None, None, False) for i in range(2)] + if case["measured"]: + calls += [(base + 10 + i, "warmup", "standard", None, None, False) + for i in range(measure["warmups"])] + calls += [(base + 100 + pair * 100 + sample, phase, variant, pair, sample, timed) + for pair in range(pairs) for sample in range(measure["samples_per_pair"]) + for phase, variant, timed in (("shape_warmup", "shape_warmup", False), + ("measurement", "standard", True))] + for seed, phase, variant, pair, sample, timed in calls: + for label in ("baseline", "candidate"): + expected[(case["id"], seed, label, phase, variant, pair, sample)] = timed + return expected + + +def _score(task: dict, ranks: list[dict], summary: dict, pairs: int) -> dict: + """Recompute from complete raw calls/samples; a rank-0 summary is insufficient.""" + if (len(ranks) != 2 or {r.get("rank") for r in ranks} != {0, 1} + or any(type(r.get("rank")) is not int or r.get("completed") is not True + or r.get("correctness") != "PASS" or r.get("error") for r in ranks)): + raise ValueError("Both ranks must complete correctly without errors") + expected = _expected_calls(task, pairs) + checks = ("shape_dtype_device", "input_parameters_preserved", "finite", "values", "nrmse_pass") + by_rank = [] + for rank in ranks: + observed = {} + for call in rank["calls"]: + key = tuple(call[k] for k in ("case_id", "seed", "label", "phase", "variant", + "pair", "sample")) + if (key in observed or key not in expected + or call["included_in_score"] is not expected[key] + or call["all_ranks_correct"] is not True + or not all(call["checks"].get(k) is True for k in checks) + or not _positive(call["local_elapsed_ms"])): + raise ValueError("Missing, duplicated or invalid forward verification") + observed[key] = call + if set(observed) != set(expected): + raise ValueError("Incomplete correctness/warmup/measurement calls") + samples = {} + for sample in rank["samples"]: + key = tuple(sample[k] for k in ("case_id", "seed", "label", "pair", "sample")) + call_key = key[:3] + ("measurement", "standard") + key[3:] + if (key in samples or call_key not in observed + or sample["local_elapsed_ms"] != observed[call_key]["local_elapsed_ms"] + or sample["input_sha256"] != observed[call_key]["input_sha256"] + or not _positive(sample["slowest_rank_elapsed_ms"])): + raise ValueError("Invalid timed sample") + samples[key] = sample + if len(samples) != sum(expected.values()): + raise ValueError("Incomplete timed samples") + by_rank.append(samples) + if by_rank[0].keys() != by_rank[1].keys(): + raise ValueError("Ranks disagree on timed calls") + for key, first in by_rank[0].items(): + second = by_rank[1][key] + maximum = max(first["local_elapsed_ms"], second["local_elapsed_ms"]) + if (first["slowest_rank_elapsed_ms"] != second["slowest_rank_elapsed_ms"] + or not math.isclose(first["slowest_rank_elapsed_ms"], maximum, rel_tol=1e-6)): + raise ValueError("Timing does not use the maximum of both ranks") + cases = [] + for case in task["cases"]: + if not case["measured"]: + continue + grouped = {label: [s["slowest_rank_elapsed_ms"] for s in by_rank[0].values() + if s["case_id"] == case["id"] and s["label"] == label] + for label in ("baseline", "candidate")} + b, c = (statistics.median(grouped[label]) for label in ("baseline", "candidate")) + cases.append({"id": case["id"], "baseline_median_ms": b, "candidate_median_ms": c, + "speedup": b / c, "samples_ms": grouped}) + speedup = math.exp(statistics.mean(math.log(c["speedup"]) for c in cases)) + score = {"completed": True, "correctness": "PASS", "cases": cases, "speedup": speedup, + "aggregation": "geometric_mean_case_speedups", + "measurement_protocol": + "shape-matched-different-values-warmup-both-labels-before-paired-timing", + "metrics": {"suite_latency_ms": sum(c["candidate_median_ms"] for c in cases)}} + if summary != score or any(r["cases"] != cases or r["speedup"] != speedup for r in ranks): + raise ValueError("Summary disagrees with raw two-rank samples") + return score + + +def _adjudicate(root: Path, execution: dict, guard: dict, identity: dict, pairs: int) -> dict: + result = {"status": "infrastructure_failure", "completed": False, + "correctness": "INCONCLUSIVE", "speedup": None, "metrics": {}, + "round_consumed": False, "process_cleanup": guard, "execution": execution} + ranks = [] + try: + malformed_ranks = [] + for rank in range(2): + path = root / "results" / f"rank-{rank}.json" + if path.exists(): + try: + record = json.loads(path.read_text(encoding="utf-8")) + if (not isinstance(record, dict) or type(record.get("rank")) is not int + or record["rank"] != rank): + raise ValueError("Rank/file identity mismatch") + except (OSError, ValueError): + malformed_ranks.append(rank) + continue + ranks.append(record) + # A peer may leave a truncated record when the job is stopped. + # Preserve observed candidate work before parsing the next rank. + phase, kind, calls = (record.get(k) for k in ("phase", "failure_kind", "calls")) + if ((isinstance(phase, str) and phase.startswith("candidate_")) + or (isinstance(kind, str) and kind.startswith("candidate_")) + or (isinstance(calls, list) and any( + isinstance(c, dict) and c.get("label") == "candidate" for c in calls))): + result["round_consumed"] = True + result["rank_status"] = [{k: r.get(k) for k in ( + "rank", "phase", "completed", "correctness", "failure_kind", "error")} for r in ranks] + failed = any(isinstance(value, str) and value.startswith("candidate_") + for r in ranks for value in (r.get("phase"), r.get("failure_kind"))) + # A later cleanup/integrity failure cannot erase observed candidate work + # and turn a consumed attempt into a free infrastructure retry. + if failed: + result.update(status="candidate_failure", correctness="FAIL", round_consumed=True) + if any(not isinstance(r.get(key, ""), str) + for r in ranks for key in ("phase", "failure_kind")): + raise ValueError("Malformed rank phase or failure_kind") + if malformed_ranks: + raise ValueError("Incomplete or malformed rank records; no usable score") + if not guard.get("verified"): + raise ValueError("Workload process cleanup was not verified") + if _hashes(root, identity) != identity: + raise ValueError("Frozen task, grader, request or candidate changed during execution") + if failed: + raise ValueError("Candidate failed or did not complete; inspect retained rank logs") + if (execution.get("status") != "completed" or execution.get("returncode") != 0 + or execution.get("metric_error") or not _positive(execution.get("metric"))): + raise ValueError("Executor did not complete with a valid result; no usable score") + summary = json.loads((root / "results" / "summary.json").read_text(encoding="utf-8")) + task = json.loads((root / "task" / "task.json").read_text(encoding="utf-8")) + score = _score(task, ranks, summary, pairs) + if execution["metric"] != score["metrics"]["suite_latency_ms"]: + raise ValueError("Execution metric disagrees with raw samples") + result.update(score, status="completed", round_consumed=True) + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError) as exc: + result["error"] = str(exc) + return result diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 02c703b3..31b50016 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -304,7 +304,8 @@ def _settle(root, plan, state, pending, execution, assess): def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, - usage: Path | None, execute, assess, final=False, reason=None) -> dict: + usage: Path | None, execute, assess, final=False, reason=None, + strategy=None, knowledge_refs=None) -> dict: with _locked(root): plan, state = _load(root) _active(state, reserve=0 if final else plan["final_reserve_seconds"]) @@ -318,6 +319,15 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, raise ValueError("Round budget consumed, run independent final evaluation") if not final and (not isinstance(hypothesis, str) or not hypothesis.strip()): raise ValueError("Declare the round hypothesis before execution") + if strategy is not None and (not isinstance(strategy, str) or not strategy.strip()): + raise ValueError("Strategy must be a nonempty declaration") + if knowledge_refs is not None: + available = {context["path"] for context in state["contexts"]} + if (not isinstance(knowledge_refs, list) + or any(not isinstance(ref, str) or ref not in available + for ref in knowledge_refs) + or len(set(knowledge_refs)) != len(knowledge_refs)): + raise ValueError("Cite unique delivered context paths from this campaign") source = root / state["best"]["path"] if final else Path(candidate) if (source.is_symlink() or (plan["candidate_kind"] == "file" and (not source.is_file() or source.suffix != ".py")) @@ -335,7 +345,8 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str, pending = {"kind": "final" if final else "round", "path": relative, "candidate_id": state["best"].get("candidate_id", "baseline") if final else f"candidate-{count + 1:03d}", - "hypothesis": hypothesis, "observations": observations, + "hypothesis": hypothesis, "strategy": strategy, + "knowledge_refs": knowledge_refs, "observations": observations, "candidate_sha256": fingerprint, "contexts": list(state["contexts"]), "started_at": time.time()} state["pending"] = pending diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index b4e8d277..2b39b7a7 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -225,6 +225,8 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N hypothesis="Independent final of the selected candidate" if final else args.hypothesis, usage=args.usage_export, execute=_session_execute, assess=_session_assess, final=final, reason=getattr(args, "reason", None), + strategy=getattr(args, "strategy", None), + knowledge_refs=getattr(args, "knowledge_ref", None), ) diff --git a/tests/test_benchmark_compare.py b/tests/test_benchmark_compare.py new file mode 100644 index 00000000..20e57324 --- /dev/null +++ b/tests/test_benchmark_compare.py @@ -0,0 +1,286 @@ +"""Real campaign subprocesses with synthetic host receipts; no Agent/device launch.""" + +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from inferrsi.apps import benchmark_compare as compare +from inferrsi.apps.cli import main +from inferrsi.benchmark import registry +from inferrsi.harness.optimization_session import _runtime_identity + + +@pytest.fixture +def experiment(tmp_path, monkeypatch, capsys): + source = tmp_path / "source/inferrsi" + source.mkdir(parents=True) + from importlib.resources import files + + actual = Path(str(files("inferrsi"))) + for name in _runtime_identity(): + target = source / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(actual / name, target) + corpus = source / "knowledge/data" + corpus.mkdir(parents=True) + (corpus / "synthetic.txt").write_text("Test knowledge", encoding="utf-8") + task = source / "benchmark/tasks/test-work" + task.mkdir(parents=True) + (task / "task.json").write_text('{"devices":1}', encoding="utf-8") + (task / "TASK.md").write_text("Synthetic task", encoding="utf-8") + (task / "baseline.py").write_text("FACTOR = 1\n", encoding="utf-8") + (task / "optimization.py").write_text( + "import json\n" + "def validate_arguments(arguments):\n assert not arguments\n return {}\n" + "def arguments(candidate, output, settings):\n" + " return [str(candidate), str(output)]\n" + "def assess(directory, settings, execution):\n" + " return json.loads((directory/'measurement/result.json').read_text())\n", + encoding="utf-8", + ) + (task / "benchmark.py").write_text( + "import json,runpy,sys\nfrom pathlib import Path\n" + "value=runpy.run_path(sys.argv[1])['FACTOR']\n" + "out=Path(sys.argv[2]);out.mkdir()\n" + "if 'final' in out.parts and Path('final-score.json').exists():\n" + " value=json.loads(Path('final-score.json').read_text())\n" + "(out/'result.json').write_text(json.dumps({'status':'completed','verdict':'PASS'," + "'score':value,'speedup':value}))\n", encoding="utf-8", + ) + monkeypatch.setattr(compare, "files", lambda _: source) + monkeypatch.setattr(compare, "task_resources", lambda _: task) + monkeypatch.setattr(registry, "task_resources", lambda _: task) + config = tmp_path / "config.json" + config.write_text(json.dumps({"tasks": ["test-work"], "rounds": 1, "max_minutes": 5, + "evaluation_timeout_seconds": 15, "final_reserve_seconds": 15, + "job_max_minutes": 10, "task_arguments": {"test-work": ""}, + "model": "synthetic-model", "effort": "high", "environment_id": "test-environment"}), + encoding="utf-8") + root = tmp_path / "comparison" + assert main(["benchmarks", "compare", "--config", str(config), "--output", str(root)]) == 0 + initial = json.loads(capsys.readouterr().out) + return root, initial, capsys + + +def operation(experiment, action, trial=None, *arguments): + root, _, capsys = experiment + argv = ["benchmarks", "compare", "--output", str(root), "--action", action] + if trial: + argv += ["--trial", trial] + assert main([*argv, *arguments]) == 0 + return json.loads(capsys.readouterr().out) + + +def finish(experiment, trial, *, factor=2, final=True, model="synthetic-model", usage=True, + final_score=None): + root, _, capsys = experiment + dispatch = operation(experiment, "dispatch", trial) + assert dispatch["may_create"] + prompt = dispatch["handoff"]["prompt"] + command = json.loads(re.search(r"Start argv .*?: (\[.*\])\n", prompt).group(1)) + receipt = root / (trial + "-receipt.json") + receipt.write_text('{"synthetic_host_creation":true}', encoding="utf-8") + operation(experiment, "bind", trial, "--host", "synthetic", "--session-id", trial, + "--receipt", str(receipt)) + assert main(command[4:]) == 0 + start = json.loads(capsys.readouterr().out) + campaign = Path(start["session"]) + candidate = Path(start["working_candidate"]) + (campaign / candidate).write_text(f"FACTOR = {factor}\n", encoding="utf-8") + assert main(["benchmarks", "session", "round", "--session", str(campaign), + "--candidate", str(campaign / candidate), "--hypothesis", "Synthetic hypothesis", + "--strategy", "Synthetic strategy"]) == 0 + capsys.readouterr() + if final: + if final_score is not None: + (campaign / "final-score.json").write_text(json.dumps(final_score)) + assert main(["benchmarks", "session", "final", "--session", str(campaign)]) == 0 + capsys.readouterr() + observation = {"host": "synthetic", "host_id": None, "session_id": trial, + "state": "completed", "native_receipt": {"synthetic_host_terminal": True}, + "benchmark_observation": {"model": model, "effort": "high", + "environment_id": "test-environment", "environment_fingerprint": "a" * 64, + "knowledge_policy_violation": False}} + if usage: + observation.update(usage_scope="session_total", usage={"input_tokens": 100, + "output_tokens": 20}) + receipt.write_text(json.dumps(observation), encoding="utf-8") + operation(experiment, "record", trial, "--observation", str(receipt)) + return campaign, operation(experiment, "collect", trial)["trial"] + + +def test_default_cli_requires_environment_and_discovers_both_tasks(capsys, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + assert main(["benchmarks", "compare"]) == 0 + result = json.loads(capsys.readouterr().out) + assert result["state"] == "configuration_required" + assert result["resolved_config"]["tasks"] == list(compare.DEFAULT_TASKS) + assert result["resolved_config"]["rounds"] == 8 + assert not list(tmp_path.iterdir()) + + +def test_real_default_preparation_and_off_runtime_boots(tmp_path, capsys): + root = tmp_path / "real-corpus-comparison" + assert main(["benchmarks", "compare", "--environment-id", "software-control", + "--output", str(root)]) == 0 + prepared = json.loads(capsys.readouterr().out) + assert len(prepared["trials"]) == 4 + assert {trial["task"] for trial in prepared["trials"]} == set(compare.DEFAULT_TASKS) + off = next(trial for trial in prepared["trials"] if trial["condition"] == "off") + runtime = compare._io_path(root / off["workspace"] / "runtime") + environment = {**os.environ, "PYTHONPATH": str(runtime), "PYTHONDONTWRITEBYTECODE": "1"} + result = subprocess.run([sys.executable, "-B", "-m", "inferrsi", "benchmarks", "--help"], + cwd=tmp_path, env=environment, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr + assert b"compare" in result.stdout + + +def test_prepare_failure_does_not_publish_partial_experiment(experiment, monkeypatch): + root, _, _ = experiment + from inferrsi.apps.cli.arguments import parser + + def interrupted(*_): + raise OSError("Synthetic copy failure") + + monkeypatch.setattr(compare, "_copy_runtime", interrupted) + output = root.parent / "interrupted" + with pytest.raises(OSError, match="copy failure"): + compare.prepare(parser().parse_args([ + "benchmarks", "compare", "--config", str(root.parent / "config.json"), + "--output", str(output)])) + assert not output.exists() + assert not list(root.parent.glob(".compare-*")) + + +def test_cli_overrides_config_and_reports_sources(tmp_path): + config = tmp_path / "config.json" + config.write_text('{"rounds":2,"policy":{"tolerances":{"speedup":0.01}}}') + from inferrsi.apps.cli.arguments import parser + + resolved, sources = compare.resolve_config(parser().parse_args([ + "benchmarks", "compare", "--config", str(config), "--rounds", "4"])) + assert resolved["rounds"] == 4 and sources["rounds"] == "cli" + assert resolved["policy"]["tolerances"]["speedup"] == 0.01 + assert resolved["policy"]["tolerances"]["wall_seconds"] == 0 + + +@pytest.mark.parametrize("field,value", [("rounds", True), ("repeats", 0), + ("max_minutes", float("inf")), ("task_arguments", []), ("unexpected", 1)]) +def test_invalid_configuration_rejected(tmp_path, field, value): + from inferrsi.apps.cli.arguments import parser + + config = tmp_path / "bad.json" + config.write_text(json.dumps({field: value})) + with pytest.raises(ValueError): + compare.resolve_config(parser().parse_args([ + "benchmarks", "compare", "--config", str(config)])) + + +def test_preparation_separates_corpus_and_preserves_shared_runtime(experiment): + root, initial, _ = experiment + assert len(initial["trials"]) == 2 + assert initial["next_action"]["operation"] == "dispatch" + for trial in initial["trials"]: + corpus = root / trial["workspace"] / "runtime/inferrsi/knowledge/data" + assert corpus.exists() is (trial["condition"] == "on") + status = operation(experiment, "status") + assert [trial["state"] for trial in status["trials"]] == ["pending", "pending"] + + +def test_dispatch_once_and_serial_order(experiment): + first, second = [trial["id"] for trial in experiment[1]["trials"]] + assert operation(experiment, "dispatch", first)["may_create"] + assert not operation(experiment, "dispatch", first)["may_create"] + with pytest.raises(ValueError, match="next declared"): + compare.dispatch(experiment[0], second) + assert operation(experiment, "dispatch", second, "--parallel")["may_create"] + assert not operation(experiment, "dispatch", second, "--parallel")["may_create"] + + +def test_full_pair_collects_independent_finals_trace_and_report(experiment): + first, second = [trial["id"] for trial in experiment[1]["trials"]] + _, on = finish(experiment, first) + assert on["eligible"] and on["metrics"]["speedup"] == 2 + assert on["trace"][0]["strategy"] == "Synthetic strategy" + assert on["trace"][0]["knowledge_refs"] is None + assert on["metrics"]["tokens"] == 120 + _, off = finish(experiment, second, factor=1.5) + assert off["eligible"] and off["knowledge_deliveries"] == [] + report = operation(experiment, "report") + assert report["assessment"]["counts"]["eligible_pairs"] == 1 + assert all(Path(path).is_file() for path in report["reports"].values()) + assert report["next_action"]["operation"] == "completed" + assert operation(experiment, "status")["next_action"]["operation"] == "completed" + + +def test_missing_final_is_not_replaced_by_best_round(experiment): + trial = experiment[1]["trials"][0]["id"] + _, result = finish(experiment, trial, final=False) + assert not result["eligible"] + assert result["metrics"]["speedup"] is None + report = operation(experiment, "report") + assert report["assessment"]["conclusion"]["kind"] == "insufficient_evidence" + + +def test_valid_final_regression_remains_a_measurement(experiment): + trial = experiment[1]["trials"][0]["id"] + _, result = finish(experiment, trial, final_score=0.98) + assert result["eligible"] + assert result["metrics"]["speedup"] == 0.98 + assert result["final"]["speedup"] == 0.98 + assert result["trace"][0]["best_speedup"] == 2 + + +def test_missing_usage_stays_unknown(experiment): + trial = experiment[1]["trials"][0]["id"] + _, result = finish(experiment, trial, usage=False) + assert result["eligible"] and result["metrics"]["tokens"] is None + + +def test_changed_evidence_after_collection_invalidates_report(experiment): + trial = experiment[1]["trials"][0]["id"] + campaign, result = finish(experiment, trial) + assert result["eligible"] + (campaign / "frozen/baseline.py").write_text("CHANGED = True\n") + result = operation(experiment, "report") + document = json.loads(Path(result["reports"]["json"]).read_text(encoding="utf-8")) + assert not document["trials"][0]["eligible"] + assert result["assessment"]["conclusion"]["kind"] == "insufficient_evidence" + + +def test_actual_model_mismatch_cannot_pass(experiment): + trial = experiment[1]["trials"][0]["id"] + _, result = finish(experiment, trial, model="other-model") + assert not result["eligible"] + assert any("model" in reason for reason in result["gaps"]) + + +def test_changed_frozen_runtime_rejects_resume(experiment): + root, initial, _ = experiment + runtime = root / initial["trials"][1]["workspace"] / "runtime/inferrsi" + (runtime / "knowledge/data").mkdir() + (runtime / "knowledge/data/injected.txt").write_text("Unexpected corpus") + with pytest.raises(ValueError, match="runtime changed"): + compare.status(root) + + +def test_round_cannot_cite_undelivered_knowledge(experiment): + root, _, capsys = experiment + campaign = root / "citation-control" + assert main(["benchmarks", "session", "start", "test-work", "--session", str(campaign), + "--rounds", "1", "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "citation-control", "--evaluation-args", ""]) == 0 + capsys.readouterr() + with pytest.raises(SystemExit) as error: + main(["benchmarks", "session", "round", "--session", str(campaign), + "--candidate", str(campaign / "work/candidate.py"), "--hypothesis", "Test", + "--knowledge-ref", "knowledge/missing.json"]) + assert error.value.code == 2 + assert "delivered context" in capsys.readouterr().err + assert json.loads((campaign / "state.json").read_text())["rounds"] == [] diff --git a/tests/test_comparison_report.py b/tests/test_comparison_report.py new file mode 100644 index 00000000..2b0d1feb --- /dev/null +++ b/tests/test_comparison_report.py @@ -0,0 +1,146 @@ +"""Comparison reports retain evidence, unknown values and escaped Agent narrative.""" + +import hashlib +import json + +import pytest + +from inferrsi.apps.views.comparison_report import write_comparison_report +from inferrsi.benchmark.comparison import DEFAULT_POLICY, summarize + + +def report(): + trials = [] + for condition, speedup in (("on", 2), ("off", 1.5)): + trials.append({ + "id": "task-1-" + condition, "task": "task", "repeat": 1, + "condition": condition, "status": "completed", "eligible": True, "gaps": [], + "metrics": {"speedup": speedup, "wall_seconds": 10, "tokens": None}, + "usage": {"tokens": {"value": None, "coverage": "unknown"}}, + "trace": [{"round": 1, "strategy": "request-invariant reuse", + "hypothesis": "Retain conditioning", "knowledge_refs": ["context-1"], + "speedup": speedup, "best_speedup": speedup, "quality": "PASS", + "adopted": True}, + {"round": 2, "strategy": "rejected fusion", "speedup": None, + "best_speedup": speedup, "quality": "FAIL", "adopted": False}], + "final": {"speedup": speedup, "verdict": "PASS", "independent": True}, + "knowledge_deliveries": ["context-1"] if condition == "on" else [], + "evidence": "trials/task-1-" + condition + "/delivery", + }) + return {"schema_version": 1, "focus": "knowledge", "tasks": ["task"], "repeats": 1, + "policy": DEFAULT_POLICY, "trials": trials, + "assessment": summarize(trials, ["task"], 1, DEFAULT_POLICY)} + + +def test_json_markdown_and_offline_html_describe_one_snapshot(tmp_path): + value = report() + value["trials"][0]["final"] = { + "speedup": 2, "binding": {"verified": True}, "delivery_accepted": True, + "selected_candidate": {"candidate_id": "candidate-001", "path": "rounds/candidate.py"}, + "assessment": {"status": "completed", "verdict": "PASS", + "case_metrics": [{"id": "compact", "baseline_median_ms": 10.5, + "candidate_median_ms": 5.25, "speedup": 2, + "samples_ms": {"baseline": [10, 11], + "candidate": [5, 5.5]}}], + "evidence_files": {"measurement/result.json": "raw-evidence-digest"}}, + } + paths = write_comparison_report(value, tmp_path / "report") + assert set(paths) == {"json", "markdown", "html"} + from pathlib import Path + + raw = Path(paths["json"]).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + assert json.loads(raw) == value + markdown = Path(paths["markdown"]).read_text(encoding="utf-8") + html = Path(paths["html"]).read_text(encoding="utf-8") + for document in (markdown, html): + assert digest in document + assert "Efficiency improved" in document + assert "rejected fusion" in document + assert "Independent final" in document + assert "unknown" in document + assert "context-1" in document + assert "Baseline median" in document + assert "compact" in document + assert "candidate-001" in document + assert "Cached input" in document + assert "raw-evidence-digest" not in document + assert "samples_ms" not in document + assert "descriptive" not in document or "statistical significance" in document + assert "Usage and evidence" in html + assert "compact10.55.252.000×2" in html + assert "
" not in html
+
+
+def test_imported_narrative_is_text_in_html_and_markdown(tmp_path):
+    value = report()
+    malicious = ' | [click](javascript:alert(2))\n# Heading'
+    value["trials"][0]["trace"][0]["strategy"] = malicious
+    value["trials"][0]["id"] = '">'
+    paths = write_comparison_report(value, tmp_path)
+    from pathlib import Path
+
+    html = Path(paths["html"]).read_text(encoding="utf-8")
+    markdown = Path(paths["markdown"]).read_text(encoding="utf-8")
+    assert "