From 8d0f3f026ebe10935118929f18aabbc337512263 Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Tue, 17 Nov 2026 15:57:09 +0800 Subject: [PATCH 1/7] Add service-reuse project campaigns Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- docs/architecture.md | 4 + docs/dashboard.md | 5 + docs/operations.md | 1 + docs/optimization-workflow-unification.md | 8 +- docs/service-reuse-campaigns.md | 220 +++++ src/inferrsi/apps/cli/arguments.py | 45 +- src/inferrsi/apps/views/dashboard.py | 9 + src/inferrsi/apps/views/dashboard_data.py | 32 + src/inferrsi/apps/views/dashboard_live.py | 8 +- src/inferrsi/apps/views/optimization_chart.py | 59 ++ src/inferrsi/harness/candidates.py | 14 +- src/inferrsi/harness/local_process.py | 39 + src/inferrsi/harness/optimization.py | 864 ++++++++++++++++- src/inferrsi/harness/optimization_session.py | 74 +- src/inferrsi/harness/project_service.py | 787 +++++++++++++++ src/inferrsi/knowledge/catalog.py | 35 +- .../knowledge/knowledge_operations.py | 94 ++ tests/fixtures/service_reuse_protocol.py | 119 +++ tests/test_project_objectives.py | 98 ++ tests/test_project_service_reuse.py | 908 ++++++++++++++++++ 20 files changed, 3401 insertions(+), 22 deletions(-) create mode 100644 docs/service-reuse-campaigns.md create mode 100644 src/inferrsi/harness/project_service.py create mode 100644 tests/fixtures/service_reuse_protocol.py create mode 100644 tests/test_project_objectives.py create mode 100644 tests/test_project_service_reuse.py diff --git a/docs/architecture.md b/docs/architecture.md index cba3b66f5..fef78e550 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,6 +28,10 @@ does not import Benchmark. Benchmark supplies task definitions and specialized rules, while Harness manages their execution and the complete optimization lifecycle. Neither infers quality criteria from a natural-language goal. Ordinary projects default to the current Agent conversation. +The optional [service-reuse protocol](service-reuse-campaigns.md) adds project +assessment and foreground service ownership within this same controller. Its +protocol and result rules live in `harness.project_service`, with process +lifetime managed by `local_process`. The [native host handoff](native-agent-sessions.md) lets the initiating Agent resolve the task request before any campaign exists, use its host's new-session tool and retain the native identity and returned artifacts. The child owns the diff --git a/docs/dashboard.md b/docs/dashboard.md index 3bbda6443..e779b4441 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -141,6 +141,11 @@ metric is replaced with zero. ## Campaign integration +For managed [service-reuse campaigns](service-reuse-campaigns.md), the dashboard +reads the control record and displays per-objective values, stage verdicts, +selections, repairs, references, reminders and final prerequisites. The compact +live view refreshes the full page. + The following optional layout associates multiple runs with candidate and round records. Create the manifest and supply your own campaign ID and optional target: diff --git a/docs/operations.md b/docs/operations.md index 9832c1479..a9444bb45 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -29,6 +29,7 @@ Effect classes used below: | --- | --- | --- | --- | | Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution | | Optimize an ordinary project in the current Agent | CLI `campaign start/knowledge/round/status/final/recover` | Experiment-executing / local records | Harness freezes explicit caller-provided checks, manages multi-file candidates, executes and assesses retained results. This path does not depend on Benchmark or infer acceptance from the goal. See [shared workflow](optimization-workflow-unification.md) | +| Reuse one project service across multi-scenario checks | CLI `campaign start --evaluation-mode service_reuse`, then `baseline/round/status/recover/final` | Experiment-executing / local records | Caller supplies the protocol, service and profiler scripts; `resume`, `refresh-references` and explicit retry retain recovery decisions. See [service reuse](service-reuse-campaigns.md) | | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | | Discover packaged tasks | CLI `benchmarks list` / `benchmarks show`; Python `benchmark.registry.builtin_tasks` / `describe_task` | Read-only | Reads the task library's registry and opaque manifests from installed resources | | Execute one packaged task | CLI `benchmarks run`; Python `apps.benchmark_operations.run_task` | Experiment-executing | Application launches the task entry, supplies the execution callback and preserves arguments and exit status. The task owns its device, measurement and assessment protocol | diff --git a/docs/optimization-workflow-unification.md b/docs/optimization-workflow-unification.md index c93ed9219..09d127a77 100644 --- a/docs/optimization-workflow-unification.md +++ b/docs/optimization-workflow-unification.md @@ -32,7 +32,7 @@ CLI `final` 返回后自动生成 `delivery//report.md`、离线 `rep ## 时序图 -以下是普通项目的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 +以下是普通项目默认 AB/BA 模式的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 先准备任务条件,再调用 start。start 内复制任务、脚本和基线并写入计划/状态,不执行测量。Agent 逐次调用 round,最后调用 final 独立重测已保存的最佳版本。 @@ -149,7 +149,7 @@ Harness 没有常驻 Agent 驱动。Agent 逐次调用这些操作,代码在 | 一次测量 | 原有 `run` 或 `benchmarks run` | 保留独立使用方式;不会自动生成多轮候选 | | 旧式手动 campaign | 原有 `campaign init/record/decide` | 兼容已有记录;不允许写进新控制器管理的 campaign 形成第二套状态 | -两个优化入口都调用 `harness.optimization_session.session_command`,再调用 **同一个** `harness.optimization.evaluate_round`。执行与评测适配也在 Harness 内。普通项目由 `harness.project` 固化调用方协议并判定留存结果,整个生命周期不导入 Benchmark。 +两个优化入口都调用 `harness.optimization_session.session_command`,使用同一个 `harness.optimization` 控制器。AB/BA 与题包使用 `evaluate_round`;显式 service_reuse 使用该控制器内的 `service_command` 和模式分支,共用锁及单一控制记录。执行与评测适配也在 Harness 内;普通项目整个生命周期不导入 Benchmark。 题库 `start` 由 CLI 经 `benchmark.registry` 定位资源目录,再交给 `harness.task_protocol.prepare` 固化;后续从冻结目录加载题包自己的 `optimization.py`,无需再次查询注册表。专用协议检查仍属于题包。两类 `assess` 都只读留存结果,不重新测量。KnowledgeReader 与 `apps/campaign.py` 的记录回调由 CLI 注入,Harness 不直接导入 Benchmark 或目标 Knowledge 模块,也不经记录模块间接引入知识实现。 @@ -161,6 +161,10 @@ Harness 没有常驻 Agent 驱动。Agent 逐次调用这些操作,代码在 ## 普通项目怎么接入 +普通项目默认使用下面的 AB/BA 流程。需要复用服务时,可显式选择 +[service_reuse 模式](service-reuse-campaigns.md),按该协议准备初始 baseline、 +多场景指标、同服务质量与 profile 脚本,以及恢复和 final 所需条件。 + 无需先登记成题库任务。Agent 准备一个协议目录,包含测量、质量脚本以及要固定的输入/配置,再调用 `campaign start --help` 按实际项目填写参数。 下面是命令结构;尖括号是需要落实的项目条件,不是可直接执行的默认值。命令以 argv 执行,不经 shell;Windows 命令参数中的路径建议用正斜杠。 diff --git a/docs/service-reuse-campaigns.md b/docs/service-reuse-campaigns.md new file mode 100644 index 000000000..2c70d8e5f --- /dev/null +++ b/docs/service-reuse-campaigns.md @@ -0,0 +1,220 @@ +# Service-reuse project campaigns + +Ordinary projects can explicitly choose `--evaluation-mode service_reuse`. +AB/BA remains the default, and packaged `benchmarks session` tasks keep their +existing protocol. This mode does not choose a model, workload, quality task, +profiler or numerical acceptance thresholds. The caller supplies them before +search. Local implementation does not settle pending repository-owner decisions +in issue #467 or qualify a real model. + +The existing `optimization_session` entry assembles the mode inside the shared +`harness.optimization` controller. There is one campaign lock, plan and state; +Apps projects those records without writing selection decisions. + +## Prepare the protocol + +Use the Git checkout root and commit the declared project inputs first. The +controller extracts those files +from the recorded commit into a retained candidate; it never runs the editable +workspace as `final`. The protocol directory contains the caller's scripts and +any declared inputs. Campaign output must be outside the project and protocol. + +Supply a JSON file with these fields; every threshold and workload is explicit: + +| Field | Contract | +| --- | --- | +| `schema_version`, `protocol_id` | Integer 1 and a nonempty caller-owned protocol ID | +| `conditions` | `id`, `workload`, `model`, `precision`, `topology`, `runtime`, `timing_scope`, and an `effective_config` object | +| `commands` | Explicit argv arrays for `precheck`, `service`, `ready`, `warmup`, `measure`, `quality`, `profile`, `stop`; all use a retained `{protocol}` script | +| `sampling` | Positive `stage_timeout_seconds`; `baseline`, `candidate`, `final` objects each declare positive integer `repetitions` and nonnegative integer `warmups`; zero warmups also require `warmup_reason` | +| `scenarios` | Objects declaring unique string `id`, `workload`, positive integer `requests`, and positive integer `required_successes` no greater than requests | +| `objectives` | Objects described below; at least one participates in gain assessment | +| `ranking` | Unique objective IDs in priority order | +| `quality` | `mode`, `dataset`, `scorer`, `key`, `direction`, positive integer `samples`, plus the corresponding quality gate | +| `stagnation` | `{"disabled": true}`, or positive integer `rounds` and `action` equal to `advisory` or `pause` | +| `screening` | Optional argv `command` and positive `estimated_gain`; the estimate never replaces a formal gate | + +An objective declares `id`, `scenario`, dotted JSON `key`, `unit`, finite inclusive +`domain: {"min": ..., "max": ...}`, `direction` (`minimize`/`maximize`), +`aggregate` (`median`/`mean`), `threshold_type`, boolean `gain`, `gain_threshold`, +`parent_regression_limit`, and `cumulative_regression_limit`. Gates must be +finite and nonnegative. Threshold types are `relative_percent`, `absolute`, +and `percentage_point`; the last uses percent values and unit `%`. Use absolute +thresholds for negative-valued domains. + +For quality, `absolute` requires `threshold`. `baseline_comparison` requires +nonnegative `tolerance` and compares with the compatible retained baseline score. +By default the baseline service collects the reference after its performance. +An optional `reference_path` points inside the retained protocol to a compatible +existing baseline quality result. In addition to the quality-result fields, +that result declares `revision`, `conditions`, `service_id`, and `run_id`. +Missing or incompatible reference evidence is INCONCLUSIVE. + +Commands are argv, not shell fragments. Put environment setup and pipelines in +reviewed project scripts. Available replacements are `{python}`, `{protocol}`, +`{candidate}`, `{baseline}`, `{attempt_dir}`, `{effective_config}`, +`{service_receipt}`, and, for finite client commands, `{result}`. Scripts also +receive `INFERRSI_ATTEMPT_REQUEST`, identifying the retained request document. +Use runtime secret injection; do not put credentials into these records. + +## Service and result interfaces + +The `service` command stays in the foreground. Harness owns its process group +or Windows Job through the complete attempt. A short-lived launcher that leaves +an unowned server behind is unsupported. `--evaluation-prefix`, when supplied, +wraps this foreground service. Remote resource ownership needs its own project +adapter; a bare pre-existing endpoint is insufficient. + +`precheck` writes `status: completed` and boolean `passed`. All other clients +write an object with `status: completed` and `binding` equal to the attempt's +request document. The adapter must derive the binding from the actual service, +loaded code and configuration, rather than simply echoing supplied paths. + +`ready` additionally returns `ready: true`, `ownership: local_process_domain`, +`owner_pid` equal to the foreground owner PID, `loaded_root` equal to the actual +execution copy, and `loaded_revision` equal to the retained revision. The caller's +startup script can write its endpoint into `{service_receipt}` for clients. +Harness checks service liveness and binding between stages. + +Each measurement result contains `scenarios[scenario_id]` with `workload`, actual +`requests`, `successes`, the declared metric fields, and `units[objective_id]`. +Every repetition covers every required scenario. Warmup results are separate. +Quality results contain `dataset`, `scorer`, `samples`, and the declared numeric +score field. An explicit `status: candidate_failure` from precheck/quality is a +rejection; crashes, absent evidence and incompatible results are INCONCLUSIVE. + +A candidate profile returns `validated: true`, nonempty attempt-relative +`artifacts`, and numeric host-clock `capture_started_at`/`capture_ended_at` within +the profile stage. Its project script owns actual profiling and validation. +Harness retains the receipt, checks nonempty artifacts, and associates them with +the same instance; it does not invent an integrated profiler. Performance samples +exclude profiling requests and time. `stop` returns `cleanup_complete: true`; +Harness also stops and verifies its owned process domain. Unknown cleanup blocks +further execution until investigated. + +## Lifecycle + +Use `campaign start --help` for shared budget/input arguments. In service mode +supply `--evaluation-config ` and omit the AB/BA single-metric, +quality-command and sampling flags. Mixing both parameter sets is an error. + +```text +inferrsi campaign start --evaluation-mode service_reuse --evaluation-config ... +inferrsi campaign baseline --session +inferrsi campaign knowledge --session --query --context-bytes +inferrsi campaign round --session --candidate --hypothesis +inferrsi campaign status --session +inferrsi campaign final --session --reason +``` + +`start` prepares records without measuring. Baseline has no profile requirement. +A candidate performs functional checks, startup/readiness, necessary warmup, +performance repetitions, quality, then profile, all on one instance. Performance +or quality rejection skips the dependent stages. Screening can schedule formal +validation for a bounded actual improvement below an estimated target; its +samples do not enter formal aggregates. + +Every constrained metric must satisfy both parent-relative and initial-baseline +cumulative regression limits, and at least one designated metric must improve +strictly and reach its formal gain threshold. Values are ranked lexicographically +in their own units, with exact ties retaining the incumbent. A passing candidate +can remain unselected; acceptance and selection are recorded separately. +Relative zero-to-zero change is zero, never a gain; zero-to-nonzero relative +change is undefined and INCONCLUSIVE. Declare absolute/percentage-point gates +before search when zero-capable metrics need those transitions. + +## Failure and recovery + +Profile failure, interruption, invalid binding or missing output leaves the +candidate attempt INCONCLUSIVE and stops campaign execution. It cannot become a +parent or selection. Keep the completed performance/quality evidence and inspect +the exact stage and cleanup result before proceeding. + +```text +inferrsi campaign recover --session +inferrsi campaign round --session --candidate \ + --hypothesis --retry-of --repair-reason +``` + +`recover` collects existing records without launching anything. An explicit retry +creates a new attempt in the same logical round, with new service/samples and the +complete performance/quality/profile sequence. It never inherits a failed +attempt's measurements or overwrites a previous selection. Use optional +`--repair-protocol ` on that retry to retain repaired execution +scripts; the numeric gates and quality declaration remain frozen. Later rounds +use the retained repair, while final uses its selected attempt's own scripts, +revision and effective configuration. Policy changes require a new approved +protocol rather than an execution-script repair. + +Before committing a recovered candidate or a final verdict, Harness rechecks the +required retained profile and its attempt/service binding; final also rechecks +the selected revision and effective configuration. Missing or incompatible +evidence leaves the attempt INCONCLUSIVE and preserves the previous selection. +The execution results and any assessment made before settlement remain recorded. +These checks use structured records and artifact presence, without file fingerprints. + +After an interrupted process with unverified cleanup, `recover`, `resume` or an +explicit retry can take `--cleanup-receipt `. The receipt names +`attempt_id`, positive numeric `observed_at`, `reason`, `cleanup_complete: true`, +and `services`, covering each recorded owner with `service_id`, `pid`, and +`domain_empty: true`. Harness checks recorded POSIX groups are empty as well. +This is explicit investigation evidence; no automatic cleanup or measurement +retry is inferred. `resume --reason ` records an advisory-pause or explicit +abandonment decision; it does not make an incomplete attempt pass or satisfy an +exact-round policy. + +If conditions change, use `refresh-references --conditions --reason `. +It measures the original baseline and comparison parent under the new conditions +(one run when they are the same version), retains both old/new records, and does +not collect another parent profile. This restores comparison references only. +The original baseline uses or collects compatible quality evidence as declared +by the quality protocol. A distinct parent's refresh runs performance only; it +does not consume the baseline quality cache or require another quality test. +At least one new candidate attempt must complete performance, quality and profile +and be accepted under the current conditions before final can be conclusive. +An incompatible old profile and successful reference refreshes do not satisfy +that requirement. Returning to earlier conditions also requires a fresh acceptance. + +Final selects only from current-condition acceptances after a condition change. +Without them it records INCONCLUSIVE and the missing-acceptance reason without +starting a final service or collecting a profile. Otherwise it independently +retests the retained selection against its acceptance parent and initial baseline, +not against itself. If baseline remains selected without a condition change, +it validates baseline evidence without requiring a gain over itself. Final never +collects another profile and requires the selected candidate's compatible retained +profile and effective configuration. + +## Records, compatibility and validation + +Plan schema 3/state schema 2 are specific to service reuse. Missing mode means +legacy AB/BA; old plans are not converted. Legacy sessions still require their recorded runtime for resume. This mode stores commit records, +configuration values, ownership/run IDs and raw evidence without calculating, +verifying or displaying file/content fingerprints. Runtime checks cover the +recorded installed package version; this is not a source-byte integrity claim. +Readonly retained inputs and value/identity checks do not provide cryptographic +tamper detection. Old AB/BA and other APIs retain their default checks. + +`dashboard --input ` displays separate objective units, stage results, +selection, repair history, reference epoch, reminders and final prerequisites. +There is no summed multi-objective speedup. Service-mode knowledge capture retains +source text and existing commit attribution without the old content-identity +reader. It records package core documents and identifies unloaded references; +checkout/store sources requiring content identity are explicitly unavailable. +Captured material proves preparation, not model consumption. + +`summary.usage` covers caller-supplied round-attempt exports, including explicit +repair attempts; `other_attempt_usage` is separate. Unknown usage stays unknown, +and recovering a settled attempt never imports or counts the same usage twice. + +Consecutive complete rejections/unselected candidates may trigger the declared +system-wide analysis reminder. Infrastructure/cancelled/incomplete attempts do +not count; recovery does not double-count. Advisory does not pause execution; +explicit pause is resumed with a reason. + +The synthetic CPU controls are `tests/test_project_objectives.py` and +`tests/test_project_service_reuse.py`. The latter guards fingerprint helpers from +before campaign creation and uses a temporary loopback service. Run it only where +loopback sockets are authorized; it does not require GPU/NPU resources. In a +workspace forbidding fingerprints, do not run conflicting legacy tests. Run +unchanged-default AB/BA/packaged-task regression in a separately authorized +environment. CPU controls cannot establish real-model quality or service gains. diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 0b0c5da18..9ef792cf6 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -55,6 +55,10 @@ def _optimization_actions(stages, *, project=False): start = stages.add_parser("start", help="Freeze explicit task conditions and search budget") start.set_defaults(session_action="start", project_task=project) if project: + start.add_argument("--evaluation-mode", choices=("abba", "service_reuse"), default="abba", + help="AB/BA (default), or one owned service per retained attempt") + start.add_argument("--evaluation-config", type=Path, + help="Explicit service-reuse JSON protocol (required in that mode)") start.add_argument("--repo", type=Path, required=True) start.add_argument("--include", action="append", required=True, help="Project-relative input file/directory; repeat for dependencies") @@ -62,11 +66,11 @@ def _optimization_actions(stages, *, project=False): start.add_argument("--instructions", type=Path, required=True) for name in ("measure-command", "quality-command", "metric-key", "metric-unit", "quality-key", "scope"): - start.add_argument("--" + name, required=True) - start.add_argument("--direction", choices=("minimize", "maximize"), required=True) - start.add_argument("--min-improvement-pct", type=float, required=True) - start.add_argument("--repetitions", type=int, required=True) - start.add_argument("--warmups", type=int, required=True) + start.add_argument("--" + name) + start.add_argument("--direction", choices=("minimize", "maximize")) + start.add_argument("--min-improvement-pct", type=float) + start.add_argument("--repetitions", type=int) + start.add_argument("--warmups", type=int) start.add_argument("--round-mode", choices=("exact", "up_to"), default="up_to") else: start.add_argument("task") @@ -82,7 +86,10 @@ def _optimization_actions(stages, *, project=False): start.add_argument("--knowledge", choices=("on", "off"), default="on") start.add_argument("--evaluation-prefix", default="", help="Quoted allocator argv prefix") start.add_argument("--python", default=sys.executable) - for name in ("status", "round", "final", "recover", "knowledge"): + actions = ("status", "round", "final", "recover", "knowledge") + if project: + actions += ("baseline", "refresh-references", "resume") + for name in actions: stage = stages.add_parser(name) stage.set_defaults(session_action=name) stage.add_argument("--session", "--directory", dest="session", type=Path, required=True) @@ -93,10 +100,32 @@ def _optimization_actions(stages, *, project=False): stage.add_argument("--reason", help="Why the caller stopped an up-to-budget search") if name in ("round", "final"): stage.add_argument("--usage-export", type=Path) + if project and name in ("round", "baseline", "refresh-references"): + stage.add_argument("--retry-of", help="Explicitly rerun this failed attempt in full") + stage.add_argument( + "--repair-protocol", + type=Path, + help="Retain repaired execution scripts for an explicit full retry", + ) + stage.add_argument( + "--repair-reason", help="Investigation and repair before a new attempt" + ) + if name == "refresh-references": + stage.add_argument("--conditions", type=Path, required=True) + stage.add_argument("--reason", required=True) + if name == "resume": + stage.add_argument("--reason", required=True) + if project and name in ("recover", "resume", "round", "baseline", "refresh-references"): + stage.add_argument( + "--cleanup-receipt", + type=Path, + help="Investigation receipt proving interrupted resources were cleaned", + ) if name == "knowledge": stage.add_argument("--query", required=True) - stage.add_argument("--relevance-method", choices=RELEVANCE_METHODS, - default="weighted-lexical-v1") + stage.add_argument( + "--relevance-method", choices=RELEVANCE_METHODS, default="weighted-lexical-v1" + ) stage.add_argument("--context-bytes", type=int, required=True) stage.add_argument("--limit", type=int, default=5) diff --git a/src/inferrsi/apps/views/dashboard.py b/src/inferrsi/apps/views/dashboard.py index 5bf5b22e1..3b65cdbf1 100644 --- a/src/inferrsi/apps/views/dashboard.py +++ b/src/inferrsi/apps/views/dashboard.py @@ -310,6 +310,10 @@ def dashboard_snapshot(source: Path, translations: Path | None = None) -> dict: """Read one view and its charts from the currently recorded evidence.""" source = Path(source).resolve() view = load_dashboard(source) + if view.get("evaluation_mode") == "service_reuse": + from .optimization_chart import render_service + + return {"data": view, "chart": render_service(view["service_optimization"]), "temporal": ""} from .retrieval_projection import project_knowledge, public_value view["knowledge_trace"] = project_knowledge(source, view) @@ -341,6 +345,11 @@ def render_dashboard(snapshot: dict, *, live: bool = False) -> str: """Render a snapshot; only the local live view enables same-origin requests.""" if "fixed_attempt" in snapshot["data"]: return render_attempt(snapshot["data"]["fixed_attempt"], title="fixed attempt Dashboard") + if snapshot["data"].get("evaluation_mode") == "service_reuse": + refresh = '' if live else "" + return ('' + refresh + + 'Service-reuse optimization
' + snapshot["chart"] + + '
') view = {**snapshot["data"], "live": live} payload = json.dumps(view, ensure_ascii=False, allow_nan=False) # The JSON data block cannot terminate its script element or create markup. diff --git a/src/inferrsi/apps/views/dashboard_data.py b/src/inferrsi/apps/views/dashboard_data.py index b5fbad929..113832e99 100644 --- a/src/inferrsi/apps/views/dashboard_data.py +++ b/src/inferrsi/apps/views/dashboard_data.py @@ -762,6 +762,38 @@ def load_dashboard(source: Path) -> dict: root = Path(source).resolve() if not root.is_dir(): raise ValueError("dashboard input must be an existing directory") + manifest = root / "campaign.json" + if manifest.is_symlink(): + raise ValueError("Campaign manifest must not be symbolic") + if ( + manifest.is_file() + and not manifest.is_symlink() + and manifest.stat().st_size <= 4 * 1024 * 1024 + ): + try: + header = json.loads(manifest.read_text(encoding="utf-8")) + except ValueError: + raise ValueError("Invalid campaign manifest") from None + if not isinstance(header, dict): + raise ValueError("Campaign manifest must be an object") + service_mode = header.get("evaluation_mode") == "service_reuse" + if header.get("managed_optimization") and (root / "plan.json").is_file(): + plan_header = json.loads((root / "plan.json").read_text(encoding="utf-8")) + service_mode |= ( + plan_header.get("evaluation_mode") == "service_reuse" + or plan_header.get("schema_version") == 3 + ) + if service_mode: + from inferrsi.harness.optimization import read_records + + from .retrieval_projection import public_value + + _, _, summary = read_records(root) + return { + "campaign_id": sanitized_text(header["campaign_id"]), + "evaluation_mode": "service_reuse", + "service_optimization": public_value(summary), + } single = (root / "run.lock.json").is_file() campaign, campaign_state = read_object(root / "campaign.json", root) if not single and campaign_state != "present": diff --git a/src/inferrsi/apps/views/dashboard_live.py b/src/inferrsi/apps/views/dashboard_live.py index 884606ee5..053622075 100644 --- a/src/inferrsi/apps/views/dashboard_live.py +++ b/src/inferrsi/apps/views/dashboard_live.py @@ -56,7 +56,7 @@ def __init__( self.source = Path(source).resolve() if source is not None else None self.translations = Path(translations).resolve() if translations else None self._fixed_attempt = deepcopy(fixed_attempt) - self.snapshot() + self._service_reuse = self.snapshot()["data"].get("evaluation_mode") == "service_reuse" super().__init__(("127.0.0.1", port), DashboardHandler) def snapshot(self) -> dict: @@ -119,7 +119,7 @@ def do_GET(self) -> None: self._reply(403, b"Invalid host") return events = re.fullmatch(r"/api/profile-events/([0-9a-f]{64})", self.path) - if events and self.server.source is not None: + if events and self.server.source is not None and not self.server._service_reuse: from inferrsi.harness.records.profile_timeline import profile_event_table payload = profile_event_table(self.server.source, events.group(1)) @@ -143,6 +143,10 @@ def do_GET(self) -> None: body, encoding = self._representation( json.dumps(snapshot, ensure_ascii=False, allow_nan=False).encode("utf-8") ) + if snapshot["data"].get("evaluation_mode") == "service_reuse": + self._reply(200, body, content_type="application/json; charset=utf-8", + encoding=encoding) + return etag = '"' + hashlib.sha256(body).hexdigest() + '"' if self.headers.get("If-None-Match") == etag: self._reply(304, etag=etag, encoding=encoding) diff --git a/src/inferrsi/apps/views/optimization_chart.py b/src/inferrsi/apps/views/optimization_chart.py index 9c2b83a2d..51d51d049 100644 --- a/src/inferrsi/apps/views/optimization_chart.py +++ b/src/inferrsi/apps/views/optimization_chart.py @@ -1,9 +1,68 @@ """Render shared Harness optimization records without filling missing measurements.""" +import json import math from html import escape +def render_service(summary): + final = summary["final_assessment"] + chunks = [ + "

Service-reuse optimization

", + "

Independent final: " + escape(final["verdict"] if final else "Not recorded") + "

", + "

Final prerequisite: " + escape(summary["final_block_reason"] or "satisfied") + "

", + "

Reference epoch: " + + str(summary["reference_epoch"]) + + "; accepted attempts: " + + str(len(summary["accepted_attempt_ids"])) + + "

", + ] + for attempt in summary["attempts"]: + execution = attempt["execution"] + chunks.extend( + [ + "

" + escape(attempt["kind"] + ": " + attempt["attempt_id"]) + "

", + "

" + + escape(attempt["assessment"]["verdict"] + ": " + attempt["assessment"]["reason"]) + + "

", + "

Retained revision: " + escape(attempt["revision"]) + "

", + "" + "" + "", + ] + ) + metrics = (attempt["assessment"].get("performance") or {}).get("metrics", {}) + for name, metric in metrics.items(): + chunks.append( + "" + + "".join( + "" + for field in ( + name, + metric["value"], + metric["unit"], + metric["parent_change"], + metric["initial_change"], + ) + ) + + "" + ) + chunks.append("
Objective values in their declared units
ObjectiveValueUnitParent changeInitial change
" + escape(str(field)) + "
    ") + for stage in execution.get("steps", []): + chunks.append( + "
  • " + + escape(stage["id"] + ": " + stage["status"] + ": " + stage.get("reason", "")) + + "
  • " + ) + chunks.append("
") + chunks.append( + "
Recorded selection, references, repairs, reminders and knowledge" + "" + "
" + escape(json.dumps(summary, ensure_ascii=False, indent=2)) + "
" + ) + return "".join(chunks) + + def render_result(summary: dict) -> str: final = summary["final_assessment"] verdict = final["verdict"] if final else "Not recorded" diff --git a/src/inferrsi/harness/candidates.py b/src/inferrsi/harness/candidates.py index 11d647ccc..75772da9c 100644 --- a/src/inferrsi/harness/candidates.py +++ b/src/inferrsi/harness/candidates.py @@ -43,22 +43,30 @@ def identity(source: Path, includes: list[str] | None = None) -> str: for name, path in members(source, includes).items()}) -def retain(source: Path, destination: Path, includes: list[str] | None = None) -> str: +def retain(source: Path, destination: Path, includes: list[str] | None = None, *, + fingerprint: bool = True) -> str | None: """Copy bytes, then reject a changing input set or partial snapshot.""" source = source.absolute() if destination.resolve().is_relative_to(source.resolve()): raise ValueError("Snapshot output must be outside the candidate inputs") - before = identity(source, includes) + selected = members(source, includes) + before = identity(source, includes) if fingerprint else None if source.is_file(): destination.write_bytes(source.read_bytes()) shutil.copymode(source, destination) else: destination.mkdir() - for name, path in members(source, includes).items(): + for name, path in selected.items(): target = destination / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(path.read_bytes()) shutil.copymode(path, target) + if not fingerprint: + if set(members(source, includes)) != set(selected) or ( + set(members(destination)) != set(selected) + ): + raise ValueError("Candidate file set changed while taking its snapshot") + return None if identity(source, includes) != before or identity(destination) != before: raise ValueError("Candidate changed while taking its snapshot") return before diff --git a/src/inferrsi/harness/local_process.py b/src/inferrsi/harness/local_process.py index 1208e5063..b14f0dec1 100644 --- a/src/inferrsi/harness/local_process.py +++ b/src/inferrsi/harness/local_process.py @@ -2,9 +2,12 @@ import signal import subprocess import time +from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path +from .atomic_json import atomic_write_json + # H local process control domain (moved from infra in H02): one command inside its # Windows Job or POSIX process group, deadline and cancellation, and tree cleanup. # Benchmark or quality results are never interpreted here. @@ -208,6 +211,42 @@ def _wait(process: subprocess.Popen, deadline: float, cancellation) -> None: raise +@contextmanager +def supervised_service(argv, cwd, environment, directory, owner): + """Own a foreground service until every client stage has finished.""" + directory.mkdir(parents=True) + receipt = {**owner, "argv": argv, "cwd": str(cwd), "pid": None, + "process_group_id": None, "started_at": None, "ended_at": None, + "domain_empty": None, "returncode": None} + path = directory / "owner.json" + atomic_write_json(path, receipt) + process, job = None, None + try: + if os.name == "nt": + job = _WindowsJob() + options = ({"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == "nt" + else {"start_new_session": True}) + with (directory / "stdout.log").open("wb") as stdout, ( + directory / "stderr.log" + ).open("wb") as stderr: + process = subprocess.Popen(argv, cwd=cwd, env=environment, stdin=subprocess.DEVNULL, + stdout=stdout, stderr=stderr, shell=False, **options) + if job is not None: + job.attach(process) + receipt.update(pid=process.pid, started_at=_utc_now(), + process_group_id=process.pid if os.name != "nt" else None) + atomic_write_json(path, receipt) + yield process, receipt + finally: + if process is not None: + _stop_tree(process, job) + receipt.update(returncode=process.poll(), ended_at=_utc_now(), + domain_empty=job.emptied if job else _group_empty(process.pid)) + elif job is not None: + job.close() + atomic_write_json(path, receipt) + + def _supervise( argv: list[str], cwd: Path, diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 02c703b30..a975b5032 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -19,6 +19,8 @@ def _locked(root): def initialize(root: Path, plan: dict) -> dict: + if plan.get("evaluation_mode") == "service_reuse": + return _service_initialize(root, plan) if type(plan["round_limit"]) is not int or plan["round_limit"] < 1: raise ValueError("rounds must be a positive integer") if (type(plan["max_seconds"]) not in (int, float) @@ -55,6 +57,29 @@ def initialize(root: Path, plan: dict) -> dict: def _control(root): plan = json.loads((root / "plan.json").read_text(encoding="utf-8")) state = json.loads((root / "state.json").read_text(encoding="utf-8")) + manifest_header = json.loads((root / "campaign.json").read_text(encoding="utf-8")) + if ( + plan.get("evaluation_mode") == "service_reuse" + or plan.get("schema_version") == 3 + or manifest_header.get("evaluation_mode") == "service_reuse" + ): + from . import project_service as service + + manifest = service.read(root / "campaign.json") + if ( + plan.get("evaluation_mode") != "service_reuse" + or plan.get("schema_version") != 3 + or state.get("schema_version") != 2 + or ( + manifest.get("evaluation_mode") != "service_reuse" + or manifest.get("protocol_id") != plan["evaluation_settings"]["protocol_id"] + or state.get("protocol_id") != manifest["protocol_id"] + ) + ): + raise ValueError("Incompatible service campaign records") + service.no_fingerprints(plan) + service.no_fingerprints(state) + return plan, state if digest(plan) != state["plan_digest"]: raise ValueError("Frozen optimization plan changed") manifest = json.loads((root / "campaign.json").read_text(encoding="utf-8")) @@ -65,6 +90,8 @@ def _control(root): def _load(root): plan, state = _control(root) + if plan.get("evaluation_mode") == "service_reuse": + return _service_load(root, plan, state) frozen = root / "frozen" paths = list(frozen.rglob("*")) if frozen.is_symlink() or any(path.is_symlink() for path in paths): @@ -97,6 +124,10 @@ def _load(root): def status(root: Path) -> dict: + plan, control = _control(root) + if plan.get("evaluation_mode") == "service_reuse": + plan, control = _load(root) + return _service_status(root, plan, control) _, control = _control(root) if control["state"] == "blocked" and control.get("integrity_error"): # A diagnostic is still readable after rejected input drift. Never expose @@ -159,18 +190,23 @@ def capture_context(root: Path, reader, *, retain_context, **request) -> dict: return {"reference": reference, "context": json.loads(path.read_text(encoding="utf-8"))} -def _usage_record(source): +def _usage_record(source, *, fingerprint=True): if source is None: return {"usage": None, "tool_calls": None, "usage_source": None}, None raw = Path(source).read_bytes() payload = json.loads(raw) if not isinstance(payload, dict): raise ValueError("Usage export must be a JSON object") + if not fingerprint: + from .project_service import no_fingerprints + + no_fingerprints(payload) usage = normalize_usage(payload) if not usage or usage.get("input_tokens") is None or usage.get("output_tokens") is None: raise ValueError("Usage export requires nonnegative input_tokens and output_tokens") return ({"usage": usage, "tool_calls": None, - "usage_source": {"kind": "caller_export", "sha256": hashlib.sha256(raw).hexdigest()}}, + "usage_source": {"kind": "caller_export", **( + {"sha256": hashlib.sha256(raw).hexdigest()} if fingerprint else {})}}, raw) @@ -181,6 +217,8 @@ def _token_totals(records): def _summary(root, plan, state): + if plan.get("evaluation_mode") == "service_reuse": + return _service_summary(root, plan, state) rounds = state["rounds"] final = state["final"] return { @@ -384,3 +422,825 @@ def recover(root: Path, assess) -> dict: raise ValueError("Execution record unavailable, inspect cleanup before recovery") execution = json.loads(path.read_text(encoding="utf-8")) return _settle(root, plan, state, pending, execution, assess) + + +def _service_initialize(root, plan): + from . import project_service as service + + service.validate(plan["evaluation_settings"]) + if ( + type(plan["round_limit"]) is not int + or plan["round_limit"] < 1 + or ( + plan["round_mode"] not in ("up_to", "exact") + or not service.positive(plan["max_seconds"]) + or not service.positive(plan["evaluation_timeout_seconds"]) + or not service.positive(plan["final_reserve_seconds"]) + or plan["final_reserve_seconds"] >= plan["max_seconds"] + ) + ): + raise ValueError("Invalid service campaign round/time budget") + started = time.time() + state = { + "schema_version": 2, + "protocol_id": plan["evaluation_settings"]["protocol_id"], + "state": "needs_baseline", + "started_at": started, + "deadline": started + plan["max_seconds"], + "pending": None, + "rounds": [], + "attempts": [], + "contexts": [], + "recovery_events": [], + "reference_epoch": 0, + "protocol_path": "frozen/protocol", + "conditions": plan["evaluation_settings"]["conditions"], + "requires_current_conditions_acceptance": False, + "accepted_attempt_ids": [], + "initial_reference": None, + "comparison_parent": None, + "best": None, + "final": None, + "stop_reason": None, + "stagnation": {"consecutive": 0, "notified": False, "events": []}, + } + atomic_write_json(root / "plan.json", plan) + atomic_write_json( + root / "campaign.json", + { + "campaign_id": root.name, + "managed_optimization": True, + "evaluation_mode": "service_reuse", + "protocol_id": state["protocol_id"], + }, + ) + atomic_write_json(root / "state.json", state) + return _service_status(root, plan, state) + + +def _service_load(root, plan, state): + from inferrsi import __version__ + + from . import project_service as service + + if plan.get("runtime_version") != __version__: + raise ValueError("Runtime package version changed; use the recorded version to resume") + service.validate(plan["evaluation_settings"]) + if service.read(root / "frozen/evaluation.json") != plan["evaluation_settings"]: + raise ValueError("Retained protocol and plan disagree") + service.validate_conditions(state["conditions"]) + candidates.members(service.safe_path(root, state["protocol_path"])) + candidates.members(service.safe_path(root, plan["baseline_path"])) + for attempt in state["attempts"]: + directory = service.safe_path(root, attempt["path"]) + if service.read(directory / "observation.json") != attempt: + raise ValueError("Retained attempt and control record disagree") + for context in state["contexts"]: + service.read(service.safe_path(root, context["path"])) + for event in state["recovery_events"]: + if event["kind"] == "cleanup_inspection": + proof = service.read(service.safe_path(root, event["path"])) + if ( + proof["attempt_id"] != event["attempt_id"] + or proof.get("cleanup_complete") is not True + ): + raise ValueError("Retained cleanup inspection is incomplete") + if state["best"] is not None: + candidates.members(service.safe_path(root, state["best"]["path"])) + return plan, state + + +def _service_profile(root, selected, conditions): + from . import project_service as service + + if selected is None: + return False + profile = selected.get("profile") + try: + if ( + not profile + or not isinstance(selected.get("binding"), dict) + or profile["conditions"] != conditions + or (profile["attempt_id"] != selected["attempt_id"]) + or profile["binding"] != selected["binding"] + or any( + selected["binding"].get(key) != selected[key] + for key in ("attempt_id", "revision", "conditions") + ) + or selected["binding"].get("effective_config") != conditions["effective_config"] + or not selected["binding"].get("service_id") + or not selected["binding"].get("run_id") + or not isinstance(profile["artifacts"], list) + or not profile["artifacts"] + ): + return False + trace = service.read(service.safe_path(root, profile["result"])) + if ( + trace.get("status") != "completed" + or trace.get("validated") is not True + or (trace.get("binding") != profile["binding"]) + or trace != profile.get("receipt") + ): + return False + return all( + service.safe_path(root, path).is_file() + and service.safe_path(root, path).stat().st_size > 0 + for path in profile["artifacts"] + ) + except (OSError, ValueError, KeyError, TypeError): + return False + + +def _service_selected_evidence_reason(root, state): + from . import project_service as service + + if not state["initial_reference"] or state["best"] is None: + return "missing_baseline_reference" + retained = next( + (item for item in state["attempts"] if item["attempt_id"] == state["best"]["attempt_id"]), + None, + ) + if ( + retained is None + or retained["assessment"]["verdict"] != "PASS" + or (_service_choice(retained, retained["execution"]) != state["best"]) + ): + return "unverifiable_selected_attempt_binding" + if state["best"]["conditions"] != state["conditions"]: + return "incompatible_selected_conditions" + try: + config = service.read(service.safe_path(root, state["best"]["effective_config_path"])) + if config != state["best"]["conditions"]["effective_config"]: + return "unverifiable_selected_configuration" + except (OSError, ValueError, KeyError): + return "unverifiable_selected_configuration" + if state["best"]["kind"] == "candidate" and not _service_profile( + root, state["best"], state["conditions"] + ): + return "missing_or_incompatible_selected_profile" + return None + + +def _service_final_reason(root, plan, state): + if state["requires_current_conditions_acceptance"] or ( + state["reference_epoch"] > 0 + and ( + state["best"] is None + or state["best"]["attempt_id"] not in state["accepted_attempt_ids"] + or state["best"]["reference_epoch"] != state["reference_epoch"] + ) + ): + return "missing_current_conditions_acceptance" + reason = _service_selected_evidence_reason(root, state) + if reason: + return reason + if state["pending"] is not None or state["state"] != "active": + return "unresolved_attempt_or_pause" + completed = sum(item["terminal"] for item in state["rounds"]) + if not completed or plan["round_mode"] == "exact" and completed != plan["round_limit"]: + return "incomplete_round_policy" + if time.time() >= state["deadline"]: + return "deadline_exhausted" + return None + + +def _service_status(root, plan, state): + reason = _service_final_reason(root, plan, state) + remaining = max(0, state["deadline"] - time.time()) + next_action = ( + "campaign_baseline" + if state["state"] == "needs_baseline" + else "recover_existing_attempt_without_relaunch" + if state["pending"] + else "investigate_then_explicit_retry_or_resume" + if state["state"] == "blocked" + else "resume_after_recorded_analysis" + if state["state"] == "paused" + else "submit_current_conditions_candidate" + if state["requires_current_conditions_acceptance"] + else "independently_retest_selected_candidate_with_final" + if len(state["rounds"]) >= plan["round_limit"] + else "edit_candidate_and_submit_round" + ) + if state["state"] == "completed": + next_action = "completed" + return { + "session": str(root), + "evaluation_mode": "service_reuse", + "state": state["state"], + "next_action": next_action, + "can_finalize": reason is None, + "final_block_reason": reason, + "completed_rounds": sum(item["terminal"] for item in state["rounds"]), + "allocated_rounds": len(state["rounds"]), + "round_limit": plan["round_limit"], + "remaining_seconds": remaining, + "remaining_search_seconds": max(0, remaining - plan["final_reserve_seconds"]), + "round_mode": plan["round_mode"], + "working_candidate": plan["working_candidate"], + "best": state["best"], + "no_candidate_selected": state["best"] is not None and state["best"]["kind"] == "baseline", + "pending": state["pending"], + "reference_epoch": state["reference_epoch"], + "conditions": state["conditions"], + "accepted_attempt_ids": state["accepted_attempt_ids"], + "stagnation": state["stagnation"], + "final": state["final"], + "latest_observation": state["attempts"][-1] if state["attempts"] else None, + "summary": str(root / "summary.json"), + } + + +def _service_summary(root, plan, state): + return { + "schema_version": 2, + "evaluation_mode": "service_reuse", + "task": plan["task"], + "condition": plan["condition"], + "state": state["state"], + "planned_rounds": plan["round_limit"], + "completed_rounds": sum(item["terminal"] for item in state["rounds"]), + "selected_candidate": state["best"], + "selection": plan["selection"], + "final_assessment": state["final"]["assessment"] if state["final"] else None, + "final_block_reason": ( + None if state["state"] == "completed" else _service_final_reason(root, plan, state) + ), + "optimization_speedup": None, + "reference_epoch": state["reference_epoch"], + "conditions": state["conditions"], + "initial_reference": state["initial_reference"], + "comparison_parent": state["comparison_parent"], + "accepted_attempt_ids": state["accepted_attempt_ids"], + "attempts": state["attempts"], + "knowledge_deliveries": state["contexts"], + "stagnation": state["stagnation"], + "usage": _token_totals([item for item in state["attempts"] if item["kind"] == "round"]), + "usage_scope": "caller_round_attempt_exports", + "other_attempt_usage": _token_totals( + [item for item in state["attempts"] if item["kind"] != "round"] + ), + "stop_reason": state["stop_reason"], + "wall_elapsed_seconds": time.time() - state["started_at"], + } + + +def _service_choice(pending, execution): + return { + "kind": "candidate" if pending["role"] == "candidate" else "baseline", + "attempt_id": pending["attempt_id"], + "candidate_id": pending["candidate_id"], + "path": pending["path"] + "/candidate", + "revision": pending["revision"], + "conditions": pending["conditions"], + "reference_epoch": pending["reference_epoch"], + "effective_config_path": pending["path"] + "/effective-config.json", + "measurements": execution["measurements"], + "quality_score": execution["quality_score"], + "profile": execution.get("profile"), + "protocol_path": pending["protocol_path"], + "binding": execution["binding"], + "parent_reference_at_acceptance": ( + { + key: value + for key, value in pending["parent"].items() + if key != "parent_reference_at_acceptance" + } + if pending.get("parent") + else None + ), + } + + +def _service_settle(root, plan, state, pending, execution): + from . import project_service as service + + path = service.safe_path(root, pending["path"]) / "observation.json" + if path.exists(): + record = service.read(path) + if record["attempt_id"] != pending["attempt_id"]: + raise ValueError("Recovery attempt identity mismatch") + else: + record = { + **pending, + "execution": execution, + "assessment": execution["assessment"], + "elapsed_seconds": execution.get("elapsed_seconds"), + } + if any(item["attempt_id"] == record["attempt_id"] for item in state["attempts"]): + state["pending"] = None + return _service_status(root, plan, state) + reason, choice = None, None + if record["kind"] == "round" and record["assessment"]["verdict"] == "PASS": + choice = _service_choice(record, record["execution"]) + if not _service_profile(root, choice, record["conditions"]): + reason = "profile: incomplete_or_unverifiable_after_cleanup" + elif record["kind"] == "final" and record["assessment"]["verdict"] != "INCONCLUSIVE": + reason = _service_selected_evidence_reason(root, state) + if reason: + choice = None + record["assessment_before_settlement"] = record["assessment"] + record["assessment"] = service.assessment( + "INCONCLUSIVE", + reason, + performance=record["assessment"].get("performance"), + quality_passed=record["assessment"].get("quality_passed"), + ) + if record["kind"] == "round": + record["selected"] = choice is not None and ( + state["best"] is None + or service.better( + plan["evaluation_settings"], + choice["measurements"]["values"], + state["best"]["measurements"]["values"], + ) + ) + record["best_after"] = choice if record["selected"] else state["best"] + atomic_write_json(path, record) + judged = record["assessment"] + verdict, kind = judged["verdict"], record["kind"] + if kind == "baseline": + if verdict == "PASS": + choice = _service_choice(record, record["execution"]) + state.update( + initial_reference=choice, comparison_parent=choice, best=choice, state="active" + ) + else: + state["state"] = "blocked" + elif kind == "refresh": + if verdict == "PASS": + initial, parent = record["execution"]["references"] + state.update( + initial_reference=initial, comparison_parent=parent, best=None, state="active" + ) + else: + state["state"] = "blocked" + elif kind == "round": + round_record = state["rounds"][record["round_number"] - 1] + round_record["attempt_ids"].append(record["attempt_id"]) + round_record["terminal"] = verdict != "INCONCLUSIVE" + selected = record["selected"] + if verdict == "PASS": + state["accepted_attempt_ids"].append(record["attempt_id"]) + if selected: + state.update( + best=choice, + comparison_parent=choice, + requires_current_conditions_acceptance=False, + ) + state["state"] = "blocked" if verdict == "INCONCLUSIVE" else "active" + if verdict != "INCONCLUSIVE": + stagnation = state["stagnation"] + if selected: + stagnation.update(consecutive=0, notified=False) + else: + stagnation["consecutive"] += 1 + policy = plan["evaluation_settings"]["stagnation"] + if ( + not policy.get("disabled") + and not stagnation["notified"] + and (stagnation["consecutive"] >= policy["rounds"]) + ): + event = { + "id": record["attempt_id"] + "-stagnation", + "round": record["round_number"], + "consecutive": stagnation["consecutive"], + "best": state["best"], + "attempt_path": record["path"], + "action": policy["action"], + "message": "Reassess system-wide critical paths, compute, communication, " + "scheduling and resources across scenarios. Use available profiles; " + "record evidence gaps, the next hypothesis and restructuring tradeoffs.", + } + stagnation["events"].append(event) + stagnation["notified"] = True + if policy["action"] == "pause": + state["state"] = "paused" + elif kind == "final": + state["final"] = record + state["stop_reason"] = record["hypothesis"] + if verdict != "INCONCLUSIVE": + state["state"] = "completed" + elif not record.get("admission_failure"): + state["state"] = "blocked" + state["attempts"].append(record) + state["pending"] = None + atomic_write_json(root / "state.json", state) + atomic_write_json(root / "summary.json", _service_summary(root, plan, state)) + return _service_status(root, plan, state) + + +def _service_execute(root, directory, plan, pending, deadline): + from . import project_service as service + + if pending["kind"] != "refresh": + return service.execute(root, directory, plan, pending, deadline) + references, runs = [], [] + started = time.monotonic() + initial = pending["refresh_initial"] + for label, reference_kind, source in ( + ("baseline", "initial", initial), + ("parent", "parent", pending["refresh_parent"]), + ): + if ( + reference_kind == "parent" + and source["revision"] == initial["revision"] + and ( + source["conditions"]["effective_config"] + == initial["conditions"]["effective_config"] + ) + ): + references.append(references[0]) + continue + relative = pending["path"] + "/" + label + subdir = service.safe_path(root, relative) + subdir.mkdir() + candidates.retain( + service.safe_path(root, source["path"]), subdir / "candidate", fingerprint=False + ) + job = { + **pending, + "path": relative, + "revision": source["revision"], + "attempt_id": pending["attempt_id"] + "-" + label, + "candidate_id": source["candidate_id"], + "role": "baseline", + "reference_kind": reference_kind, + "parent": None, + "initial": None, + } + result = service.execute(root, subdir, plan, job, deadline) + runs.append(result) + if result["assessment"]["verdict"] != "PASS": + return { + "assessment": service.assessment("INCONCLUSIVE", "reference_refresh_incomplete"), + "runs": runs, + "cleanup_verified": result["cleanup_verified"], + "elapsed_seconds": time.monotonic() - started, + } + references.append(_service_choice(job, result)) + return { + "assessment": service.assessment( + "PASS", "references_refreshed_not_candidate_acceptance", eligible=False + ), + "retained_parent_profile": pending["refresh_parent"].get("profile"), + "parent_profile_applicability": "not_applicable" + if pending["refresh_parent"].get("profile") + else "not_available", + "references": references, + "runs": runs, + "cleanup_verified": True, + "elapsed_seconds": time.monotonic() - started, + } + + +def _service_cleanup_confirmed(state, attempt): + return attempt.get("execution", {}).get("cleanup_verified") is True or any( + event["kind"] == "cleanup_inspection" and event["attempt_id"] == attempt["attempt_id"] + for event in state["recovery_events"] + ) + + +def _service_cleanup_receipt(root, state, path): + from . import project_service as service + from .local_process import _group_empty + + latest = state["pending"] or (state["attempts"][-1] if state["attempts"] else None) + proof = service.read(path) + if ( + latest is None + or proof.get("attempt_id") != latest["attempt_id"] + or not proof.get("reason") + or ( + proof.get("cleanup_complete") is not True + or not service.positive(proof.get("observed_at")) + or not isinstance(proof.get("services"), list) + or any(not isinstance(item, dict) for item in proof.get("services", [])) + ) + ): + raise ValueError( + "Cleanup inspection must identify the attempt, time, reason and all services" + ) + directory = service.safe_path(root, latest["path"]) + owners = [ + service.read(service.safe_path(directory, name)) + for name in ( + "service/owner.json", + "baseline/service/owner.json", + "parent/service/owner.json", + ) + if service.safe_path(directory, name).is_file() + ] + if len(owners) != len(proof["services"]): + raise ValueError("Cleanup inspection must cover every recorded service owner") + for owner in owners: + matches = [ + item for item in proof["services"] if item.get("service_id") == owner["service_id"] + ] + if ( + len(matches) != 1 + or matches[0].get("pid") != owner["pid"] + or (matches[0].get("domain_empty") is not True) + ): + raise ValueError("Cleanup inspection service owner mismatch") + if ( + owner.get("process_group_id") is not None + and _group_empty(owner["process_group_id"]) is not True + ): + raise ValueError("Recorded process domain is still live or unobservable") + relative = "recoveries/cleanup-" + uuid.uuid4().hex + ".json" + (root / "recoveries").mkdir(exist_ok=True) + atomic_write_json(service.safe_path(root, relative), proof) + state["recovery_events"].append( + {"kind": "cleanup_inspection", "attempt_id": latest["attempt_id"], "path": relative} + ) + atomic_write_json(root / "state.json", state) + + +def service_command(root, args, *, knowledge_reader=None): + """Service-mode operations inside the shared controller's lock and control record.""" + from . import project_service as service + + with _locked(root): + plan, state = _load(root) + action = args.session_action + if getattr(args, "cleanup_receipt", None) is not None: + _service_cleanup_receipt(root, state, args.cleanup_receipt) + if action == "status": + return _service_status(root, plan, state) + if action == "recover": + if state["pending"] is None: + return _service_status(root, plan, state) + pending = state["pending"] + directory = service.safe_path(root, pending["path"]) + observation = directory / "observation.json" + if observation.exists(): + result = service.read(observation)["execution"] + else: + try: + result = service.read(directory / "execution/execution.json") + except (OSError, ValueError): + if not _service_cleanup_confirmed(state, pending): + raise ValueError( + "No execution receipt; inspect cleanup before recovery" + ) from None + result = {"status": "interrupted", "available_evidence_path": pending["path"]} + if result.get("status") == "running" or result.get("assessment") is None: + if not _service_cleanup_confirmed(state, pending): + raise ValueError( + "Execution incomplete; inspect cleanup and supply --cleanup-receipt" + ) + result.update( + status="interrupted", + cleanup_verified=True, + assessment=service.assessment( + "INCONCLUSIVE", "interrupted_execution_requires_full_retry" + ), + ) + return _service_settle(root, plan, state, pending, result) + if action == "knowledge": + _active( + { + **state, + "state": "active" if state["state"] == "needs_baseline" else state["state"], + }, + reserve=plan["final_reserve_seconds"], + ) + if not plan["knowledge_allowed"] or knowledge_reader is None: + raise ValueError("Knowledge delivery is disabled or unavailable") + context = knowledge_reader.collect_service_context( + args.query, + relevance_method=args.relevance_method, + limit=args.limit, + context_bytes=args.context_bytes, + ) + service.no_fingerprints(context) + relative = "knowledge/context-" + uuid.uuid4().hex + ".json" + (root / "knowledge").mkdir(exist_ok=True) + atomic_write_json(service.safe_path(root, relative), context) + reference = { + "path": relative, + "kind": "delivered_context", + "prepared_not_consumed": True, + } + state["contexts"].append(reference) + atomic_write_json(root / "state.json", state) + return {"reference": reference, "context": context} + if action == "resume": + if not args.reason.strip() or state["pending"] is not None: + raise ValueError( + "Record investigation before resume; resolve pending execution first" + ) + latest = state["attempts"][-1] if state["attempts"] else None + if ( + state["state"] == "active" + and state["recovery_events"] + and ( + state["recovery_events"][-1].get("kind") == "resume" + and state["recovery_events"][-1].get("reason") == args.reason + ) + ): + return _service_status(root, plan, state) + if ( + state["state"] not in ("paused", "blocked") + or latest is None + or not _service_cleanup_confirmed(state, latest) + or latest["kind"] in ("baseline", "refresh") + ): + raise ValueError("Cannot resume without verified cleanup and usable references") + state["recovery_events"].append( + {"kind": "resume", "reason": args.reason, "attempt_id": latest["attempt_id"]} + ) + state["state"] = "active" + atomic_write_json(root / "state.json", state) + return _service_status(root, plan, state) + retry = getattr(args, "retry_of", None) + repair = getattr(args, "repair_reason", None) + repaired_protocol = getattr(args, "repair_protocol", None) + if repaired_protocol and not retry: + raise ValueError("Execution script repairs require an explicit --retry-of") + old = next((item for item in state["attempts"] if item["attempt_id"] == retry), None) + if retry or repair: + if ( + not old + or not repair + or not repair.strip() + or old != state["attempts"][-1] + or ( + old["assessment"]["verdict"] != "INCONCLUSIVE" + or not _service_cleanup_confirmed(state, old) + or old["kind"] + != { + "round": "round", + "baseline": "baseline", + "refresh-references": "refresh", + }.get(action) + ) + ): + raise ValueError( + "Retry needs the latest inconclusive attempt, a reason and verified cleanup" + ) + elif state["state"] in ("blocked", "paused"): + raise ValueError( + "Investigate and explicitly retry or resume before launching more work" + ) + if state["pending"] is not None or state["state"] == "completed": + raise ValueError("Campaign completed or attempt unresolved") + reserve = 0 if action == "final" else plan["final_reserve_seconds"] + if time.time() >= state["deadline"] - reserve: + raise ValueError("Campaign budget exhausted; no automatic retry or reduced sampling") + kind = { + "round": "round", + "baseline": "baseline", + "final": "final", + "refresh-references": "refresh", + }.get(action) + if kind is None: + raise ValueError("Unknown service campaign action") + initial, parent = state["initial_reference"], state["comparison_parent"] + admission = None + if kind == "baseline": + if initial is not None: + raise ValueError("Initial baseline is already measured") + source, commit = root / plan["baseline_path"], plan["baseline_revision"] + candidate_id, number = "baseline", None + elif kind == "round": + if initial is None or parent is None: + raise ValueError("Complete baseline and compatible reference refresh first") + if ( + state["best"] is not None + and state["best"]["kind"] == "candidate" + and not _service_profile(root, state["best"], state["conditions"]) + ): + raise ValueError( + "Selected parent profile is missing or incompatible; investigate first" + ) + if ( + initial["conditions"] != state["conditions"] + or parent["conditions"] != state["conditions"] + ): + raise ValueError("Refresh both incompatible performance references first") + number = old["round_number"] if old else len(state["rounds"]) + 1 + if number > plan["round_limit"] or not args.hypothesis.strip(): + raise ValueError("Round budget exhausted or hypothesis missing") + if args.candidate.is_symlink(): + raise ValueError("Candidate root cannot be symbolic") + source = args.candidate.resolve() + commit = service.revision(source, plan["candidate_includes"]) + candidate_id = f"candidate-{number:03d}" + if not old: + state["rounds"].append({"number": number, "terminal": False, "attempt_ids": []}) + elif kind == "refresh": + if initial is None or parent is None or not args.reason.strip(): + raise ValueError("Initial references and a change reason required") + conditions = service.read(args.conditions) + service.validate_conditions(conditions) + if not old: + if conditions == state["conditions"]: + raise ValueError("No changed conditions; reuse compatible retained references") + if conditions["id"] == state["conditions"]["id"]: + raise ValueError("Changed conditions require a new explicit ID") + state.update( + conditions=conditions, + reference_epoch=state["reference_epoch"] + 1, + requires_current_conditions_acceptance=True, + accepted_attempt_ids=[], + ) + elif conditions != old["conditions"]: + raise ValueError("Reference retry must retain its declared conditions") + source, commit = root / plan["baseline_path"], plan["baseline_revision"] + candidate_id, number = "reference-refresh", None + else: + if plan["round_mode"] == "up_to" and not (args.reason or "").strip(): + raise ValueError("Record why search stopped") + admission = _service_final_reason(root, plan, state) + selected = state["best"] + source = ( + service.safe_path(root, selected["path"]) + if selected + else root / plan["baseline_path"] + ) + commit = selected["revision"] if selected else plan["baseline_revision"] + candidate_id, number = selected["candidate_id"] if selected else "none", None + parent = selected.get("parent_reference_at_acceptance") if selected else None + attempt_id = uuid.uuid4().hex + prefix = ( + f"rounds/round-{number:02d}" + if kind == "round" + else (f"references/epoch-{state['reference_epoch']}" if kind == "refresh" else kind) + ) + relative = prefix + "/attempt-" + attempt_id + directory = service.safe_path(root, relative) + directory.mkdir(parents=True) + protocol_path = state["protocol_path"] + if kind == "final" and state["best"] is not None: + protocol_path = state["best"]["protocol_path"] + if repaired_protocol: + candidates.retain(repaired_protocol, directory / "protocol", fingerprint=False) + protocol_path = relative + "/protocol" + state["protocol_path"] = protocol_path + observations, raw = _usage_record(getattr(args, "usage_export", None), fingerprint=False) + if raw is not None: + (directory / "agent-usage.json").write_bytes(raw) + if kind == "round": + service.retain_revision( + source, directory / "candidate", plan["candidate_includes"], commit + ) + else: + candidates.retain(source, directory / "candidate", fingerprint=False) + if kind == "round" and service.revision(source, plan["candidate_includes"]) != commit: + raise ValueError("Candidate revision changed while retaining inputs") + pending = { + "attempt_id": attempt_id, + "protocol_path": protocol_path, + "kind": kind, + "path": relative, + "candidate_id": candidate_id, + "revision": commit, + "round_number": number, + "started_at": time.time(), + "hypothesis": getattr(args, "hypothesis", None) + or getattr(args, "reason", None) + or "Initial baseline", + "retry_of": retry, + "repair_reason": repair, + "role": "candidate" if kind == "round" else "final" if kind == "final" else "baseline", + "conditions": state["conditions"], + "reference_epoch": state["reference_epoch"], + "observations": observations, + "parent": parent, + "initial": initial, + "contexts": list(state["contexts"]), + "admission_failure": admission, + } + if kind == "refresh": + pending.update(refresh_initial=initial, refresh_parent=parent) + state["pending"] = pending + atomic_write_json(root / "state.json", state) + try: + if admission: + result = { + "assessment": service.assessment("INCONCLUSIVE", admission), + "cleanup_verified": True, + "elapsed_seconds": 0, + } + else: + result = _service_execute( + root, directory, plan, pending, state["deadline"] - reserve + ) + except (Exception, KeyboardInterrupt) as error: + receipt = directory / "execution/execution.json" + try: + result = service.read(receipt) + except (OSError, ValueError): + result = {"cleanup_verified": None} + result["assessment"] = service.assessment( + "INCONCLUSIVE", + f"coordinator: {error}", + performance=(result.get("assessment") or {}).get("performance"), + quality_passed=(result.get("assessment") or {}).get("quality_passed"), + ) + execution_dir = directory / "execution" + execution_dir.mkdir(exist_ok=True) + atomic_write_json(execution_dir / "execution.json", result) + return _service_settle(root, plan, state, pending, result) diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index b4e8d2775..9030624cd 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -151,6 +151,54 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N raise ValueError("Reserve time for final within the total budget") evaluation_prefix = shlex.split(args.evaluation_prefix) if args.project_task: + names = ( + "metric_key", + "metric_unit", + "quality_key", + "scope", + "direction", + "min_improvement_pct", + "repetitions", + "warmups", + "measure_command", + "quality_command", + ) + if args.evaluation_mode == "service_reuse": + from inferrsi.harness.project_service import prepare as prepare_service + + if args.evaluation_config is None or any( + getattr(args, key) is not None for key in names + ): + raise ValueError( + "Service reuse requires --evaluation-config without AB/BA options" + ) + settings = prepare_service(root, args) + plan = { + "schema_version": 3, + "condition": args.condition, + "round_limit": args.rounds, + "max_seconds": args.max_minutes * 60, + "round_mode": args.round_mode, + "final_reserve_seconds": reserve, + "knowledge_allowed": args.knowledge == "on", + "evaluation_timeout_seconds": args.evaluation_timeout_seconds, + "evaluation_prefix": evaluation_prefix, + "python": args.python, + **settings, + } + result = optimization.initialize(root, plan) + result["instructions"] = ( + "Run campaign baseline before candidate search. Use campaign round for each " + "committed candidate; investigate INCONCLUSIVE results before explicit retry. " + "Finish with campaign final; final does not collect a profile." + ) + return result + if args.evaluation_config is not None or any( + getattr(args, key) is None for key in names + ): + raise ValueError( + "AB/BA requires its existing metric, quality, command and sampling options" + ) from inferrsi.harness.project import prepare as prepare_project repo = args.repo.resolve() @@ -205,6 +253,21 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N import json plan = json.loads((root / "plan.json").read_text(encoding="utf-8")) + manifest = json.loads((root / "campaign.json").read_text(encoding="utf-8")) + if ( + plan.get("evaluation_mode") == "service_reuse" + or plan.get("schema_version") == 3 + or manifest.get("evaluation_mode") == "service_reuse" + ): + return optimization.service_command(root, args, knowledge_reader=knowledge_reader) + if ( + args.session_action in ("baseline", "refresh-references", "resume") + or getattr(args, "retry_of", None) + or getattr(args, "repair_reason", None) + or getattr(args, "cleanup_receipt", None) + or getattr(args, "repair_protocol", None) + ): + raise ValueError("Service lifecycle/retry operations require a service-reuse campaign") if plan["runtime_identity"] != _runtime_identity(): raise ValueError("Optimization runtime changed, use its frozen runtime to resume") if args.session_action == "status": @@ -212,9 +275,14 @@ def session_command(args, *, task_source: Path | None = None, knowledge_reader=N if args.session_action == "knowledge": if knowledge_reader is None or retain_context is None: raise ValueError("A knowledge reader and capture callback are required") - return optimization.capture_context(root, knowledge_reader, retain_context=retain_context, - query="", relevance=args.query, - relevance_method=args.relevance_method, context_bytes=args.context_bytes, + return optimization.capture_context( + root, + knowledge_reader, + retain_context=retain_context, + query="", + relevance=args.query, + relevance_method=args.relevance_method, + context_bytes=args.context_bytes, limit=args.limit, ) if args.session_action == "recover": diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py new file mode 100644 index 000000000..4f7f2707d --- /dev/null +++ b/src/inferrsi/harness/project_service.py @@ -0,0 +1,787 @@ +"""Project-owned service protocols and their retained, multi-objective evidence. + +No file/content fingerprints are calculated here. Revision, configuration and +actual service receipts establish the explicitly declared execution binding. +""" + +import io +import json +import math +import os +import re +import subprocess +import tarfile +import time +import uuid +from pathlib import Path +from statistics import mean, median + +from .atomic_json import atomic_write_json +from .candidates import members, retain + + +def safe_path(root, relative): + if not isinstance(relative, str) or not relative or "\\" in relative or ":" in relative: + raise ValueError("Expected a campaign-relative path") + part = Path(relative) + if part.is_absolute() or ".." in part.parts: + raise ValueError("Path must stay inside the campaign") + target = root / part + if not target.resolve().is_relative_to(root.resolve()) or any( + p.is_symlink() for p in (target, *target.parents) if p.is_relative_to(root) + ): + raise ValueError("Symbolic or escaping campaign path") + return target + + +def no_fingerprints(value): + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str) or any( + name in key.lower() for name in ("sha256", "fingerprint", "digest") + ): + raise ValueError("File fingerprints are not part of the service-reuse protocol") + no_fingerprints(item) + elif isinstance(value, list): + for item in value: + no_fingerprints(item) + elif isinstance(value, float) and not math.isfinite(value): + raise ValueError("Nonfinite JSON value") + + +def read(path): + if path.is_symlink() or not path.is_file() or path.stat().st_size > 4 * 1024 * 1024: + raise ValueError(f"Missing, symbolic or oversized JSON: {path.name}") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Result must be a JSON object") + no_fingerprints(value) + return value + + +def number(value): + return type(value) in (int, float) and math.isfinite(value) + + +def positive(value): + return number(value) and value > 0 + + +def validate_conditions(conditions): + if not isinstance(conditions, dict): + raise ValueError("Explicit test conditions required") + if not isinstance(conditions.get("id"), str) or not conditions["id"].strip(): + raise ValueError("Conditions require a nonempty string ID") + for key in ("id", "workload", "model", "precision", "topology", "runtime", "timing_scope"): + if not conditions.get(key): + raise ValueError(f"conditions.{key} is required") + if not isinstance(conditions.get("effective_config"), dict): + raise ValueError("conditions.effective_config must be an object") + no_fingerprints(conditions) + + +def validate(settings): + no_fingerprints(settings) + for name in ("conditions", "commands", "sampling", "quality", "stagnation"): + if not isinstance(settings.get(name), dict): + raise ValueError(f"{name} must be an explicit object") + if type(settings.get("schema_version")) is not int or settings["schema_version"] != 1: + raise ValueError("Service evaluation schema_version must be 1") + if not isinstance(settings.get("protocol_id"), str) or not settings["protocol_id"].strip(): + raise ValueError("Explicit protocol_id required") + validate_conditions(settings.get("conditions")) + commands = settings.get("commands", {}) + for name in ("precheck", "service", "ready", "warmup", "measure", "quality", "profile", "stop"): + command = commands.get(name) + if ( + not isinstance(command, list) + or not command + or any(not isinstance(arg, str) or not arg for arg in command) + ): + raise ValueError(f"commands.{name} must be explicit argv") + if not any("{protocol}" in arg for arg in command): + raise ValueError(f"commands.{name} must use a retained protocol script") + if name != "service" and not any("{result}" in arg for arg in command): + raise ValueError(f"commands.{name} must write the supplied result") + sampling = settings.get("sampling", {}) + if not positive(sampling.get("stage_timeout_seconds")): + raise ValueError("Positive stage timeout required") + for role in ("baseline", "candidate", "final"): + config = sampling.get(role, {}) + if not isinstance(config, dict): + raise ValueError("Sampling policies must be objects") + if type(config.get("repetitions")) is not int or config["repetitions"] < 1: + raise ValueError(f"Positive {role} repetitions required") + if type(config.get("warmups")) is not int or config["warmups"] < 0: + raise ValueError(f"Nonnegative {role} warmups required") + if config["warmups"] == 0 and not config.get("warmup_reason"): + raise ValueError("Explain why no warmup is necessary") + scenarios = settings.get("scenarios") + if not isinstance(scenarios, list) or not scenarios: + raise ValueError("Declare scenarios") + ids = [] + for item in scenarios: + if not isinstance(item, dict) or not isinstance(item.get("id"), str): + raise ValueError("Scenario must declare a string ID") + if ( + not item.get("id") + or not item.get("workload") + or (type(item.get("requests")) is not int or item["requests"] < 1) + or type(item.get("required_successes")) is not int + or not (0 < item["required_successes"] <= item["requests"]) + ): + raise ValueError("Declare scenario workload, request and success counts") + ids.append(item["id"]) + if len(set(ids)) != len(ids): + raise ValueError("Duplicate scenario") + objectives = settings.get("objectives") + if not isinstance(objectives, list) or not objectives: + raise ValueError("Declare objectives") + names = [] + for obj in objectives: + if not isinstance(obj, dict) or not isinstance(obj.get("domain"), dict): + raise ValueError("Objective and domain must be objects") + for key in ("id", "key", "unit"): + if not isinstance(obj.get(key), str) or not obj[key].strip(): + raise ValueError(f"Objective {key} required") + names.append(obj["id"]) + if obj.get("scenario") not in ids or obj.get("direction") not in ("minimize", "maximize"): + raise ValueError("Invalid objective scenario or direction") + if ( + obj.get("aggregate") not in ("median", "mean") + or obj.get("threshold_type") not in ("relative_percent", "absolute", "percentage_point") + or type(obj.get("gain")) is not bool + ): + raise ValueError("Declare aggregation, threshold type and gain participation") + domain = obj.get("domain", {}) + if ( + not number(domain.get("min")) + or not number(domain.get("max")) + or (domain["min"] > domain["max"]) + ): + raise ValueError("Finite ordered objective domain required") + if obj["threshold_type"] == "percentage_point" and obj["unit"] != "%": + raise ValueError("Percentage-point objectives use percent values and unit %") + if obj["threshold_type"] == "relative_percent" and domain["min"] < 0: + raise ValueError("Use absolute thresholds for negative-valued metrics") + for key in ("gain_threshold", "parent_regression_limit", "cumulative_regression_limit"): + if not number(obj.get(key)) or obj[key] < 0: + raise ValueError(f"Nonnegative finite {key} required") + ranking = settings.get("ranking") + if len(set(names)) != len(names) or not any(obj["gain"] for obj in objectives): + raise ValueError("Unique objectives and at least one gain objective required") + if ( + not isinstance(ranking, list) + or not ranking + or any(not isinstance(name, str) for name in ranking) + or len(set(ranking)) != len(ranking) + or any(name not in names for name in ranking) + ): + raise ValueError("Declare an ordered, unique ranking of known objectives") + quality = settings.get("quality", {}) + if ( + quality.get("mode") not in ("absolute", "baseline_comparison") + or quality.get("direction") not in ("minimize", "maximize") + or not all( + isinstance(quality.get(key), str) and quality[key].strip() + for key in ("key", "dataset", "scorer") + ) + ): + raise ValueError("Declare quality mode, direction, dataset, scorer and key") + gate = "threshold" if quality["mode"] == "absolute" else "tolerance" + if type(quality.get("samples")) is not int or quality["samples"] < 1: + raise ValueError("Declare the required quality sample count") + if not number(quality.get(gate)) or (gate == "tolerance" and quality[gate] < 0): + raise ValueError("Explicit quality gate required") + reminder = settings.get("stagnation", {}) + if reminder != {"disabled": True} and ( + type(reminder.get("rounds")) is not int + or reminder["rounds"] < 1 + or reminder.get("action") not in ("advisory", "pause") + ): + raise ValueError("Explicit stagnation threshold/action or disabled required") + screen = settings.get("screening") + if screen is not None: + if not isinstance(screen, dict) or ( + not isinstance(screen.get("command"), list) + or not screen["command"] + or any(not isinstance(arg, str) or not arg for arg in screen["command"]) + or not positive(screen.get("estimated_gain")) + ): + raise ValueError("Screening command and positive estimated_gain required") + if not any("{result}" in arg for arg in screen["command"]) or not any( + "{protocol}" in arg for arg in screen["command"] + ): + raise ValueError("Screening must use retained protocol and result") + + +def value(document, key): + for part in key.split("."): + if not isinstance(document, dict) or part not in document: + raise ValueError(f"Missing metric: {key}") + document = document[part] + return document + + +def aggregate(settings, documents, role): + if len(documents) != settings["sampling"][role]["repetitions"]: + raise ValueError("Incomplete performance repetitions") + samples = {obj["id"]: [] for obj in settings["objectives"]} + for document in documents: + for scenario in settings["scenarios"]: + result = document.get("scenarios", {}).get(scenario["id"], {}) + if ( + result.get("workload") != scenario["workload"] + or type(result.get("requests")) is not int + or result["requests"] != scenario["requests"] + or type(result.get("successes")) is not int + or not scenario["required_successes"] <= result["successes"] <= (result["requests"]) + ): + raise ValueError("Incomplete or incompatible scenario requests") + for obj in settings["objectives"]: + result = document["scenarios"][obj["scenario"]] + metric = value(result, obj["key"]) + if ( + result.get("units", {}).get(obj["id"]) != obj["unit"] + or not number(metric) + or not (obj["domain"]["min"] <= metric <= obj["domain"]["max"]) + ): + raise ValueError(f"Invalid metric domain or unit: {obj['id']}") + samples[obj["id"]].append(metric) + aggregates = { + obj["id"]: (median if obj["aggregate"] == "median" else mean)(samples[obj["id"]]) + for obj in settings["objectives"] + } + if not all(number(v) for v in aggregates.values()): + raise ValueError("Nonfinite aggregate") + return {"values": aggregates, "samples": samples} + + +def change(obj, candidate, reference): + sign = 1 if obj["direction"] == "maximize" else -1 + delta = sign * (candidate - reference) + if obj["threshold_type"] == "relative_percent": + if reference == 0: + if candidate != 0: + raise ValueError(f"Undefined relative change from zero: {obj['id']}") + return 0 + delta = 100 * delta / abs(reference) + if not number(delta): + raise ValueError("Nonfinite derived change") + return delta + + +def performance(settings, measured, parent, initial, *, screening=False): + metrics, limits, gain, actual_gain = {}, True, False, False + for obj in settings["objectives"]: + name = obj["id"] + p = change(obj, measured["values"][name], parent["values"][name]) + b = change(obj, measured["values"][name], initial["values"][name]) + bounded = p >= -obj["parent_regression_limit"] and b >= -obj["cumulative_regression_limit"] + improved = obj["gain"] and p > 0 + metrics[name] = { + "value": measured["values"][name], + "unit": obj["unit"], + "parent_change": p, + "initial_change": b, + "limits_passed": bounded, + "gain_passed": improved and p >= obj["gain_threshold"], + "thresholds": { + key: obj[key] + for key in ( + "threshold_type", + "gain_threshold", + "parent_regression_limit", + "cumulative_regression_limit", + ) + }, + } + limits &= bounded + actual_gain |= improved + gain |= metrics[name]["gain_passed"] + passed = limits and (actual_gain if screening else gain) + return { + "passed": passed, + "metrics": metrics, + "reason": "performance_gate_satisfied" if passed else "performance_gate_not_met", + } + + +def better(settings, left, right): + by_id = {obj["id"]: obj for obj in settings["objectives"]} + for name in settings["ranking"]: + if left[name] != right[name]: + return ( + left[name] < right[name] + if by_id[name]["direction"] == "minimize" + else left[name] > right[name] + ) + return False + + +def quality_score(settings, document): + config = settings["quality"] + if document.get("dataset") != config["dataset"] or document.get("scorer") != config["scorer"]: + raise ValueError("Incompatible quality dataset/scorer") + score = value(document, config["key"]) + if ( + not number(score) + or type(document.get("samples")) is not int + or document["samples"] != config["samples"] + ): + raise ValueError("Missing numerical quality evidence") + return score + + +def quality_passed(settings, score, reference=None): + config = settings["quality"] + sign = 1 if config["direction"] == "maximize" else -1 + if config["mode"] == "absolute": + return sign * (score - config["threshold"]) >= 0 + if reference is None or not number(reference): + raise ValueError("Missing compatible baseline quality reference") + return sign * (score - reference) >= -config["tolerance"] + + +def revision(source, includes): + """Use an existing commit, never a file fingerprint, for retained project inputs.""" + + def git(*args): + return subprocess.check_output(["git", "-C", str(source), *args], text=True).strip() + + if Path(git("rev-parse", "--show-toplevel")).resolve() != source.resolve(): + raise ValueError("Use the Git checkout root as the service-reuse project") + commit = git("rev-parse", "HEAD") + if not re.fullmatch(r"[a-f0-9]{40,64}", commit): + raise ValueError("Candidate requires an existing Git commit") + selected = members(source, includes) + tracked = set(git("ls-files", "-z").split("\0")) + if any(name not in tracked for name in selected) or git( + "status", "--porcelain", "--", *selected + ): + raise ValueError("Commit declared candidate inputs before service-reuse evaluation") + return commit + + +def retain_revision(source, destination, includes, commit): + selected = members(source, includes) + archive = subprocess.check_output( + ["git", "-C", str(source), "archive", "--format=tar", commit, "--", *includes] + ) + destination.mkdir() + copied = set() + with tarfile.open(fileobj=io.BytesIO(archive)) as stream: + for item in stream.getmembers(): + if item.isdir(): + continue + if not item.isfile(): + raise ValueError("Candidate archive contains symbolic or special files") + if item.name not in selected: + continue + target = safe_path(destination, item.name) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(stream.extractfile(item).read()) + target.chmod(item.mode & ~0o222) + copied.add(item.name) + if copied != set(selected): + raise ValueError("Declared candidate files are missing from the retained commit") + + +def prepare(root, args): + from inferrsi import __version__ + + settings = read(args.evaluation_config) + validate(settings) + source, protocol = args.repo.resolve(), args.protocol.resolve() + if args.repo.is_symlink() or args.protocol.is_symlink(): + raise ValueError("Project and protocol roots cannot be symbolic") + if root.exists() or root.is_relative_to(source) or root.is_relative_to(protocol): + raise ValueError("Use a new campaign directory outside project/protocol") + commit = revision(source, args.include) + members(protocol) + if not args.instructions.is_file() or args.instructions.is_symlink(): + raise ValueError("Regular task instructions required") + (root / "frozen").mkdir(parents=True) + retain_revision(source, root / "frozen/baseline", args.include, commit) + if revision(source, args.include) != commit: + raise ValueError("Candidate revision changed while retaining inputs") + retain(protocol, root / "frozen/protocol", fingerprint=False) + (root / "frozen/TASK.md").write_bytes(args.instructions.read_bytes()) + atomic_write_json(root / "frozen/evaluation.json", settings) + for path in (root / "frozen").rglob("*"): + if path.is_file(): + path.chmod(path.stat().st_mode & ~0o222) + return { + "runtime_version": __version__, + "evaluation_settings": settings, + "baseline_revision": commit, + "baseline_path": "frozen/baseline", + "candidate_name": "candidate", + "candidate_kind": "directory", + "candidate_includes": args.include, + "working_candidate": str(source), + "task_source": "project", + "task": "project:" + source.name, + "evaluation_mode": "service_reuse", + "selection": {"ranking": settings["ranking"]}, + } + + +def assessment(verdict, reason, **fields): + return { + "status": "infrastructure_failure" if verdict == "INCONCLUSIVE" else "completed", + "verdict": verdict, + "eligible": verdict == "PASS", + "reason": reason, + **fields, + } + + +def execute(root, directory, plan, pending, deadline): + """One retained attempt, one owned service, ordered client stages, no silent retry.""" + from .local_process import supervised_service + from .optimization_session import runtime_environment + from .scenarios.external import execute_trial + + settings = plan["evaluation_settings"] + conditions = pending["conditions"] + config = conditions["effective_config"] + source = directory / "candidate" + workspace = directory / "workspace" + retain(source, workspace, fingerprint=False) + expected = { + "attempt_id": pending["attempt_id"], + "service_id": uuid.uuid4().hex, + "run_id": uuid.uuid4().hex, + "revision": pending["revision"], + "conditions": conditions, + "effective_config": config, + } + atomic_write_json(directory / "request.json", expected) + atomic_write_json(directory / "effective-config.json", config) + receipt = directory / "service.json" + values = { + "python": plan["python"], + "protocol": str(safe_path(root, pending.get("protocol_path", "frozen/protocol"))), + "candidate": str(workspace), + "baseline": str(root / plan["baseline_path"]), + "service_receipt": str(receipt), + "effective_config": str(directory / "effective-config.json"), + "attempt_dir": str(directory), + } + environment = runtime_environment() + environment.update( + INFERRSI_ATTEMPT_REQUEST=str(directory / "request.json"), PYTHONDONTWRITEBYTECODE="1" + ) + environment["PYTHONPATH"] = str(workspace) + os.pathsep + environment["PYTHONPATH"] + end = time.monotonic() + min(plan["evaluation_timeout_seconds"], max(0, deadline - time.time())) + record = { + "status": "running", + "steps": [], + "cleanup_verified": None, + "binding": expected, + "assessment": None, + } + output = directory / "execution" + output.mkdir() + started, process, owner = time.monotonic(), None, None + + def save(): + atomic_write_json(output / "execution.json", record) + + def command(argv): + for key, val in values.items(): + argv = [arg.replace("{" + key + "}", val) for arg in argv] + return argv + + def binding(document): + if document.get("binding") != expected: + raise ValueError("Service/revision/configuration binding mismatch") + + def stage(name, *, bound=True, argv=None): + definition = { + "id": name, + "status": "running", + "started_at": time.time(), + "execution_status": "not_started", + } + record["steps"].append(definition) + save() + if bound and (process is None or process.poll() is not None): + raise ValueError(f"Service unavailable before {name}") + executed = execute_trial( + tuple(command(argv or settings["commands"][name.split("-")[0]])), + workspace, + output / name, + min(end, time.monotonic() + settings["sampling"]["stage_timeout_seconds"]), + None, + environment=environment, + ) + definition.update(executed, execution_status=executed["status"], ended_at=time.time()) + save() + if executed["status"] != "completed": + raise ValueError(f"{name}: {executed['status']}: {executed.get('error', '')}") + document = read(output / name / "result.json") + if bound: + binding(document) + if process.poll() is not None: + raise ValueError(f"Service exited during {name}") + if document.get("status") == "candidate_failure" and name.startswith( + ("precheck", "quality") + ): + return document + if document.get("status") != "completed": + raise ValueError(f"{name}: missing completed stage evidence") + definition["result"] = document + save() + return document + + def skipped(names, reason): + record["steps"].extend( + {"id": name, "status": "skipped", "reason": reason} for name in names + ) + save() + + measured, gate, score, profile, quality_ok = None, None, None, None, None + try: + precheck = stage("precheck", bound=False) + if precheck.get("status") == "candidate_failure" or precheck.get("passed") is False: + skipped(("service", "measurement", "quality", "profile"), "precheck_rejected") + record["assessment"] = assessment("FAIL", "functional_check_rejected") + record["cleanup_verified"] = True + elif precheck.get("passed") is not True: + raise ValueError("Missing functional/numerical check evidence") + else: + service_step = {"id": "service", "status": "running", "started_at": time.time()} + record["steps"].append(service_step) + save() + with supervised_service( + [*plan["evaluation_prefix"], *command(settings["commands"]["service"])], + workspace, + environment, + directory / "service", + expected, + ) as (process, owner): + service_step.update(pid=process.pid, owner_path="service/owner.json") + save() + try: + ready = stage("ready") + if ( + ready.get("ready") is not True + or ready.get("ownership") != "local_process_domain" + or ready.get("owner_pid") != process.pid + or ready.get("loaded_revision") != expected["revision"] + or ready.get("loaded_root") != str(workspace.resolve()) + ): + raise ValueError("Service readiness/ownership unverified") + role = pending["role"] + for index in range(settings["sampling"][role]["warmups"]): + stage(f"warmup-{index + 1:03d}") + parent, initial = pending.get("parent"), pending.get("initial") + screen = settings.get("screening") + screened = True + if role == "candidate" and screen: + docs = [ + stage(f"screening-{i + 1:03d}", argv=screen["command"]) + for i in range(settings["sampling"][role]["repetitions"]) + ] + preview = aggregate(settings, docs, role) + record["screening"] = performance( + settings, + preview, + parent["measurements"], + initial["measurements"], + screening=True, + ) + record["screening"].update( + measurements=preview, estimated_gain=screen["estimated_gain"] + ) + screened = record["screening"]["passed"] + if screened: + docs = [ + stage(f"measure-{i + 1:03d}") + for i in range(settings["sampling"][role]["repetitions"]) + ] + measured = aggregate(settings, docs, role) + gate = ( + performance( + settings, measured, parent["measurements"], initial["measurements"] + ) + if parent is not None + else {"passed": True, "metrics": {}} + ) + if not screened or not gate["passed"]: + skipped(("quality", "profile"), "performance_gate_not_met") + record["assessment"] = assessment( + "FAIL", "performance_gate_not_met", performance=gate + ) + elif pending.get("reference_kind") == "parent": + skipped(("quality", "profile"), "performance_reference_only") + record["assessment"] = assessment( + "PASS", "performance_reference_refreshed", performance=gate, + eligible=False, + ) + else: + reference_path = settings["quality"].get("reference_path") + if role == "baseline" and reference_path: + reference_step = { + "id": "quality", + "status": "running", + "source": "retained_baseline_reference", + "path": reference_path, + "started_at": time.time(), + } + record["steps"].append(reference_step) + save() + quality = read( + safe_path( + safe_path( + root, pending.get("protocol_path", "frozen/protocol") + ), + reference_path, + ) + ) + if ( + quality.get("revision") != pending["revision"] + or quality.get("conditions") != conditions + or not quality.get("service_id") + or not quality.get("run_id") + ): + raise ValueError( + "quality: missing/stale/incompatible baseline reference" + ) + quality_score(settings, quality) + reference_step.update( + status="completed", ended_at=time.time(), result=quality + ) + save() + else: + quality = stage("quality") + if quality.get("status") == "candidate_failure": + quality_ok = False + else: + score = quality_score(settings, quality) + quality_ok = quality_passed( + settings, + score, + (initial.get("quality_score") if initial else score), + ) + if not quality_ok: + skipped(("profile",), "quality_gate_not_met") + record["assessment"] = assessment( + "FAIL", + "quality_gate_not_met", + quality_passed=False, + performance=gate, + ) + else: + if role == "candidate": + trace = stage("profile") + artifacts = trace.get("artifacts") + if not isinstance(artifacts, list) or not artifacts: + raise ValueError("profile: missing artifacts") + for relative in artifacts: + path = safe_path(directory, relative) + if not path.is_file() or path.stat().st_size == 0: + raise ValueError("profile: missing or empty artifact") + if trace.get("validated") is not True: + raise ValueError("profile: project validation missing") + start, finish = ( + trace.get("capture_started_at"), + trace.get("capture_ended_at"), + ) + if ( + not number(start) + or not number(finish) + or not ( + record["steps"][-1]["started_at"] + <= start + <= finish + <= record["steps"][-1]["ended_at"] + ) + ): + raise ValueError( + "profile: missing or incompatible capture interval" + ) + profile = { + "attempt_id": pending["attempt_id"], + "conditions": conditions, + "binding": expected, + "receipt": trace, + "artifacts": [ + str((directory / item).relative_to(root)) + for item in artifacts + ], + "result": str( + (output / "profile/result.json").relative_to(root) + ), + } + else: + skipped( + ("profile",), + "not_required" + if role == "baseline" + else "reuse_selected_candidate_profile", + ) + record["assessment"] = assessment( + "PASS", + "fixed_checks_satisfied", + quality_passed=True, + performance=gate, + ) + except BaseException: + record["failed_stage"] = record["steps"][-1]["id"] + raise + finally: + try: + cleanup = stage("stop", bound=False) + binding(cleanup) + record["external_cleanup_verified"] = ( + cleanup.get("cleanup_complete") is True + ) + except (OSError, ValueError, KeyError) as error: + record["external_cleanup_verified"] = False + record["cleanup_error"] = str(error) + record["cleanup_verified"] = ( + owner["domain_empty"] is True and record.get("external_cleanup_verified") is True + ) + if not record["cleanup_verified"]: + raise ValueError("Service cleanup is unverified") + except (Exception, KeyboardInterrupt) as error: + failed = record.get( + "failed_stage", record["steps"][-1]["id"] if record["steps"] else "startup" + ) + record["assessment"] = assessment( + "INCONCLUSIVE", + f"{failed}: {type(error).__name__}: {error}", + performance=gate, + quality_passed=quality_ok, + ) + for definition in record["steps"]: + if definition["id"] == failed: + definition.setdefault("execution_status", definition["status"]) + if definition["status"] in ("running", "completed"): + definition["status"] = "failed" + definition["reason"] = str(error) + if owner is not None: + record["cleanup_verified"] = ( + owner["domain_empty"] is True and record.get("external_cleanup_verified") is True + ) + for name in ("quality", "profile"): + if not any(step["id"] == name for step in record["steps"]): + skipped((name,), "prerequisite_incomplete") + if owner is not None: + service_step.update( + status="completed" if owner["domain_empty"] is True else "failed", + ended_at=time.time(), + returncode=owner["returncode"], + ) + record.update( + status="completed", + measurements=measured, + quality_score=score, + profile=profile, + elapsed_seconds=time.monotonic() - started, + ) + save() + return record diff --git a/src/inferrsi/knowledge/catalog.py b/src/inferrsi/knowledge/catalog.py index a8c1fc4a3..2c760e379 100644 --- a/src/inferrsi/knowledge/catalog.py +++ b/src/inferrsi/knowledge/catalog.py @@ -88,7 +88,7 @@ def _validate_record( raise ValueError(f"Imported knowledge must state locally_verified=false: {record['id']}") -def knowledge_records() -> list[dict[str, Any]]: +def knowledge_records(*, file_fingerprints: bool = True) -> list[dict[str, Any]]: data = files("inferrsi").joinpath("knowledge", "data") records = [] seen = set() @@ -116,12 +116,32 @@ def knowledge_records() -> list[dict[str, Any]]: ): if snapshot is None: raise ValueError(f"Knowledge record lacks its source snapshot: {repo}") - record = _source_snapshot.resolve(record, snapshot) + if file_fingerprints: + record = _source_snapshot.resolve(record, snapshot) + else: + revision = snapshot.get("source_revision") + if not isinstance(revision, str) or not re.fullmatch( + r"[0-9a-f]{40}", revision + ): + raise ValueError("Knowledge source commit is unavailable") + record = { + **record, + "schema_version": 1, + "sources": [ + { + "type": "git", + "commit": revision, + "url": _source_snapshot.REPOSITORY + "/tree/" + revision, + } + ], + } elif snapshot is not None: raise ValueError( f"Source-snapshot pack holds a record that is not schema " f"{_source_snapshot.STORED_SCHEMA}: {repo}/{'/'.join(parts)}" ) + if not file_fingerprints: + record = without_file_fingerprints(record) _validate_record(record, repo, parts, tier) if record["id"] in seen: raise ValueError(f"Duplicate knowledge ID: {record['id']}") @@ -130,6 +150,17 @@ def knowledge_records() -> list[dict[str, Any]]: return records +def without_file_fingerprints(value): + """Project source attribution without computing or validating file identities.""" + if isinstance(value, dict): + return {key: without_file_fingerprints(item) for key, item in value.items() + if key != "source_files" and not any( + part in key.lower() for part in ("sha256", "fingerprint", "digest"))} + if isinstance(value, list): + return [without_file_fingerprints(item) for item in value] + return value + + def search_knowledge( query: str = "", *, diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 9c5e1e7b3..b5262e0c9 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -264,6 +264,100 @@ def _tree( class KnowledgeReader: + def collect_service_context(self, cue, *, relevance_method, limit, context_bytes): + """Package knowledge for service campaigns; no Bundle/content identity path.""" + from .catalog import without_file_fingerprints + from .context_projection import assemble_context_text + + if ( + not words(cue) + or relevance_method not in RELEVANCE_METHODS + or ( + type(limit) is not int + or limit < 1 + or type(context_bytes) is not int + or context_bytes < 1 + ) + ): + raise ValueError("Valid cue, relevance method, positive limit/context budget required") + ranked, omissions = [], [] + package = files("inferrsi").joinpath("knowledge") + for record in knowledge_records(file_fingerprints=False): + documents = record.get("skill_documents", {}) + path = documents.get("original") if documents else None + if path is not None: + safe_path(path) + if not path.startswith("data/skill_library/"): + raise ValueError("Service knowledge resource must stay inside the library") + node = package.joinpath(*path.split("/")) + if isinstance(node, Path) and node.is_symlink(): + raise ValueError("Symbolic knowledge resource") + body = node.read_bytes() + else: + body = _json(record) + path = "record.json" + scorer = task_score if relevance_method == TASK_RELEVANCE else skill_score + score = ( + scorer(cue, record, body, None) + if relevance_method == TASK_RELEVANCE + else scorer(cue, record, body, method=relevance_method) + ) + if score: + ranked.append((score, record["id"], record, path, body)) + contexts, used = [], 0 + for score, identity, record, path, body in sorted( + ranked, key=lambda item: (-item[0], item[1]) + ): + ref = "package:" + identity + if len(contexts) >= limit or used + len(body) > context_bytes: + omissions.append( + { + "ref": ref, + "reason": "context_limit" + if used + len(body) > context_bytes + else "item_limit", + } + ) + continue + sources = record["sources"] + contexts.append( + { + "ref": ref, + "revision": sources[0]["commit"], + "sources": sources, + "content_bytes": len(body), + "score": score, + "resources": [{"path": path, "role": "core", "content": body.decode("utf-8")}], + "omissions": [ + {"path": target, "reason": "linked_document_not_loaded"} + for target in record.get("skill_metadata", {}).get("references", []) + ], + "limitations": [ + "Prepared source text; no content identity or model-consumption claim." + ], + } + ) + used += len(body) + if self.skill_root or self.store: + omissions.append({"reason": "checkout_and_store_require_content_identity_not_used"}) + return without_file_fingerprints( + { + "schema_version": 2, + "kind": "service_knowledge_context", + "request": { + "relevance": cue, + "relevance_method": relevance_method, + "limit": limit, + "context_bytes": context_bytes, + }, + "contexts": contexts, + "omissions": omissions, + "context_bytes": used, + "assembled_text": assemble_context_text(contexts), + "prepared_not_consumed": True, + } + ) + def __init__(self, *, skill_root: Path | None = None, store=None): self.skill_root = skill_root.resolve() if skill_root is not None else None if store is not None and store.skill_root != self.skill_root: diff --git a/tests/fixtures/service_reuse_protocol.py b/tests/fixtures/service_reuse_protocol.py new file mode 100644 index 000000000..aef72d820 --- /dev/null +++ b/tests/fixtures/service_reuse_protocol.py @@ -0,0 +1,119 @@ +"""Synthetic HTTP service exercising real process ownership, never model performance.""" + +import json +import os +import runpy +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from urllib.request import urlopen + +phase, candidate, receipt_path, result_path = sys.argv[1:] +candidate, receipt_path, result_path = map(Path, (candidate, receipt_path, result_path)) +attempt = Path(os.environ["INFERRSI_ATTEMPT_REQUEST"]).parent +request = json.loads((attempt / "request.json").read_text()) +config = runpy.run_path(str(candidate / "engine.py"))["CONFIG"] + + +def write(path, data): + path.write_text(json.dumps(data), encoding="utf-8") + + +if phase == "service": + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + stage = self.path.lstrip("/") + with (attempt / "events.jsonl").open("a") as stream: + stream.write( + json.dumps( + { + "stage": stage, + "version": config["version"], + "service_id": request["service_id"], + } + ) + + "\n" + ) + body = {"status": "completed", "binding": request} + if stage == "ready": + body.update( + ready=True, + ownership="local_process_domain", + owner_pid=os.getpid(), + loaded_root=str(candidate.resolve()), + loaded_revision=request["revision"], + ) + elif stage in ("measure", "warmup", "screening"): + body["scenarios"] = { + name: { + "workload": "synthetic-" + name, + "requests": 2, + "successes": 2, + "latency": config[name], + "units": {name: "ms"}, + } + for name in ("a", "b") + } + if config.get("missing_metric"): + del body["scenarios"]["b"]["latency"] + elif stage == "quality": + body.update( + dataset="synthetic", + scorer="exact-v1", + samples=2, + quality=config.get("quality", 1), + ) + if config.get("quality_rejected"): + body = {"status": "candidate_failure", "binding": request} + if config.get("quality_missing"): + del body["quality"] + elif stage == "profile": + if config.get("profile_mode") == "failed": + body["status"] = "error" + elif config.get("profile_mode") == "missing": + body.update(validated=True, artifacts=["absent.trace"]) + else: + (attempt / "profile.trace").write_text( + "" if config.get("profile_mode") == "empty" else "synthetic trace" + ) + body.update( + validated=True, + artifacts=["profile.trace"], + capture_started_at=time.time(), + capture_ended_at=time.time(), + ) + elif stage == "stop": + body["cleanup_complete"] = True + threading.Thread(target=self.server.shutdown).start() + if config.get("foreign_stage") == stage: + body["binding"] = {**request, "service_id": "another-service"} + encoded = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + server = HTTPServer(("127.0.0.1", 0), Handler) + write(receipt_path, {"endpoint": f"http://127.0.0.1:{server.server_port}"}) + server.serve_forever() + server.server_close() +elif phase == "precheck": + write( + result_path, {"status": "completed", "passed": not config.get("functional_failure", False)} + ) +else: + if phase == "profile" and config.get("profile_mode") == "timeout": + time.sleep(30) + if phase == "ready": + until = time.monotonic() + 5 + while not receipt_path.is_file() and time.monotonic() < until: + time.sleep(0.01) + endpoint = json.loads(receipt_path.read_text())["endpoint"] + with urlopen(endpoint + "/" + phase, timeout=5) as response: + write(result_path, json.load(response)) diff --git a/tests/test_project_objectives.py b/tests/test_project_objectives.py new file mode 100644 index 000000000..1a2c7270f --- /dev/null +++ b/tests/test_project_objectives.py @@ -0,0 +1,98 @@ +"""Pure objective boundaries; these numbers are test controls, never campaign defaults.""" + +import copy + +import pytest + +from inferrsi.harness.project_service import better, change, performance, quality_passed + + +def objective(direction="minimize", kind="relative_percent"): + return { + "id": "a", + "unit": "ms", + "direction": direction, + "threshold_type": kind, + "gain": True, + "gain_threshold": 4, + "parent_regression_limit": 3, + "cumulative_regression_limit": 3, + } + + +@pytest.mark.parametrize("direction,sign", [("minimize", -1), ("maximize", 1)]) +def test_zero_thresholds_and_both_directions(direction, sign): + obj = objective(direction) + assert change(obj, 0, 0) == 0 + with pytest.raises(ValueError, match="Undefined"): + change(obj, 1, 0) + for kind in ("absolute", "percentage_point"): + obj["threshold_type"] = kind + assert change(obj, 2, 0) == sign * 2 + assert change(obj, 0, 0) == 0 + obj.update(gain_threshold=0, parent_regression_limit=0, cumulative_regression_limit=0) + config = {"objectives": [obj]} + assert not performance( + config, {"values": {"a": 1}}, {"values": {"a": 1}}, {"values": {"a": 1}} + )["passed"] + + +def test_combined_gain_and_dual_limits_include_exact_boundary(): + a, b = objective(), objective() + b["id"] = "b" + config = {"objectives": [a, b], "ranking": ["a", "b"]} + base = {"values": {"a": 100, "b": 100}} + assert performance(config, {"values": {"a": 95, "b": 103}}, base, base)["passed"] + assert not performance(config, {"values": {"a": 95, "b": 104}}, base, base)["passed"] + parent = {"values": {"a": 95, "b": 102}} + assert not performance(config, {"values": {"a": 90, "b": 104}}, parent, base)["passed"] + assert better(config, {"a": 95, "b": 103}, base["values"]) + assert not better(config, base["values"], base["values"]) + assert not better(config, {"a": 101, "b": 10}, base["values"]) + + +def test_screening_actual_gain_is_not_an_estimated_or_formal_threshold(): + obj = objective() + config = {"objectives": [obj]} + reference, measured = {"values": {"a": 100}}, {"values": {"a": 99}} + assert performance(config, measured, reference, reference, screening=True)["passed"] + assert not performance(config, measured, reference, reference)["passed"] + obj["gain_threshold"] = 0 + assert performance(config, measured, reference, reference)["passed"] + + +def test_quality_requires_reference_and_has_absolute_semantics(): + config = {"quality": {"mode": "absolute", "direction": "maximize", "threshold": 0.9}} + assert quality_passed(config, 0.9) + comparison = copy.deepcopy(config) + comparison["quality"].update(mode="baseline_comparison", tolerance=0) + with pytest.raises(ValueError, match="Missing compatible"): + quality_passed(comparison, 0.9) + assert quality_passed(comparison, 0.9, 0.9) + assert not quality_passed(comparison, 0.8, 0.9) + + +@pytest.mark.parametrize("bad", [None, True, float("nan"), float("inf"), -1, 101]) +def test_metric_evidence_requires_finite_domain_valid_numbers(bad): + from inferrsi.harness.project_service import aggregate + + obj = objective() + obj.update(scenario="s", key="metric", aggregate="mean", domain={"min": 0, "max": 100}) + config = { + "sampling": {"candidate": {"repetitions": 1}}, + "objectives": [obj], + "scenarios": [{"id": "s", "workload": "fixed", "requests": 2, "required_successes": 2}], + } + doc = { + "scenarios": { + "s": { + "workload": "fixed", + "requests": 2, + "successes": 2, + "metric": bad, + "units": {"a": "ms"}, + } + } + } + with pytest.raises(ValueError): + aggregate(config, [doc], "candidate") diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py new file mode 100644 index 000000000..3de55e9e5 --- /dev/null +++ b/tests/test_project_service_reuse.py @@ -0,0 +1,908 @@ +"""Fingerprint-free CPU orchestration controls; no real-model qualification.""" + +import copy +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from inferrsi.apps.views import dashboard_data +from inferrsi.apps.cli import main +from inferrsi.apps.cli.arguments import parser +from inferrsi.harness import candidates, optimization, optimization_session, project_service +from inferrsi.harness.records import evidence, knowledge_capture +from inferrsi.infra import artifacts +from inferrsi.knowledge import knowledge_operations + + +@pytest.fixture(autouse=True) +def no_fingerprints(monkeypatch): + calls = [] + + def forbidden(*args, **kwargs): + calls.append(True) + raise AssertionError("This test must not calculate or verify a file fingerprint") + + for module, name in ( + (candidates, "identity"), + (artifacts, "sha256"), + (artifacts, "digest"), + (optimization, "sha256"), + (optimization, "digest"), + (optimization_session, "_runtime_identity"), + (knowledge_operations, "_hash"), + (hashlib, "sha256"), + ): + monkeypatch.setattr(module, name, forbidden) + for module, names in ( + (dashboard_data, ("file_ref", "read_object", "read_snapshot")), + (evidence, ("file_ref", "read_object", "read_snapshot")), + (knowledge_capture, ("store_capture", "verify_capture")), + ): + for name in names: + monkeypatch.setattr(module, name, forbidden) + yield + assert not calls, "A fingerprint call was attempted, even if execution caught its exception" + + +def commit(repo, **changes): + config = {"version": "baseline", "a": 100, "b": 100, "quality": 1} + config.update(changes) + (repo / "engine.py").write_text("CONFIG = " + repr(config) + "\n", encoding="utf-8") + for args in ( + ("add", "engine.py"), + ( + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "-qm", + "Synthetic version", + ), + ): + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) + + +def settings(): + return { + "schema_version": 1, + "protocol_id": "synthetic-service-control", + "conditions": { + "id": "c0", + "workload": "synthetic", + "model": "synthetic", + "precision": "integers", + "topology": "one-cpu-process", + "runtime": "stdlib", + "timing_scope": "synthetic-value", + "effective_config": {"mode": "fixture"}, + }, + "commands": { + name: [ + "{python}", + "{protocol}/protocol.py", + name, + "{candidate}", + "{service_receipt}", + "{result}", + ] + for name in ( + "precheck", + "service", + "ready", + "warmup", + "measure", + "quality", + "profile", + "stop", + ) + }, + "sampling": { + "stage_timeout_seconds": 8, + **{ + role: {"repetitions": 2, "warmups": 1} + for role in ("baseline", "candidate", "final") + }, + }, + "scenarios": [ + {"id": name, "workload": "synthetic-" + name, "requests": 2, "required_successes": 2} + for name in ("a", "b") + ], + "objectives": [ + { + "id": name, + "scenario": name, + "key": "latency", + "unit": "ms", + "domain": {"min": 0, "max": 10000}, + "direction": "minimize", + "aggregate": "median", + "threshold_type": "relative_percent", + "gain": True, + "gain_threshold": 4, + "parent_regression_limit": 3, + "cumulative_regression_limit": 3, + } + for name in ("a", "b") + ], + "ranking": ["a", "b"], + "quality": { + "mode": "absolute", + "dataset": "synthetic", + "scorer": "exact-v1", + "samples": 2, + "key": "quality", + "direction": "maximize", + "threshold": 1, + }, + "stagnation": {"rounds": 2, "action": "advisory"}, + } + + +@pytest.fixture +def campaign(tmp_path, capsys): + repo, protocol, root = (tmp_path / name for name in ("repo", "protocol", "campaign")) + repo.mkdir() + subprocess.run(["git", "-C", str(repo), "init", "-q"], check=True) + commit(repo) + protocol.mkdir() + shutil.copyfile( + Path(__file__).parent / "fixtures/service_reuse_protocol.py", protocol / "protocol.py" + ) + instructions = tmp_path / "TASK.md" + instructions.write_text("Synthetic service control; no model claims.") + configuration = tmp_path / "evaluation.json" + configuration.write_text(json.dumps(settings())) + args = [ + "campaign", + "start", + "--session", + str(root), + "--repo", + str(repo), + "--include", + "engine.py", + "--protocol", + str(protocol), + "--instructions", + str(instructions), + "--evaluation-mode", + "service_reuse", + "--evaluation-config", + str(configuration), + "--rounds", + "8", + "--max-minutes", + "10", + "--evaluation-timeout-seconds", + "25", + "--final-reserve-seconds", + "30", + ] + return {"repo": repo, "root": root, "start": args, "config": configuration, "capsys": capsys} + + +def call(campaign, action, *extra, expected=0): + args = ( + campaign["start"] + if action == "start" + else ["campaign", action, "--session", str(campaign["root"]), *map(str, extra)] + ) + assert main(args) == expected + return json.loads(campaign["capsys"].readouterr().out) + + +def begin(campaign): + result = call(campaign, "start") + assert result["next_action"] == "campaign_baseline" + assert not (campaign["root"] / "baseline").exists() + result = call(campaign, "baseline") + assert result["best"]["kind"] == "baseline" + return result + + +def submit(campaign, *, expected=0, retry=None, **changes): + commit(campaign["repo"], **changes) + args = ["--candidate", campaign["repo"], "--hypothesis", "Synthetic committed candidate"] + if retry: + args += ["--retry-of", retry, "--repair-reason", "Investigated and repaired fixture"] + return call(campaign, "round", *args, expected=expected) + + +def events(campaign, attempt): + path = campaign["root"] / attempt["path"] / "events.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] + + +def test_one_service_order_and_independent_final_ignores_workspace(campaign): + begin(campaign) + result = submit(campaign, version="accepted", a=95, b=102) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == "PASS" + log = events(campaign, attempt) + assert [row["stage"] for row in log] == [ + "ready", + "warmup", + "measure", + "measure", + "quality", + "profile", + "stop", + ] + assert len({row["service_id"] for row in log}) == 1 + assert attempt["execution"]["cleanup_verified"] is True + (campaign["repo"] / "engine.py").write_text("raise RuntimeError('unsubmitted workspace')") + final = call(campaign, "final", "--reason", "Synthetic search complete") + assert final["state"] == "completed" + assert final["final"]["assessment"]["verdict"] == "PASS" + assert {row["version"] for row in events(campaign, final["final"])} == {"accepted"} + assert "profile" not in [row["stage"] for row in events(campaign, final["final"])] + project_service.no_fingerprints(optimization.read_records(campaign["root"])[2]) + + +@pytest.mark.parametrize("failure", ["failed", "missing"]) +def test_profile_failure_and_failed_repair_preserve_prior_selection(campaign, failure): + begin(campaign) + selected = submit(campaign, version="selected", a=90, b=100)["best"] + broken = submit( + campaign, version="profile-broken", a=80, b=100, profile_mode=failure, expected=2 + ) + old = broken["latest_observation"] + assert old["assessment"]["verdict"] == "INCONCLUSIVE" + assert old["assessment"]["quality_passed"] is True + assert "profile" in old["assessment"]["reason"] + assert broken["best"] == selected + assert len(events(campaign, old)) == 7 + repaired = submit(campaign, version="repair-failed", a=70, b=120, retry=old["attempt_id"]) + assert repaired["latest_observation"]["assessment"]["verdict"] == "FAIL" + assert repaired["best"] == selected + assert project_service.read(campaign["root"] / old["path"] / "observation.json") == old + final = call(campaign, "final", "--reason", "Repair rejected") + assert final["final"]["revision"] == selected["revision"] + assert {row["version"] for row in events(campaign, final["final"])} == {"selected"} + + +def test_conditions_refresh_cannot_finalize_without_new_acceptance(campaign): + begin(campaign) + selected = submit(campaign, version="old-selected", a=90)["best"] + new = copy.deepcopy(settings()["conditions"]) + new.update(id="c1", runtime="changed-stdlib") + path = campaign["root"].parent / "conditions.json" + path.write_text(json.dumps(new)) + refreshed = call( + campaign, "refresh-references", "--conditions", path, "--reason", "New test conditions" + ) + assert refreshed["can_finalize"] is False + assert refreshed["accepted_attempt_ids"] == [] + assert refreshed["best"] is None + rejected = call(campaign, "final", "--reason", "References alone are insufficient") + assert rejected["final"]["assessment"]["verdict"] == "INCONCLUSIVE" + assert rejected["final"]["assessment"]["reason"] == "missing_current_conditions_acceptance" + assert not (campaign["root"] / rejected["final"]["path"] / "service").exists() + accepted = submit(campaign, version="new-selected", a=80) + assert accepted["best"]["conditions"] == new + assert accepted["best"]["revision"] != selected["revision"] + final = call(campaign, "final", "--reason", "New conditions accepted") + assert final["final"]["assessment"]["verdict"] == "PASS" + + +def test_stagnation_is_advisory_and_recover_does_not_retrigger(campaign): + begin(campaign) + submit(campaign, version="reject1", a=110) + result = submit(campaign, version="reject2", a=111) + assert result["state"] == "active" + assert len(result["stagnation"]["events"]) == 1 + recovered = call(campaign, "recover") + assert recovered["stagnation"] == result["stagnation"] + improved = submit(campaign, version="gain", a=90) + assert improved["stagnation"]["consecutive"] == 0 + + +def test_baseline_selected_final_has_no_self_gain_requirement(campaign): + begin(campaign) + submit(campaign, version="no-improvement", a=100) + final = call(campaign, "final", "--reason", "No candidate selected") + assert final["final"]["assessment"]["verdict"] == "PASS" + assert final["best"]["kind"] == "baseline" + + +def test_missing_performance_evidence_blocks_without_quality_or_profile(campaign): + begin(campaign) + result = submit(campaign, version="missing", a=90, missing_metric=True, expected=2) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == "INCONCLUSIVE" + assert "quality" not in [row["stage"] for row in events(campaign, attempt)] + assert "profile" not in [row["stage"] for row in events(campaign, attempt)] + + +def test_service_mode_cli_does_not_require_legacy_metric_arguments(campaign): + args = parser().parse_args(campaign["start"]) + assert args.evaluation_mode == "service_reuse" and args.metric_key is None + with pytest.raises(SystemExit): + parser().parse_args(["benchmarks", "session", "baseline", "--session", "unused"]) + + +def test_knowledge_and_dashboard_never_enter_legacy_fingerprint_readers(campaign): + begin(campaign) + context = call( + campaign, "knowledge", "--query", "optimization", "--context-bytes", "65536", "--limit", "1" + ) + assert context["context"]["contexts"] + project_service.no_fingerprints(context) + submit(campaign, version="selected", a=90) + from inferrsi.apps.views.dashboard import dashboard_snapshot, render_dashboard + + snapshot = dashboard_snapshot(campaign["root"]) + project_service.no_fingerprints(snapshot) + page = render_dashboard(snapshot) + assert "Service-reuse optimization" in page and "Objective values" in page + assert "sha256" not in page.lower() + + +@pytest.mark.parametrize( + "changes,expected", + [ + ({"quality": 0}, 0), + ({"quality_rejected": True}, 0), + ({"quality_missing": True}, 2), + ({"functional_failure": True}, 0), + ({"profile_mode": "empty"}, 2), + ({"foreign_stage": "quality"}, 2), + ], +) +def test_rejections_and_missing_stage_evidence_are_distinct(campaign, changes, expected): + begin(campaign) + result = submit(campaign, version="rejected", a=90, expected=expected, **changes) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == ("FAIL" if expected == 0 else "INCONCLUSIVE") + assert result["best"]["kind"] == "baseline" + if not changes.get("profile_mode"): + assert not any( + row["id"] == "profile" and row["status"] == "completed" + for row in attempt["execution"]["steps"] + ) + + +def test_profile_timeout_retains_quality_and_stops_without_silent_retry(campaign): + config = settings() + config["sampling"]["stage_timeout_seconds"] = 1 + campaign["config"].write_text(json.dumps(config)) + begin(campaign) + result = submit(campaign, version="timeout", a=90, profile_mode="timeout", expected=2) + attempt = result["latest_observation"] + stage = next(row for row in attempt["execution"]["steps"] if row["id"] == "profile") + assert stage["status"] == "timeout" + assert attempt["assessment"]["quality_passed"] is True + assert attempt["execution"]["cleanup_verified"] is True + recovered = call(campaign, "recover", expected=2) + assert recovered["latest_observation"] == attempt + + +def test_profile_interruption_is_inconclusive_with_verified_cleanup(campaign, monkeypatch): + from inferrsi.harness.scenarios import external + + begin(campaign) + original = external.execute_trial + + def interrupted(command, *args, **kwargs): + if "profile" in command: + raise KeyboardInterrupt() + return original(command, *args, **kwargs) + + monkeypatch.setattr(external, "execute_trial", interrupted) + result = submit(campaign, version="interrupted", a=90, expected=2) + assert result["latest_observation"]["assessment"]["verdict"] == "INCONCLUSIVE" + assert result["latest_observation"]["assessment"]["quality_passed"] is True + assert result["latest_observation"]["execution"]["cleanup_verified"] is True + + +def test_incomplete_baseline_prevents_candidate_search(campaign): + commit(campaign["repo"], missing_metric=True) + call(campaign, "start") + result = call(campaign, "baseline", expected=2) + assert result["best"] is None and result["state"] == "blocked" + assert ( + rejected( + [ + "campaign", + "round", + "--session", + str(campaign["root"]), + "--candidate", + str(campaign["repo"]), + "--hypothesis", + "must not start", + ] + ) + == 2 + ) + + +def test_baseline_comparison_quality_and_pause_resume(campaign): + config = settings() + config["quality"].pop("threshold") + config["quality"].update(mode="baseline_comparison", tolerance=0) + config["stagnation"] = {"rounds": 1, "action": "pause"} + campaign["config"].write_text(json.dumps(config)) + begin(campaign) + result = submit(campaign, version="worse-quality", a=90, quality=0) + assert result["state"] == "paused" + assert result["latest_observation"]["assessment"]["quality_passed"] is False + resumed = call(campaign, "resume", "--reason", "Reviewed system-wide bottlenecks") + repeated = call(campaign, "resume", "--reason", "Reviewed system-wide bottlenecks") + assert resumed["stagnation"] == repeated["stagnation"] + assert resumed["state"] == "active" + + +def test_accepted_but_unselected_candidate_keeps_its_verdict(campaign): + begin(campaign) + result = submit(campaign, version="second-objective-gain", a=102, b=90) + attempt = result["latest_observation"] + assert attempt["assessment"]["eligible"] is True + assert attempt["selected"] is False + assert result["best"]["kind"] == "baseline" + + +def test_screening_below_estimate_runs_formal_quality_and_profile_on_same_service(campaign): + config = settings() + config["screening"] = { + "command": [arg.replace("measure", "screening") for arg in config["commands"]["measure"]], + "estimated_gain": 50, + } + campaign["config"].write_text(json.dumps(config)) + begin(campaign) + result = submit(campaign, version="actual-gain", a=95) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == "PASS" + log = events(campaign, attempt) + assert [r["stage"] for r in log].count("screening") == 2 + assert len({r["service_id"] for r in log}) == 1 + + +@pytest.mark.parametrize("damage", [None, "missing", "empty", "receipt", "binding", "artifacts"]) +def test_recover_commits_existing_observation_once_without_remeasurement( + campaign, monkeypatch, damage +): + begin(campaign) + selected = submit(campaign, version="previous-selection", a=95)["best"] + previous = project_service.read( + campaign["root"] / selected["path"].removesuffix("/candidate") / "observation.json" + ) + original = optimization.atomic_write_json + + def fail_state_commit(path, data): + if path.name == "state.json" and data.get("pending") is None and data.get("attempts"): + if data["attempts"][-1]["kind"] == "round": + raise OSError("synthetic coordinator state-write failure") + return original(path, data) + + monkeypatch.setattr(optimization, "atomic_write_json", fail_state_commit) + commit(campaign["repo"], version="recovery", a=90) + assert ( + rejected( + [ + "campaign", + "round", + "--session", + str(campaign["root"]), + "--candidate", + str(campaign["repo"]), + "--hypothesis", + "Injected settlement interruption", + ] + ) + == 2 + ) + campaign["capsys"].readouterr() + monkeypatch.setattr(optimization, "atomic_write_json", original) + pending = project_service.read(campaign["root"] / "state.json")["pending"] + observation_path = campaign["root"] / pending["path"] / "observation.json" + observation = project_service.read(observation_path) + profile = observation["execution"]["profile"] + artifact = campaign["root"] / profile["artifacts"][0] + if damage == "missing": + artifact.unlink() + elif damage == "empty": + artifact.write_text("") + elif damage == "receipt": + (campaign["root"] / profile["result"]).write_text('{"status":"incomplete"}') + elif damage == "binding": + profile["binding"]["service_id"] = "another-service" + profile["receipt"]["binding"] = profile["binding"] + (campaign["root"] / profile["result"]).write_text(json.dumps(profile["receipt"])) + observation_path.write_text(json.dumps(observation)) + elif damage == "artifacts": + profile["artifacts"] = [] + observation_path.write_text(json.dumps(observation)) + result = call(campaign, "recover", expected=2 if damage else 0) + attempt = result["latest_observation"] + log = events(campaign, attempt) + assert result["completed_rounds"] == (1 if damage else 2) + assert len(log) == 7 + if damage: + assert result["state"] == "blocked" + assert attempt["assessment"]["verdict"] == "INCONCLUSIVE" + assert attempt["assessment_before_settlement"]["verdict"] == "PASS" + assert attempt["execution"]["assessment"]["verdict"] == "PASS" + assert attempt["selected"] is False + assert result["best"] == selected + assert result["accepted_attempt_ids"] == [selected["attempt_id"]] + else: + assert attempt["assessment"]["verdict"] == "PASS" + assert result["best"]["attempt_id"] == attempt["attempt_id"] + assert project_service.read( + campaign["root"] / previous["path"] / "observation.json" + ) == previous + again = call(campaign, "recover", expected=2 if damage else 0) + assert again["latest_observation"] == attempt + assert len(events(campaign, attempt)) == len(log) + + +@pytest.mark.parametrize("interrupted", [False, True]) +@pytest.mark.parametrize("damage", ["profile", "configuration"]) +def test_final_rechecks_selected_evidence_before_settlement( + campaign, monkeypatch, interrupted, damage +): + begin(campaign) + selected = submit(campaign, version="selected", a=90)["best"] + selected_attempt = project_service.read( + campaign["root"] / selected["path"].removesuffix("/candidate") / "observation.json" + ) + execute, write = optimization._service_execute, optimization.atomic_write_json + + def execute_then_damage(*args): + result = execute(*args) + assert result["assessment"]["verdict"] == "PASS" + if not interrupted: + damage_evidence() + return result + + def damage_evidence(): + if damage == "profile": + (campaign["root"] / selected["profile"]["artifacts"][0]).unlink() + else: + (campaign["root"] / selected["effective_config_path"]).write_text('{"mode":"altered"}') + + def fail_state_commit(path, data): + if path.name == "state.json" and data.get("final") is not None: + raise OSError("synthetic final state-write failure") + return write(path, data) + + monkeypatch.setattr(optimization, "_service_execute", execute_then_damage) + if interrupted: + monkeypatch.setattr(optimization, "atomic_write_json", fail_state_commit) + assert rejected( + ["campaign", "final", "--session", str(campaign["root"]), "--reason", "Stop search"] + ) == 2 + campaign["capsys"].readouterr() + monkeypatch.setattr(optimization, "atomic_write_json", write) + pending = project_service.read(campaign["root"] / "state.json")["pending"] + assert project_service.read( + campaign["root"] / pending["path"] / "observation.json" + )["assessment"]["verdict"] == "PASS" + damage_evidence() + result = call(campaign, "recover", expected=2) + else: + result = call(campaign, "final", "--reason", "Stop search", expected=2) + final = result["final"] + assert result["state"] == "blocked" + assert result["best"] == selected + assert final["assessment"]["verdict"] == "INCONCLUSIVE" + assert final["assessment"]["reason"] == ( + "missing_or_incompatible_selected_profile" if damage == "profile" + else "unverifiable_selected_configuration" + ) + assert final["assessment_before_settlement"]["verdict"] == "PASS" + assert final["execution"]["assessment"]["verdict"] == "PASS" + assert project_service.read( + campaign["root"] / selected_attempt["path"] / "observation.json" + ) == selected_attempt + log = events(campaign, final) + assert [row["stage"] for row in log] == [ + "ready", "warmup", "measure", "measure", "quality", "stop" + ] + again = call(campaign, "recover", expected=2) + assert again["final"] == final + assert events(campaign, final) == log + + +def test_final_cannot_use_deleted_profile_or_changed_retained_configuration(campaign): + begin(campaign) + selected = submit(campaign, version="selected", a=90)["best"] + config_path = campaign["root"] / selected["effective_config_path"] + config_path.write_text('{"mode":"altered"}') + result = call(campaign, "final", "--reason", "Cannot establish selected configuration") + assert result["final"]["assessment"]["reason"] == "unverifiable_selected_configuration" + config_path.write_text(json.dumps(settings()["conditions"]["effective_config"])) + (campaign["root"] / selected["profile"]["artifacts"][0]).unlink() + result = call(campaign, "final", "--reason", "Cannot establish selected profile") + assert result["final"]["assessment"]["reason"] == "missing_or_incompatible_selected_profile" + + +@pytest.mark.parametrize("block", ["commands", "sampling", "conditions", "quality", "stagnation"]) +def test_invalid_protocol_containers_rejected_before_campaign_writes(campaign, block): + config = settings() + config[block] = [] + campaign["config"].write_text(json.dumps(config)) + assert rejected(campaign["start"]) == 2 + assert not campaign["root"].exists() + + +def rejected(arguments): + with pytest.raises(SystemExit) as raised: + main(arguments) + return raised.value.code + + +def test_live_dashboard_state_has_no_content_fingerprint(campaign): + import threading + from urllib.request import urlopen + + from inferrsi.apps.views.dashboard_live import DashboardServer + + call(campaign, "start") + server = DashboardServer(campaign["root"], None, 0) + worker = threading.Thread(target=server.serve_forever) + worker.start() + try: + with urlopen(f"http://127.0.0.1:{server.server_port}/api/state") as response: + assert response.status == 200 and response.headers.get("ETag") is None + project_service.no_fingerprints(json.load(response)) + with urlopen(f"http://127.0.0.1:{server.server_port}/") as response: + assert b"Service-reuse optimization" in response.read() + finally: + server.shutdown() + worker.join() + server.server_close() + + +@pytest.mark.parametrize("compatible", [True, False]) +def test_existing_quality_reference_is_reused_only_when_compatible(campaign, compatible): + config = settings() + config["quality"].pop("threshold") + config["quality"].update( + mode="baseline_comparison", tolerance=0, reference_path="reference.json" + ) + revision = subprocess.check_output( + ["git", "-C", str(campaign["repo"]), "rev-parse", "HEAD"], text=True + ).strip() + reference = { + "status": "completed", + "revision": revision, + "conditions": config["conditions"], + "service_id": "previous-service", + "run_id": "previous-run", + "dataset": "synthetic", + "scorer": "exact-v1" if compatible else "incompatible-scorer", + "samples": 2, + "quality": 1, + } + protocol = Path(campaign["start"][campaign["start"].index("--protocol") + 1]) + (protocol / "reference.json").write_text(json.dumps(reference)) + campaign["config"].write_text(json.dumps(config)) + call(campaign, "start") + result = call(campaign, "baseline", expected=0 if compatible else 2) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == ("PASS" if compatible else "INCONCLUSIVE") + assert "quality" not in [row["stage"] for row in events(campaign, attempt)] + + +@pytest.mark.parametrize("same_version", [False, True]) +def test_reference_refresh_uses_baseline_cache_without_testing_parent_quality( + campaign, same_version +): + config = settings() + config["quality"].pop("threshold") + config["quality"].update( + mode="baseline_comparison", tolerance=0, reference_path="reference.json" + ) + revision = subprocess.check_output( + ["git", "-C", str(campaign["repo"]), "rev-parse", "HEAD"], text=True + ).strip() + reference = { + "status": "completed", "revision": revision, "conditions": config["conditions"], + "service_id": "previous-service", "run_id": "previous-run", + "dataset": "synthetic", "scorer": "exact-v1", "samples": 2, "quality": 1, + } + protocol = Path(campaign["start"][campaign["start"].index("--protocol") + 1]) + (protocol / "reference.json").write_text(json.dumps(reference)) + campaign["config"].write_text(json.dumps(config)) + selected = begin(campaign)["best"] + if not same_version: + selected = submit(campaign, version="selected-parent", a=90)["best"] + assert selected["revision"] != revision + conditions = {**config["conditions"], "id": "c1", "runtime": "changed-stdlib"} + conditions_path = campaign["root"].parent / "conditions.json" + conditions_path.write_text(json.dumps(conditions)) + failed = call( + campaign, "refresh-references", "--conditions", conditions_path, + "--reason", "Changed conditions", expected=2, + )["latest_observation"] + assert failed["assessment"]["verdict"] == "INCONCLUSIVE" + assert "incompatible baseline reference" in ( + failed["execution"]["runs"][0]["assessment"]["reason"] + ) + repaired = campaign["root"].parent / "repaired-protocol" + shutil.copytree(protocol, repaired) + reference.update(conditions=conditions, service_id="new-service", run_id="new-run") + (repaired / "reference.json").write_text(json.dumps(reference)) + result = call( + campaign, "refresh-references", "--conditions", conditions_path, + "--reason", "Refresh compatible references", "--retry-of", failed["attempt_id"], + "--repair-reason", "Supplied compatible baseline quality evidence", + "--repair-protocol", repaired, + ) + attempt = result["latest_observation"] + assert attempt["assessment"]["verdict"] == "PASS" + references = attempt["execution"]["references"] + assert [item["revision"] for item in references] == [revision, selected["revision"]] + assert all(item["conditions"] == conditions for item in references) + assert references[0]["quality_score"] == 1 + assert len(attempt["execution"]["runs"]) == (1 if same_version else 2) + if same_version: + assert references[0] == references[1] + else: + assert references[1]["quality_score"] is None + assert references[1]["profile"] is None + parent_run = attempt["execution"]["runs"][1] + for name in ("quality", "profile"): + assert next(step for step in parent_run["steps"] if step["id"] == name) == { + "id": name, "status": "skipped", "reason": "performance_reference_only" + } + for reference in references: + log_path = campaign["root"] / reference["path"].removesuffix("/candidate") / "events.jsonl" + stages = [json.loads(line)["stage"] for line in log_path.read_text().splitlines()] + assert stages == ["ready", "warmup", "measure", "measure", "stop"] + assert result["best"] is None and result["can_finalize"] is False + assert project_service.read(campaign["root"] / failed["path"] / "observation.json") == failed + accepted = submit(campaign, version="current-conditions", a=80) + assert accepted["latest_observation"]["assessment"]["verdict"] == "PASS" + assert accepted["best"]["conditions"] == conditions + + +def test_execution_script_repair_is_retained_and_final_uses_selected_scripts(campaign): + begin(campaign) + selected = submit(campaign, version="selected", a=90)["best"] + broken = submit(campaign, version="broken-profile", a=80, profile_mode="failed", expected=2) + old = broken["latest_observation"] + protocol = Path(campaign["start"][campaign["start"].index("--protocol") + 1]) + repaired = campaign["root"].parent / "repaired-protocol" + shutil.copytree(protocol, repaired) + path = repaired / "protocol.py" + path.write_text( + path.read_text().replace( + 'if phase == "service":', + 'raise RuntimeError("repaired script rejected")\nif phase == "service":', + ) + ) + attempt = call( + campaign, + "round", + "--candidate", + campaign["repo"], + "--hypothesis", + "Execution repair", + "--retry-of", + old["attempt_id"], + "--repair-reason", + "Investigated fixture", + "--repair-protocol", + repaired, + expected=2, + ) + assert attempt["best"] == selected + assert attempt["latest_observation"]["protocol_path"] != selected["protocol_path"] + inspection = campaign["root"].parent / "cleanup.json" + inspection.write_text( + json.dumps( + { + "attempt_id": attempt["latest_observation"]["attempt_id"], + "observed_at": 1, + "reason": "Inspected failed precheck; no service started", + "cleanup_complete": True, + "services": [], + } + ) + ) + call( + campaign, + "resume", + "--reason", + "Abandon failed execution repair", + "--cleanup-receipt", + inspection, + ) + result = call(campaign, "final", "--reason", "Retest retained selected script set") + assert result["final"]["assessment"]["verdict"] == "PASS" + assert result["final"]["protocol_path"] == selected["protocol_path"] + + +def test_round_usage_export_is_retained_without_fingerprint(campaign): + begin(campaign) + export = campaign["root"].parent / "usage.json" + export.write_text('{"input_tokens":10,"output_tokens":2}') + commit(campaign["repo"], version="usage", a=90) + result = call( + campaign, + "round", + "--candidate", + campaign["repo"], + "--hypothesis", + "Usage evidence", + "--usage-export", + export, + ) + summary = optimization.read_records(campaign["root"])[2] + assert summary["usage"]["total_tokens"] == 12 + assert result["latest_observation"]["observations"]["usage_source"] == {"kind": "caller_export"} + call(campaign, "recover") + assert optimization.read_records(campaign["root"])[2]["usage"]["total_tokens"] == 12 + + +def test_exact_round_policy_and_legacy_argument_validation(campaign): + campaign["start"] += ["--round-mode", "exact"] + begin(campaign) + submit(campaign, version="one-of-eight", a=90) + result = call(campaign, "final", "--reason", "Incomplete exact budget") + assert result["final"]["assessment"]["reason"] == "incomplete_round_policy" + assert result["state"] == "active" + from inferrsi.harness.project import steps + + assert [ + step["side"] + for step in steps( + { + "quality_command": ["quality"], + "measure_command": ["measure"], + "warmups": 0, + "repetitions": 2, + } + ) + if step["phase"] == "measurement" + ] == ["baseline", "candidate", "candidate", "baseline"] + + +def test_current_epoch_candidate_can_seed_selection_below_old_parent_rank(campaign): + begin(campaign) + old = submit(campaign, version="old-parent", a=90)["best"] + conditions = copy.deepcopy(settings()["conditions"]) + conditions.update(id="c1", runtime="new-runtime") + path = campaign["root"].parent / "conditions.json" + path.write_text(json.dumps(conditions)) + call(campaign, "refresh-references", "--conditions", path, "--reason", "New runtime") + result = submit(campaign, version="c1-accepted", a=92, b=90) + assert result["best"]["revision"] != old["revision"] + assert result["best"]["measurements"]["values"]["a"] > old["measurements"]["values"]["a"] + assert result["best"]["attempt_id"] in result["accepted_attempt_ids"] + + +def test_missing_mode_marker_rejects_without_falling_back_to_file_fingerprints(campaign): + call(campaign, "start") + path = campaign["root"] / "plan.json" + plan = json.loads(path.read_text()) + del plan["evaluation_mode"] + path.write_text(json.dumps(plan)) + assert rejected(["campaign", "status", "--session", str(campaign["root"])]) == 2 + from inferrsi.apps.views.dashboard import dashboard_snapshot + + with pytest.raises(ValueError, match="Incompatible service campaign"): + dashboard_snapshot(campaign["root"]) + + +def test_final_checks_selected_revision_against_original_attempt(campaign): + begin(campaign) + submit(campaign, version="selected", a=90) + path = campaign["root"] / "state.json" + state = json.loads(path.read_text()) + state["best"]["revision"] = state["initial_reference"]["revision"] + path.write_text(json.dumps(state)) + result = call(campaign, "final", "--reason", "Selected revision cannot be established") + assert result["final"]["assessment"]["reason"] == "unverifiable_selected_attempt_binding" + assert result["final"]["assessment"]["verdict"] == "INCONCLUSIVE" + assert not (campaign["root"] / result["final"]["path"] / "service").exists() From 8770343fd71b000fb938ba516ea74874426e2d37 Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Tue, 17 Nov 2026 16:55:02 +0800 Subject: [PATCH 2/7] Fix service-reuse revision binding and cleanup Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- docs/service-reuse-campaigns.md | 18 +++++- src/inferrsi/harness/optimization.py | 15 +++-- src/inferrsi/harness/project_service.py | 26 +++++--- tests/fixtures/service_reuse_protocol.py | 4 +- tests/test_project_service_reuse.py | 66 ++++++++++++++++++++- tests/test_service_fingerprint_isolation.py | 27 +++++++++ 6 files changed, 137 insertions(+), 19 deletions(-) create mode 100644 tests/test_service_fingerprint_isolation.py diff --git a/docs/service-reuse-campaigns.md b/docs/service-reuse-campaigns.md index 2c70d8e5f..a5e20a2df 100644 --- a/docs/service-reuse-campaigns.md +++ b/docs/service-reuse-campaigns.md @@ -14,9 +14,13 @@ Apps projects those records without writing selection decisions. ## Prepare the protocol Use the Git checkout root and commit the declared project inputs first. The -controller extracts those files -from the recorded commit into a retained candidate; it never runs the editable -workspace as `final`. The protocol directory contains the caller's scripts and +controller records the source repository and commit, and rebuilds each execution +workspace from that commit's Git objects, including baseline, reference refresh +and final. Keep those repositories and commits available throughout the campaign. +Replacing a retained directory does not replace the executed version; an unavailable +source commit makes execution INCONCLUSIVE. Older records without a source repository +cannot establish that binding and must start a new campaign. +The protocol directory contains the caller's scripts and any declared inputs. Campaign output must be outside the project and protocol. Supply a JSON file with these fields; every threshold and workload is explicit: @@ -92,6 +96,14 @@ exclude profiling requests and time. `stop` returns `cleanup_complete: true`; Harness also stops and verifies its owned process domain. Unknown cleanup blocks further execution until investigated. +Cleanup has a separate bounded window of `sampling.stage_timeout_seconds`, starting +when `stop` is invoked even if the evaluation deadline was exhausted. This window +is additional to the evaluation/campaign budget and is recorded as +`cleanup_timeout_seconds` with `cleanup_budget_scope: separate_from_evaluation`. +It applies per service, including each service in a reference refresh. Owned process +termination still runs afterward. A successful cleanup does not change a timed-out +evaluation's INCONCLUSIVE verdict. + ## Lifecycle Use `campaign start --help` for shared budget/input arguments. In service mode diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index a975b5032..511f54f1c 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -692,6 +692,7 @@ def _service_choice(pending, execution): "candidate_id": pending["candidate_id"], "path": pending["path"] + "/candidate", "revision": pending["revision"], + "source_repo": pending.get("source_repo"), "conditions": pending["conditions"], "reference_epoch": pending["reference_epoch"], "effective_config_path": pending["path"] + "/effective-config.json", @@ -855,13 +856,11 @@ def _service_execute(root, directory, plan, pending, deadline): relative = pending["path"] + "/" + label subdir = service.safe_path(root, relative) subdir.mkdir() - candidates.retain( - service.safe_path(root, source["path"]), subdir / "candidate", fingerprint=False - ) job = { **pending, "path": relative, "revision": source["revision"], + "source_repo": source.get("source_repo"), "attempt_id": pending["attempt_id"] + "-" + label, "candidate_id": source["candidate_id"], "role": "baseline", @@ -1186,10 +1185,13 @@ def service_command(root, args, *, knowledge_reader=None): service.retain_revision( source, directory / "candidate", plan["candidate_includes"], commit ) + if service.revision(source, plan["candidate_includes"]) != commit: + raise ValueError("Candidate revision changed while retaining inputs") + source_repo = str(source) + elif kind == "final" and selected: + source_repo = selected.get("source_repo") else: - candidates.retain(source, directory / "candidate", fingerprint=False) - if kind == "round" and service.revision(source, plan["candidate_includes"]) != commit: - raise ValueError("Candidate revision changed while retaining inputs") + source_repo = plan["working_candidate"] pending = { "attempt_id": attempt_id, "protocol_path": protocol_path, @@ -1197,6 +1199,7 @@ def service_command(root, args, *, knowledge_reader=None): "path": relative, "candidate_id": candidate_id, "revision": commit, + "source_repo": source_repo, "round_number": number, "started_at": time.time(), "hypothesis": getattr(args, "hypothesis", None) diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py index 4f7f2707d..bf8eb6d40 100644 --- a/src/inferrsi/harness/project_service.py +++ b/src/inferrsi/harness/project_service.py @@ -364,26 +364,25 @@ def git(*args): def retain_revision(source, destination, includes, commit): - selected = members(source, includes) + if not isinstance(commit, str) or not re.fullmatch(r"[a-f0-9]{40,64}", commit): + raise ValueError("Candidate requires an existing Git commit") archive = subprocess.check_output( ["git", "-C", str(source), "archive", "--format=tar", commit, "--", *includes] ) destination.mkdir() - copied = set() + copied = False with tarfile.open(fileobj=io.BytesIO(archive)) as stream: for item in stream.getmembers(): if item.isdir(): continue if not item.isfile(): raise ValueError("Candidate archive contains symbolic or special files") - if item.name not in selected: - continue target = safe_path(destination, item.name) target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(stream.extractfile(item).read()) target.chmod(item.mode & ~0o222) - copied.add(item.name) - if copied != set(selected): + copied = True + if not copied: raise ValueError("Declared candidate files are missing from the retained commit") @@ -448,7 +447,6 @@ def execute(root, directory, plan, pending, deadline): config = conditions["effective_config"] source = directory / "candidate" workspace = directory / "workspace" - retain(source, workspace, fingerprint=False) expected = { "attempt_id": pending["attempt_id"], "service_id": uuid.uuid4().hex, @@ -481,6 +479,8 @@ def execute(root, directory, plan, pending, deadline): "cleanup_verified": None, "binding": expected, "assessment": None, + "cleanup_timeout_seconds": settings["sampling"]["stage_timeout_seconds"], + "cleanup_budget_scope": "separate_from_evaluation", } output = directory / "execution" output.mkdir() @@ -509,11 +509,14 @@ def stage(name, *, bound=True, argv=None): save() if bound and (process is None or process.poll() is not None): raise ValueError(f"Service unavailable before {name}") + stage_end = time.monotonic() + settings["sampling"]["stage_timeout_seconds"] + if name != "stop": + stage_end = min(end, stage_end) executed = execute_trial( tuple(command(argv or settings["commands"][name.split("-")[0]])), workspace, output / name, - min(end, time.monotonic() + settings["sampling"]["stage_timeout_seconds"]), + stage_end, None, environment=environment, ) @@ -544,6 +547,13 @@ def skipped(names, reason): measured, gate, score, profile, quality_ok = None, None, None, None, None try: + # Rebuild from Git objects; retained directories can be replaced after acceptance. + repository = pending.get("source_repo") + if not repository: + raise ValueError("Authoritative source repository is unavailable") + retain_revision(repository, workspace, plan["candidate_includes"], pending["revision"]) + if not source.exists(): + retain(workspace, source, fingerprint=False) precheck = stage("precheck", bound=False) if precheck.get("status") == "candidate_failure" or precheck.get("passed") is False: skipped(("service", "measurement", "quality", "profile"), "precheck_rejected") diff --git a/tests/fixtures/service_reuse_protocol.py b/tests/fixtures/service_reuse_protocol.py index aef72d820..697d9db1a 100644 --- a/tests/fixtures/service_reuse_protocol.py +++ b/tests/fixtures/service_reuse_protocol.py @@ -100,7 +100,9 @@ def do_GET(self): self.wfile.write(encoded) server = HTTPServer(("127.0.0.1", 0), Handler) - write(receipt_path, {"endpoint": f"http://127.0.0.1:{server.server_port}"}) + temporary_receipt = receipt_path.with_suffix(".tmp") + write(temporary_receipt, {"endpoint": f"http://127.0.0.1:{server.server_port}"}) + os.replace(temporary_receipt, receipt_path) server.serve_forever() server.server_close() elif phase == "precheck": diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index 3de55e9e5..0cdcf4de5 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -12,7 +12,13 @@ from inferrsi.apps.views import dashboard_data from inferrsi.apps.cli import main from inferrsi.apps.cli.arguments import parser -from inferrsi.harness import candidates, optimization, optimization_session, project_service +from inferrsi.harness import ( + candidates, + optimization, + optimization_session, + project, + project_service, +) from inferrsi.harness.records import evidence, knowledge_capture from inferrsi.infra import artifacts from inferrsi.knowledge import knowledge_operations @@ -33,6 +39,7 @@ def forbidden(*args, **kwargs): (optimization, "sha256"), (optimization, "digest"), (optimization_session, "_runtime_identity"), + (project, "sha256"), (knowledge_operations, "_hash"), (hashlib, "sha256"), ): @@ -906,3 +913,60 @@ def test_final_checks_selected_revision_against_original_attempt(campaign): assert result["final"]["assessment"]["reason"] == "unverifiable_selected_attempt_binding" assert result["final"]["assessment"]["verdict"] == "INCONCLUSIVE" assert not (campaign["root"] / result["final"]["path"] / "service").exists() + + +@pytest.mark.parametrize("role", ["baseline", "candidate"]) +def test_final_rebuilds_changed_retained_code_from_recorded_commit(campaign, role): + begin(campaign) + if role == "candidate": + alternate = campaign["repo"].with_name("alternate-repo") + shutil.copytree(campaign["repo"], alternate) + campaign["repo"] = alternate + result = submit(campaign, version="accepted", a=90 if role == "candidate" else 100) + selected = result["best"] + saved = campaign["root"] / selected["path"] / "engine.py" + saved.unlink() + saved.write_text("raise RuntimeError('changed retained source must never run')\n") + # The current checkout and file set are not the authority for an older commit. + (campaign["repo"] / "engine.py").unlink() + result = call(campaign, "final", "--reason", "Verify original committed version") + final = result["final"] + assert final["assessment"]["verdict"] == "PASS" + assert final["revision"] == selected["revision"] + expected_version = "accepted" if role == "candidate" else "baseline" + assert {row["version"] for row in events(campaign, final)} == {expected_version} + assert "changed retained" not in ( + campaign["root"] / final["path"] / "candidate/engine.py" + ).read_text() + + +def test_missing_authoritative_repository_cannot_run_retained_selection(campaign): + begin(campaign) + selected = submit(campaign, version="selected", a=90)["best"] + shutil.rmtree(campaign["repo"] / ".git") + result = call(campaign, "final", "--reason", "Source commit unavailable", expected=2) + final = result["final"] + assert final["assessment"]["verdict"] == "INCONCLUSIVE" + assert result["best"] == selected + assert not (campaign["root"] / final["path"] / "service").exists() + + +def test_whole_attempt_timeout_still_runs_bounded_stop(campaign): + config = settings() + config["sampling"]["stage_timeout_seconds"] = 8 + campaign["config"].write_text(json.dumps(config)) + index = campaign["start"].index("--evaluation-timeout-seconds") + campaign["start"][index + 1] = "6" + begin(campaign) + result = submit(campaign, version="deadline", a=90, profile_mode="timeout", expected=2) + attempt = result["latest_observation"] + execution = attempt["execution"] + steps = {row["id"]: row for row in execution["steps"]} + assert steps["profile"]["status"] == "timeout" + assert steps["stop"]["status"] == "completed" + assert execution["cleanup_timeout_seconds"] == 8 + assert execution["cleanup_budget_scope"] == "separate_from_evaluation" + assert execution["cleanup_verified"] is True + assert attempt["assessment"]["verdict"] == "INCONCLUSIVE" + assert attempt["assessment"]["quality_passed"] is True + assert [row["stage"] for row in events(campaign, attempt)][-1] == "stop" diff --git a/tests/test_service_fingerprint_isolation.py b/tests/test_service_fingerprint_isolation.py new file mode 100644 index 000000000..21f4a1f8c --- /dev/null +++ b/tests/test_service_fingerprint_isolation.py @@ -0,0 +1,27 @@ +"""Check cross-file fixture isolation without executing a fingerprint workflow.""" + +import os +import subprocess +import sys +from pathlib import Path + + +def test_service_guard_restores_lazy_import_aliases(tmp_path): + check = tmp_path / "test_aliases.py" + check.write_text( + "def test_restored_alias():\n" + " from inferrsi.harness import project\n" + " from inferrsi.infra import artifacts\n" + " assert project.sha256 is artifacts.sha256\n" + " assert project.sha256.__module__ == 'inferrsi.infra.artifacts'\n", + encoding="utf-8", + ) + result = subprocess.run( + [sys.executable, "-B", "-m", "pytest", "-p", "no:cacheprovider", "-q", + "tests/test_project_service_reuse.py::test_exact_round_policy_and_legacy_argument_validation", + str(check)], + cwd=Path(__file__).resolve().parents[1], + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1", "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, + capture_output=True, text=True, timeout=90, + ) + assert result.returncode == 0, result.stdout + result.stderr From 92a4658a3eb209f6e0c810e347cfd197d54f66a9 Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Tue, 17 Nov 2026 18:52:03 +0800 Subject: [PATCH 3/7] Fix snapshot fidelity and campaign stagnation Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- docs/service-reuse-campaigns.md | 15 ++-- src/inferrsi/harness/optimization.py | 57 ++++++++-------- src/inferrsi/harness/project_service.py | 45 ++++++------ tests/test_project_service_reuse.py | 91 +++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 51 deletions(-) diff --git a/docs/service-reuse-campaigns.md b/docs/service-reuse-campaigns.md index a5e20a2df..1c63e7779 100644 --- a/docs/service-reuse-campaigns.md +++ b/docs/service-reuse-campaigns.md @@ -16,7 +16,10 @@ Apps projects those records without writing selection decisions. Use the Git checkout root and commit the declared project inputs first. The controller records the source repository and commit, and rebuilds each execution workspace from that commit's Git objects, including baseline, reference refresh -and final. Keep those repositories and commits available throughout the campaign. +and final. It reads declared tree members and raw blobs directly, preserving their +executable modes while making copies read-only; archive export attributes cannot +omit or rewrite inputs. Keep those repositories and commits available throughout +the campaign. Replacing a retained directory does not replace the executed version; an unavailable source commit makes execution INCONCLUSIVE. Older records without a source repository cannot establish that binding and must start a new campaign. @@ -176,6 +179,8 @@ abandonment decision; it does not make an incomplete attempt pass or satisfy an exact-round policy. If conditions change, use `refresh-references --conditions --reason `. +This requires room for a new candidate round; when the round budget is exhausted, +the command rejects the refresh before changing conditions or starting a service. It measures the original baseline and comparison parent under the new conditions (one run when they are the same version), retains both old/new records, and does not collect another parent profile. This restores comparison references only. @@ -218,9 +223,11 @@ Captured material proves preparation, not model consumption. repair attempts; `other_attempt_usage` is separate. Unknown usage stays unknown, and recovering a settled attempt never imports or counts the same usage twice. -Consecutive complete rejections/unselected candidates may trigger the declared -system-wide analysis reminder. Infrastructure/cancelled/incomplete attempts do -not count; recovery does not double-count. Advisory does not pause execution; +Consecutive complete rejections may trigger the declared system-wide analysis +reminder. Every accepted gain resets the streak, even when ranking leaves the +candidate unselected. A new reference epoch resets the streak and notification +flag while retaining earlier events. Infrastructure/cancelled/incomplete attempts +do not count; recovery does not double-count. Advisory does not pause execution; explicit pause is resumed with a reason. The synthetic CPU controls are `tests/test_project_objectives.py` and diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index 511f54f1c..242634cc6 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -780,43 +780,41 @@ def _service_settle(root, plan, state, pending, execution): round_record = state["rounds"][record["round_number"] - 1] round_record["attempt_ids"].append(record["attempt_id"]) round_record["terminal"] = verdict != "INCONCLUSIVE" - selected = record["selected"] if verdict == "PASS": state["accepted_attempt_ids"].append(record["attempt_id"]) - if selected: + if record["selected"]: state.update( best=choice, comparison_parent=choice, requires_current_conditions_acceptance=False, ) state["state"] = "blocked" if verdict == "INCONCLUSIVE" else "active" - if verdict != "INCONCLUSIVE": + if verdict == "PASS": + state["stagnation"].update(consecutive=0, notified=False) + elif verdict != "INCONCLUSIVE": stagnation = state["stagnation"] - if selected: - stagnation.update(consecutive=0, notified=False) - else: - stagnation["consecutive"] += 1 - policy = plan["evaluation_settings"]["stagnation"] - if ( - not policy.get("disabled") - and not stagnation["notified"] - and (stagnation["consecutive"] >= policy["rounds"]) - ): - event = { - "id": record["attempt_id"] + "-stagnation", - "round": record["round_number"], - "consecutive": stagnation["consecutive"], - "best": state["best"], - "attempt_path": record["path"], - "action": policy["action"], - "message": "Reassess system-wide critical paths, compute, communication, " - "scheduling and resources across scenarios. Use available profiles; " - "record evidence gaps, the next hypothesis and restructuring tradeoffs.", - } - stagnation["events"].append(event) - stagnation["notified"] = True - if policy["action"] == "pause": - state["state"] = "paused" + stagnation["consecutive"] += 1 + policy = plan["evaluation_settings"]["stagnation"] + if ( + not policy.get("disabled") + and not stagnation["notified"] + and (stagnation["consecutive"] >= policy["rounds"]) + ): + event = { + "id": record["attempt_id"] + "-stagnation", + "round": record["round_number"], + "consecutive": stagnation["consecutive"], + "best": state["best"], + "attempt_path": record["path"], + "action": policy["action"], + "message": "Reassess system-wide critical paths, compute, communication, " + "scheduling and resources across scenarios. Use available profiles; " + "record evidence gaps, the next hypothesis and restructuring tradeoffs.", + } + stagnation["events"].append(event) + stagnation["notified"] = True + if policy["action"] == "pause": + state["state"] = "paused" elif kind == "final": state["final"] = record state["stop_reason"] = record["hypothesis"] @@ -1132,6 +1130,8 @@ def service_command(root, args, *, knowledge_reader=None): elif kind == "refresh": if initial is None or parent is None or not args.reason.strip(): raise ValueError("Initial references and a change reason required") + if len(state["rounds"]) >= plan["round_limit"]: + raise ValueError("No round budget for current-conditions acceptance") conditions = service.read(args.conditions) service.validate_conditions(conditions) if not old: @@ -1145,6 +1145,7 @@ def service_command(root, args, *, knowledge_reader=None): requires_current_conditions_acceptance=True, accepted_attempt_ids=[], ) + state["stagnation"].update(consecutive=0, notified=False) elif conditions != old["conditions"]: raise ValueError("Reference retry must retain its declared conditions") source, commit = root / plan["baseline_path"], plan["baseline_revision"] diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py index bf8eb6d40..ca9e64486 100644 --- a/src/inferrsi/harness/project_service.py +++ b/src/inferrsi/harness/project_service.py @@ -4,13 +4,11 @@ actual service receipts establish the explicitly declared execution binding. """ -import io import json import math import os import re import subprocess -import tarfile import time import uuid from pathlib import Path @@ -366,24 +364,33 @@ def git(*args): def retain_revision(source, destination, includes, commit): if not isinstance(commit, str) or not re.fullmatch(r"[a-f0-9]{40,64}", commit): raise ValueError("Candidate requires an existing Git commit") - archive = subprocess.check_output( - ["git", "-C", str(source), "archive", "--format=tar", commit, "--", *includes] - ) + entries = {} + for name in includes or ["."]: + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("Candidate includes must be relative paths inside the project") + listing = subprocess.check_output( + ["git", "-C", str(source), "ls-tree", "-rz", "--full-tree", + "--format=%(objectmode) %(objecttype)%x09%(path)", commit, "--", + relative.as_posix()] + ) + if not listing: + raise ValueError("Declared candidate files are missing from the retained commit") + for entry in listing.rstrip(b"\0").split(b"\0"): + metadata, path = entry.split(b"\t", 1) + mode, kind = metadata.split() + if kind != b"blob" or mode not in (b"100644", b"100755"): + raise ValueError("Candidate tree contains symbolic or special files") + entries[os.fsdecode(path)] = int(mode, 8) & 0o555 destination.mkdir() - copied = False - with tarfile.open(fileobj=io.BytesIO(archive)) as stream: - for item in stream.getmembers(): - if item.isdir(): - continue - if not item.isfile(): - raise ValueError("Candidate archive contains symbolic or special files") - target = safe_path(destination, item.name) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(stream.extractfile(item).read()) - target.chmod(item.mode & ~0o222) - copied = True - if not copied: - raise ValueError("Declared candidate files are missing from the retained commit") + for name, mode in entries.items(): + target = safe_path(destination, name) + target.parent.mkdir(parents=True, exist_ok=True) + # Read raw tree content; archive export attributes can omit or rewrite inputs. + target.write_bytes(subprocess.check_output( + ["git", "-C", str(source), "cat-file", "blob", f"{commit}:{name}"] + )) + target.chmod(mode) def prepare(root, args): diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index 0cdcf4de5..ea57f0b3b 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -448,12 +448,22 @@ def test_baseline_comparison_quality_and_pause_resume(campaign): def test_accepted_but_unselected_candidate_keeps_its_verdict(campaign): + config = settings() + config["stagnation"] = {"rounds": 1, "action": "pause"} + campaign["config"].write_text(json.dumps(config)) begin(campaign) + rejected_round = submit(campaign, version="rejected", a=110) + assert rejected_round["state"] == "paused" + call(campaign, "resume", "--reason", "Try a second-objective gain") result = submit(campaign, version="second-objective-gain", a=102, b=90) attempt = result["latest_observation"] assert attempt["assessment"]["eligible"] is True assert attempt["selected"] is False assert result["best"]["kind"] == "baseline" + assert result["state"] == "active" + assert result["stagnation"]["consecutive"] == 0 + assert result["stagnation"]["notified"] is False + assert result["stagnation"]["events"] == rejected_round["stagnation"]["events"] def test_screening_below_estimate_runs_formal_quality_and_profile_on_same_service(campaign): @@ -970,3 +980,84 @@ def test_whole_attempt_timeout_still_runs_bounded_stop(campaign): assert attempt["assessment"]["verdict"] == "INCONCLUSIVE" assert attempt["assessment"]["quality_passed"] is True assert [row["stage"] for row in events(campaign, attempt)][-1] == "stop" + + +@pytest.mark.parametrize("attribute", ["export-ignore", "export-subst"]) +def test_execution_preserves_tree_inputs_despite_export_attributes(campaign, attribute): + repo = campaign["repo"] + payload = repo / "inputs/payload.txt" + payload.parent.mkdir() + content = b"literal $Format:%s$\r\n\x00payload\n" + payload.write_bytes(content) + payload.chmod(0o755) + (repo / ".gitattributes").write_text(f"inputs/payload.txt {attribute}\n") + subprocess.run(["git", "-C", str(repo), "add", ".gitattributes", "inputs"], check=True) + commit(repo) + campaign["start"] += ["--include", "inputs"] + baseline = begin(campaign)["latest_observation"] + candidate = submit(campaign, version="accepted", a=90)["latest_observation"] + final = call(campaign, "final", "--reason", "Verify exact committed inputs")["final"] + assert final["assessment"]["verdict"] == "PASS" + for attempt in (baseline, candidate, final): + for directory in ("candidate", "workspace"): + retained = campaign["root"] / attempt["path"] / directory / "inputs/payload.txt" + assert retained.read_bytes() == content + assert retained.stat().st_mode & 0o777 == 0o555 + + +@pytest.mark.parametrize("invalid", ["missing", "symlink"]) +def test_tree_snapshot_rejects_missing_or_special_declared_input(campaign, invalid): + repo = campaign["repo"] + if invalid == "symlink": + (repo / invalid).symlink_to("engine.py") + subprocess.run(["git", "-C", str(repo), "add", invalid], check=True) + commit(repo) + revision = subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + destination = repo.parent / "snapshot" + with pytest.raises(ValueError, match="missing|symbolic"): + project_service.retain_revision(repo, destination, ["engine.py", invalid], revision) + assert not destination.exists() + + +@pytest.mark.parametrize("rejections", [1, 2]) +def test_refresh_resets_stagnation_without_erasing_history(campaign, rejections): + begin(campaign) + for index in range(rejections): + before = submit(campaign, version=f"rejected-{index}", a=110) + conditions = {**settings()["conditions"], "id": "new-conditions"} + path = campaign["root"].parent / "conditions.json" + path.write_text(json.dumps(conditions)) + refreshed = call( + campaign, "refresh-references", "--conditions", path, "--reason", "Changed workload" + ) + assert refreshed["stagnation"]["consecutive"] == 0 + assert refreshed["stagnation"]["notified"] is False + assert refreshed["stagnation"]["events"] == before["stagnation"]["events"] + after = submit(campaign, version="new-conditions-rejected", a=110) + assert after["stagnation"]["consecutive"] == 1 + assert after["stagnation"]["events"] == before["stagnation"]["events"] + + +@pytest.mark.parametrize("round_mode", ["exact", "up_to"]) +def test_refresh_without_round_capacity_preserves_finalizable_state(campaign, round_mode): + campaign["start"][campaign["start"].index("--rounds") + 1] = "1" + campaign["start"] += ["--round-mode", round_mode] + begin(campaign) + submit(campaign, version="selected", a=90) + before = project_service.read(campaign["root"] / "state.json") + conditions = {**settings()["conditions"], "id": "new-conditions"} + path = campaign["root"].parent / "conditions.json" + path.write_text(json.dumps(conditions)) + assert rejected([ + "campaign", "refresh-references", "--session", str(campaign["root"]), + "--conditions", str(path), "--reason", "No acceptance capacity", + ]) == 2 + assert "No round budget for current-conditions acceptance" in ( + campaign["capsys"].readouterr().err + ) + assert project_service.read(campaign["root"] / "state.json") == before + assert not (campaign["root"] / "references").exists() + result = call(campaign, "final", "--reason", "Complete original conditions") + assert result["final"]["assessment"]["verdict"] == "PASS" From b1785eac165d58cdaaab2f2fc52c0be41d59c1ad Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Tue, 17 Nov 2026 19:42:40 +0800 Subject: [PATCH 4/7] Isolate fingerprint guards and align CI assertions Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- tests/test_harness_execute.py | 4 ++- tests/test_project_service_reuse.py | 33 ++++----------------- tests/test_service_fingerprint_isolation.py | 25 ++++++++++++---- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/tests/test_harness_execute.py b/tests/test_harness_execute.py index be0819e44..efd054515 100644 --- a/tests/test_harness_execute.py +++ b/tests/test_harness_execute.py @@ -29,13 +29,15 @@ # The new H closure: the H01/H02 modules, the canonical encoder, and the package init # residue (the mixed ``inferrsi.harness`` __init__ loads ``inferrsi.contracts.claim``). # execution_spec landed on main with the #397 squash 8150ee04, which also -# added it to this H02 closure. The set matches current main. +# added it to this H02 closure. Service ownership also uses the Harness-only +# atomic JSON writer, whose dependencies are all in the standard library. # #451 C1: the encoder and the claim types live in the contracts package, so importing # either runs the ``inferrsi.contracts`` initializer, whose single import is the RunSpec # module. The last two names are that exact residue; a ``contracts`` prefix is never allowed. NEW_H_CLOSURE = { "inferrsi.harness.api", "inferrsi.harness.execution_service", "inferrsi.harness.input_binding", "inferrsi.harness.local_process", + "inferrsi.harness.atomic_json", "inferrsi.harness.run_store", "inferrsi.harness.errors", "inferrsi.harness.execution_spec", "inferrsi.contracts.json_v1", "inferrsi.harness", "inferrsi.contracts.claim", diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index ea57f0b3b..82f761d7f 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -1,24 +1,17 @@ """Fingerprint-free CPU orchestration controls; no real-model qualification.""" import copy -import hashlib import json import shutil import subprocess from pathlib import Path +from types import SimpleNamespace import pytest -from inferrsi.apps.views import dashboard_data from inferrsi.apps.cli import main from inferrsi.apps.cli.arguments import parser -from inferrsi.harness import ( - candidates, - optimization, - optimization_session, - project, - project_service, -) +from inferrsi.harness import artifact_primitives, optimization, project_service from inferrsi.harness.records import evidence, knowledge_capture from inferrsi.infra import artifacts from inferrsi.knowledge import knowledge_operations @@ -32,25 +25,11 @@ def forbidden(*args, **kwargs): calls.append(True) raise AssertionError("This test must not calculate or verify a file fingerprint") - for module, name in ( - (candidates, "identity"), - (artifacts, "sha256"), - (artifacts, "digest"), - (optimization, "sha256"), - (optimization, "digest"), - (optimization_session, "_runtime_identity"), - (project, "sha256"), - (knowledge_operations, "_hash"), - (hashlib, "sha256"), - ): - monkeypatch.setattr(module, name, forbidden) - for module, names in ( - (dashboard_data, ("file_ref", "read_object", "read_snapshot")), - (evidence, ("file_ref", "read_object", "read_snapshot")), - (knowledge_capture, ("store_capture", "verify_capture")), + # Keep imported functions intact; only their defining modules see the guard. + for module in ( + artifact_primitives, artifacts, evidence, knowledge_capture, knowledge_operations, ): - for name in names: - monkeypatch.setattr(module, name, forbidden) + monkeypatch.setattr(module, "hashlib", SimpleNamespace(sha256=forbidden)) yield assert not calls, "A fingerprint call was attempted, even if execution caught its exception" diff --git a/tests/test_service_fingerprint_isolation.py b/tests/test_service_fingerprint_isolation.py index 21f4a1f8c..31c86a398 100644 --- a/tests/test_service_fingerprint_isolation.py +++ b/tests/test_service_fingerprint_isolation.py @@ -1,27 +1,42 @@ -"""Check cross-file fixture isolation without executing a fingerprint workflow.""" +"""Verify the service guard and its isolation from legacy campaign tests.""" import os import subprocess import sys from pathlib import Path +import pytest + def test_service_guard_restores_lazy_import_aliases(tmp_path): check = tmp_path / "test_aliases.py" check.write_text( "def test_restored_alias():\n" - " from inferrsi.harness import project\n" - " from inferrsi.infra import artifacts\n" - " assert project.sha256 is artifacts.sha256\n" - " assert project.sha256.__module__ == 'inferrsi.infra.artifacts'\n", + " from inferrsi.harness import artifact_primitives, project, task_protocol\n" + " assert project.sha256 is task_protocol.sha256 is artifact_primitives.sha256\n" + " assert project.sha256.__module__ == 'inferrsi.harness.artifact_primitives'\n", encoding="utf-8", ) result = subprocess.run( [sys.executable, "-B", "-m", "pytest", "-p", "no:cacheprovider", "-q", "tests/test_project_service_reuse.py::test_exact_round_policy_and_legacy_argument_validation", + "tests/test_optimization_session.py::test_started_task_uses_frozen_protocol_without_registry", str(check)], cwd=Path(__file__).resolve().parents[1], env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1", "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1"}, capture_output=True, text=True, timeout=90, ) assert result.returncode == 0, result.stdout + result.stderr + + +def test_guard_blocks_lazily_imported_fingerprints_even_when_caught(monkeypatch, tmp_path): + from tests.test_project_service_reuse import no_fingerprints + + guard = no_fingerprints.__wrapped__(monkeypatch) + next(guard) + from inferrsi.harness.task_protocol import sha256 + + with pytest.raises(AssertionError, match="must not calculate"): + sha256(tmp_path / "unread-file") + with pytest.raises(AssertionError, match="fingerprint call was attempted"): + next(guard) From e03ff9f3d87f429a0619ddc89c662362bc112a61 Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Tue, 17 Nov 2026 20:30:16 +0800 Subject: [PATCH 5/7] Reject uncommitted deletions in declared inputs Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- src/inferrsi/harness/project_service.py | 3 +- tests/test_project_service_reuse.py | 40 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py index ca9e64486..3265458af 100644 --- a/src/inferrsi/harness/project_service.py +++ b/src/inferrsi/harness/project_service.py @@ -354,8 +354,9 @@ def git(*args): raise ValueError("Candidate requires an existing Git commit") selected = members(source, includes) tracked = set(git("ls-files", "-z").split("\0")) + # Deleted tracked files are absent from members but still belong to the inputs. if any(name not in tracked for name in selected) or git( - "status", "--porcelain", "--", *selected + "status", "--porcelain", "--", *(includes or ["."]) ): raise ValueError("Commit declared candidate inputs before service-reuse evaluation") return commit diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index 82f761d7f..f21d50a9d 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -1000,6 +1000,46 @@ def test_tree_snapshot_rejects_missing_or_special_declared_input(campaign, inval assert not destination.exists() +@pytest.mark.parametrize("staged", [False, True]) +@pytest.mark.parametrize("includes", [["inputs"], None]) +def test_revision_rejects_deleted_files_inside_declared_directory(campaign, staged, includes): + repo = campaign["repo"] + inputs = repo / "inputs" + inputs.mkdir() + (inputs / "keep.txt").write_text("keep") + deleted = inputs / "delete.txt" + deleted.write_text("delete") + subprocess.run(["git", "-C", str(repo), "add", "inputs"], check=True) + commit(repo) + project_service.revision(repo, includes) + + deleted.unlink() + if staged: + subprocess.run(["git", "-C", str(repo), "add", "-u"], check=True) + with pytest.raises(ValueError, match="Commit declared candidate inputs"): + project_service.revision(repo, includes) + + subprocess.run(["git", "-C", str(repo), "add", "-u"], check=True) + commit(repo) + revision = project_service.revision(repo, includes) + destination = repo.parent / "snapshot" + project_service.retain_revision(repo, destination, includes, revision) + assert (destination / "inputs/keep.txt").read_text() == "keep" + assert not (destination / "inputs/delete.txt").exists() + + +def test_revision_ignores_deletions_outside_declared_inputs(campaign): + repo = campaign["repo"] + inputs = repo / "inputs" + inputs.mkdir() + (inputs / "keep.txt").write_text("keep") + subprocess.run(["git", "-C", str(repo), "add", "inputs"], check=True) + commit(repo) + revision = project_service.revision(repo, ["inputs"]) + (repo / "engine.py").unlink() + assert project_service.revision(repo, ["inputs"]) == revision + + @pytest.mark.parametrize("rejections", [1, 2]) def test_refresh_resets_stagnation_without_erasing_history(campaign, rejections): begin(campaign) From 6315d33373150235c18a94fbb3ceb9a7eb72271d Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Wed, 18 Nov 2026 08:55:34 +0800 Subject: [PATCH 6/7] Ignore untracked caches when checking service inputs Agent-assisted-by: Codex (author) Signed-off-by: lirx-pd <616517220@qq.com> --- src/inferrsi/harness/project_service.py | 2 +- tests/test_project_service_reuse.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py index 3265458af..507392076 100644 --- a/src/inferrsi/harness/project_service.py +++ b/src/inferrsi/harness/project_service.py @@ -356,7 +356,7 @@ def git(*args): tracked = set(git("ls-files", "-z").split("\0")) # Deleted tracked files are absent from members but still belong to the inputs. if any(name not in tracked for name in selected) or git( - "status", "--porcelain", "--", *(includes or ["."]) + "status", "--porcelain", "--untracked-files=no", "--", *(includes or ["."]) ): raise ValueError("Commit declared candidate inputs before service-reuse evaluation") return commit diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index f21d50a9d..26dc73565 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -1028,6 +1028,25 @@ def test_revision_rejects_deleted_files_inside_declared_directory(campaign, stag assert not (destination / "inputs/delete.txt").exists() +def test_revision_ignores_untracked_cache_but_rejects_untracked_input(campaign): + repo = campaign["repo"] + inputs = repo / "inputs" + inputs.mkdir() + (inputs / "keep.txt").write_text("keep") + subprocess.run(["git", "-C", str(repo), "add", "inputs"], check=True) + commit(repo) + revision = project_service.revision(repo, ["inputs"]) + + cache = inputs / "__pycache__" + cache.mkdir() + (cache / "entry").touch() + assert project_service.revision(repo, ["inputs"]) == revision + + (inputs / "new.txt").touch() + with pytest.raises(ValueError, match="Commit declared candidate inputs"): + project_service.revision(repo, ["inputs"]) + + def test_revision_ignores_deletions_outside_declared_inputs(campaign): repo = campaign["repo"] inputs = repo / "inputs" From b7cb4dcb644cb8ff274a57018f8d13f379ff7f7b Mon Sep 17 00:00:00 2001 From: lirx-pd <616517220@qq.com> Date: Wed, 18 Nov 2026 19:43:11 +0800 Subject: [PATCH 7/7] Store campaign profile paths with forward slashes Signed-off-by: lirx-pd <616517220@qq.com> Agent-assisted-by: Codex (author) --- src/inferrsi/harness/project_service.py | 8 ++++---- tests/test_project_service_reuse.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/harness/project_service.py b/src/inferrsi/harness/project_service.py index 507392076..d507116f6 100644 --- a/src/inferrsi/harness/project_service.py +++ b/src/inferrsi/harness/project_service.py @@ -727,12 +727,12 @@ def skipped(names, reason): "binding": expected, "receipt": trace, "artifacts": [ - str((directory / item).relative_to(root)) + (directory / item).relative_to(root).as_posix() for item in artifacts ], - "result": str( - (output / "profile/result.json").relative_to(root) - ), + "result": ( + output / "profile/result.json" + ).relative_to(root).as_posix(), } else: skipped( diff --git a/tests/test_project_service_reuse.py b/tests/test_project_service_reuse.py index 26dc73565..39c342f2a 100644 --- a/tests/test_project_service_reuse.py +++ b/tests/test_project_service_reuse.py @@ -223,6 +223,9 @@ def test_one_service_order_and_independent_final_ignores_workspace(campaign): ] assert len({row["service_id"] for row in log}) == 1 assert attempt["execution"]["cleanup_verified"] is True + profile = attempt["execution"]["profile"] + for relative in [profile["result"], *profile["artifacts"]]: + assert project_service.safe_path(campaign["root"], relative).is_file() (campaign["repo"] / "engine.py").write_text("raise RuntimeError('unsubmitted workspace')") final = call(campaign, "final", "--reason", "Synthetic search complete") assert final["state"] == "completed"