From 853c92f35f3b4d3df841958ba33568f690f86c5d Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 12:52:47 +0800 Subject: [PATCH 1/6] Add offline candidate trees and verification completion menus Separate editable lineage from candidate-specific verification evidence. Retain scoped attempts and dependency epochs, select complete development candidates, and enumerate bounded completion plans with shared preparation and cost caps. Cover failed-parent repair, stale evidence, deterministic selection and planner feasibility with CPU controls; document the shared execution integration boundary. Agent-assisted-by: Codex (implementation, tests and documentation) Signed-off-by: amy-why-3459 --- docs/architecture.md | 5 + docs/operations.md | 1 + docs/speculative-search.md | 140 ++++++++ src/inferrsi/harness/search_tree.py | 361 +++++++++++++++++++++ src/inferrsi/harness/verification_plans.py | 145 +++++++++ tests/fixtures/speculative_search.py | 67 ++++ tests/test_speculative_search.py | 263 +++++++++++++++ tests/test_verification_plans.py | 151 +++++++++ 8 files changed, 1133 insertions(+) create mode 100644 docs/speculative-search.md create mode 100644 src/inferrsi/harness/search_tree.py create mode 100644 src/inferrsi/harness/verification_plans.py create mode 100644 tests/fixtures/speculative_search.py create mode 100644 tests/test_speculative_search.py create mode 100644 tests/test_verification_plans.py diff --git a/docs/architecture.md b/docs/architecture.md index 4a14f040..e2f05e8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,6 +28,11 @@ does not import Benchmark. Benchmark supplies task definitions and specialized rules, while Harness manages their execution and the complete optimization lifecycle. Neither infers quality criteria from a natural-language goal. Ordinary projects default to the current Agent conversation. +The [offline speculative-search primitives](speculative-search.md) add immutable +candidate trees, scoped evidence transitions and bounded completion menus in +Harness. They expose pure control operations for future VATS assembly; the +existing session controller continues to own execution. Production adapter +integration, learned role selection and GVO remain separate delivery work. The [native host handoff](native-agent-sessions.md) lets the initiating Agent resolve the task request before any campaign exists, use its host's new-session tool and retain the native identity and returned artifacts. The child owns the diff --git a/docs/operations.md b/docs/operations.md index 806b2602..176e0c8c 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -65,6 +65,7 @@ Effect classes used below: | Qualify an artifact verifier host | [Manual reviewer qualification](skill-eval-reviewer-qualification.md), `python -m tools.skill_eval_qualify` | Trusted-control execution / report-generating | Manual main-only workflow; fixed software controls, no model or external artifact input; unavailable isolation fails rather than skips | | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | +| Build an offline candidate tree and completion menu | Python `harness.search_tree` / `harness.verification_plans.completion_menu`; [control contract](speculative-search.md) | Read-only / in-memory transitions | Immutable candidates, scoped caller-supplied receipts and bounded exact plans; execution, durable state, accounting, learned roles and final acceptance require separate adapters | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new | diff --git a/docs/speculative-search.md b/docs/speculative-search.md new file mode 100644 index 00000000..96d6e38a --- /dev/null +++ b/docs/speculative-search.md @@ -0,0 +1,140 @@ +# Offline candidate trees and verification plans + +This is the first implementation slice of +[the SpecRSI proposal](https://github.com/InferMatrixLab/InferRSI/issues/363). +`harness/search_tree.py` provides immutable candidate and evidence transitions; +`harness/verification_plans.py` constructs bounded completion menus. Both are +pure Python control primitives with no execution, storage or model access. +The existing project and benchmark session commands retain their current behavior. + +The design follows [SpecRSI v2, Section 3](https://arxiv.org/html/2610.16491v2#S3): +editing an incomplete or unsuccessful parent is allowed, while adopting a version +requires complete admissible evidence for that version. Completion bundles keep +complementary checks together. This slice supplies building blocks for the inner +search; a learned Scorer/Policy, automatic Drafter and GVO updates remain future +integration work. + +## State and evidence + +`Contract` freezes the search identity, verification context, required checks, +objective, direction, improver snapshot and candidate/depth/check-attempt caps. +The context identifies the plan, validator, environment, workload, original +baseline and quality reference. Checks may encode separately required repetitions. +The current selector supports a scalar objective plus hard PASS requirements; +the Skill lane's quality-first lexicographic selector needs a separate adapter. + +`initialize` requires one complete, passing receipt per root check. Root receipts +count against the attempt cap. Their acquisition/reuse cost must already be in +the caller's authoritative accounting. A passing objective check carries a finite +value; other checks and rejected/inconclusive outcomes cannot supply a value. + +`add_candidate` records an immutable artifact digest and edit parent, with the +frozen producer snapshot, producer revision, context/history digests and actual +generation dependencies. Callers retain and verify artifacts with the existing +`harness.candidates.retain` / `identity` helpers before admission. This core does +not authenticate digests or decide whether an edit was permitted. Parent quality +failures do not invalidate the edit lineage. Real generation-source dependencies +are inherited and must still be current; a child cannot drop them to escape an +invalidation. Rebase or composition is a new candidate with its own evidence. +This offline core predeclares dependency keys at initialization; introducing new +source authorities dynamically requires later controller integration. + +`register_attempt` binds a caller-owned attempt/execution key, reservation and +lease to one candidate, check and complete verification context. Registering an +attempt grants no execution permission. Unknown work on the same check cannot be +replaced, even after dependency invalidation. Valid outcomes cannot be resampled +until they pass; an INCONCLUSIVE outcome may be retried with a fresh identity +under the caller's frozen retry rules and a new reservation. + +`record_receipt` requires the entire registered binding. Identical delivery is +idempotent and conflicting delivery is rejected. History retains late results; +`evidence` admits only the latest attempt with current dependency epochs and +valid candidate provenance. FAIL is an observed outcome, whereas INCONCLUSIVE +leaves a check missing. Identical candidate bytes do not transfer evidence. + +`accept` selects only fully passing candidates, preferring the root on an +objective tie and otherwise using lexicographic candidate ID order. The result is +a **development selection**. Independent final/holdout acceptance and production +promotion remain caller-owned. `verified_gain` measures this selection relative +to the root. Invalidation recomputes eligibility; invalid root evidence blocks +acceptance and further drafting/check registration, while late archival remains +available. + +## Completion planning + +`completion_menu(state, jobs, available_budget=..., available_jobs=..., +max_targets=..., max_jobs=...)` takes a supplied integer-cost budget view and +returns immutable `Plan` values. The caller has already accounted for planning, +generation, final reserve, in-flight obligations and unknown spend when deriving +that view. Costs must be known positive conservative bounds in the same unit. + +Each `Job` names either a candidate/check pair or a shared preparation step. +Prerequisites form an acyclic graph. The solver takes each candidate's complete +missing-check set, closes its prerequisites, and enumerates unions of at most +`max_targets` candidate seeds. Every feasible singleton completion remains in +the menu. Shared jobs are charged once per plan, conflicts exclude combinations, +and available job slots and remaining check attempts bound each proposal. +`max_jobs` bounds the input graph; enumeration is intended for small offline +controls, not an unbounded production optimizer. No Scorer prediction is used +as evidence or as a substitute for a completion seed. + +Plans bind the search ID and state revision and carry a deterministic topological +job order. The caller retains the full job definitions and costs with its frozen +plan; job IDs alone do not authenticate them. The adapter rebuilds job +relations from current observations at each synchronous decision boundary; +already observed checks are not silently scheduled again. An in-flight attempt, +including an invalidated one, blocks replanning until its execution is settled. +An empty menu is a valid outcome under a tight budget; the root remains available +when its evidence is valid. Feasibility says nothing about whether checks pass. + +## Shared execution and persistence boundary + +These modules introduce no second runner, budget ledger, holdout tracker or +storage backend. A future application adapter must: + +1. Read current authoritative state and the frozen contract, settle the previous + work block, and supply a conservative available-budget view. +2. Generate and retain permitted edits, or select an affordable check plan using + the frozen roles. Account for generation, planning and failed work. +3. Use shared execution controls for atomic claim/reservation, absolute deadline, + resource isolation and actual execution. Bind their identities to the tree. +4. Authenticate and assess receipts, apply a transition against the expected + search revision, and persist it using the authoritative storage CAS/lock. + A pure function's revision check is **not a durable or cross-worker CAS**. + On conflict, reread and recompute; an ambiguous claim never grants a retry. +5. Persist real dependency invalidation before cancellation. Reconcile costs and + retain leases until process cleanup is confirmed. Receipt archival itself + neither charges again nor releases a resource. +6. Freeze the selected version for independent final acceptance, with holdout + access and interruptions tracked separately from development feedback. + +There is no production adapter in this slice. In particular, local immutable +values and revision checks do not establish crash recovery, external exactly-once, +multi-worker resource isolation, budget transactions or holdout non-reuse. + +## Verification and remaining delivery + +Run the CPU controls with: + +```bash +python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py +``` + +Tests cover retained file/directory artifacts for synthetic Engine/Skill shapes, +failed-parent repair, incomplete ancestors, failure versus missing evidence, +dependency/result order, identical bytes with separate attempts, stale revision +rejection, immutable role snapshots, deterministic selection, invalid roots and +bounded search. The planner is compared with an independent exhaustive job-subset +oracle across budgets and target limits, including shared preparation and conflicts. + +| Issue acceptance area | This slice | Remaining evidence | +| --- | --- | --- | +| AC-01/02 | Frozen scalar contract and offline candidate tree | Production support matrix and role/execution adapters | +| AC-03/05 | Scoped evidence, epoch invalidation, development selection and revision guards | Authoritative durable state integration and restart/race tests | +| AC-04 | Bounded exact completion menu and exhaustive controls | Scorer refinement, Policy selection and scalable solver integration | +| AC-06 | Frozen snapshot identity only | GVO data collection, objectives, parameter updates and new searches | +| AC-07/08 | References to caller-owned execution identities | Actual budget/resource transactions, cancellation and independent final/holdout | +| AC-09/10 | Synthetic control fixtures | Both real adapters, model generation, training, matched experiments and lifecycle costs | + +These tests do not establish model quality, engine speedup or SpecRSI efficiency. +The parent issue remains open for its full dual-loop delivery. diff --git a/src/inferrsi/harness/search_tree.py b/src/inferrsi/harness/search_tree.py new file mode 100644 index 00000000..d6623835 --- /dev/null +++ b/src/inferrsi/harness/search_tree.py @@ -0,0 +1,361 @@ +"""Pure VATS candidate/evidence transitions; execution and durable CAS belong to the caller. + +Inputs are trusted controller records, not authenticated executor messages. This module +neither starts work nor grants a claim, releases a lease, spends budget or promotes a +production version. See docs/speculative-search.md for the adapter boundary. +""" + +from dataclasses import dataclass, fields, replace +from math import isfinite + +from inferrsi.contracts.claim import Verdict + + +def _text(value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("Identity must be a nonempty string") + + +def _integer(value, minimum=0): + if type(value) is not int or value < minimum: + raise ValueError("Expected a bounded nonnegative integer") + + +def _sha(value): + if not isinstance(value, str) or len(value) != 64 or set(value) - set("0123456789abcdef"): + raise ValueError("Expected a lowercase SHA-256 digest") + + +@dataclass(frozen=True) +class Context: + contract: str + plan: str + validator: str + environment: str + workload: str + baseline: str + quality_reference: str + + def __post_init__(self): + for field in fields(self): + _text(getattr(self, field.name)) + + +@dataclass(frozen=True, order=True) +class Dependency: + key: str + epoch: int + + def __post_init__(self): + _text(self.key) + _integer(self.epoch) + + +def _dependencies(values): + values = tuple(values) + if any(not isinstance(value, Dependency) for value in values): + raise ValueError("Expected dependency identities") + if len({value.key for value in values}) != len(values): + raise ValueError("Duplicate dependency key") + return tuple(sorted(values)) + + +@dataclass(frozen=True) +class Producer: + snapshot: str + revision: str + context_digest: str + history_digest: str + + def __post_init__(self): + _text(self.snapshot) + _text(self.revision) + _sha(self.context_digest) + _sha(self.history_digest) + + +@dataclass(frozen=True) +class Contract: + search_id: str + context: Context + improver_snapshot: str + checks: tuple[str, ...] + objective_check: str + direction: str + max_candidates: int + max_depth: int + max_attempts: int + + def __post_init__(self): + _text(self.search_id) + _text(self.improver_snapshot) + if not isinstance(self.context, Context): + raise ValueError("Expected verification context") + if not isinstance(self.checks, (tuple, list)): + raise ValueError("Required checks must be an ordered sequence") + checks = tuple(self.checks) + for check in checks: + _text(check) + if not checks or len(set(checks)) != len(checks) or self.objective_check not in checks: + raise ValueError("Required checks must be unique and include the objective") + object.__setattr__(self, "checks", checks) + if self.direction not in ("maximize", "minimize"): + raise ValueError("Unknown objective direction") + for value in (self.max_candidates, self.max_depth, self.max_attempts): + _integer(value, 1) + + +@dataclass(frozen=True) +class Candidate: + candidate_id: str + artifact_digest: str + parent_id: str | None + producer: Producer + dependencies: tuple[Dependency, ...] = () + + def __post_init__(self): + _text(self.candidate_id) + _sha(self.artifact_digest) + if self.parent_id is not None: + _text(self.parent_id) + if not isinstance(self.producer, Producer): + raise ValueError("Expected producer identity") + object.__setattr__(self, "dependencies", _dependencies(self.dependencies)) + + +@dataclass(frozen=True) +class Attempt: + attempt_id: str + execution_key: str + reservation_id: str + lease_id: str + search_id: str + candidate_id: str + artifact_digest: str + check_id: str + context: Context + dependencies: tuple[Dependency, ...] = () + + def __post_init__(self): + for name in ("attempt_id", "execution_key", "reservation_id", "lease_id", "search_id", + "candidate_id", "check_id"): + _text(getattr(self, name)) + _sha(self.artifact_digest) + if not isinstance(self.context, Context): + raise ValueError("Expected verification context") + object.__setattr__(self, "dependencies", _dependencies(self.dependencies)) + + +@dataclass(frozen=True) +class Receipt: + attempt: Attempt + verdict: Verdict + evidence_digest: str + objective: float | None = None + + def __post_init__(self): + if not isinstance(self.attempt, Attempt) or not isinstance(self.verdict, Verdict): + raise ValueError("Expected bound attempt and verdict") + _sha(self.evidence_digest) + if self.objective is not None and ( + type(self.objective) not in (int, float) or not isfinite(self.objective) + ): + raise ValueError("Objective must be a finite number, not missing-value imputation") + if self.verdict != Verdict.PASS and self.objective is not None: + raise ValueError("Rejected or inconclusive evidence cannot supply an objective") + + +@dataclass(frozen=True) +class SearchState: + contract: Contract + root_id: str + candidates: tuple[Candidate, ...] + attempts: tuple[Attempt, ...] + receipts: tuple[Receipt, ...] + dependencies: tuple[Dependency, ...] + revision: int = 0 + + +def candidate(state, candidate_id): + for value in state.candidates: + if value.candidate_id == candidate_id: + return value + raise ValueError("Unknown candidate") + + +def _current(state, dependencies): + epochs = {item.key: item.epoch for item in state.dependencies} + return all(epochs.get(item.key) == item.epoch for item in dependencies) + + +def candidate_valid(state, candidate_id): + return _current(state, candidate(state, candidate_id).dependencies) + + +def _binding(state, attempt): + node = candidate(state, attempt.candidate_id) + if (attempt.search_id != state.contract.search_id + or attempt.artifact_digest != node.artifact_digest + or attempt.context != state.contract.context + or attempt.check_id not in state.contract.checks + or not set(node.dependencies).issubset(attempt.dependencies)): + raise ValueError("Attempt does not bind this candidate and verification contract") + + +def latest_attempt(state, candidate_id, check_id): + return next((item for item in reversed(state.attempts) + if (item.candidate_id, item.check_id) == (candidate_id, check_id)), None) + + +def evidence(state, candidate_id): + """Current, scoped outcomes, including FAIL; INCONCLUSIVE remains missing.""" + if not candidate_valid(state, candidate_id): + return {} + receipts = {item.attempt.attempt_id: item for item in state.receipts} + result = {} + for check in state.contract.checks: + attempt = latest_attempt(state, candidate_id, check) + if attempt is None or not _current(state, attempt.dependencies): + continue + receipt = receipts.get(attempt.attempt_id) + if receipt is not None and receipt.verdict != Verdict.INCONCLUSIVE: + result[check] = receipt + return result + + +def missing_checks(state, candidate_id): + observed = evidence(state, candidate_id) + return tuple(check for check in state.contract.checks if check not in observed) + + +def complete_candidates(state): + result = [] + for node in state.candidates: + records = evidence(state, node.candidate_id) + if len(records) == len(state.contract.checks) and all( + record.verdict == Verdict.PASS for record in records.values() + ): + result.append(node) + return tuple(result) + + +def accept(state): + """Select a development candidate only; the root wins ties, then stable ID order.""" + complete = complete_candidates(state) + if state.root_id not in {item.candidate_id for item in complete}: + raise ValueError("Search root no longer has complete admissible evidence") + sign = -1 if state.contract.direction == "maximize" else 1 + return min(complete, key=lambda item: ( + sign * evidence(state, item.candidate_id)[state.contract.objective_check].objective, + item.candidate_id != state.root_id, item.candidate_id, + )) + + +def verified_gain(state): + selected = accept(state) + check = state.contract.objective_check + gain = (evidence(state, selected.candidate_id)[check].objective + - evidence(state, state.root_id)[check].objective) + return gain if state.contract.direction == "maximize" else -gain + + +def _receipt_shape(state, receipt): + objective = receipt.attempt.check_id == state.contract.objective_check + if (receipt.verdict == Verdict.PASS and objective) != (receipt.objective is not None): + raise ValueError("Only a passing objective check must supply an objective") + + +def initialize(contract, root, receipts, *, dependencies=()): + """Import previously acquired root evidence; its execution/cost remains caller-owned.""" + receipts = tuple(receipts) + if root.parent_id is not None: + raise ValueError("Search root cannot have an edit parent in this search") + state = SearchState(contract, root.candidate_id, (root,), + tuple(item.attempt for item in receipts), receipts, + _dependencies(dependencies)) + if len(receipts) != len(contract.checks) or len(receipts) > contract.max_attempts: + raise ValueError("Root requires exactly one receipt per required check within the cap") + if len({item.attempt.check_id for item in receipts}) != len(receipts): + raise ValueError("Duplicate root check") + for field in ("attempt_id", "execution_key"): + if len({getattr(item.attempt, field) for item in receipts}) != len(receipts): + raise ValueError("Root execution identities must be unique") + for receipt in receipts: + _binding(state, receipt.attempt) + _receipt_shape(state, receipt) + accept(state) + return state + + +def _guard(state, expected_revision, *, require_root=True): + _integer(expected_revision) + if state.revision != expected_revision: + raise ValueError("Stale search revision; reread and recompute before storage CAS") + if require_root: + accept(state) + + +def add_candidate(state, node, *, expected_revision): + _guard(state, expected_revision) + if len(state.candidates) >= state.contract.max_candidates: + raise ValueError("Candidate limit exhausted") + if any(item.candidate_id == node.candidate_id for item in state.candidates): + raise ValueError("Candidate identity is immutable") + parent = candidate(state, node.parent_id) + if (node.producer.snapshot != state.contract.improver_snapshot + or not _current(state, node.dependencies) + or not set(parent.dependencies).issubset(node.dependencies)): + raise ValueError("Invalid generation dependencies or changed improver snapshot") + depth = 1 + while parent.parent_id is not None: + parent = candidate(state, parent.parent_id) + depth += 1 + if depth > state.contract.max_depth: + raise ValueError("Draft depth limit exhausted") + return replace(state, candidates=(*state.candidates, node), revision=state.revision + 1) + + +def register_attempt(state, attempt, *, expected_revision): + """Record a caller-owned claim. A return value is NOT permission to execute.""" + _guard(state, expected_revision) + _binding(state, attempt) + if not _current(state, attempt.dependencies): + raise ValueError("Attempt depends on invalidated evidence or generation sources") + if len(state.attempts) >= state.contract.max_attempts: + raise ValueError("Check attempt limit exhausted") + if any(item.attempt_id == attempt.attempt_id or item.execution_key == attempt.execution_key + for item in state.attempts): + raise ValueError("Attempt and execution identities cannot be reused") + previous = latest_attempt(state, attempt.candidate_id, attempt.check_id) + if previous is not None: + receipt = next((item for item in state.receipts if item.attempt == previous), None) + if receipt is None: + raise ValueError("Unsettled attempt cannot be replaced or implicitly replayed") + if _current(state, previous.dependencies) and receipt.verdict != Verdict.INCONCLUSIVE: + raise ValueError("A valid outcome cannot be retried until it passes") + return replace(state, attempts=(*state.attempts, attempt), revision=state.revision + 1) + + +def record_receipt(state, receipt, *, expected_revision): + """Retain late receipts as history; evidence() separately decides current admission.""" + _guard(state, expected_revision, require_root=False) + if receipt.attempt not in state.attempts: + raise ValueError("Receipt does not match a registered attempt") + _receipt_shape(state, receipt) + previous = next((item for item in state.receipts + if item.attempt.attempt_id == receipt.attempt.attempt_id), None) + if previous is not None: + if previous != receipt: + raise ValueError("Conflicting receipt for an immutable attempt") + return state + return replace(state, receipts=(*state.receipts, receipt), revision=state.revision + 1) + + +def invalidate(state, dependency_key, *, expected_revision): + """Invalidate a real source, not a parent quality outcome; no cancel/release is issued.""" + _guard(state, expected_revision, require_root=False) + if dependency_key not in {item.key for item in state.dependencies}: + raise ValueError("Unknown dependency") + dependencies = tuple(replace(item, epoch=item.epoch + 1) if item.key == dependency_key + else item for item in state.dependencies) + return replace(state, dependencies=dependencies, revision=state.revision + 1) diff --git a/src/inferrsi/harness/verification_plans.py b/src/inferrsi/harness/verification_plans.py new file mode 100644 index 00000000..81d39f86 --- /dev/null +++ b/src/inferrsi/harness/verification_plans.py @@ -0,0 +1,145 @@ +"""Bounded exact completion-menu construction for offline VATS controls. + +Costs are caller-supplied conservative integer bounds in one declared unit. A plan +is a proposal against a budget view, never a resource reservation or execution grant. +""" + +from dataclasses import dataclass +from itertools import combinations + +from inferrsi.contracts.claim import Verdict + +from .search_tree import accept, candidate_valid, evidence, missing_checks + + +@dataclass(frozen=True) +class Job: + job_id: str + cost: int + candidate_id: str | None = None + check_id: str | None = None + prerequisites: tuple[str, ...] = () + conflicts: tuple[str, ...] = () + + def __post_init__(self): + if not isinstance(self.job_id, str) or not self.job_id.strip(): + raise ValueError("Job identity must be nonempty") + if type(self.cost) is not int or self.cost <= 0: + raise ValueError("Job cost must be a known positive integer upper bound") + if (self.candidate_id is None) != (self.check_id is None): + raise ValueError("A check job needs both candidate and check identities") + for value in (self.candidate_id, self.check_id): + if value is not None and (not isinstance(value, str) or not value.strip()): + raise ValueError("Check identity must be nonempty") + for name in ("prerequisites", "conflicts"): + if not isinstance(getattr(self, name), (tuple, list)): + raise ValueError("Job relations must be ordered sequences") + values = tuple(getattr(self, name)) + if (any(not isinstance(value, str) or not value.strip() for value in values) + or len(set(values)) != len(values) or self.job_id in values): + raise ValueError("Job relations must be unique, nonempty, non-self identities") + object.__setattr__(self, name, values) + + +@dataclass(frozen=True) +class Plan: + candidate_ids: tuple[str, ...] + job_ids: tuple[str, ...] + cost: int + search_id: str + state_revision: int + + +def completion_menu(state, jobs, *, available_budget, available_jobs, max_targets, max_jobs): + """Enumerate unions of up to max_targets completion seeds, retaining every feasible seed. + +The caller must settle a synchronous work block first. Preparation jobs have no +candidate/check pair. Shared prerequisites are charged once per proposed plan; +conflicts prohibit co-scheduling even when only one side declares the relation. +""" + for value, minimum in ((available_budget, 0), (available_jobs, 0), + (max_targets, 1), (max_jobs, 1)): + if type(value) is not int or value < minimum: + raise ValueError("Menu limits and available budget must be known integers") + accept(state) + receipts = {item.attempt.attempt_id for item in state.receipts} + # An invalidated in-flight job is still real work; only its owner can settle it. + if any(item.attempt_id not in receipts for item in state.attempts): + raise ValueError("Settle the current work block before constructing another menu") + jobs = tuple(jobs) + if len(jobs) > max_jobs: + raise ValueError("Job menu exceeds the declared enumeration bound") + by_id = {job.job_id: job for job in jobs} + if len(by_id) != len(jobs): + raise ValueError("Duplicate job identity") + checks = {} + for job in jobs: + if (set(job.prerequisites) | set(job.conflicts)) - by_id.keys(): + raise ValueError("Unknown job relation") + if job.candidate_id is not None: + candidate_valid(state, job.candidate_id) # Reject unknown candidate IDs. + if job.check_id not in state.contract.checks: + raise ValueError("Unknown required check") + pair = (job.candidate_id, job.check_id) + if pair in checks: + raise ValueError("Ambiguous jobs for one candidate check") + checks[pair] = job.job_id + + closures = {} + + def closure(job_id, visiting=frozenset()): + if job_id in visiting: + raise ValueError("Cyclic job prerequisites") + if job_id not in closures: + result = {job_id} + for parent in by_id[job_id].prerequisites: + result.update(closure(parent, visiting | {job_id})) + closures[job_id] = frozenset(result) + return closures[job_id] + + for job in jobs: + closure(job.job_id) + + seeds = {} + for node in state.candidates: + cid = node.candidate_id + if not candidate_valid(state, cid) or any( + item.verdict == Verdict.FAIL for item in evidence(state, cid).values() + ): + continue + missing = missing_checks(state, cid) + if not missing or any((cid, check) not in checks for check in missing): + continue + seeds[cid] = frozenset().union(*(closure(checks[(cid, check)]) for check in missing)) + + plans = {} + remaining_attempts = state.contract.max_attempts - len(state.attempts) + for size in range(1, min(max_targets, len(seeds)) + 1): + for targets in combinations(sorted(seeds), size): + union = frozenset().union(*(seeds[cid] for cid in targets)) + if len(union) > available_jobs: + continue + check_jobs = [by_id[jid] for jid in union if by_id[jid].candidate_id is not None] + if len(check_jobs) > remaining_attempts: + continue + # A cross-candidate prerequisite must itself be a legal outstanding check. + if any(job.check_id not in missing_checks(state, job.candidate_id) + or not candidate_valid(state, job.candidate_id) for job in check_jobs): + continue + if any(set(by_id[jid].conflicts) & union for jid in union): + continue + cost = sum(by_id[jid].cost for jid in union) + if cost > available_budget: + continue + # Stable topological order, so an adapter never receives a child before setup. + ordered = [] + remaining = set(union) + while remaining: + ready = sorted(jid for jid in remaining + if not (set(by_id[jid].prerequisites) & remaining)) + ordered.extend(ready) + remaining.difference_update(ready) + completed = tuple(sorted(cid for cid, seed in seeds.items() if seed <= union)) + plan = Plan(completed, tuple(ordered), cost, state.contract.search_id, state.revision) + plans[plan.job_ids] = plan + return tuple(sorted(plans.values(), key=lambda plan: (plan.cost, plan.job_ids))) diff --git a/tests/fixtures/speculative_search.py b/tests/fixtures/speculative_search.py new file mode 100644 index 00000000..10493820 --- /dev/null +++ b/tests/fixtures/speculative_search.py @@ -0,0 +1,67 @@ +"""Synthetic observations and identities; no model, hardware or billing evidence.""" + +from dataclasses import replace +from hashlib import sha256 + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import search_tree as tree + + +def digest(text): + return sha256(text.encode()).hexdigest() + + +def contract(**changes): + return replace(tree.Contract( + "search", tree.Context("contract", "plan", "grader", "cpu-fixture", "development", + "original-baseline", "quality-reference"), + "improver-0", ("quality", "objective"), "objective", "maximize", 20, 5, 40, + ), **changes) + + +def producer(snapshot="improver-0"): + return tree.Producer(snapshot, "synthetic-producer-1", digest("context"), digest("history")) + + +def attempt(state, cid, check, *, dependencies=(), tag=None): + node = tree.candidate(state, cid) + tag = tag or str(len(state.attempts)) + return tree.Attempt( + f"attempt-{tag}", f"execution-{tag}", f"reservation-{tag}", f"lease-{tag}", + state.contract.search_id, cid, node.artifact_digest, check, state.contract.context, + tuple(sorted(set(node.dependencies) | set(dependencies))), + ) + + +def receipt(binding, verdict=Verdict.PASS, objective=None): + return tree.Receipt(binding, verdict, digest(f"receipt:{binding.attempt_id}"), objective) + + +def start(*, configuration=None, dependencies=(), root_dependencies=(), score=1): + configuration = configuration or contract() + root = tree.Candidate("root", digest("root"), None, producer(), root_dependencies) + seed = tree.SearchState(configuration, "root", (root,), (), (), tuple(dependencies)) + receipts = tuple(receipt(attempt(seed, "root", check, tag=f"root-{check}"), + objective=score if check == configuration.objective_check else None) + for check in configuration.checks) + return tree.initialize(configuration, root, receipts, dependencies=dependencies) + + +def add(state, cid, parent="root", *, dependencies=None, artifact_digest=None, snapshot=None): + node = tree.Candidate(cid, artifact_digest or digest(cid), parent, + producer(snapshot or state.contract.improver_snapshot), + tree.candidate(state, parent).dependencies if dependencies is None + else dependencies) + return tree.add_candidate(state, node, expected_revision=state.revision) + + +def observe(state, cid, check, verdict=Verdict.PASS, objective=None, *, dependencies=()): + binding = attempt(state, cid, check, dependencies=dependencies) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + return tree.record_receipt(state, receipt(binding, verdict, objective), + expected_revision=state.revision) + + +def complete(state, cid, score): + state = observe(state, cid, "quality") + return observe(state, cid, "objective", objective=score) diff --git a/tests/test_speculative_search.py b/tests/test_speculative_search.py new file mode 100644 index 00000000..997e95f5 --- /dev/null +++ b/tests/test_speculative_search.py @@ -0,0 +1,263 @@ +from dataclasses import FrozenInstanceError, fields, replace + +import pytest + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import candidates +from inferrsi.harness import search_tree as tree +from tests.fixtures.speculative_search import ( + add, + attempt, + complete, + contract, + digest, + observe, + producer, + receipt, + start, +) + + +@pytest.mark.parametrize("lane,filename", [("engine", "kernel.py"), ("skill", "SKILL.md")]) +def test_failed_parent_can_be_repaired_with_its_own_snapshot_and_complete_evidence( + tmp_path, lane, filename, +): + # File retention is real; grades are synthetic controls, not either production adapter. + work = tmp_path / "work" + work.mkdir() + source = work / filename + state = start(configuration=contract(search_id=lane)) + for cid, value, parent in (("broken", -1, "root"), ("repaired", 3, "broken")): + source.write_text(str(value)) + retained = tmp_path / cid + identity = candidates.retain(work, retained) + state = add(state, cid, parent, artifact_digest=identity) + actual = int((retained / filename).read_text()) + state = observe(state, cid, "quality", Verdict.PASS if actual > 0 else Verdict.FAIL) + if actual > 0: + # A high objective alone never compensates for missing or failed quality. + state = observe(state, cid, "objective", objective=actual) + source.write_text("999") + assert candidates.identity(tmp_path / "repaired") == tree.accept(state).artifact_digest + assert tree.accept(state).candidate_id == "repaired" + assert tree.verified_gain(state) == 2 + assert tree.missing_checks(state, "broken") == ("objective",) + assert tree.evidence(state, "broken")["quality"].verdict == Verdict.FAIL + assert tree.candidate_valid(state, "broken") + + +def test_incomplete_parent_can_be_extended_without_fabricating_parent_results(): + state = add(add(start(), "parent"), "child", "parent") + state = complete(state, "child", 4) + assert tree.accept(state).candidate_id == "child" + assert tree.evidence(state, "parent") == {} + + +def test_objective_prediction_partial_results_and_failed_quality_cannot_win(): + state = add(start(), "candidate") + state = observe(state, "candidate", "objective", objective=1000) + assert tree.accept(state).candidate_id == "root" + state = observe(state, "candidate", "quality", Verdict.FAIL) + assert tree.missing_checks(state, "candidate") == () # Known failure is not missing. + assert tree.accept(state).candidate_id == "root" + assert tree.verified_gain(state) == 0 + + +@pytest.mark.parametrize("direction", ["maximize", "minimize"]) +@pytest.mark.parametrize("order", [("z", "a"), ("a", "z")]) +def test_selection_is_independent_of_receipt_order_and_root_wins_ties(direction, order): + state = start(configuration=contract(direction=direction), score=10) + for cid in order: + state = complete(add(state, cid), cid, 11 if direction == "maximize" else 9) + assert tree.accept(state).candidate_id == "a" + assert tree.verified_gain(state) == 1 + tied = complete(add(start(score=10), "a"), "a", 10) + assert tree.accept(tied).candidate_id == "root" + + +@pytest.mark.parametrize("late", [True, False]) +def test_real_dependency_invalidation_and_success_race_cannot_leave_stale_incumbent(late): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + state = observe(state, "candidate", "quality") + binding = attempt(state, "candidate", "objective", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + result = receipt(binding, objective=5) + if not late: + state = tree.record_receipt(state, result, expected_revision=state.revision) + assert tree.accept(state).candidate_id == "candidate" + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + if late: + state = tree.record_receipt(state, result, expected_revision=state.revision) + assert result in state.receipts # Archive is distinct from evidence admission. + assert tree.accept(state).candidate_id == "root" + assert tree.missing_checks(state, "candidate") == ("objective",) + + +def test_old_receipt_after_new_attempt_cannot_override_current_outcome_or_be_reused(): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + state = observe(state, "candidate", "quality") + state = observe(state, "candidate", "objective", objective=9, dependencies=(dependency,)) + old = state.receipts[-1] + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + new_dependency = state.dependencies[0] + binding = attempt(state, "candidate", "objective", dependencies=(new_dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.record_receipt(state, old, expected_revision=state.revision) + assert tree.accept(state).candidate_id == "root" + state = tree.record_receipt(state, receipt(binding, Verdict.FAIL), + expected_revision=state.revision) + before = state + state = tree.record_receipt(state, old, expected_revision=state.revision) + assert state is before + assert tree.evidence(state, "candidate")["objective"].verdict == Verdict.FAIL + assert tree.accept(state).candidate_id == "root" + + +def test_generation_invalidation_affects_actual_descendants_but_not_other_branch(): + dependency = tree.Dependency("retrieved-context", 0) + state = start(dependencies=(dependency,)) + state = add(state, "parent", dependencies=(dependency,)) + state = complete(add(state, "child", "parent"), "child", 10) + state = complete(add(state, "other"), "other", 3) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + assert not tree.candidate_valid(state, "child") + assert tree.accept(state).candidate_id == "other" + with pytest.raises(ValueError, match="dependencies"): + add(state, "grandchild", "child", dependencies=()) + + +def test_invalid_root_blocks_search_and_acceptance_but_allows_late_archival(): + dependency = tree.Dependency("root-source", 0) + state = add(start(dependencies=(dependency,), root_dependencies=(dependency,)), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + state = tree.record_receipt(state, receipt(binding), expected_revision=state.revision) + with pytest.raises(ValueError, match="root"): + tree.accept(state) + with pytest.raises(ValueError, match="root"): + add(state, "another") + + +def test_inconclusive_is_missing_and_may_be_retried_with_fresh_identity(): + state = add(start(), "candidate") + state = observe(state, "candidate", "quality", Verdict.INCONCLUSIVE) + assert tree.missing_checks(state, "candidate") == ("quality", "objective") + state = complete(state, "candidate", 2) + assert tree.accept(state).candidate_id == "candidate" + assert len([item for item in state.attempts if item.check_id == "quality"]) == 3 + + +@pytest.mark.parametrize("verdict", [Verdict.PASS, Verdict.FAIL]) +def test_valid_outcomes_cannot_be_retried_until_they_pass(verdict): + state = observe(add(start(), "candidate"), "candidate", "quality", verdict) + with pytest.raises(ValueError, match="retried"): + observe(state, "candidate", "quality") + + +def test_unsettled_attempt_blocks_replacement_even_after_invalidation(): + dependency = tree.Dependency("measurement-source", 0) + state = add(start(dependencies=(dependency,)), "candidate") + binding = attempt(state, "candidate", "quality", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + replacement = attempt(state, "candidate", "quality", dependencies=state.dependencies) + with pytest.raises(ValueError, match="Unsettled"): + tree.register_attempt(state, replacement, expected_revision=state.revision) + + +@pytest.mark.parametrize("field", [field.name for field in fields(tree.Context)]) +def test_receipts_from_changed_contract_plan_grader_environment_or_data_are_rejected(field): + state = add(start(), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + wrong = replace(binding, context=replace(binding.context, **{field: "different"})) + with pytest.raises(ValueError, match="registered"): + tree.record_receipt(state, receipt(wrong), expected_revision=state.revision) + + +@pytest.mark.parametrize("field,value", [ + ("search_id", "other-search"), ("candidate_id", "root"), + ("artifact_digest", digest("different")), ("check_id", "objective"), + ("attempt_id", "other-attempt"), ("execution_key", "other-execution"), + ("reservation_id", "other-reservation"), ("lease_id", "other-lease"), +]) +def test_receipt_must_match_the_entire_registered_binding(field, value): + state = add(start(), "candidate") + binding = attempt(state, "candidate", "quality") + state = tree.register_attempt(state, binding, expected_revision=state.revision) + with pytest.raises(ValueError, match="registered"): + tree.record_receipt(state, receipt(replace(binding, **{field: value})), + expected_revision=state.revision) + + +def test_same_bytes_do_not_transfer_evidence_and_conflicting_receipts_are_rejected(): + state = complete(add(start(), "first", artifact_digest=digest("same")), "first", 5) + old = state.receipts[-1] + state = add(state, "second", "first", artifact_digest=digest("same")) + assert tree.evidence(state, "second") == {} + with pytest.raises(ValueError, match="Conflicting"): + tree.record_receipt(state, replace(old, objective=6), expected_revision=state.revision) + with pytest.raises(ValueError, match="reused"): + tree.register_attempt(state, replace(old.attempt, candidate_id="second"), + expected_revision=state.revision) + + +def test_generation_snapshot_candidates_and_history_are_immutable(): + state = start() + with pytest.raises(ValueError, match="snapshot"): + add(state, "candidate", snapshot="updated-mid-search") + advanced = add(state, "candidate") + assert len(state.candidates) == 1 + with pytest.raises(FrozenInstanceError): + advanced.candidates[1].artifact_digest = digest("mutated") + with pytest.raises(ValueError, match="immutable"): + add(advanced, "candidate", artifact_digest=digest("changed")) + with pytest.raises(ValueError, match="Stale"): + tree.add_candidate(advanced, tree.Candidate("other", digest("other"), "root", producer()), + expected_revision=state.revision) + + +@pytest.mark.parametrize("limit", ["max_candidates", "max_depth", "max_attempts"]) +def test_bounded_search_still_returns_the_qualified_root(limit): + config = contract(**{limit: 2 if limit == "max_attempts" else 1}) + state = start(configuration=config) + with pytest.raises(ValueError, match="limit"): + if limit == "max_candidates": + add(state, "first") + elif limit == "max_depth": + add(add(state, "first"), "second", "first") + else: + observe(add(state, "first"), "first", "quality") + assert tree.accept(state).candidate_id == "root" + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), True, "10"]) +def test_invalid_objective_never_becomes_evidence(value): + state = add(start(), "candidate") + with pytest.raises(ValueError, match="finite"): + receipt(attempt(state, "candidate", "objective"), objective=value) + + +def test_root_bootstrap_requires_complete_scoped_passing_evidence(): + state = start() + for receipts in (state.receipts[:1], (*state.receipts, state.receipts[0]), + (replace(state.receipts[0], verdict=Verdict.FAIL), state.receipts[1])): + with pytest.raises(ValueError): + tree.initialize(state.contract, state.candidates[0], receipts) + with pytest.raises(ValueError, match="objective"): + tree.initialize(state.contract, state.candidates[0], + (state.receipts[0], replace(state.receipts[1], objective=None))) + + +def test_contract_copies_ordered_checks_and_rejects_strings_and_unknown_dependencies(): + checks = ["quality", "objective"] + configuration = contract(checks=checks) + checks.clear() + assert configuration.checks == ("quality", "objective") + with pytest.raises(ValueError, match="ordered"): + contract(checks="quality") + with pytest.raises(ValueError, match="dependencies"): + add(start(), "candidate", dependencies=(tree.Dependency("undeclared", 0),)) diff --git a/tests/test_verification_plans.py b/tests/test_verification_plans.py new file mode 100644 index 00000000..d87028f4 --- /dev/null +++ b/tests/test_verification_plans.py @@ -0,0 +1,151 @@ +from dataclasses import replace +from itertools import combinations + +import pytest + +from inferrsi.contracts.claim import Verdict +from inferrsi.harness import search_tree as tree +from inferrsi.harness.verification_plans import Job, completion_menu +from tests.fixtures.speculative_search import add, attempt, contract, observe, start + + +def menu(state, jobs, **changes): + options = dict(available_budget=100, available_jobs=20, max_targets=3, max_jobs=20) + return completion_menu(state, jobs, **(options | changes)) + + +def fixture(): + state = start() + jobs = [Job("setup", 3)] + for cid in ("a", "b", "c"): + state = add(state, cid) + jobs.extend((Job(f"{cid}-quality", 2, cid, "quality", ("setup",)), + Job(f"{cid}-objective", 4, cid, "objective", (f"{cid}-quality",)))) + return state, tuple(jobs) + + +def test_low_standalone_value_checks_are_preserved_as_completion_bundles(): + state, jobs = fixture() + plans = menu(state, jobs, available_budget=9) + assert len(plans) == 3 + assert {plan.candidate_ids for plan in plans} == {("a",), ("b",), ("c",)} + assert all(len(plan.job_ids) == 3 and plan.cost == 9 for plan in plans) + assert all(plan.job_ids[0] == "setup" for plan in plans) + # No partial check subset can be presented as a completed candidate. + assert menu(state, jobs, available_budget=8) == () + + +def test_cross_branch_completion_counts_shared_preparation_once(): + state, jobs = fixture() + plans = menu(state, jobs, available_budget=15, max_targets=2) + pairs = [plan for plan in plans if len(plan.candidate_ids) == 2] + assert len(pairs) == 3 + assert all(plan.cost == 15 and plan.job_ids.count("setup") == 1 for plan in pairs) + assert all(len(plan.job_ids) == 5 for plan in pairs) + + +def exhaustive_oracle(jobs, budget, job_limit, target_limit): + # Independent oracle enumerates all job subsets, not candidate-completion unions. + result = {} + for size in range(len(jobs) + 1): + for subset in combinations(jobs, size): + ids = {job.job_id for job in subset} + if (len(ids) > job_limit or sum(job.cost for job in subset) > budget + or any(set(job.prerequisites) - ids or set(job.conflicts) & ids + for job in subset)): + continue + targets = tuple(cid for cid in ("a", "b", "c") + if {f"{cid}-quality", f"{cid}-objective"} <= ids) + if not 0 < len(targets) <= target_limit: + continue + cost = sum(job.cost for job in subset) + result[targets] = min(cost, result.get(targets, cost)) + return result + + +@pytest.mark.parametrize("budget", [0, 8, 9, 14, 15, 20, 21]) +@pytest.mark.parametrize("target_limit", [1, 2, 3]) +def test_exact_menu_matches_independent_exhaustive_job_subset_oracle(budget, target_limit): + state, jobs = fixture() + expected = exhaustive_oracle(jobs, budget, 7, target_limit) + actual = {plan.candidate_ids: plan.cost for plan in menu( + state, jobs, available_budget=budget, available_jobs=7, max_targets=target_limit, + )} + assert actual == expected + + +def test_conflicts_job_caps_and_attempt_caps_cannot_be_overridden_by_budget(): + state, jobs = fixture() + jobs = tuple(replace(job, conflicts=("b-objective",)) if job.job_id == "a-objective" + else job for job in jobs) + assert all(not {"a", "b"} <= set(plan.candidate_ids) for plan in menu(state, jobs)) + assert menu(state, jobs, available_jobs=2) == () + limited = replace(state, contract=replace(state.contract, max_attempts=3)) + assert menu(limited, jobs) == () # Root used two; completing another needs two. + with pytest.raises(ValueError, match="enumeration"): + menu(state, jobs, max_jobs=6) + + +def test_partial_evidence_reduces_work_but_failure_is_not_a_completion_target(): + state, jobs = fixture() + state = observe(state, "a", "quality") + state = observe(state, "b", "quality", Verdict.FAIL) + # The adapter rebuilds the menu from remaining work and scopes prerequisite reuse. + jobs = tuple(replace(job, prerequisites=("setup",)) if job.job_id == "a-objective" + else job for job in jobs if job.job_id != "a-quality") + plans = menu(state, jobs, available_budget=7) + assert [(plan.candidate_ids, plan.cost) for plan in plans] == [(("a",), 7)] + + +def test_unknown_cost_and_invalid_dependency_graphs_are_rejected(): + state, jobs = fixture() + for cost in (0, -1, True, None, float("nan")): + with pytest.raises(ValueError, match="cost"): + Job("unknown", cost) + with pytest.raises(ValueError, match="Unknown job relation"): + menu(state, (*jobs, Job("orphan", 1, prerequisites=("absent",)))) + with pytest.raises(ValueError, match="Cyclic"): + menu(state, (*jobs, Job("x", 1, prerequisites=("y",)), + Job("y", 1, prerequisites=("x",)))) + with pytest.raises(ValueError, match="Ambiguous"): + menu(state, (*jobs, Job("duplicate", 1, "a", "quality"))) + with pytest.raises(ValueError, match="ordered"): + Job("x", 1, prerequisites="setup") + + +def test_inflight_work_prevents_replanning_even_if_source_is_invalidated(): + dependency = tree.Dependency("source", 0) + state = add(start(dependencies=(dependency,)), "a") + binding = attempt(state, "a", "quality", dependencies=(dependency,)) + state = tree.register_attempt(state, binding, expected_revision=state.revision) + state = tree.invalidate(state, dependency.key, expected_revision=state.revision) + with pytest.raises(ValueError, match="Settle"): + menu(state, []) + + +def test_inconclusive_outcome_remains_a_completion_job(): + state, jobs = fixture() + state = observe(state, "a", "quality", Verdict.INCONCLUSIVE) + assert ("a",) in {plan.candidate_ids for plan in menu(state, jobs, available_budget=9)} + + +def test_planning_never_changes_evidence_or_spends_the_supplied_budget(): + state, jobs = fixture() + before = state + plans = menu(state, jobs) + assert plans + assert all((plan.search_id, plan.state_revision) == (state.contract.search_id, state.revision) + for plan in plans) + assert state is before + assert tree.accept(state).candidate_id == "root" + assert tree.verified_gain(state) == 0 + with pytest.raises(ValueError, match="known integers"): + menu(state, jobs, available_budget=None) + + +def test_all_jobs_must_name_known_candidates_and_frozen_required_checks(): + state = start(configuration=contract()) + with pytest.raises(ValueError, match="Unknown candidate"): + menu(state, [Job("q", 1, "absent", "quality")]) + with pytest.raises(ValueError, match="Unknown required check"): + menu(state, [Job("q", 1, "root", "unregistered-check")]) From abe72f633747e86c8a2b59e4b6e117fe39bf204c Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:12:59 +0800 Subject: [PATCH 2/6] harness: assemble bounded VATS and GVO with durable work control Signed-off-by: amy-why-3459 --- docs/architecture.md | 11 +- docs/module-layout.md | 3 + docs/operations.md | 3 + docs/speculative-search.md | 356 ++++++---- src/inferrsi/apps/cli/__init__.py | 5 + src/inferrsi/apps/cli/arguments.py | 3 + src/inferrsi/apps/search_benchmarks.py | 159 +++++ src/inferrsi/apps/speculative_search.py | 218 ++++++ src/inferrsi/harness/gvo.py | 213 ++++++ src/inferrsi/harness/local_process.py | 7 +- src/inferrsi/harness/scenarios/external.py | 3 + src/inferrsi/harness/search_adapter.py | 283 ++++++++ src/inferrsi/harness/search_campaign.py | 341 ++++++++++ src/inferrsi/harness/search_continuations.py | 155 +++++ src/inferrsi/harness/search_records.py | 60 ++ src/inferrsi/harness/search_tree.py | 2 + src/inferrsi/harness/vats.py | 659 +++++++++++++++++++ src/inferrsi/harness/work_control.py | 278 ++++++++ src/inferrsi/model/search_roles.py | 215 ++++++ tests/test_search_benchmarks.py | 115 ++++ tests/test_search_roles.py | 71 ++ tests/test_speculative_search.py | 8 + tests/test_vats_runtime.py | 310 +++++++++ tests/test_work_control.py | 152 +++++ tools/speculative_skill.py | 165 +++++ 25 files changed, 3655 insertions(+), 140 deletions(-) create mode 100644 src/inferrsi/apps/search_benchmarks.py create mode 100644 src/inferrsi/apps/speculative_search.py create mode 100644 src/inferrsi/harness/gvo.py create mode 100644 src/inferrsi/harness/search_adapter.py create mode 100644 src/inferrsi/harness/search_campaign.py create mode 100644 src/inferrsi/harness/search_continuations.py create mode 100644 src/inferrsi/harness/search_records.py create mode 100644 src/inferrsi/harness/vats.py create mode 100644 src/inferrsi/harness/work_control.py create mode 100644 src/inferrsi/model/search_roles.py create mode 100644 tests/test_search_benchmarks.py create mode 100644 tests/test_search_roles.py create mode 100644 tests/test_vats_runtime.py create mode 100644 tests/test_work_control.py create mode 100644 tools/speculative_skill.py diff --git a/docs/architecture.md b/docs/architecture.md index e2f05e8e..7bc0a272 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -28,11 +28,12 @@ does not import Benchmark. Benchmark supplies task definitions and specialized rules, while Harness manages their execution and the complete optimization lifecycle. Neither infers quality criteria from a natural-language goal. Ordinary projects default to the current Agent conversation. -The [offline speculative-search primitives](speculative-search.md) add immutable -candidate trees, scoped evidence transitions and bounded completion menus in -Harness. They expose pure control operations for future VATS assembly; the -existing session controller continues to own execution. Production adapter -integration, learned role selection and GVO remain separate delivery work. +The [experimental speculative search](speculative-search.md) composes immutable +candidate trees, bounded completion menus, synchronous VATS and CPU GVO roles. +Its work controller uses shared RunStore transactions and the existing local +process executor. Apps supplies Engine/Skill grading; the support matrix identifies +remaining production accounting and pilot evidence. Serial session commands keep +their existing controller and behavior. The [native host handoff](native-agent-sessions.md) lets the initiating Agent resolve the task request before any campaign exists, use its host's new-session tool and retain the native identity and returned artifacts. The child owns the diff --git a/docs/module-layout.md b/docs/module-layout.md index bfd68633..589d6202 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -114,6 +114,9 @@ Several flat modules remain shared seams until their slice starts. | `harness/optimization_session.py`, `harness/project.py`, `harness/task_protocol.py` | Shared preparation, execution and retained-result assessment | CLI supplies task resources and knowledge access; ordinary project lifecycle does not import Benchmark | | `harness/optimization.py`, `harness/candidates.py`, `harness/usage.py` | Shared project and packaged-task lifecycle | Injected execution/assessment; declared file/directory candidates, selection direction and exact/up-to rounds | | `harness/agent_sessions.py` | Native host handoff records | Freezes requests and retains native IDs and exported observations; the initiating Agent calls host tools | +| `harness/search_tree.py`, `verification_plans.py`, `vats.py`, `gvo.py`, `search_campaign.py`, `search_continuations.py` | Experimental candidate search and group learning | Frozen contracts and injected evaluators; CPU roles live in `model/search_roles.py` | +| `harness/work_control.py`, `search_records.py`, `search_adapter.py` | Search persistence, accounting and artifact adapters | Shared RunStore transactions and existing local process executor; trusted local host profile | +| `apps/search_benchmarks.py`, `apps/speculative_search.py` | Search composition and Engine CLI | Join task-owned assessment with Harness control; no benchmark acceptance rules embedded in the search algorithm | | `harness/skill_evaluation.py`, `harness/artifact_sandbox.py`, `harness/improvement.py`, `harness/atomic_json.py` | Skill-evaluation host adapter, trial execution and retention, sandbox replay, search progression | Verdict rules and task definitions arrive as injected values; no Model, Infra, Knowledge, Benchmark or Apps import | | `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py`, `benchmark/improvement.py` | Skill-evaluation criteria, task definitions and acceptance rules | Pure judgments over retained facts; no Harness import and no process start | | `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py`, `apps/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | diff --git a/docs/operations.md b/docs/operations.md index 176e0c8c..c750af2f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -66,6 +66,9 @@ Effect classes used below: | Replay a cloud-generated parser pair | [Isolated cloud replay](skill-eval-cloud-replay.md), `python -m tools.skill_eval_cloud_replay` | Untrusted-artifact execution / report-generating | Manual pinned data ingress; same-host controls before bubblewrap-only replay; no models or credentials | | Validate one harness-improvement step | [Offline improvement](harness-improvement.md), Python `tools.skill_eval_improve.prepare` / `run_offline` | Snapshot-writing / offline experiment-executing | Supplied declarative INV/DOM edits, shared total limits, rerun and one holdout gate; trusted fixed-output clients only; no real-model or Store execution | | Build an offline candidate tree and completion menu | Python `harness.search_tree` / `harness.verification_plans.completion_menu`; [control contract](speculative-search.md) | Read-only / in-memory transitions | Immutable candidates, scoped caller-supplied receipts and bounded exact plans; execution, durable state, accounting, learned roles and final acceptance require separate adapters | +| Run bounded speculative Engine search and role updates | `inferrsi search engine`; [support and configuration](speculative-search.md) | Execution / candidate writes / CPU model updates | Explicit shared RunStore, permissions, checkpoint, budgets, domain and final gate; actual device allocation is caller-owned | +| Inspect/export speculative search | `inferrsi search status` / `search export` | Read / output artifact | Reads the existing store; never restarts search, training or final evaluation | +| Compose Skill speculative search | Python `apps.search_benchmarks.SkillEvaluator` and checkout `tools.speculative_skill.Comparator` | Explicit provider execution through existing comparator | Frozen original baseline, split descriptors, independent grader and bounded provider; see campaign accounting limits in the support matrix | | Plan and report skill PR evaluations | [Skill PR integration](skill-eval-pr.md), `python -m tools.skill_eval_pr` | Read-only planning / authorized experiment-executing / PR-publishing | Trusted mapping, exact operator approval, consumed execution identity and credential-separated publication from retained evidence; no automatic paid trials | | Recover goal progress and the next proposal | [Offline goal checkpoints](goal-checkpoints.md), `python -m tools.goal_checkpoint` | Read-only | Explicit scope/criteria, pinned owner decisions and existing comparison receipts; a disposable view, never execution or acceptance authority | | Summarize campaign gains | `tools/campaign_summary_chart.py`; `report`/`run` figure | Figure-generating | Renders recorded values into one gain figure plus a matching table; validates the specification and computes nothing new | diff --git a/docs/speculative-search.md b/docs/speculative-search.md index 96d6e38a..d7639bb5 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -1,140 +1,228 @@ -# Offline candidate trees and verification plans - -This is the first implementation slice of -[the SpecRSI proposal](https://github.com/InferMatrixLab/InferRSI/issues/363). -`harness/search_tree.py` provides immutable candidate and evidence transitions; -`harness/verification_plans.py` constructs bounded completion menus. Both are -pure Python control primitives with no execution, storage or model access. -The existing project and benchmark session commands retain their current behavior. - -The design follows [SpecRSI v2, Section 3](https://arxiv.org/html/2610.16491v2#S3): -editing an incomplete or unsuccessful parent is allowed, while adopting a version -requires complete admissible evidence for that version. Completion bundles keep -complementary checks together. This slice supplies building blocks for the inner -search; a learned Scorer/Policy, automatic Drafter and GVO updates remain future -integration work. - -## State and evidence - -`Contract` freezes the search identity, verification context, required checks, -objective, direction, improver snapshot and candidate/depth/check-attempt caps. -The context identifies the plan, validator, environment, workload, original -baseline and quality reference. Checks may encode separately required repetitions. -The current selector supports a scalar objective plus hard PASS requirements; -the Skill lane's quality-first lexicographic selector needs a separate adapter. - -`initialize` requires one complete, passing receipt per root check. Root receipts -count against the attempt cap. Their acquisition/reuse cost must already be in -the caller's authoritative accounting. A passing objective check carries a finite -value; other checks and rejected/inconclusive outcomes cannot supply a value. - -`add_candidate` records an immutable artifact digest and edit parent, with the -frozen producer snapshot, producer revision, context/history digests and actual -generation dependencies. Callers retain and verify artifacts with the existing -`harness.candidates.retain` / `identity` helpers before admission. This core does -not authenticate digests or decide whether an edit was permitted. Parent quality -failures do not invalidate the edit lineage. Real generation-source dependencies -are inherited and must still be current; a child cannot drop them to escape an -invalidation. Rebase or composition is a new candidate with its own evidence. -This offline core predeclares dependency keys at initialization; introducing new -source authorities dynamically requires later controller integration. - -`register_attempt` binds a caller-owned attempt/execution key, reservation and -lease to one candidate, check and complete verification context. Registering an -attempt grants no execution permission. Unknown work on the same check cannot be -replaced, even after dependency invalidation. Valid outcomes cannot be resampled -until they pass; an INCONCLUSIVE outcome may be retried with a fresh identity -under the caller's frozen retry rules and a new reservation. - -`record_receipt` requires the entire registered binding. Identical delivery is -idempotent and conflicting delivery is rejected. History retains late results; -`evidence` admits only the latest attempt with current dependency epochs and -valid candidate provenance. FAIL is an observed outcome, whereas INCONCLUSIVE -leaves a check missing. Identical candidate bytes do not transfer evidence. - -`accept` selects only fully passing candidates, preferring the root on an -objective tie and otherwise using lexicographic candidate ID order. The result is -a **development selection**. Independent final/holdout acceptance and production -promotion remain caller-owned. `verified_gain` measures this selection relative -to the root. Invalidation recomputes eligibility; invalid root evidence blocks -acceptance and further drafting/check registration, while late archival remains -available. - -## Completion planning - -`completion_menu(state, jobs, available_budget=..., available_jobs=..., -max_targets=..., max_jobs=...)` takes a supplied integer-cost budget view and -returns immutable `Plan` values. The caller has already accounted for planning, -generation, final reserve, in-flight obligations and unknown spend when deriving -that view. Costs must be known positive conservative bounds in the same unit. - -Each `Job` names either a candidate/check pair or a shared preparation step. -Prerequisites form an acyclic graph. The solver takes each candidate's complete -missing-check set, closes its prerequisites, and enumerates unions of at most -`max_targets` candidate seeds. Every feasible singleton completion remains in -the menu. Shared jobs are charged once per plan, conflicts exclude combinations, -and available job slots and remaining check attempts bound each proposal. -`max_jobs` bounds the input graph; enumeration is intended for small offline -controls, not an unbounded production optimizer. No Scorer prediction is used -as evidence or as a substitute for a completion seed. - -Plans bind the search ID and state revision and carry a deterministic topological -job order. The caller retains the full job definitions and costs with its frozen -plan; job IDs alone do not authenticate them. The adapter rebuilds job -relations from current observations at each synchronous decision boundary; -already observed checks are not silently scheduled again. An in-flight attempt, -including an invalidated one, blocks replanning until its execution is settled. -An empty menu is a valid outcome under a tight budget; the root remains available -when its evidence is valid. Feasibility says nothing about whether checks pass. - -## Shared execution and persistence boundary - -These modules introduce no second runner, budget ledger, holdout tracker or -storage backend. A future application adapter must: - -1. Read current authoritative state and the frozen contract, settle the previous - work block, and supply a conservative available-budget view. -2. Generate and retain permitted edits, or select an affordable check plan using - the frozen roles. Account for generation, planning and failed work. -3. Use shared execution controls for atomic claim/reservation, absolute deadline, - resource isolation and actual execution. Bind their identities to the tree. -4. Authenticate and assess receipts, apply a transition against the expected - search revision, and persist it using the authoritative storage CAS/lock. - A pure function's revision check is **not a durable or cross-worker CAS**. - On conflict, reread and recompute; an ambiguous claim never grants a retry. -5. Persist real dependency invalidation before cancellation. Reconcile costs and - retain leases until process cleanup is confirmed. Receipt archival itself - neither charges again nor releases a resource. -6. Freeze the selected version for independent final acceptance, with holdout - access and interruptions tracked separately from development feedback. - -There is no production adapter in this slice. In particular, local immutable -values and revision checks do not establish crash recovery, external exactly-once, -multi-worker resource isolation, budget transactions or holdout non-reuse. - -## Verification and remaining delivery - -Run the CPU controls with: +# Speculative candidate search and group learning + +The experimental implementation of [issue #363](https://github.com/InferMatrixLab/InferRSI/issues/363) +composes bounded VATS, trainable CPU roles, GVO, durable work accounting and +independent final selection. It follows the two loops in +[SpecRSI v2 §3](https://arxiv.org/html/2610.16491v2#S3). It does not reproduce the +paper's language-model training or establish its reported gains. Existing +`campaign` and `benchmarks session` commands keep their serial lifecycle. + +## Supported profile and authority + +| Component | Implementation | Boundary | +| --- | --- | --- | +| Candidate/evidence model | `harness.search_tree`, `search_records` | Immutable artifacts, actual dependency epochs and complete-candidate selection | +| Solver | `harness.verification_plans`, `vats.menu` | Bounded exhaustive completion plans; conservative integer millisecond caps | +| Drafter | `model.search_roles`, `harness.search_adapter` | Trainable softmax over supplied exact text edits; actual retained changes, not free-form LLM generation | +| Policy | `model.search_roles` | Independent eight-feature linear softmax, recorded behavior probabilities, deterministic deployment actions | +| Scorer | `model.search_roles` | Separate linear model; actual suffix MSE and optional paid continuation differences | +| Work/state authority | `harness.work_control` | Transactions in the shared `RunStore` SQLite database | +| Execution | Existing `scenarios.external` and `local_process` | Absolute deadline and process-domain cleanup; no new subprocess runner | +| Engine | `apps.search_benchmarks.EngineEvaluator` | Existing packaged `optimization.py` as one whole check; currently the Qwen denoiser NPU task | +| Skill | `apps.search_benchmarks.SkillEvaluator`, checkout `tools.speculative_skill.Comparator` | Explicit provider configuration, original baseline, independent grader, separate development/holdout descriptors | +| Final | `harness.search_campaign.finalize` | One-time scope claim before access; receipt-only recovery/export; task promotion only | + +This is a trusted local host controller. Resource domains exclude **cooperating +controllers sharing one RunStore**. They are not scheduler/GPU reservations, +filesystem sandboxes, distributed leases, provider cancellation guarantees or +cryptographic attestations. The caller supplies authorized resources and frozen +edit/check permissions. Candidate code cannot enforce its own isolation. + +Execution uses one worker. The offline planner represents shared setup and +conflicts; supplied runtime adapters package setup into each whole check. They +do not execute arbitrary standalone shared-setup jobs. Multi-worker performance +qualification remains outstanding. + +## Candidate and evidence semantics + +`Contract` freezes search ID, context, roles, checks, objective/direction and +candidate/depth/attempt caps. Context includes plan, validator, workload, +environment, original baseline and reference. Objectives must be finite; +overflowing the difference between two finite scores is also rejected. Quality +checks are hard requirements, never compensable by a better score. + +`qualify_root` obtains root evidence through claimed real checks. Unknown root +acquisition can only recover its original receipt. `import_qualified_root` +explicitly re-scopes root receipts under identical artifact/check/context +conditions, retaining physical execution and reservation identities. Root check +attempts count against each search's cap; actual acquisition is charged once to +the campaign, separately from independent search returns. + +In `A → B → C`, B may be unchecked or fail quality while C is edited and checked. +C needs every required receipt for **C's own artifact** to win. Task failure does +not invalidate descendants. Genuine environment/source dependency invalidation +does invalidate dependent evidence. Dependencies are predeclared; dynamic +introduction of new authorities is unsupported. + +`record_receipt` archives registered late results; `evidence` admits only current, +matching receipts. Equal bytes alone do not authorize reuse. Duplicate delivery +is idempotent; conflicts fail. FAIL is an observation; INCONCLUSIVE leaves a +check missing. A valid outcome cannot be repeatedly sampled until it passes. +Permitted retries need new identities and paid reservations. + +`Search.invalidate` persists the epoch change and incumbent recomputation +atomically. An invalid root blocks search; late evidence can still be archived +and charged. `ACCEPT` prefers the root on ties, then candidate ID order. It +selects a development result without changing the stable version. + +## VATS execution + +`Search.start` retains a qualified root and frozen role/settings/adapter hashes. +Each decision claims and meters Solver/Scorer/Policy preparation, constructs +bounded DRAFT/VERIFY/ACCEPT actions and records the behavior law and paid state. +Completion bundles remain in the menu; Scorer predictions never become receipts. +Optional refinement adds unions without removing base completions. An undersized +menu cap fails explicitly. Collection samples Policy; deployment uses argmax. +Drafter edits remain sampled from the supplied permitted vocabulary. + +The chosen work block is persisted before execution. Each edit waits for its +parent artifact, without waiting for parent quality. Each check binds candidate, +contract, attempt, execution, reservation and domain. Settlement commits elapsed +time, cleanup facts and evidence/incumbent changes in one transaction. Recovery +continues the retained block cursor before making another Policy decision. + +ACCEPT and limits return only complete admissible versions. Unstarted work stops +when its conservative cap no longer fits. Unknown launched operations block +continuation until reconciled. Candidate, depth, step, menu and attempt limits +bound the search independently of its time budget. + +The planner closes prerequisites, counts shared costs once and checks conflicts. +`max_targets` limits completion **seeds**, not candidates incidentally completed +by cross-candidate prerequisites. `max_jobs` bounds the whole supplied job graph. +The exact algorithm targets small menus. `draft_ahead=False` and `bundled=False` +are explicit ablation switches; they are not claims of comparative efficiency. + +## GVO and paid continuations + +`collect_group` requires at least two independent complete searches with the same +root, contract, budget, snapshot and adapter. Each owns a tree and seed. The +campaign must be able to provide equal declared budgets before collection. +Leaves from one tree are not independent group members. + +`gvo.examples` computes whole-search returns and leave-one-out advantages. +Natural sampled Drafter/Policy choices supply the clipped PPO-style objective +with KL regularization against a frozen reference. Scorer targets use actual +remaining gains and future costs from each paid decision boundary. Current +preparation is sunk for that suffix but stays in total campaign cost. Missing +labels/cost observations are never zero-filled. Penalties and training settings +are explicit frozen conditions. + +`train_group` checks retained settled records and replayed behavior probabilities, +updates separate actual parameter arrays and commits checkpoint plus training +receipt. `learn` supports Reset/Carry and reuses completed updates on resume; +unknown training cannot be relaunched. An initial checkpoint with explicit +provenance is required. This profile provides neither a pretrained model nor a +supervised warmup dataset. + +`search_continuations.continue_from` copies a retained paid state, charges the +copy, forces one legal first action and continues the frozen policy. The entire +suffix is Scorer-only. `paired_targets` requires the same source state, settings, +snapshot, adapter and continuation seed with different first actions. Both +absolute suffix targets and observed relative differences train the Scorer. +Caller-authored unreceipted pair targets are rejected. + +These are real updates to a constrained CPU operator model. They are not LLM +finetuning, prompt summarization or evidence of pretrained-model improvement. + +## Accounting, recovery and final + +`WorkControl` extends the existing RunStore database with campaign, domain and +holdout tables. It is the authority for this experimental runtime, not a mirror +of serial-session manifests. One transaction reserves work/domains and registers +the pending tree attempt. Controllers using the same resource or holdout must +use the **same shared store**. + +Launch intent is durable before dispatch; an ambiguous crash never authorizes a +second launch. Known elapsed time and cleanup release unused reservation. +Unknown usage/cleanup retains the entire reservation and domain. Over-cap work +is charged and blocks promotion. Absolute deadlines include queueing and cleanup. +In-process bounded planning, editing and training are checked after return; +they are not forcibly preempted like subprocesses. Preflight freezing and initial +setup are outside this campaign meter, a remaining lifecycle-cost limitation. + +Receipts bind candidate/adapter identities and content hashes. They detect +accidental mutation under a trusted host, not malicious-host forgery. Recovery +checks retained candidate bytes and never launches new work. Lost preparation, +training or copy receipts remain unresolved; no guessed-cost retry is available. +External process-liveness reconciliation needs a qualified authority. + +`finalize` consumes a unique scope in the shared store before final data access, +freezes selection and closes search/training. Engine compares again with the +original baseline. For Skill, supply a **holdout-bound adapter** and the declared +holdout scope, never the development adapter. Exact bounded radix utility keeps +quality before token use; reduced tokens cannot offset a quality regression. + +After interruption, final consumes existing receipts only. Even an unstarted +remaining check cannot be newly dispatched by recovery. Existing final results +are immutable; export never reruns them. Only complete accepted final evidence, +known cleanup and valid budget can update `stable`. Role checkpoints are never +automatically promoted to a production/default improver. Such promotion needs +separate independent data and acceptance authority. + +Enforced spend is in milliseconds. Skill reports retain provider token/currency +facts, and the existing comparator enforces its per-comparison limits. A +**campaign-wide provider token/currency reservation bridge is not implemented**. +Elapsed-time caps must not be presented as that guarantee. + +## Entry points + +`inferrsi search engine --help` lists explicit Engine bounds and permissions. +Supply a fresh `--session`, shared `--store`, unique `--campaign`, packaged task, +initial checkpoint, edit vocabulary, authorized resource domain, time/search/ +training bounds, independent group count, rounds, final scope and final threshold. +Task protocol flags follow `--`, for example `--pairs 1 --samples 1 --warmups 0` +when the campaign owner chooses one measurement per arm. No gate is inferred. +A domain name does not allocate a device. + +Checkpoint JSON contains numeric arrays `drafter`, `policy`, `scorer` and nonempty +`origin`. Drafter dimensions match edits; Policy/Scorer each have eight dimensions. +Freeze initial parameters and provenance in the campaign plan. +`--edit TOKEN FILE BEFORE AFTER` permits one exact replacement in a candidate +relative file; candidates cannot change the evaluator or roles. + +Read/export without execution: + +```bash +inferrsi search status --session +inferrsi search export --session --output +``` + +Python composition uses `qualify_root`, `collect_group`, `learn`, `continue_from` +and `finalize`. Recovery requires the original frozen adapters. The CLI has no +blind whole-campaign restart command. Skill's checkout `Comparator` takes the +existing comparator configuration with an explicit bounded provider, original +Git baseline, editable bundle paths and two split descriptors (`path`, `sha256`, +`scope`, `task_count`). It uses the existing private Git-index materializer, +executes the existing comparator under the shared process executor and revalidates +its independent retained report. Construction does not load holdout bytes. +Provider readiness/credentials/authorization remain existing explicit requirements. + +## Validation and issue completion ```bash -python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py +python -m pytest tests/test_speculative_search.py tests/test_verification_plans.py \ + tests/test_work_control.py tests/test_search_roles.py tests/test_vats_runtime.py \ + tests/test_search_benchmarks.py ``` -Tests cover retained file/directory artifacts for synthetic Engine/Skill shapes, -failed-parent repair, incomplete ancestors, failure versus missing evidence, -dependency/result order, identical bytes with separate attempts, stale revision -rejection, immutable role snapshots, deterministic selection, invalid roots and -bounded search. The planner is compared with an independent exhaustive job-subset -oracle across budgets and target limits, including shared preparation and conflicts. +Controls cover exhaustive planning, failed-parent repair, finite-score overflow, +concurrent domain claims, unknown reservations, atomic evidence/accounting, +invalid roots, process-backed CPU searches, numerical actor-gradient checks, +parameter updates, fresh searches, paid continuations, recovery and final export. +The numeric process grader and Skill report fixtures test software behavior; +they are not real task/provider efficacy evidence. -| Issue acceptance area | This slice | Remaining evidence | +| Issue area | Delivered software | Still required for full acceptance | | --- | --- | --- | -| AC-01/02 | Frozen scalar contract and offline candidate tree | Production support matrix and role/execution adapters | -| AC-03/05 | Scoped evidence, epoch invalidation, development selection and revision guards | Authoritative durable state integration and restart/race tests | -| AC-04 | Bounded exact completion menu and exhaustive controls | Scorer refinement, Policy selection and scalable solver integration | -| AC-06 | Frozen snapshot identity only | GVO data collection, objectives, parameter updates and new searches | -| AC-07/08 | References to caller-owned execution identities | Actual budget/resource transactions, cancellation and independent final/holdout | -| AC-09/10 | Synthetic control fixtures | Both real adapters, model generation, training, matched experiments and lifecycle costs | - -These tests do not establish model quality, engine speedup or SpecRSI efficiency. -The parent issue remains open for its full dual-loop delivery. +| AC-01/02 | Support/permission boundaries and immutable trees | Production role/environment qualification | +| AC-03/05 | Durable transitions, late receipts, invalidation, bounded synchronous VATS | Production adapter fault/race evidence | +| AC-04 | Exact planner, independent oracle and Scorer refinement | Scaling and real complementary-check evidence | +| AC-06 | Independent groups, actor/value updates, paid continuations, checkpoints, Reset/Carry | Actual task training results and role qualification | +| AC-07 | Shared local claims, elapsed-time caps and cleanup-aware recovery | Host callback preemption, provider/device total accounting and production resource integration | +| AC-08 | One-time final scope and task promotion gate | Real independent final evidence and separate default-improver qualification | +| AC-09 | Packaged Engine evaluator and explicit Skill comparator bridge | Both complete real VATS/GVO/final pilots | +| AC-10 | Retained trajectories/costs and ablation switches | Serial/VATS, bundle, learning, Reset/Carry and worker experiments with full lifecycle costs | + +The parent issue remains open. Software tests, CPU mathematical checks or an +available adapter do not establish full acceptance, NPU speedup, provider quality, +search efficiency or universal SpecRSI gains. diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a..68f716a4 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -349,6 +349,11 @@ def main(argv: list[str] | None = None) -> int: result = getattr(agent_sessions, args.agent_action)(args.job) _print(result) return 0 + if args.command == "search": + from ..speculative_search import command as search_command + + _print(search_command(args)) + return 0 if args.command == "benchmarks": if args.benchmark_action == "session": from ...harness.optimization_session import session_command diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 0b0c5da1..e8ec8f57 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -107,6 +107,9 @@ def parser() -> argparse.ArgumentParser: ) root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + from ..speculative_search import add_arguments as search_arguments + + search_arguments(commands) mindie = commands.add_parser("mindie", help="Use project-approved, local MindIE knowledge") mindie_actions = mindie.add_subparsers(dest="mindie_action", required=True) enable = mindie_actions.add_parser("enable", help="Approve a MindIE domain for this checkout") diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py new file mode 100644 index 00000000..eb7a1172 --- /dev/null +++ b/src/inferrsi/apps/search_benchmarks.py @@ -0,0 +1,159 @@ +"""Benchmark-owned evaluators composed with Harness speculative search. + +The engine protocol remains one indivisible check; no CPU or partial-case run +can impersonate its NPU performance qualification. Skill grading stays in the +existing independent comparator and quality-first rules. +""" + +import math +import sys +import time +from pathlib import Path + +from inferrsi.apps.improvement import assessment +from inferrsi.benchmark.improvement import SelectionPolicy, holdout_accepts +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import candidates, task_protocol +from inferrsi.harness.artifact_primitives import digest, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import runtime_environment +from inferrsi.harness.scenarios.external import execute_trial + + +class EngineEvaluator: + def __init__(self, package, frozen, *, arguments, python=sys.executable, prefix=()): + self.frozen = Path(frozen).resolve() + self.plan = task_protocol.prepare( + Path(str(task_resources(package))), self.frozen, arguments + ) + self.python, self.prefix = python, tuple(prefix) + self.identity = digest( + { + "lane": "engine", + "task": package, + "plan": self.plan, + "python": python, + "prefix": self.prefix, + } + ) + + def evaluate(self, candidate, check, output, deadline): + if check != "evaluation": + raise ValueError("Packaged engine protocol requires one full evaluation job") + actual = { + p.relative_to(self.frozen).as_posix(): sha256(p) + for p in self.frozen.rglob("*") + if p.is_file() + } + if actual != self.plan["frozen_files"]: + raise ValueError("Frozen benchmark resources changed") + output.mkdir(parents=True) + candidates.retain(candidate / "candidate.py", output / "candidate.py") + protocol = task_protocol.protocol(self.frozen) + args = protocol.arguments( + output / "candidate.py", output / "measurement", self.plan["evaluation_settings"] + ) + from inferrsi.harness import optimization_session + + command = ( + *self.prefix, + self.python, + str(Path(optimization_session.__file__).resolve()), + str(self.frozen / "benchmark.py"), + *args, + ) + environment = runtime_environment() + environment["PYTHONDONTWRITEBYTECODE"] = "1" + execution = execute_trial( + command, + output, + output / "execution", + time.monotonic() + max(0, deadline - time.time()), + None, + environment=environment, + include_cleanup=True, + ) + assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) + if execution["cleanup_verified"] is not True: + assessed = {"verdict": "INCONCLUSIVE", "score": None} + atomic_write_json(output / "assessment.json", assessed) + return { + "verdict": assessed["verdict"], + "objective": assessed.get("score"), + "cleanup_verified": execution["cleanup_verified"], + "assessment_sha256": sha256(output / "assessment.json"), + "execution_sha256": sha256(output / "execution/execution.json"), + } + + +class SkillEvaluator: + """Use an explicitly bound independent comparator, with an exact lexicographic utility. + + The comparator's run method returns a verified report and cleanup facts; it + must run the existing Skill comparison for the supplied candidate and split. + No generic model client or fixed-output fixture is silently selected here. + """ + + def __init__(self, comparator, *, split, policy, token_cap, task_count): + if not isinstance(policy, SelectionPolicy) or split not in ("development", "holdout"): + raise ValueError("A frozen Skill selection policy and explicit split are required") + if ( + type(token_cap) is not int + or token_cap < 1 + or type(task_count) is not int + or task_count < 1 + or task_count * (token_cap + 1) > 2**53 - 1 + ): + raise ValueError("Skill utility must be exactly representable within declared bounds") + self.comparator, self.split, self.policy = comparator, split, policy + self.holdout_scope = ( + getattr(comparator, "holdout_scope", None) if split == "holdout" else None + ) + self.token_cap, self.task_count = token_cap, task_count + self.identity = digest( + { + "lane": "skill", + "comparator": comparator.identity, + "split": split, + "min_success_rate": policy.min_success_rate, + "max_regressions": policy.max_regressions, + "token_cap": token_cap, + "task_count": task_count, + "utility": "successes*(token_cap+1)-tokens", + } + ) + + def evaluate(self, candidate, check, output, deadline): + if check != "evaluation": + raise ValueError("Skill comparison is an indivisible evaluation job") + report, cleanup = self.comparator.run(candidate, output, deadline, split=self.split) + score = assessment(report, self.policy) + tokens = score["usage"]["candidate"]["total_tokens"] + if ( + cleanup is not True + or not score["complete"] + or score["reason"] == "unknown_usage" + or tokens is None + or type(tokens) is not int + or not 0 <= tokens <= self.token_cap + or score["trials"] != self.task_count + ): + verdict, objective = "INCONCLUSIVE", None + elif score["reason"] not in ("eligible", "no_improvement"): + verdict, objective = "FAIL", None + else: + # One extra success dominates the entire possible token range. + # This exact radix preserves the existing quality-then-token order. + verdict = "PASS" + objective = score["successes"] * (self.token_cap + 1) - tokens + if not math.isfinite(objective): + raise ValueError("Invalid Skill utility") + if self.split == "holdout" and not holdout_accepts(score): + verdict, objective = "FAIL", None + return { + "verdict": verdict, + "objective": objective, + "cleanup_verified": cleanup, + "report_digest": digest(report), + "assessment": score, + } diff --git a/src/inferrsi/apps/speculative_search.py b/src/inferrsi/apps/speculative_search.py new file mode 100644 index 00000000..318939f0 --- /dev/null +++ b/src/inferrsi/apps/speculative_search.py @@ -0,0 +1,218 @@ +"""CLI composition for the supported packaged Engine VATS/GVO profile.""" + +import json +import sys +import time +from dataclasses import asdict, replace +from pathlib import Path + +from inferrsi.harness import candidates, search_tree +from inferrsi.harness.artifact_primitives import digest +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.run_store import RunStore, RunStoreReader +from inferrsi.harness.search_adapter import ArtifactAdapter, Edit +from inferrsi.harness.search_campaign import collect_group, finalize, learn, qualify_root +from inferrsi.harness.search_records import decode +from inferrsi.harness.vats import Settings +from inferrsi.harness.work_control import WorkControl +from inferrsi.model.search_roles import Snapshot + +from .search_benchmarks import EngineEvaluator + + +def add_arguments(commands): + search = commands.add_parser("search", help="Bounded SpecRSI search and role training") + actions = search.add_subparsers(dest="search_action", required=True) + run = actions.add_parser("engine", help="Run a supplied packaged Engine protocol") + run.add_argument("--session", type=Path, required=True) + run.add_argument("--store", type=Path, required=True, help="Shared resource/holdout RunStore") + run.add_argument("--campaign", required=True, help="Unique campaign ID in the shared store") + run.add_argument("--task", required=True) + run.add_argument( + "--edit", + nargs=4, + action="append", + required=True, + metavar=("TOKEN", "FILE", "BEFORE", "AFTER"), + ) + run.add_argument( + "--checkpoint", + type=Path, + required=True, + help="Initial trainable finite-edit role snapshot with provenance", + ) + run.add_argument("--domain", required=True, help="Caller-owned exclusive measurement domain") + run.add_argument("--max-seconds", type=int, required=True) + run.add_argument("--evaluation-seconds", type=int, required=True) + run.add_argument("--final-reserve-seconds", type=int, required=True) + run.add_argument("--search-seconds", type=int, required=True) + run.add_argument("--preparation-seconds", type=int, required=True) + run.add_argument("--draft-seconds", type=int, required=True) + run.add_argument("--training-seconds", type=int, required=True) + run.add_argument("--candidates", type=int, required=True) + run.add_argument("--depth", type=int, required=True) + run.add_argument("--attempts", type=int, required=True) + run.add_argument("--steps", type=int, required=True) + run.add_argument("--plans", type=int, required=True) + run.add_argument("--targets", type=int, required=True) + run.add_argument("--groups", type=int, required=True, help="Independent searches per round") + run.add_argument("--rounds", type=int, required=True) + run.add_argument("--learning-rate", type=float, required=True) + run.add_argument("--seed", type=int, default=0) + run.add_argument("--mode", choices=("reset", "carry"), default="reset") + run.add_argument("--final-scope", required=True) + run.add_argument("--minimum-final-score", type=float, required=True) + run.add_argument("--python", default=sys.executable) + run.add_argument("arguments", nargs="*", help="Task protocol arguments after --") + for action in ("status", "export"): + read = actions.add_parser(action) + read.add_argument("--session", type=Path, required=True) + if action == "export": + read.add_argument("--output", type=Path, required=True) + + +def command(args): + root = args.session.resolve() + if args.search_action != "engine": + location = json.loads((root / "control.json").read_text(encoding="utf-8")) + control = WorkControl(RunStoreReader(location["store"], "speculative")) + result = control.read(location["campaign"]) + if args.search_action == "export": + atomic_write_json(args.output, result) + return result + if root.exists(): + raise ValueError("Use a new session directory; status/export never restart work") + if args.groups < 2 or args.rounds < 1: + raise ValueError("GVO needs at least two independent searches and one training round") + if args.final_reserve_seconds < args.evaluation_seconds: + raise ValueError("Final reserve must cover the full evaluation cap") + required = ( + args.evaluation_seconds + + args.final_reserve_seconds + + (args.groups * args.rounds + 2) * args.search_seconds + + args.rounds * args.training_seconds + ) + if args.max_seconds < required: + raise ValueError( + "Campaign cap must fund qualification, all search groups, training and final" + ) + snapshot = Snapshot(**json.loads(args.checkpoint.read_text(encoding="utf-8"))) + if ( + len(snapshot.drafter) != len(args.edit) + or len(snapshot.policy) != 8 + or len(snapshot.scorer) != 8 + ): + raise ValueError( + "Checkpoint needs one Drafter weight per edit and eight Policy/Scorer weights" + ) + settings = Settings( + args.steps, + 1, + args.depth, + args.targets, + args.candidates, + args.plans, + args.preparation_seconds * 1000, + args.draft_seconds * 1000, + args.search_seconds * 1000, + ) + root.mkdir(parents=True) + evaluator = EngineEvaluator( + args.task, root / "frozen", arguments=args.arguments, python=args.python + ) + adapter = ArtifactAdapter( + root / "artifacts", + (Edit(*raw) for raw in args.edit), + {"evaluation": args.evaluation_seconds * 1000}, + evaluator, + domain=args.domain, + ) + source = root / "initial" + source.mkdir() + candidates.retain(root / "frozen/baseline.py", source / "candidate.py") + node = adapter.import_root(source, snapshot=digest(snapshot.document()), context={}) + control = WorkControl(RunStore(args.store.resolve(), "speculative")) + campaign = args.campaign + atomic_write_json( + root / "control.json", {"store": str(args.store.resolve()), "campaign": campaign} + ) + specification = { + "adapter": adapter.identity, + "settings": asdict(settings), + "groups": args.groups, + "rounds": args.rounds, + "mode": args.mode, + "learning_rate": args.learning_rate, + "minimum_final_score": args.minimum_final_score, + "initial_snapshot": snapshot.document(), + } + contract = search_tree.Contract( + "qualification", + search_tree.Context( + digest(specification), + digest(specification), + evaluator.identity, + args.domain, + args.task, + node.artifact_digest, + evaluator.identity, + ), + digest(snapshot.document()), + ("evaluation",), + "evaluation", + "maximize", + args.candidates, + args.depth, + args.attempts, + ) + control.create( + campaign, + specification, + {"searches": {}, "checkpoint": snapshot.document(), "stable": asdict(node)}, + cap_ms=args.max_seconds * 1000, + final_reserve_ms=args.final_reserve_seconds * 1000, + deadline=time.time() + args.max_seconds, + ) + initial = qualify_root(control, campaign, adapter, node, contract) + trained = learn( + control, + campaign, + adapter, + initial, + snapshot, + settings, + rounds=args.rounds, + group_size=args.groups, + training_cap_ms=args.training_seconds * 1000, + learning_rate=args.learning_rate, + seed=args.seed, + mode=args.mode, + ) + # A fresh post-update search demonstrates the new frozen checkpoint. Selection + # is development-only, and never uses final results to choose another version. + records = collect_group( + control, + campaign, + adapter, + initial, + trained, + settings, + group_id="deployment", + size=2, + seed=args.seed, + collect=False, + ) + states = [decode(record["tree"]) for record in records] + selected_state = max(states, key=search_tree.verified_gain) + selected = replace(search_tree.accept(selected_state), parent_id=None) + finalize( + control, + campaign, + adapter, + selected, + holdout_scope=args.final_scope, + accept_result=lambda values: all( + v["objective"] >= args.minimum_final_score for v in values + ), + ) + return control.read(campaign) diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py new file mode 100644 index 00000000..10e95096 --- /dev/null +++ b/src/inferrsi/harness/gvo.py @@ -0,0 +1,213 @@ +"""GVO data separation and metered group updates of frozen search roles.""" + +import math +import time +import uuid +from dataclasses import replace + +from inferrsi.model.search_roles import Choice, finite, update + +from .artifact_primitives import digest +from .search_records import decode + + +def choice_from_document(raw): + return Choice( + **( + raw + | { + "options": tuple(raw["options"]), + "features": tuple(tuple(row) for row in raw["features"]), + "probabilities": tuple(raw["probabilities"]), + } + ) + ) + + +def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): + records = tuple(records) + if len(records) < 2: + raise ValueError("GVO requires at least two independent complete searches") + if finite(expense_weight) < 0 or finite(elapsed_weight) < 0: + raise ValueError("Return penalties cannot be negative") + identities, roots, contracts, snapshots, settings, adapters = ( + [], + set(), + set(), + set(), + set(), + set(), + ) + returns = [] + for record in records: + if record["status"] != "complete" or record["pending"] is not None: + raise ValueError("Incomplete searches cannot supply GVO returns") + state = decode(record["tree"]) + identities.append(state.contract.search_id) + roots.add( + next(n.artifact_digest for n in state.candidates if n.candidate_id == state.root_id) + ) + frozen = record["tree"]["contract"].copy() + frozen.pop("search_id") + contracts.add(digest(frozen)) + snapshots.add(record["snapshot"]) + settings.add(record["settings"]) + adapters.add(record["adapter"]) + returns.append( + finite( + record["gain"] + - record["initial_gain"] + - expense_weight * finite(record["spent_ms"]) / 1000 + - elapsed_weight * finite(record["elapsed_ms"]) / 1000 + ) + ) + if len(set(identities)) != len(identities) or any( + len(values) != 1 for values in (roots, contracts, snapshots, settings, adapters) + ): + raise ValueError("Search group must have distinct IDs and identical starting conditions") + if any(record["forced"] for record in records): + raise ValueError("Forced continuations are Scorer-only, never actor group members") + advantages = [ + finite(value - math.fsum(v for j, v in enumerate(returns) if j != i) / (len(returns) - 1)) + for i, value in enumerate(returns) + ] + actors, values = [], [] + for record, advantage in zip(records, advantages): + values.extend( + suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + ) + if not record["collect"]: + raise ValueError("Deployment searches cannot supply on-policy actor data") + for step in record["trace"]: + if step["forced"]: + raise ValueError("Forced action leaked into a natural trajectory") + for raw in (step["choice"], *step["draft_choices"]): + choice = choice_from_document(raw) + if choice.sampled: + actors.append((choice, advantage)) + return actors, values, {"search_ids": identities, "returns": returns, "advantages": advantages} + + +def suffix_examples(record, *, expense_weight, elapsed_weight): + if record["status"] != "complete" or record["pending"] is not None: + raise ValueError("Suffix target requires a completed and settled search") + end = record["started_at"] + record["elapsed_ms"] / 1000 + result = [] + for step in record["trace"]: + future_cost = record["spent_ms"] - step["spent_before"] + if future_cost < 0 or end < step["at"]: + raise ValueError("Suffix accounting precedes its paid decision state") + target = finite( + record["gain"] + - step["gain_before"] + - expense_weight * future_cost / 1000 + - elapsed_weight * (end - step["at"]) + ) + result.append((tuple(step["features"]), target)) + return result + + +def train_group( + control, + campaign, + records, + snapshot, + reference, + *, + cap_ms, + learning_rate, + expense_weight=0.0, + elapsed_weight=0.0, + scorer_continuations=(), + scorer_pairs=(), + pair_examples=(), + **training, +): + """One actual parameter update, durably checkpointed inside the campaign budget.""" + records = tuple(records) + if any(record["snapshot"] != digest(snapshot.document()) for record in records): + raise ValueError("Training checkpoint differs from collection checkpoint") + doc = control.read(campaign) + if doc["final_claim"] is not None: + raise ValueError("Training is closed before final evaluation") + if digest(doc["state"]["checkpoint"]) != digest(snapshot.document()): + raise ValueError("Training checkpoint is no longer current") + if any( + op["category"] == "training" and op["status"] != "settled" + for op in doc["operations"].values() + ): + raise ValueError("Unresolved training execution cannot be relaunched") + for record in (*records, *scorer_continuations, *(r for pair in scorer_pairs for r in pair)): + search_id = decode(record["tree"]).contract.search_id + retained = doc["state"]["searches"].get(search_id) + if retained is None or digest(retained) != digest(record): + raise ValueError("Training requires the retained, settled search record") + if pair_examples: + raise ValueError("Unreceipted pair targets are unsupported; use paid suffix records") + from .search_continuations import paired_targets + + pair_examples = tuple( + paired_targets(pair, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + for pair in scorer_pairs + ) + operation = "gvo-" + uuid.uuid4().hex + control.claim( + campaign, + doc["revision"], + operation, + category="training", + cap_ms=cap_ms, + binding={ + "snapshot": digest(snapshot.document()), + "reference": digest(reference.document()), + "training": { + "learning_rate": learning_rate, + "expense_weight": expense_weight, + "elapsed_weight": elapsed_weight, + **training, + }, + "searches": [decode(r["tree"]).contract.search_id for r in records], + }, + ) + claim = control.launch(campaign, operation) + started = time.monotonic() + actors, values, summary = examples( + records, expense_weight=expense_weight, elapsed_weight=elapsed_weight + ) + for record in scorer_continuations: + if not record["forced"]: + raise ValueError("Additional Scorer trajectories must be explicitly marked forced") + values.extend( + suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) + ) + trained = update( + snapshot, + reference, + actors, + values, + pair_examples, + learning_rate=learning_rate, + groups=len(records), + **training, + ) + trained = replace(trained, origin=digest({"parent": snapshot.document(), "group": summary})) + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def checkpoint(state): + state["checkpoint"] = trained.document() + state.setdefault("training", []).append( + {**summary, "operation": operation, "checkpoint": digest(trained.document())} + ) + return state + + control.settle( + campaign, + operation, + {**summary, "checkpoint": trained.document()}, + actual_ms=elapsed, + cleanup_verified=True, + transition=checkpoint, + ) + if elapsed > cap_ms or time.time() >= claim["deadline"]: + raise ValueError("Training exceeded its frozen budget") + return trained diff --git a/src/inferrsi/harness/local_process.py b/src/inferrsi/harness/local_process.py index 1208e506..a12ad938 100644 --- a/src/inferrsi/harness/local_process.py +++ b/src/inferrsi/harness/local_process.py @@ -329,7 +329,7 @@ def _supervise( def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadline: float, *, - environment: dict | None = None) -> dict: + environment: dict | None = None, include_cleanup: bool = False) -> dict: """Legacy trial record: execute argv under a whole-run deadline in trial_dir.""" trial_dir.mkdir(parents=True) result_path = trial_dir / "result.json" @@ -358,4 +358,9 @@ def execute_command(command: tuple[str, ...], cwd: Path, trial_dir: Path, deadli if run["error"] is not None: record["error"] = run["error"] record["elapsed_seconds"] = run["elapsed_seconds"] + if include_cleanup: + record.update(cleanup_verified=(run["domain_empty"] if run["pid"] is not None else True), + termination_requested=run["termination_requested"], + process_exited=run["returncode"] is not None, + process_started_at=run["started_at"], process_ended_at=run["ended_at"]) return record diff --git a/src/inferrsi/harness/scenarios/external.py b/src/inferrsi/harness/scenarios/external.py index ba532614..b98826ad 100644 --- a/src/inferrsi/harness/scenarios/external.py +++ b/src/inferrsi/harness/scenarios/external.py @@ -17,6 +17,7 @@ def execute_trial( *, metric: Callable[[dict, str], float] | None = None, environment: dict | None = None, + include_cleanup: bool = False, ) -> dict: """Run one command and retain its record. @@ -27,6 +28,8 @@ def execute_trial( if metric_key is not None and metric is None: raise TypeError("execute_trial needs the caller's metric validation for metric_key") options = {"environment": environment} if environment is not None else {} + if include_cleanup: + options["include_cleanup"] = True record = execute_command(command, cwd, trial_dir, deadline, **options) result_path = Path(record["result_path"]) if metric_key is not None: diff --git a/src/inferrsi/harness/search_adapter.py b/src/inferrsi/harness/search_adapter.py new file mode 100644 index 00000000..4896f8a0 --- /dev/null +++ b/src/inferrsi/harness/search_adapter.py @@ -0,0 +1,283 @@ +"""Artifact-backed finite-edit generation and authenticated command evaluations. + +Commands and graders are trusted host inputs frozen before search. Candidates +must run in a sandbox supplied by the application when their code is untrusted; +the existing local process domain supplies cleanup, not filesystem isolation. +""" + +import json +import math +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +from inferrsi.contracts.claim import Verdict +from inferrsi.model.search_roles import choose + +from . import candidates +from . import search_tree as tree +from .artifact_primitives import digest, sha256 +from .atomic_json import atomic_write_json +from .optimization_session import runtime_environment +from .scenarios.external import execute_trial +from .verification_plans import Job + + +@dataclass(frozen=True) +class Edit: + token: str + path: str + before: str + after: str + + def __post_init__(self): + path = Path(self.path) + if ( + not self.token + or not self.before + or self.before == self.after + or path.is_absolute() + or ".." in path.parts + or not path.parts + ): + raise ValueError("Edit must name one permitted relative file and a nonempty change") + + +class ArtifactAdapter: + def __init__(self, root, edits, checks, evaluator, *, domain): + self.root = Path(root).resolve() + self.root.mkdir(parents=True, exist_ok=True) + self.edits, self.checks = tuple(edits), dict(checks) + if ( + not self.edits + or len({e.token for e in self.edits}) != len(self.edits) + or not self.checks + or any(type(v) is not int or v <= 0 for v in self.checks.values()) + or not domain + ): + raise ValueError("Frozen edit vocabulary, check caps and domain are required") + self.evaluator, self.domain = evaluator, domain + self.identity = digest( + { + "edits": [asdict(e) for e in self.edits], + "checks": self.checks, + "evaluator": evaluator.identity, + "domain": domain, + "generation": "finite-edit-softmax-v1", + } + ) + + def path(self, node): + path = self.root / "candidates" / node.candidate_id + if path.parent != self.root / "candidates" or path.is_symlink(): + raise ValueError("Candidate ID must name one retained snapshot") + if candidates.identity(path) != node.artifact_digest: + raise ValueError("Retained candidate artifact changed") + return path + + def import_root(self, source, *, snapshot, context): + destination = self.root / "candidates" / "root" + destination.parent.mkdir(exist_ok=True) + fingerprint = candidates.retain(Path(source), destination) + return tree.Candidate( + "root", + fingerprint, + None, + tree.Producer(snapshot, self.identity, digest(context), digest([])), + ) + + def jobs(self, state): + return tuple( + Job(f"{node.candidate_id}:{check}", self.checks[check], node.candidate_id, check) + for node in state.candidates + if tree.candidate_valid(state, node.candidate_id) + and not any( + r.verdict == Verdict.FAIL for r in tree.evidence(state, node.candidate_id).values() + ) + for check in tree.missing_checks(state, node.candidate_id) + ) + + def domains(self, kind, node, check): + return (self.domain,) if kind == "verify" else () + + def _save(self, operation, binding, result): + if Path(operation).name != operation or operation in (".", ".."): + raise ValueError("Operation must name one retained execution") + binding = binding | {"adapter": self.identity} + path = self.root / "receipts" / (operation + ".json") + path.parent.mkdir(exist_ok=True) + atomic_write_json( + path, + { + "binding": binding, + "result": result, + "digest": digest({"binding": binding, "result": result}), + }, + ) + return result + + def draft(self, parent, state, snapshot, rng, operation, deadline): + started = time.monotonic() + source = self.path(parent) + files = candidates.members(source) + legal = [ + (index, edit) + for index, edit in enumerate(self.edits) + if edit.path in files + and files[edit.path].read_text(encoding="utf-8").count(edit.before) == 1 + ] + choice, node = None, None + if legal and time.time() < deadline: + vectors = [ + tuple(float(i == index) for i in range(len(self.edits))) for index, _ in legal + ] + choice = choose( + snapshot, + "drafter", + [edit.token for _, edit in legal], + vectors, + rng=rng, + context=digest(asdict(parent)), + collect=True, + ) + edit = legal[choice.selected][1] + destination = self.root / "candidates" / operation + candidates.retain(source, destination) + path = destination / edit.path + text = path.read_text(encoding="utf-8") + path.write_text(text.replace(edit.before, edit.after, 1), encoding="utf-8") + node = tree.Candidate( + operation, + candidates.identity(destination), + parent.candidate_id, + tree.Producer( + digest(snapshot.document()), + self.identity, + digest(asdict(parent)), + digest([asdict(r) for r in state.receipts]), + ), + parent.dependencies, + ) + result = { + "candidate": asdict(node) if node else None, + "choice": asdict(choice) if choice else None, + "cleanup_verified": True, + "elapsed_ms": max(1, math.ceil((time.monotonic() - started) * 1000)), + } + if time.time() >= deadline: + result["candidate"] = None + result["error"] = "draft_deadline" + return self._save(operation, {"candidate": asdict(parent), "kind": "draft"}, result) + + def verify(self, node, check, operation, deadline): + started = time.monotonic() + source = self.path(node) + output = self.root / "executions" / operation + result = self.evaluator.evaluate(source, check, output, deadline) + if candidates.identity(source) != node.artifact_digest: + raise ValueError("Evaluation changed the retained candidate") + verdict = Verdict(result["verdict"]) + objective = result.get("objective") + if objective is not None and ( + type(objective) not in (int, float) or not math.isfinite(objective) + ): + raise ValueError("Nonfinite benchmark score") + result = { + **result, + "verdict": verdict.value, + "elapsed_ms": max(1, math.ceil((time.monotonic() - started) * 1000)), + "evidence_digest": digest(result), + "objective": objective, + } + return self._save( + operation, {"candidate": asdict(node), "check": check, "kind": "verify"}, result + ) + + def recover(self, operation, pending): + if Path(operation).name != operation or operation in (".", ".."): + raise ValueError("Operation must name one retained execution") + path = self.root / "receipts" / (operation + ".json") + if not path.is_file(): + return None + receipt = json.loads(path.read_text(encoding="utf-8")) + if ( + digest({"binding": receipt["binding"], "result": receipt["result"]}) + != receipt["digest"] + ): + raise ValueError("Recovery receipt changed") + expected = { + "candidate": pending["binding"]["candidate"], + "kind": pending["kind"], + "adapter": self.identity, + } + if pending["kind"] == "verify": + expected["check"] = pending["binding"]["check"] + if digest(expected) != digest(receipt["binding"]): + raise ValueError("Recovery receipt belongs to a different operation") + for raw in (expected["candidate"], receipt["result"].get("candidate")): + if raw is not None: + self.path( + tree.Candidate( + **( + raw + | { + "producer": tree.Producer(**raw["producer"]), + "dependencies": tuple( + tree.Dependency(**d) for d in raw["dependencies"] + ), + } + ) + ) + ) + return receipt["result"] + + +class CommandEvaluator: + """Frozen independent evaluator command with explicit JSON result protocol. + + The grader writes {verdict, objective?} to {result}; the shared executor + independently supplies process/cleanup facts. Useful for project protocols + and CPU integration controls, not an implicit production qualification. + """ + + def __init__(self, commands, resources): + self.commands = {key: tuple(value) for key, value in commands.items()} + self.resources = {Path(path).resolve(): sha256(Path(path)) for path in resources} + self.identity = digest( + {"commands": self.commands, "resources": {str(k): v for k, v in self.resources.items()}} + ) + + def evaluate(self, candidate, check, output, deadline): + if any(sha256(path) != expected for path, expected in self.resources.items()): + raise ValueError("Frozen evaluator resources changed") + command = tuple(arg.replace("{candidate}", str(candidate)) for arg in self.commands[check]) + output.parent.mkdir(parents=True, exist_ok=True) + record = execute_trial( + command, + candidate, + output, + time.monotonic() + max(0, deadline - time.time()), + None, + environment=runtime_environment(), + include_cleanup=True, + ) + result = { + "verdict": "INCONCLUSIVE", + "objective": None, + "cleanup_verified": record["cleanup_verified"], + "execution_sha256": sha256(output / "execution.json"), + } + if ( + record["status"] == "completed" + and record["returncode"] == 0 + and record["cleanup_verified"] is True + ): + document = json.loads(Path(record["result_path"]).read_text(encoding="utf-8")) + result.update( + verdict=Verdict(document["verdict"]).value, + objective=document.get("objective"), + result_sha256=sha256(Path(record["result_path"])), + ) + if any(sha256(path) != expected for path, expected in self.resources.items()): + raise ValueError("Evaluator resources changed during execution") + return result diff --git a/src/inferrsi/harness/search_campaign.py b/src/inferrsi/harness/search_campaign.py new file mode 100644 index 00000000..fea8bce9 --- /dev/null +++ b/src/inferrsi/harness/search_campaign.py @@ -0,0 +1,341 @@ +"""Campaign assembly: real root acquisition, independent searches, GVO and final gating.""" + +import time +from dataclasses import asdict, replace + +from inferrsi.contracts.claim import Verdict +from inferrsi.model.search_roles import Snapshot + +from . import search_tree as tree +from .artifact_primitives import digest +from .gvo import train_group +from .search_records import decode, encode +from .vats import Search + + +def qualify_root(control, campaign, adapter, root, contract): + """Acquire root evidence using real checks and the same authoritative accounting.""" + retained = control.read(campaign)["state"].get("qualified_root") + if retained is not None: + state = decode(retained) + if state.contract != contract or tree.candidate(state, state.root_id) != root: + raise ValueError("Root qualification is bound to another frozen context") + tree.accept(state) + return state + receipts = [] + for check in contract.checks: + doc = control.read(campaign) + binding = { + "candidate": asdict(root), + "context": asdict(contract.context), + "check": check, + "adapter": adapter.identity, + } + operation = "root-" + digest(binding) + previous = doc["operations"].get(operation) + if previous is None: + control.claim( + campaign, + doc["revision"], + operation, + category="root", + cap_ms=adapter.checks[check], + domains=(adapter.domain,), + binding=binding, + ) + claim = control.launch(campaign, operation) + result = adapter.verify(root, check, operation, claim["deadline"]) + elif previous["status"] == "settled": + result = previous["receipt"] + else: + result = adapter.recover(operation, {"kind": "verify", "binding": binding}) + if result is None: + raise ValueError("Root execution unknown; recover its receipt without relaunch") + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + if ( + result["cleanup_verified"] is not True + or result["verdict"] != "PASS" + or control.read(campaign)["blocked"] + ): + raise ValueError("Root lacks complete admissible real evidence") + attempt = tree.Attempt( + operation, + operation, + operation, + operation, + contract.search_id, + root.candidate_id, + root.artifact_digest, + check, + contract.context, + root.dependencies, + ) + receipts.append( + tree.Receipt(attempt, Verdict.PASS, result["evidence_digest"], result.get("objective")) + ) + state = tree.initialize(contract, root, receipts, dependencies=root.dependencies) + doc = control.read(campaign) + control.update(campaign, doc["revision"], lambda body: body | {"qualified_root": encode(state)}) + return state + + +def import_qualified_root(source, contract): + """Explicit root-only evidence import under identical artifact/check/context scope. + + No new physical execution is asserted: execution/reservation/lease keys and + evidence digests retain their original identity. The receiving attempt is a + scoped import recorded in the campaign; arbitrary child evidence is not reused. + """ + if ( + source.contract.context != contract.context + or source.contract.checks != contract.checks + or source.contract.objective_check != contract.objective_check + or source.contract.direction != contract.direction + ): + raise ValueError("Root evidence reuse requires an identical verification contract") + root = tree.candidate(source, source.root_id) + observed = tree.evidence(source, source.root_id) + if len(observed) != len(contract.checks) or any( + r.verdict != Verdict.PASS for r in observed.values() + ): + raise ValueError("Cannot import invalid root evidence") + receipts = [ + replace( + record, + attempt=replace( + record.attempt, + search_id=contract.search_id, + attempt_id=f"{contract.search_id}-import-{check}", + ), + ) + for check, record in observed.items() + ] + return tree.initialize(contract, root, receipts, dependencies=source.dependencies) + + +def collect_group( + control, campaign, adapter, root, snapshot, settings, *, group_id, size, seed=0, collect=True +): + if type(size) is not int or size < 2: + raise ValueError("A group needs two or more independent searches") + doc = control.read(campaign) + outstanding = sum( + max( + 0, + settings.search_ms + - doc["state"]["searches"].get(f"{group_id}-{i}", {}).get("spent_ms", 0), + ) + for i in range(size) + if doc["state"]["searches"].get(f"{group_id}-{i}", {}).get("status") != "complete" + ) + if outstanding > min(control.available(doc), (doc["deadline"] - time.time()) * 1000): + raise ValueError("Group cannot provide identical search budgets within the campaign cap") + result = [] + for index in range(size): + search_id = f"{group_id}-{index}" + contract = replace( + root.contract, search_id=search_id, improver_snapshot=digest(snapshot.document()) + ) + initial = import_qualified_root(root, contract) + search = Search(control, campaign, search_id, snapshot, settings, adapter) + doc = control.read(campaign) + if search_id not in doc["state"]["searches"]: + search.start(initial, seed=seed + index, collect=collect) + else: + _, retained, retained_tree = search.read() + if ( + retained["seed"] != seed + index + or retained["collect"] != collect + or retained_tree.contract != contract + or tree.candidate(retained_tree, retained_tree.root_id) + != tree.candidate(initial, initial.root_id) + ): + raise ValueError("Resumed group has different frozen starting conditions") + result.append(search.run()) + return tuple(result) + + +def learn( + control, + campaign, + adapter, + root, + snapshot, + settings, + *, + rounds, + group_size, + training_cap_ms, + learning_rate, + seed=0, + mode="reset", + **training, +): + if type(rounds) is not int or rounds < 1 or mode not in ("reset", "carry"): + raise ValueError("Frozen positive outer rounds and Reset/Carry mode are required") + reference = snapshot + for index in range(rounds): + records = collect_group( + control, + campaign, + adapter, + root, + snapshot, + settings, + group_id=f"round-{index}", + size=group_size, + seed=seed + index * group_size, + ) + ids = [decode(r["tree"]).contract.search_id for r in records] + doc = control.read(campaign) + retained = [ + op + for op in doc["operations"].values() + if op["category"] == "training" and op["binding"]["searches"] == ids + ] + if retained: + if len(retained) != 1 or retained[0]["status"] != "settled": + raise ValueError("Unknown training outcome; no second update is permitted") + op = retained[0] + expected = { + "learning_rate": learning_rate, + "expense_weight": 0.0, + "elapsed_weight": 0.0, + **training, + } + if ( + op["binding"]["snapshot"] != digest(snapshot.document()) + or op["binding"]["reference"] != digest(reference.document()) + or digest(op["binding"]["training"]) != digest(expected) + ): + raise ValueError("Frozen outer-loop training configuration changed") + snapshot = Snapshot(**op["receipt"]["checkpoint"]) + else: + snapshot = train_group( + control, + campaign, + records, + snapshot, + reference, + cap_ms=training_cap_ms, + learning_rate=learning_rate, + **training, + ) + if mode == "carry": + best = min( + (decode(record["tree"]) for record in records), + key=lambda state: (-tree.verified_gain(state), state.contract.search_id), + ) + selected = tree.accept(best) + # Carry the complete selected artifact, retaining its actual receipts. + selected = replace(selected, parent_id=None) + root = tree.initialize( + best.contract, + selected, + tree.evidence(best, selected.candidate_id).values(), + dependencies=best.dependencies, + ) + return snapshot + + +def finalize(control, campaign, adapter, selected, *, holdout_scope, accept_result): + """Freeze once; a failed/unknown holdout never reopens search or changes selection. + + accept_result is a fixed independent final gate. Only this function records + stable promotion; search acceptance and trained checkpoints do not promote. + """ + doc = control.read(campaign) + if doc["final_result"] is not None: + return doc["final_result"] + required_scope = getattr(adapter.evaluator, "holdout_scope", None) + if required_scope is not None and holdout_scope != required_scope: + raise ValueError("Final scope must match the evaluator's frozen holdout data identity") + binding = { + "candidate": asdict(selected), + "adapter": adapter.identity, + "checks": adapter.checks, + "scope": holdout_scope, + } + recovering = doc["final_claim"] is not None + if recovering: + if digest(doc["final_claim"]["binding"]) != digest(binding): + raise ValueError("Cannot change the final candidate or validation scope") + else: + control.claim_final(campaign, doc["revision"], holdout_scope, binding) + results = [] + for check, cap in adapter.checks.items(): + doc = control.read(campaign) + operation = "final-" + digest({"campaign": campaign, "binding": binding, "check": check}) + if recovering: + op = doc["operations"].get(operation) + if op is None: + raise ValueError("Interrupted holdout may be consumed; no new execution") + if op["status"] == "settled": + result = op["receipt"] + else: + result = adapter.recover( + operation, + {"kind": "verify", "binding": {"candidate": asdict(selected), "check": check}}, + ) + if result is None: + raise ValueError("Unknown final execution; recover its receipt without rerun") + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + else: + control.claim( + campaign, + doc["revision"], + operation, + category="final", + cap_ms=cap, + domains=(adapter.domain,), + binding=binding | {"check": check}, + final=True, + ) + claim = control.launch(campaign, operation) + result = adapter.verify(selected, check, operation, claim["deadline"]) + control.settle( + campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + ) + results.append(result) + doc = control.read(campaign) + unresolved = doc["blocked"] or any( + op["status"] != "settled" for op in doc["operations"].values() + ) + passed = ( + not unresolved + and all(r["verdict"] == "PASS" and r["cleanup_verified"] is True for r in results) + and accept_result(results) + ) + result = { + "candidate": selected.candidate_id, + "verdict": "PASS" if passed else "FAIL", + "results": results, + "promoted": passed, + } + if unresolved or any( + r["verdict"] == "INCONCLUSIVE" or r["cleanup_verified"] is not True for r in results + ): + result.update(verdict="INCONCLUSIVE", promoted=False) + control.finish_final( + campaign, + binding, + result, + transition=(lambda body: body | {"stable": asdict(selected)}) if passed else None, + ) + return result diff --git a/src/inferrsi/harness/search_continuations.py b/src/inferrsi/harness/search_continuations.py new file mode 100644 index 00000000..665243f7 --- /dev/null +++ b/src/inferrsi/harness/search_continuations.py @@ -0,0 +1,155 @@ +"""Paid-state continuation sampling, isolated from natural GVO actor groups.""" + +import copy +import math +import time +from dataclasses import asdict, replace + +from .artifact_primitives import digest +from .search_records import decode, encode +from .vats import Action, Search + + +def continue_from(search, step_index, action_key, search_id, *, copy_cap_ms, seed): + """Clone a retained paid decision state and force exactly its first action. + + Original observations are explicit imports with their physical execution IDs. + Copying is charged; all suffix edits/checks use ordinary WorkControl claims. + No holdout or unobserved outcome is used. The entire suffix is Scorer-only. + """ + doc, source, _ = search.read() + if doc["final_claim"] is not None or source["status"] != "complete": + raise ValueError("Continuation source must be a completed development search") + if ( + source["forced"] + or type(step_index) is not int + or not 0 <= step_index < len(source["trace"]) + ): + raise ValueError("Select a retained natural paid decision state") + step = source["trace"][step_index] + actions = [Action(**raw) for raw in step["menu"]] + selected = next((i for i, action in enumerate(actions) if action.key == action_key), None) + if selected is None: + raise ValueError("Forced action is outside the original paid menu") + action = actions[selected] + fork = Search( + search.control, search.campaign, search_id, search.snapshot, search.settings, search.adapter + ) + if search_id in doc["state"]["searches"]: + _, retained, _ = fork.read() + if retained.get("continuation") != { + "source": search.search_id, + "step": step_index, + "action": action_key, + "paid_state": digest(step["paid_state"]), + "seed": seed, + }: + raise ValueError("Continuation identity already belongs to another paid state") + return fork.run() + operation = "copy-" + search_id + search.control.claim( + search.campaign, + doc["revision"], + operation, + category="continuation_copy", + cap_ms=copy_cap_ms, + binding={"search": search_id, "source": search.search_id, "step": step_index}, + ) + search.control.launch(search.campaign, operation) + started = time.monotonic() + current = decode(copy.deepcopy(step["paid_state"])) + # Change the receiving scope, retaining immutable physical source identities. + imports = { + attempt.attempt_id: replace( + attempt, search_id=search_id, attempt_id=f"{search_id}-import-{attempt.attempt_id}" + ) + for attempt in current.attempts + } + current = replace( + current, + contract=replace(current.contract, search_id=search_id), + attempts=tuple(imports[a.attempt_id] for a in current.attempts), + receipts=tuple(replace(r, attempt=imports[r.attempt.attempt_id]) for r in current.receipts), + ) + record = copy.deepcopy(source) + forced_step = copy.deepcopy(step) + now = time.time() + forced_step.update( + action=asdict(action), + features=step["vectors"][selected], + forced=True, + draft_choices=[], + at=now, + ) + record.update( + tree=encode(current), + seed=seed, + forced=True, + trace=[forced_step], + status="active", + pending=None, + spent_ms=step["spent_before"], + started_at=now, + elapsed_ms=0, + draft_attempts=step["draft_attempts"], + initial_gain=step["gain_before"], + initial_spent_ms=step["spent_before"], + continuation={ + "source": search.search_id, + "step": step_index, + "action": action_key, + "paid_state": digest(step["paid_state"]), + "seed": seed, + }, + active_action=None + if action.kind == "accept" + else { + "action": asdict(action), + "cursor": 0, + "last_child": None, + "jobs": step["jobs"], + }, + ) + record.pop("gain", None) + record.pop("stop_reason", None) + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def commit(state): + if search_id in state["searches"]: + raise ValueError("Continuation search was concurrently created") + state["searches"][search_id] = record + return state + + search.control.settle( + search.campaign, + operation, + {"record": record}, + actual_ms=elapsed, + cleanup_verified=True, + transition=commit, + ) + if action.kind == "accept": + return fork.finish("forced_accept") + return fork.run() + + +def paired_targets(records, *, expense_weight, elapsed_weight): + """Produce differences only for two settled continuations of the same paid state.""" + from .gvo import suffix_examples + + if len(records) != 2: + raise ValueError("A matched comparison needs exactly two continuations") + left, right = records + keys = ("source", "step", "paid_state", "seed") + if ( + not all(r["forced"] and r.get("continuation") for r in records) + or any(left["continuation"][k] != right["continuation"][k] for k in keys) + or any(left[k] != right[k] for k in ("snapshot", "settings", "adapter")) + or left["continuation"]["action"] == right["continuation"]["action"] + ): + raise ValueError("Scorer pair must share a paid state and frozen continuation policy") + first = [ + suffix_examples(r, expense_weight=expense_weight, elapsed_weight=elapsed_weight)[0] + for r in records + ] + return (first[0][0], first[1][0], first[0][1] - first[1][1]) diff --git a/src/inferrsi/harness/search_records.py b/src/inferrsi/harness/search_records.py new file mode 100644 index 00000000..8377a358 --- /dev/null +++ b/src/inferrsi/harness/search_records.py @@ -0,0 +1,60 @@ +"""Explicit JSON encoding of immutable search controls, without executable payloads.""" + +from dataclasses import asdict + +from inferrsi.contracts.claim import Verdict + +from . import search_tree as tree + + +def encode(state): + return asdict(state) + + +def decode(raw): + def dependencies(values): + return tuple(tree.Dependency(**value) for value in values) + + def attempt(value): + return tree.Attempt( + **( + value + | { + "context": tree.Context(**value["context"]), + "dependencies": dependencies(value["dependencies"]), + } + ) + ) + + contract = tree.Contract( + **(raw["contract"] | {"context": tree.Context(**raw["contract"]["context"])}) + ) + nodes = tuple( + tree.Candidate( + **( + value + | { + "producer": tree.Producer(**value["producer"]), + "dependencies": dependencies(value["dependencies"]), + } + ) + ) + for value in raw["candidates"] + ) + return tree.SearchState( + contract, + raw["root_id"], + nodes, + tuple(attempt(value) for value in raw["attempts"]), + tuple( + tree.Receipt( + attempt(value["attempt"]), + Verdict(value["verdict"]), + value["evidence_digest"], + value["objective"], + ) + for value in raw["receipts"] + ), + dependencies(raw["dependencies"]), + raw["revision"], + ) diff --git a/src/inferrsi/harness/search_tree.py b/src/inferrsi/harness/search_tree.py index d6623835..e2fa8fe4 100644 --- a/src/inferrsi/harness/search_tree.py +++ b/src/inferrsi/harness/search_tree.py @@ -256,6 +256,8 @@ def verified_gain(state): check = state.contract.objective_check gain = (evidence(state, selected.candidate_id)[check].objective - evidence(state, state.root_id)[check].objective) + if not isfinite(gain): + raise ValueError("Verified gain exceeds the finite objective domain") return gain if state.contract.direction == "maximize" else -gain diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py new file mode 100644 index 00000000..bf79022f --- /dev/null +++ b/src/inferrsi/harness/vats.py @@ -0,0 +1,659 @@ +"""Bounded synchronous VATS over immutable candidates and injected real operations. + +The controller owns search transitions; WorkControl owns persistent claims and +accounting; the adapter owns permitted edits and authenticated real observations. +No value prediction is admitted as evidence. There is no background re-dispatch. +""" + +import math +import random +import time +import uuid +from dataclasses import asdict, dataclass, replace +from itertools import combinations + +from inferrsi.model.search_roles import choose, dot + +from . import search_tree as tree +from .artifact_primitives import digest +from .search_records import decode, encode +from .verification_plans import completion_menu + + +@dataclass(frozen=True) +class Settings: + max_steps: int + max_parents: int + draft_depth: int + max_targets: int + max_jobs: int + max_plans: int + preparation_ms: int + draft_ms: int + search_ms: int + refine_queries: int = 0 + temperature: float = 1.0 + exploration: float = 0.1 + draft_ahead: bool = True + bundled: bool = True + + def __post_init__(self): + for key in ( + "max_steps", + "max_parents", + "draft_depth", + "max_targets", + "max_jobs", + "max_plans", + "preparation_ms", + "draft_ms", + "search_ms", + ): + if type(getattr(self, key)) is not int or getattr(self, key) <= 0: + raise ValueError("VATS bounds must be positive integers") + if type(self.refine_queries) is not int or self.refine_queries < 0: + raise ValueError("Invalid Scorer query cap") + if not math.isfinite(self.temperature) or self.temperature <= 0: + raise ValueError("Invalid policy temperature") + if not 0 <= self.exploration < 1: + raise ValueError("Invalid policy exploration") + + +@dataclass(frozen=True) +class Action: + kind: str + parents: tuple[str, ...] = () + depth: int = 0 + jobs: tuple[str, ...] = () + cost_ms: int = 0 + + @property + def key(self): + return digest(asdict(self)) + + +def depth(state, node): + result = 0 + while node.parent_id is not None: + result += 1 + node = tree.candidate(state, node.parent_id) + return result + + +def features(state, action, available, settings): + """Fixed public features; ACCEPT has a fixed zero Policy logit.""" + if action.kind == "accept": + return (0.0,) * 8 + return ( + 1.0, + float(action.kind == "draft"), + float(action.kind == "verify"), + action.cost_ms / settings.search_ms, + len(state.candidates) / state.contract.max_candidates, + available / settings.search_ms, + action.depth / state.contract.max_depth, + len(action.jobs) / settings.max_jobs, + ) + + +def menu(state, jobs, available, settings, snapshot): + """Preserve singleton completion seeds before optionally refining their unions.""" + available = min(available, settings.search_ms) + plans = completion_menu( + state, + jobs, + available_budget=max(0, available), + available_jobs=settings.max_jobs, + max_targets=settings.max_targets, + max_jobs=settings.max_jobs, + ) + singleton_plans = completion_menu( + state, + jobs, + available_budget=max(0, available), + available_jobs=settings.max_jobs, + max_targets=1, + max_jobs=settings.max_jobs, + ) + seeds = [Action("verify", jobs=plan.job_ids, cost_ms=plan.cost) for plan in singleton_plans] + unions = [Action("verify", jobs=plan.job_ids, cost_ms=plan.cost) for plan in plans] + allowed = {frozenset(a.jobs): a for a in unions} + # Refine priority among feasible bounded unions without evicting completion + # seeds or bypassing Solver's frozen target/job/cost/attempt constraints. + by_job = {job.job_id: job for job in jobs} + queries = 0 + extra = [] + for left, right in combinations(seeds, 2): + if queries + 2 > settings.refine_queries: + break + union = tuple(dict.fromkeys((*left.jobs, *right.jobs))) + if frozenset(union) not in allowed: + continue + cost = sum(by_job[jid].cost for jid in union) + attempts = sum(by_job[jid].candidate_id is not None for jid in union) + if ( + cost > available + or len(union) > settings.max_jobs + or attempts + len(state.attempts) > state.contract.max_attempts + or any(set(by_job[jid].conflicts) & set(union) for jid in union) + ): + continue + trial = allowed[frozenset(union)] + queries += 2 + if dot(snapshot.scorer, features(state, trial, available, settings)) > dot( + snapshot.scorer, features(state, left, available, settings) + ): + extra.append(trial) + if not settings.bundled: + # Ablation: individual executable jobs, explicitly retaining incomplete evidence. + seeds = [ + Action("verify", jobs=(job.job_id,), cost_ms=job.cost) + for job in jobs + if not job.prerequisites + and job.cost <= available + and job.candidate_id is not None + and job.check_id in tree.missing_checks(state, job.candidate_id) + ] + extra = [] + unions = [] + parents = [ + node + for node in state.candidates + if tree.candidate_valid(state, node.candidate_id) + and (settings.draft_ahead or node in tree.complete_candidates(state)) + ] + drafts = [] + slots = state.contract.max_candidates - len(state.candidates) + for count in range(1, min(len(parents), settings.max_parents) + 1): + for selected in combinations(parents, count): + for height in range(1, settings.draft_depth + 1): + cost = count * height * settings.draft_ms + if ( + cost <= available + and count * height <= slots + and all( + depth(state, node) + height <= state.contract.max_depth for node in selected + ) + ): + drafts.append( + Action( + "draft", + tuple(node.candidate_id for node in selected), + height, + cost_ms=cost, + ) + ) + if len(drafts) >= settings.max_plans: + break + if len(drafts) >= settings.max_plans: + break + if len(drafts) >= settings.max_plans: + break + required = [Action("accept"), *seeds] + if len(required) > settings.max_plans: + raise ValueError("Menu cap cannot preserve all feasible completion seeds") + unique = {action.key: action for action in required} + for action in (*extra, *drafts, *unions): + if len(unique) >= settings.max_plans: + break + unique[action.key] = action + return tuple(unique.values()) + + +class Search: + """A resumable search bound to one immutable role snapshot and adapter contract. + + Adapter methods: jobs(tree), draft(parent, choice_context, snapshot, rng, + destination, deadline), verify(node, check, destination, deadline), recover(...). + Draft returns (Candidate, Choice|None, receipt); verify returns Receipt facts. + Both return authenticated elapsed_ms and cleanup_verified in their receipt. + """ + + def __init__(self, control, campaign, search_id, snapshot, settings, adapter): + self.control, self.campaign, self.search_id = control, campaign, search_id + self.snapshot, self.settings, self.adapter = snapshot, settings, adapter + self.snapshot_id = digest(snapshot.document()) + self.settings_id = digest(asdict(settings)) + + def read(self): + document = self.control.read(self.campaign) + search = document["state"]["searches"][self.search_id] + if ( + search["snapshot"] != self.snapshot_id + or search["settings"] != self.settings_id + or search["adapter"] != self.adapter.identity + ): + raise ValueError("Frozen search roles, menu or adapter changed") + return document, search, decode(search["tree"]) + + def start(self, initial, *, seed, collect=True, forced=False): + if ( + initial.contract.search_id != self.search_id + or initial.contract.improver_snapshot != self.snapshot_id + ): + raise ValueError("Search root must bind the frozen search and improver") + tree.accept(initial) + if not forced and len(initial.candidates) != 1: + raise ValueError("Independent natural searches must start from one qualified root") + doc = self.control.read(self.campaign) + + def transition(state): + if self.search_id in state["searches"]: + raise ValueError("Search already exists; resume it") + state["searches"][self.search_id] = { + "tree": encode(initial), + "snapshot": self.snapshot_id, + "settings": self.settings_id, + "adapter": self.adapter.identity, + "seed": seed, + "collect": collect, + "forced": forced, + "trace": [], + "status": "active", + "pending": None, + "spent_ms": 0, + "started_at": time.time(), + "elapsed_ms": 0, + "draft_attempts": 0, + "active_action": None, + "initial_gain": tree.verified_gain(initial), + "selected": initial.root_id, + } + return state + + self.control.update(self.campaign, doc["revision"], transition) + + def _change(self, state, operation, result): + search = state["searches"][self.search_id] + pending = search["pending"] + if pending is None or pending["operation"] != operation: + raise ValueError("Late result does not match the active work identity") + current = decode(search["tree"]) + if pending["kind"] == "draft": + search["draft_attempts"] += 1 + if result.get("candidate") is not None: + raw = result["candidate"] + node = tree.Candidate( + **( + raw + | { + "producer": tree.Producer(**raw["producer"]), + "dependencies": tuple( + tree.Dependency(**d) for d in raw["dependencies"] + ), + } + ) + ) + try: + tree.accept(current) + root_valid = True + except ValueError: + root_valid = False + if root_valid and tree.candidate_valid(current, node.parent_id): + current = tree.add_candidate(current, node, expected_revision=current.revision) + else: + search.setdefault("rejected_drafts", []).append(operation) + else: + bound = next(a for a in current.attempts if a.attempt_id == operation) + from inferrsi.contracts.claim import Verdict + + receipt = tree.Receipt( + bound, + Verdict(result["verdict"]), + result["evidence_digest"], + result.get("objective"), + ) + current = tree.record_receipt(current, receipt, expected_revision=current.revision) + search["tree"] = encode(current) + if tree.complete_candidates(current) and tree.evidence(current, current.root_id): + try: + search["selected"] = tree.accept(current).candidate_id + except ValueError: + search.update(status="blocked", selected=None) + else: + search.update(status="blocked", selected=None) + search["spent_ms"] += result["elapsed_ms"] + search["pending"] = None + active = search["active_action"] + if active is not None: + active["cursor"] += 1 + if pending["kind"] == "draft": + active["last_child"] = ( + result["candidate"]["candidate_id"] + if result.get("candidate") + and any( + n.candidate_id == result["candidate"]["candidate_id"] + for n in current.candidates + ) + else None + ) + if result.get("choice") is not None: + search["trace"][-1]["draft_choices"].append(result["choice"]) + return state + + def invalidate(self, dependency): + """Persist invalidation and incumbent replacement before any cancellation. + + In-flight work remains charged to its original claim; late receipts can + still be archived by recover(), but can never revive invalid evidence. + """ + doc, _, _ = self.read() + + def transition(state): + record = state["searches"][self.search_id] + current = decode(record["tree"]) + current = tree.invalidate(current, dependency, expected_revision=current.revision) + record["tree"] = encode(current) + try: + record["selected"] = tree.accept(current).candidate_id + except ValueError: + record.update(status="blocked", selected=None) + record.setdefault("invalidations", []).append(dependency) + return state + + self.control.update(self.campaign, doc["revision"], transition) + + def recover(self): + _, search, _ = self.read() + pending = search["pending"] + if pending is None: + return + # Receipt-only recovery. Never call draft/verify from this path. + result = self.adapter.recover(pending["operation"], pending) + if result is None: + raise ValueError("Unknown execution: retain reservation and do not relaunch") + self.control.settle( + self.campaign, + pending["operation"], + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + transition=lambda state: self._change(state, pending["operation"], result), + ) + + def _execute(self, kind, node, check=None): + doc, search, current = self.read() + operation = self.search_id + "-" + uuid.uuid4().hex + cap = self.settings.draft_ms if kind == "draft" else check.cost + binding = { + "search": self.search_id, + "candidate": asdict(node), + "check": check.check_id if check else None, + "snapshot": self.snapshot_id, + "adapter": self.adapter.identity, + } + + def pending(state): + search = state["searches"][self.search_id] + current = decode(search["tree"]) + if search["pending"] is not None: + raise ValueError("An operation is already pending") + if kind == "verify": + attempt = tree.Attempt( + operation, + operation, + operation, + operation, + self.search_id, + node.candidate_id, + node.artifact_digest, + check.check_id, + current.contract.context, + node.dependencies, + ) + current = tree.register_attempt( + current, attempt, expected_revision=current.revision + ) + search["tree"] = encode(current) + search["pending"] = {"operation": operation, "kind": kind, "binding": binding} + return state + + self.control.claim( + self.campaign, + doc["revision"], + operation, + category=kind, + cap_ms=cap, + binding=binding, + domains=self.adapter.domains(kind, node, check), + transition=pending, + ) + claim = self.control.launch(self.campaign, operation) + # Any exception retains the claim. The adapter's authenticated receipt is + # the only recovery route, including when writing the state below fails. + if kind == "draft": + rng = random.Random( + f"{search['seed']}:{len(current.candidates)}:{len(search['trace'])}" + ) + result = self.adapter.draft( + node, current, self.snapshot, rng, operation, claim["deadline"] + ) + else: + result = self.adapter.verify(node, check.check_id, operation, claim["deadline"]) + self.control.settle( + self.campaign, + operation, + result, + actual_ms=result.get("elapsed_ms"), + cleanup_verified=result.get("cleanup_verified"), + transition=lambda state: self._change(state, operation, result), + ) + _, updated, _ = self.read() + if updated["pending"] is not None: + raise ValueError("Execution or cleanup remains unknown") + return result + + def run(self, *, forced_action=None): + self.recover() + while True: + doc, search, current = self.read() + if search["status"] == "complete": + return search + if search["status"] == "blocked": + raise ValueError("Search root evidence is invalid; search remains blocked") + if doc["blocked"] or doc["final_claim"] is not None: + raise ValueError("Campaign blocked or final selection frozen") + if any( + op["status"] != "settled" and op["binding"].get("search") == self.search_id + for op in doc["operations"].values() + ): + raise ValueError("Unresolved claim: recover without relaunch") + if search["active_action"] is not None: + self._continue_action() + continue + remaining = min( + self.control.available(doc), self.settings.search_ms - search["spent_ms"] + ) + attempts_left = ( + current.contract.max_attempts - len(current.attempts) - search["draft_attempts"] + ) + if ( + len(search["trace"]) >= self.settings.max_steps + or attempts_left <= 0 + or remaining <= self.settings.preparation_ms + or time.time() >= doc["deadline"] + ): + return self.finish("limit") + op = self.search_id + "-prepare-" + uuid.uuid4().hex + self.control.claim( + self.campaign, + doc["revision"], + op, + category="preparation", + cap_ms=self.settings.preparation_ms, + binding={"search": self.search_id}, + ) + claim = self.control.launch(self.campaign, op) + started = time.monotonic() + jobs = self.adapter.jobs(current) + planning_state = replace( + current, + contract=replace( + current.contract, + max_attempts=current.contract.max_attempts - search["draft_attempts"], + ), + ) + actions = menu( + planning_state, + jobs, + remaining - self.settings.preparation_ms, + self.settings, + self.snapshot, + ) + actions = tuple( + a for a in actions if a.kind != "draft" or len(a.parents) * a.depth <= attempts_left + ) + vectors = tuple( + features(current, action, remaining, self.settings) for action in actions + ) + rng = random.Random(f"{search['seed']}:policy:{len(search['trace'])}") + choice = choose( + self.snapshot, + "policy", + [action.key for action in actions], + vectors, + rng=rng, + context=digest(encode(current)), + collect=search["collect"], + temperature=self.settings.temperature, + exploration=self.settings.exploration if search["collect"] else 0, + ) + selected = choice.selected + forced = forced_action is not None and not search["trace"] + if forced: + if not search["forced"]: + raise ValueError("Forced continuation must be isolated from actor training") + selected = next( + i for i, action in enumerate(actions) if action.key == forced_action + ) + action = actions[selected] + elapsed = max(1, math.ceil((time.monotonic() - started) * 1000)) + + def prepared(state): + record = state["searches"][self.search_id] + record["spent_ms"] += elapsed + record["trace"].append( + { + "action": asdict(action), + "choice": asdict(choice), + "forced": forced, + "features": vectors[selected], + "draft_choices": [], + "gain_before": tree.verified_gain(current), + "spent_before": record["spent_ms"], + "at": time.time(), + "paid_state": encode(current), + "menu": [asdict(a) for a in actions], + "vectors": vectors, + "jobs": [asdict(job) for job in jobs], + "draft_attempts": record["draft_attempts"], + } + ) + if action.kind != "accept": + record["active_action"] = { + "action": asdict(action), + "cursor": 0, + "last_child": None, + "jobs": [asdict(job) for job in jobs], + } + return state + + self.control.settle( + self.campaign, + op, + {"menu": [asdict(a) for a in actions], "elapsed_ms": elapsed}, + actual_ms=elapsed, + cleanup_verified=True, + transition=prepared, + ) + if time.time() >= claim["deadline"]: + raise ValueError("Preparation deadline exceeded") + if action.kind == "accept": + return self.finish("accept") + + def _continue_action(self): + from .verification_plans import Job + + while True: + doc, search, current = self.read() + active = search["active_action"] + action = Action(**active["action"]) + cursor = active["cursor"] + limit = ( + len(action.parents) * action.depth if action.kind == "draft" else len(action.jobs) + ) + if cursor >= limit: + + def done(state): + state["searches"][self.search_id]["active_action"] = None + return state + + self.control.update(self.campaign, doc["revision"], done) + return + next_cap = ( + self.settings.draft_ms + if action.kind == "draft" + else next( + job["cost"] for job in active["jobs"] if job["job_id"] == action.jobs[cursor] + ) + ) + if ( + time.time() >= doc["deadline"] + or next_cap > self.control.available(doc) + or next_cap > self.settings.search_ms - search["spent_ms"] + ): + + def exhausted(state): + record = state["searches"][self.search_id] + record["active_action"] = None + record["block_stop_reason"] = "budget_or_deadline" + return state + + self.control.update(self.campaign, doc["revision"], exhausted) + return + if action.kind == "draft": + branch, level = divmod(cursor, action.depth) + if level and active["last_child"] is None: + + def skip(state): + state["searches"][self.search_id]["active_action"]["cursor"] = ( + branch + 1 + ) * action.depth + return state + + self.control.update(self.campaign, doc["revision"], skip) + continue + parent = active["last_child"] if level else action.parents[branch] + self._execute("draft", tree.candidate(current, parent)) + else: + by_id = {raw["job_id"]: Job(**raw) for raw in active["jobs"]} + job = by_id[action.jobs[cursor]] + if job.candidate_id is None: + raise ValueError("Adapter must package shared setup into its real check") + self._execute("verify", tree.candidate(current, job.candidate_id), job) + + def finish(self, reason): + doc, search, current = self.read() + if search["pending"] is not None: + raise ValueError("Cannot accept unsettled work") + if search["active_action"] is not None: + raise ValueError("Cannot accept an unfinished bounded work block") + if any( + op["status"] != "settled" and op["binding"].get("search") == self.search_id + for op in doc["operations"].values() + ): + raise ValueError("Cannot accept an unresolved claim") + + def transition(state): + record = state["searches"][self.search_id] + record.update( + status="complete", + stop_reason=reason, + selected=tree.accept(current).candidate_id, + gain=tree.verified_gain(current), + elapsed_ms=max(1, math.ceil((time.time() - record["started_at"]) * 1000)), + ) + return state + + self.control.update(self.campaign, doc["revision"], transition) + return self.read()[1] diff --git a/src/inferrsi/harness/work_control.py b/src/inferrsi/harness/work_control.py new file mode 100644 index 00000000..3a0993c4 --- /dev/null +++ b/src/inferrsi/harness/work_control.py @@ -0,0 +1,278 @@ +"""Campaign accounting and CAS in the shared RunStore database. + +This is a controller for trusted host operations, not an executor or sandbox. +All cooperating controllers must share the same RunStore for resource domains +and holdout scopes to be exclusive. Unknown work retains its entire reservation. +""" + +import json +import math +import time +from contextlib import contextmanager + +from .artifact_primitives import digest +from .run_store import _transaction + + +def _json(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _positive(value): + if type(value) is not int or value <= 0: + raise ValueError("Work limits must be positive integer milliseconds") + + +class WorkControl: + def __init__(self, store): + self.store = store + if store._read_only: + return + with store._connection() as db, _transaction(db): + db.execute( + "CREATE TABLE IF NOT EXISTS work_campaigns " + "(id TEXT PRIMARY KEY, revision INTEGER NOT NULL, body TEXT NOT NULL)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS work_domains " + "(id TEXT PRIMARY KEY, campaign TEXT NOT NULL, operation TEXT NOT NULL)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS work_holdouts " + "(scope TEXT PRIMARY KEY, campaign TEXT NOT NULL, binding TEXT NOT NULL)" + ) + + @contextmanager + def _transaction(self, campaign, revision=None): + with self.store._connection() as db, _transaction(db): + row = db.execute( + "SELECT revision, body FROM work_campaigns WHERE id=?", (campaign,) + ).fetchone() + if row is None: + raise ValueError("Unknown campaign") + if revision is not None and row[0] != revision: + raise ValueError("Stale campaign revision") + body = json.loads(row[1]) + yield db, body + db.execute( + "UPDATE work_campaigns SET revision=?, body=? WHERE id=?", + (row[0] + 1, _json(body), campaign), + ) + + def create(self, campaign, contract, state, *, cap_ms, final_reserve_ms, deadline): + _positive(cap_ms) + _positive(final_reserve_ms) + if ( + not campaign + or final_reserve_ms >= cap_ms + or not math.isfinite(deadline) + or deadline <= time.time() + ): + raise ValueError("Invalid campaign budget or deadline") + body = { + "contract": contract, + "contract_digest": digest(contract), + "state": state, + "cap_ms": cap_ms, + "final_reserve_ms": final_reserve_ms, + "deadline": deadline, + "spent_ms": 0, + "operations": {}, + "events": [], + "final_claim": None, + "final_result": None, + "blocked": False, + } + with self.store._connection() as db, _transaction(db): + db.execute("INSERT INTO work_campaigns VALUES (?, 0, ?)", (campaign, _json(body))) + return self.read(campaign) + + def read(self, campaign): + with self.store._connection() as db: + row = db.execute( + "SELECT revision, body FROM work_campaigns WHERE id=?", (campaign,) + ).fetchone() + if row is None: + raise ValueError("Unknown campaign") + body = json.loads(row[1]) + if digest(body["contract"]) != body["contract_digest"]: + raise ValueError("Frozen campaign contract changed") + return {"revision": row[0], **body} + + @staticmethod + def available(body, *, final=False): + held = sum(op["cap_ms"] for op in body["operations"].values() if op["status"] != "settled") + return body["cap_ms"] - body["spent_ms"] - held - (0 if final else body["final_reserve_ms"]) + + def update(self, campaign, revision, transition): + """Persist a pure state transition and incumbent recomputation in one CAS.""" + with self._transaction(campaign, revision) as (_, body): + if body["blocked"] or body["final_claim"] is not None: + raise ValueError("Campaign transitions are blocked or final selection is frozen") + body["state"] = transition(body["state"]) + return self.read(campaign) + + def claim( + self, + campaign, + revision, + operation, + *, + category, + cap_ms, + domains=(), + binding, + final=False, + transition=None, + ): + _positive(cap_ms) + domains = tuple(sorted(set(domains))) + if not operation or not category or any(not item for item in domains): + raise ValueError("Missing operation or resource identity") + if final != (category == "final"): + raise ValueError("Final reservations require the final work category") + with self._transaction(campaign, revision) as (db, body): + if operation in body["operations"]: + raise ValueError("Operation already claimed; recover without relaunch") + if body["blocked"] or (body["final_claim"] is not None and not final): + raise ValueError("Campaign is blocked or final selection is frozen") + if final and body["final_claim"] is None: + raise ValueError("Final work requires a durable one-time claim") + if time.time() >= body["deadline"] or cap_ms > self.available(body, final=final): + raise ValueError("Work exceeds remaining budget or deadline") + for domain in domains: + if db.execute("SELECT 1 FROM work_domains WHERE id=?", (domain,)).fetchone(): + raise ValueError("Measurement domain is already leased") + db.execute( + "INSERT INTO work_domains VALUES (?, ?, ?)", (domain, campaign, operation) + ) + body["operations"][operation] = { + "category": category, + "binding": binding, + "cap_ms": cap_ms, + "domains": domains, + "status": "claimed", + "receipt": None, + "deadline": min(body["deadline"], time.time() + cap_ms / 1000), + "execution_key": operation, + } + body["events"].append({"kind": "claimed", "operation": operation, "at": time.time()}) + if transition is not None: + body["state"] = transition(body["state"]) + return self.read(campaign) + + def launch(self, campaign, operation): + """Persist intent before external launch. A crash here must never relaunch.""" + expired = False + with self._transaction(campaign) as (db, body): + op = body["operations"][operation] + if op["status"] != "claimed": + raise ValueError("Execution intent already recorded") + expired = time.time() >= op["deadline"] + if expired: + op.update(status="settled", actual_ms=0, receipt={"not_started": "deadline"}) + db.execute( + "DELETE FROM work_domains WHERE campaign=? AND operation=?", + (campaign, operation), + ) + else: + op.update(status="launched", started_at=time.time()) + body["events"].append( + { + "kind": "not_started" if expired else "launch_intent", + "operation": operation, + "at": time.time(), + } + ) + if expired: + raise ValueError("Deadline expired before launch") + return self.read(campaign)["operations"][operation] + + def settle(self, campaign, operation, receipt, *, actual_ms, cleanup_verified, transition=None): + """Only authenticated, owner-scoped receipts may be passed by the adapter. + + Receipt replay is idempotent. Evidence and accounting commit together. + Unknown usage/cleanup does not release the domain or budget reservation. + """ + if actual_ms is not None and (type(actual_ms) is not int or actual_ms < 0): + raise ValueError("Usage must be nonnegative integer milliseconds or unknown") + with self._transaction(campaign) as (db, body): + op = body["operations"][operation] + settlement = { + "receipt": receipt, + "actual_ms": actual_ms, + "cleanup_verified": cleanup_verified, + } + if op["status"] == "settled": + if _json(op.get("settlement")) != _json(settlement): + raise ValueError("Conflicting duplicate settlement") + return + if op["status"] not in ("launched", "unknown"): + raise ValueError("Cannot settle an unlaunched operation") + op["settlement"] = settlement + if actual_ms is None or cleanup_verified is not True: + op["status"] = "unknown" + body["events"].append( + {"kind": "unknown", "operation": operation, "at": time.time()} + ) + else: + op.update(status="settled", actual_ms=actual_ms, receipt=receipt) + body["spent_ms"] += actual_ms + executable_ms = min(op["cap_ms"], (op["deadline"] - op["started_at"]) * 1000) + if actual_ms > executable_ms or body["spent_ms"] > body["cap_ms"]: + body["blocked"] = True + if transition is not None and not body["blocked"]: + body["state"] = transition(body["state"]) + db.execute( + "DELETE FROM work_domains WHERE campaign=? AND operation=?", + (campaign, operation), + ) + body["events"].extend( + [ + {"kind": "cleanup_confirmed", "operation": operation, "at": time.time()}, + {"kind": "released", "operation": operation, "at": time.time()}, + ] + ) + return self.read(campaign) + + def claim_final(self, campaign, revision, scope, binding): + """Consume a shared holdout scope before any access, including interrupted access.""" + if not isinstance(scope, str) or not scope.strip(): + raise ValueError("Holdout scope must have a nonempty identity") + with self._transaction(campaign, revision) as (db, body): + if ( + body["final_claim"] is not None + or body["blocked"] + or any(op["status"] != "settled" for op in body["operations"].values()) + ): + raise ValueError("Final is already claimed or work remains unresolved") + if time.time() >= body["deadline"]: + raise ValueError("Final deadline exhausted") + if db.execute("SELECT 1 FROM work_holdouts WHERE scope=?", (scope,)).fetchone(): + raise ValueError("Holdout scope already consumed in this RunStore") + db.execute( + "INSERT INTO work_holdouts VALUES (?, ?, ?)", (scope, campaign, _json(binding)) + ) + body["final_claim"] = {"scope": scope, "binding": binding} + return self.read(campaign) + + def finish_final(self, campaign, binding, result, *, transition=None): + with self._transaction(campaign) as (_, body): + if body["final_claim"] is None or _json(body["final_claim"]["binding"]) != _json( + binding + ): + raise ValueError("Final result does not match the frozen selection") + if body["final_result"] is not None: + if _json(body["final_result"]) != _json(result): + raise ValueError("Final result is immutable; export the existing result") + return + if transition is not None and ( + body["blocked"] + or any(op["status"] != "settled" for op in body["operations"].values()) + or result.get("verdict") != "PASS" + ): + raise ValueError("Unresolved or over-budget work cannot promote a candidate") + body["final_result"] = result + if transition is not None: + body["state"] = transition(body["state"]) + return self.read(campaign) diff --git a/src/inferrsi/model/search_roles.py b/src/inferrsi/model/search_roles.py new file mode 100644 index 00000000..3e5df28c --- /dev/null +++ b/src/inferrsi/model/search_roles.py @@ -0,0 +1,215 @@ +"""Trainable finite-edit Drafter, independent Policy and linear continuation Scorer. + +The reference CPU model samples tokens from a frozen vocabulary of permitted edit +operators. It is not a language-model fine-tuner. Hosts may supply another role +implementation, but must preserve observed contexts, masks and behavior laws. +""" + +from dataclasses import asdict, dataclass, replace +from math import exp, fsum, isfinite, log + + +def finite(value): + if type(value) not in (float, int) or not isfinite(value): + raise ValueError("Expected a finite numeric observation") + return value + + +def dot(left, right): + if len(left) != len(right) or not left: + raise ValueError("Role feature dimension mismatch") + return finite(fsum(finite(a) * finite(b) for a, b in zip(left, right))) + + +def distribution(weights, features, temperature=1.0, exploration=0.0): + if finite(temperature) <= 0 or not 0 <= finite(exploration) < 1 or not features: + raise ValueError("Invalid sampling law") + logits = [dot(weights, row) / temperature for row in features] + scale = max(logits) + raw = [exp(value - scale) for value in logits] + total = fsum(raw) + soft = tuple(value / total for value in raw) + probabilities = tuple((1 - exploration) * p + exploration / len(soft) for p in soft) + if any(p <= 0 for p in probabilities): + raise ValueError("Sampling law underflows; rescale model or increase exploration") + return probabilities, soft + + +@dataclass(frozen=True) +class Snapshot: + drafter: tuple[float, ...] + policy: tuple[float, ...] + scorer: tuple[float, ...] + origin: str + + def __post_init__(self): + for name in ("drafter", "policy", "scorer"): + values = tuple(getattr(self, name)) + if not values: + raise ValueError("Empty role parameters") + for value in values: + finite(value) + object.__setattr__(self, name, values) + if not self.origin: + raise ValueError("Checkpoint provenance is required") + + def document(self): + return { + **asdict(self), + "drafter": list(self.drafter), + "policy": list(self.policy), + "scorer": list(self.scorer), + } + + +@dataclass(frozen=True) +class Choice: + role: str + options: tuple[str, ...] + features: tuple[tuple[float, ...], ...] + selected: int + probabilities: tuple[float, ...] + temperature: float + exploration: float + context: str + sampled: bool = True + + def __post_init__(self): + if ( + self.role not in ("drafter", "policy") + or not self.context + or len(set(self.options)) != len(self.options) + or not 0 <= self.selected < len(self.options) + or len(self.options) != len(self.features) + or len(self.options) != len(self.probabilities) + or abs(fsum(self.probabilities) - 1) > 1e-10 + or any(not 0 < finite(p) <= 1 for p in self.probabilities) + ): + raise ValueError("Invalid recorded role choice") + + +def choose( + snapshot, role, options, features, *, rng, context, collect, temperature=1.0, exploration=0.0 +): + probabilities, _ = distribution(getattr(snapshot, role), features, temperature, exploration) + if collect or role == "drafter": + selected = rng.choices(range(len(options)), weights=probabilities, k=1)[0] + else: + selected = max(range(len(options)), key=lambda index: probabilities[index]) + return Choice( + role, + tuple(options), + tuple(tuple(row) for row in features), + selected, + probabilities, + temperature, + exploration, + context, + sampled=collect or role == "drafter", + ) + + +def actor_gradient(weights, reference, choice, advantage, clip, kl_weight): + """Gradient ascent for the clipped surrogate minus KL to a frozen reference.""" + probabilities, soft = distribution( + weights, choice.features, choice.temperature, choice.exploration + ) + refs, _ = distribution(reference, choice.features, choice.temperature, choice.exploration) + ratio = finite(probabilities[choice.selected] / choice.probabilities[choice.selected]) + active = not ((advantage > 0 and ratio > 1 + clip) or (advantage < 0 and ratio < 1 - clip)) + means = [fsum(p * row[d] for p, row in zip(soft, choice.features)) for d in range(len(weights))] + jacobian = [ + tuple( + (1 - choice.exploration) * p * (row[d] - means[d]) / choice.temperature + for d in range(len(weights)) + ) + for p, row in zip(soft, choice.features) + ] + return tuple( + finite( + ( + advantage / choice.probabilities[choice.selected] * jacobian[choice.selected][d] + if active + else 0 + ) + - kl_weight + * fsum( + jacobian[i][d] * (log(probabilities[i] / refs[i]) + 1) + for i in range(len(probabilities)) + ) + ) + for d in range(len(weights)) + ) + + +def update( + snapshot, + reference, + actor_examples, + value_examples, + pair_examples=(), + *, + learning_rate, + clip=0.2, + kl_weight=0.01, + pair_weight=1.0, + epochs=1, + groups=1, +): + """Update actual parameters; forced trajectories belong only in value/pair examples. + + Actor examples are (Choice, whole-search advantage), value examples are + (features, realized suffix return), pairs are (left features, right features, + paid observed return difference). Missing targets are rejected, never imputed. + """ + if ( + finite(learning_rate) <= 0 + or not 0 < finite(clip) < 1 + or finite(kl_weight) < 0 + or finite(pair_weight) < 0 + or type(epochs) is not int + or epochs < 1 + or type(groups) is not int + or groups < 1 + ): + raise ValueError("Invalid frozen training settings") + actors, values, pairs = tuple(actor_examples), tuple(value_examples), tuple(pair_examples) + for choice, advantage in actors: + finite(advantage) + if not choice.sampled: + raise ValueError("Deterministic actions cannot enter the actor objective") + replay, _ = distribution( + getattr(snapshot, choice.role), choice.features, choice.temperature, choice.exploration + ) + if any(abs(a - b) > 1e-10 for a, b in zip(replay, choice.probabilities)): + raise ValueError("Behavior probabilities do not match the frozen checkpoint") + current = snapshot + for _ in range(epochs): + changed = {} + for role in ("drafter", "policy"): + weights = getattr(current, role) + gradients = [ + actor_gradient( + weights, getattr(reference, role), choice, advantage, clip, kl_weight + ) + for choice, advantage in actors + if choice.role == role + ] + changed[role] = tuple( + finite(w + learning_rate * fsum(g[d] for g in gradients) / groups) + for d, w in enumerate(weights) + ) + weights = current.scorer + gradient = [0.0] * len(weights) + for features, target in values: + error = dot(weights, features) - finite(target) + for d in range(len(weights)): + gradient[d] += 2 * error * features[d] / len(values) + for left, right, target in pairs: + delta = tuple(a - b for a, b in zip(left, right)) + error = dot(weights, delta) - finite(target) + for d in range(len(weights)): + gradient[d] += pair_weight * 2 * error * delta[d] / len(pairs) + changed["scorer"] = tuple(finite(w - learning_rate * g) for w, g in zip(weights, gradient)) + current = replace(current, **changed) + return current diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py new file mode 100644 index 00000000..5cc1ba6b --- /dev/null +++ b/tests/test_search_benchmarks.py @@ -0,0 +1,115 @@ +import json +import time +from types import SimpleNamespace + +import pytest + +from inferrsi.apps.cli.arguments import parser +from inferrsi.apps.search_benchmarks import EngineEvaluator, SkillEvaluator +from inferrsi.apps.speculative_search import command +from inferrsi.benchmark.improvement import SelectionPolicy + + +def skill_report(base_tokens=100, tokens=80, *, verdict="PASS", outcome="unchanged"): + def trial(count, result): + return { + "verdict": result, + "elapsed_seconds": 1, + "observations": { + "usage": None + if count is None + else {"input_tokens": count, "output_tokens": 0, "cached_input_tokens": 0}, + "tool_calls": 0, + }, + } + + return { + "status": {"execution_state": "completed"}, + "pairs": [ + { + "baseline": trial(base_tokens, "PASS"), + "candidate": trial(tokens, verdict), + "outcome": outcome, + } + ], + } + + +@pytest.mark.parametrize( + "report,split,verdict", + [ + (skill_report(), "development", "PASS"), + (skill_report(tokens=1, verdict="FAIL", outcome="regression"), "development", "FAIL"), + (skill_report(base_tokens=None), "development", "INCONCLUSIVE"), + (skill_report(tokens=None), "development", "INCONCLUSIVE"), + (skill_report(tokens=101), "holdout", "FAIL"), + (skill_report(tokens=100), "holdout", "PASS"), + ], +) +def test_skill_quality_unknown_usage_and_independent_holdout_gates(report, split, verdict): + comparator = SimpleNamespace(identity="explicit-comparator", run=lambda *a, **k: (report, True)) + evaluator = SkillEvaluator( + comparator, split=split, policy=SelectionPolicy(1, 0), token_cap=1000, task_count=1 + ) + result = evaluator.evaluate(None, "evaluation", None, time.time() + 1) + assert result["verdict"] == verdict + assert (result["objective"] is not None) == (verdict == "PASS") + + +def test_engine_uses_frozen_complete_task_protocol_and_rejects_mutation(tmp_path): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", + tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + assert evaluator.plan["evaluation_settings"] == {"pairs": 1, "samples": 1, "warmups": 0} + (evaluator.frozen / "reference.py").write_text("changed reference") + with pytest.raises(ValueError, match="resources changed"): + evaluator.evaluate(tmp_path, "evaluation", tmp_path / "output", time.time() + 1) + assert not (tmp_path / "output").exists() + + +def test_cli_requires_explicit_shared_store_and_read_commands_do_not_create_one(tmp_path): + args = parser().parse_args(["search", "status", "--session", str(tmp_path / "absent")]) + with pytest.raises(FileNotFoundError): + command(args) + assert not (tmp_path / "absent").exists() + with pytest.raises(SystemExit): + parser().parse_args(["search", "engine", "--session", str(tmp_path / "run")]) + + +def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): + from tests.fixtures.harness_improvement_demo import configuration + from tools.speculative_skill import Comparator + + config, repo = configuration(tmp_path) + comparison = {k: config[k] for k in ("skill", "skill_root", "model", "limits")} + comparison.update( + mode="update", + client={"explicit": "preparation-only-test"}, + snapshots={"baseline": {"git_revision": config["baseline"]}}, + ) + def descriptor(split): + return { + "path": str(tmp_path / (split + ".json")), + "sha256": split, + "scope": split, + "task_count": 3, + } + bound = Comparator( + repo, + { + "comparison": comparison, + "allowed_paths": config["allowed_paths"], + "splits": {s: descriptor(s) for s in ("development", "holdout")}, + }, + tmp_path / "baseline-snapshot", + ) + exported = bound.export_root(tmp_path / "candidate") + assert (exported / "selected/SKILL.md").is_file() + assert not (tmp_path / "holdout.json").exists() + assert not (tmp_path / "development.json").exists() + # The prepared adapter still fails before provider access if its split is missing. + with pytest.raises(FileNotFoundError): + bound.run(exported, tmp_path / "run", time.time() + 1, split="development") + json.dumps(bound.config) diff --git a/tests/test_search_roles.py b/tests/test_search_roles.py new file mode 100644 index 00000000..3378e7be --- /dev/null +++ b/tests/test_search_roles.py @@ -0,0 +1,71 @@ +import random + +import pytest + +from inferrsi.model.search_roles import Snapshot, actor_gradient, choose, distribution, update + + +def test_actor_gradient_matches_finite_difference_with_exploration_and_kl(): + model = Snapshot((0.2, -0.1), (0.3, -0.2), (0.0, 0.0), "initial") + choice = choose( + model, + "policy", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(2), + context="observed", + collect=True, + exploration=0.1, + ) + reference = (0.0, 0.0) + gradient = actor_gradient(model.policy, reference, choice, 2.0, 0.2, 0.1) + from math import log + + def objective(weights): + p, _ = distribution(weights, choice.features, exploration=0.1) + q, _ = distribution(reference, choice.features, exploration=0.1) + return 2 * p[choice.selected] / choice.probabilities[choice.selected] - 0.1 * sum( + a * log(a / b) for a, b in zip(p, q) + ) + + for i in range(2): + plus, minus = list(model.policy), list(model.policy) + plus[i] += 1e-6 + minus[i] -= 1e-6 + assert gradient[i] == pytest.approx((objective(plus) - objective(minus)) / 2e-6) + + +def test_actual_updates_keep_policy_scorer_independent_and_reject_false_behavior(): + model = Snapshot((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), "initial") + choice = choose( + model, + "drafter", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(1), + context="observed", + collect=True, + ) + trained = update(model, model, [(choice, 2.0)], [((1.0, 0.0), 4.0)], learning_rate=0.1) + assert trained.drafter != model.drafter + assert trained.scorer != model.scorer + assert trained.policy == model.policy + with pytest.raises(ValueError, match="Behavior"): + update(trained, model, [(choice, 2.0)], [], learning_rate=0.1) + with pytest.raises(ValueError, match="finite"): + update(model, model, [], [((1.0, 0.0), None)], learning_rate=0.1) + + +def test_clipped_actor_does_not_reinforce_outside_positive_ratio_bound(): + model = Snapshot((0.0, 0.0), (0.0, 0.0), (0.0, 0.0), "initial") + choice = choose( + model, + "policy", + ("a", "b"), + ((1.0, 0.0), (0.0, 1.0)), + rng=random.Random(1), + context="observed", + collect=True, + ) + weights = (10.0, -10.0) if choice.selected == 0 else (-10.0, 10.0) + assert actor_gradient(weights, model.policy, choice, 1.0, 0.2, 0.0) == (0.0, 0.0) diff --git a/tests/test_speculative_search.py b/tests/test_speculative_search.py index 997e95f5..ad686fbf 100644 --- a/tests/test_speculative_search.py +++ b/tests/test_speculative_search.py @@ -63,6 +63,14 @@ def test_objective_prediction_partial_results_and_failed_quality_cannot_win(): assert tree.verified_gain(state) == 0 +@pytest.mark.parametrize("direction,sign", [("maximize", 1), ("minimize", -1)]) +def test_finite_objectives_cannot_publish_an_infinite_gain(direction, sign): + state = start(configuration=contract(direction=direction), score=-sign * 1e308) + state = complete(add(state, "candidate"), "candidate", sign * 1e308) + with pytest.raises(ValueError, match="finite objective"): + tree.verified_gain(state) + + @pytest.mark.parametrize("direction", ["maximize", "minimize"]) @pytest.mark.parametrize("order", [("z", "a"), ("a", "z")]) def test_selection_is_independent_of_receipt_order_and_root_wins_ties(direction, order): diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py new file mode 100644 index 00000000..bd3a793f --- /dev/null +++ b/tests/test_vats_runtime.py @@ -0,0 +1,310 @@ +import json +import sys +import time +from dataclasses import asdict, replace + +import pytest + +from inferrsi.harness import search_tree as tree +from inferrsi.harness.artifact_primitives import digest +from inferrsi.harness.gvo import examples, train_group +from inferrsi.harness.run_store import RunStore +from inferrsi.harness.search_adapter import ArtifactAdapter, CommandEvaluator, Edit +from inferrsi.harness.search_campaign import collect_group, finalize, qualify_root +from inferrsi.harness.search_records import decode +from inferrsi.harness.vats import Search, Settings +from inferrsi.harness.work_control import WorkControl +from inferrsi.model.search_roles import Snapshot +from tests.fixtures.speculative_search import contract + + +@pytest.fixture +def runtime(tmp_path): + grader = tmp_path / "grader.py" + grader.write_text("""import json, pathlib, sys +value = int((pathlib.Path(sys.argv[1]) / "candidate.txt").read_text()) +check = sys.argv[2] +out = {"verdict": "PASS" if value > 0 else "FAIL"} +if value > 0 and check == "objective": + out["objective"] = value +pathlib.Path(sys.argv[3]).write_text(json.dumps(out)) +""") + source = tmp_path / "baseline" + source.mkdir() + (source / "candidate.txt").write_text("1") + evaluator = CommandEvaluator( + { + check: (sys.executable, str(grader), "{candidate}", check, "{result}") + for check in ("quality", "objective") + }, + [grader], + ) + adapter = ArtifactAdapter( + tmp_path / "artifacts", + ( + Edit("improve", "candidate.txt", "1", "3"), + Edit("break", "candidate.txt", "1", "-1"), + Edit("repair", "candidate.txt", "-1", "4"), + ), + {"quality": 3000, "objective": 3000}, + evaluator, + domain="cpu-tests", + ) + snapshot = Snapshot( + (1.0, 0.0, 0.0), + (-1.0, 3.0, 6.0, 0.0, 0.0, 0.0, 0.0, 0.0), + (0.0,) * 8, + "test-initial-checkpoint", + ) + settings = Settings(5, 1, 2, 2, 40, 50, 3000, 3000, 40000) + control = WorkControl(RunStore(tmp_path / "store", "search-test")) + control.create( + "campaign", + {"adapter": adapter.identity, "settings": asdict(settings)}, + {"searches": {}, "checkpoint": snapshot.document()}, + cap_ms=120000, + final_reserve_ms=12000, + deadline=time.time() + 120, + ) + root = adapter.import_root(source, snapshot=digest(snapshot.document()), context={}) + initial = qualify_root( + control, "campaign", adapter, root, contract(improver_snapshot=digest(snapshot.document())) + ) + return control, adapter, snapshot, settings, initial + + +def test_real_process_search_group_training_checkpoint_and_fresh_search(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="first", size=2, seed=12 + ) + assert all(record["status"] == "complete" for record in records) + assert all(tree.accept(decode(record["tree"])) for record in records) + assert any(len(record["tree"]["candidates"]) > 1 for record in records) + actors, values, group = examples(records) + assert actors and values and len(group["advantages"]) == 2 + trained = train_group( + control, "campaign", records, snapshot, snapshot, cap_ms=5000, learning_rate=0.01 + ) + assert trained.document() == control.read("campaign")["state"]["checkpoint"] + assert trained.scorer != snapshot.scorer + next_records = collect_group( + control, "campaign", adapter, root, trained, settings, group_id="next", size=2, seed=42 + ) + assert all(record["snapshot"] == digest(trained.document()) for record in next_records) + assert control.read("campaign")["spent_ms"] > 0 + + +def test_final_access_is_frozen_and_export_retry_does_not_execute(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="first", size=2, seed=4 + ) + selected = tree.accept(decode(records[0]["tree"])) + result = finalize( + control, + "campaign", + adapter, + selected, + holdout_scope="test-final", + accept_result=lambda results: all(r["verdict"] == "PASS" for r in results), + ) + before = len(list((adapter.root / "executions").iterdir())) + assert ( + finalize( + control, + "campaign", + adapter, + selected, + holdout_scope="test-final", + accept_result=lambda results: False, + ) + == result + ) + assert len(list((adapter.root / "executions").iterdir())) == before + assert control.read("campaign")["state"]["stable"]["candidate_id"] == selected.candidate_id + + +def test_resume_after_receipt_before_state_commit_never_executes_twice(runtime, monkeypatch): + control, adapter, snapshot, settings, root = runtime + search = Search(control, "campaign", root.contract.search_id, snapshot, settings, adapter) + search.start(root, seed=2, collect=False) + original = control.settle + failed = False + + def crash(campaign, operation, receipt, **kwargs): + nonlocal failed + if receipt.get("candidate") and not failed: + failed = True + raise RuntimeError("crash before commit") + return original(campaign, operation, receipt, **kwargs) + + monkeypatch.setattr(control, "settle", crash) + with pytest.raises(RuntimeError, match="crash"): + search.run() + assert search.read()[1]["pending"] is not None + monkeypatch.setattr(control, "settle", original) + before = len(list((adapter.root / "candidates").iterdir())) + search.recover() + assert search.read()[1]["pending"] is None + assert len(list((adapter.root / "candidates").iterdir())) == before + + +def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="group", size=2, seed=2 + ) + with pytest.raises(ValueError, match="distinct"): + examples((records[0], records[0])) + with pytest.raises(ValueError, match="Forced"): + examples((records[0] | {"forced": True}, records[1])) + with pytest.raises(ValueError, match="identical"): + examples((records[0] | {"settings": "other"}, records[1])) + + +def test_snapshot_change_on_resume_is_rejected(runtime): + control, adapter, snapshot, settings, root = runtime + Search(control, "campaign", "search", snapshot, settings, adapter).start(root, seed=0) + changed = replace(snapshot, policy=(2.0,) * 8) + with pytest.raises(ValueError, match="Frozen"): + Search(control, "campaign", "search", changed, settings, adapter).run() + + +def test_persisted_search_has_only_real_check_results(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="group", size=2, seed=5 + ) + serialized = json.dumps(control.read("campaign"), allow_nan=False) + assert serialized + for record in records: + state = decode(record["tree"]) + for node in tree.complete_candidates(state): + assert all( + r.verdict.value == "PASS" for r in tree.evidence(state, node.candidate_id).values() + ) + + +def test_paid_continuations_are_metered_matched_and_excluded_from_actors(runtime): + from inferrsi.harness.search_continuations import continue_from, paired_targets + from inferrsi.harness.vats import Action + + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="natural", size=2, seed=12 + ) + source = Search(control, "campaign", "natural-0", snapshot, settings, adapter) + menu = [Action(**a) for a in records[0]["trace"][0]["menu"]] + accept = next(a for a in menu if a.kind == "accept") + draft = next(a for a in menu if a.kind == "draft") + before = control.read("campaign")["spent_ms"] + suffixes = [ + continue_from(source, 0, a.key, f"suffix-{i}", copy_cap_ms=3000, seed=91) + for i, a in enumerate((accept, draft)) + ] + assert control.read("campaign")["spent_ms"] > before + assert all(s["forced"] and s["status"] == "complete" for s in suffixes) + pair = paired_targets(suffixes, expense_weight=0.1, elapsed_weight=0) + assert len(pair) == 3 + assert any( + op["category"] == "continuation_copy" + for op in control.read("campaign")["operations"].values() + ) + with pytest.raises(ValueError, match="Forced"): + examples(suffixes) + trained = train_group( + control, + "campaign", + records, + snapshot, + snapshot, + cap_ms=5000, + learning_rate=0.01, + scorer_continuations=suffixes, + scorer_pairs=(suffixes,), + ) + assert trained.scorer != snapshot.scorer + + +def test_root_invalidation_is_persisted_and_cannot_return_an_invalid_root(runtime): + control, adapter, snapshot, settings, root = runtime + dependency = tree.Dependency("environment", 0) + nodes = tuple(replace(n, dependencies=(dependency,)) for n in root.candidates) + attempts = tuple(replace(a, dependencies=(dependency,)) for a in root.attempts) + by_id = {a.attempt_id: a for a in attempts} + receipts = tuple(replace(r, attempt=by_id[r.attempt.attempt_id]) for r in root.receipts) + initial = replace( + root, candidates=nodes, attempts=attempts, receipts=receipts, dependencies=(dependency,) + ) + search = Search(control, "campaign", "search", snapshot, settings, adapter) + search.start(initial, seed=1) + search.invalidate("environment") + assert search.read()[1]["selected"] is None + assert search.read()[1]["status"] == "blocked" + with pytest.raises(ValueError, match="root evidence"): + search.run() + + +def test_late_draft_after_root_receipt_invalidation_is_charged_without_admission( + runtime, monkeypatch +): + control, adapter, snapshot, settings, root = runtime + dependency = tree.Dependency("root-proof", 0) + attempts = tuple(replace(a, dependencies=(dependency,)) for a in root.attempts) + by_id = {a.attempt_id: a for a in attempts} + initial = replace( + root, + attempts=attempts, + dependencies=(dependency,), + receipts=tuple(replace(r, attempt=by_id[r.attempt.attempt_id]) for r in root.receipts), + ) + search = Search(control, "campaign", "search", snapshot, settings, adapter) + search.start(initial, seed=2, collect=False) + original = control.settle + injected = False + + def invalidate_first(campaign, operation, receipt, **kwargs): + nonlocal injected + if receipt.get("candidate") and not injected: + injected = True + search.invalidate("root-proof") + return original(campaign, operation, receipt, **kwargs) + + monkeypatch.setattr(control, "settle", invalidate_first) + with pytest.raises(ValueError, match="root evidence"): + search.run() + assert injected + doc, record, state = search.read() + assert record["pending"] is None and record["selected"] is None + assert len(state.candidates) == 1 + assert all(op["status"] == "settled" for op in doc["operations"].values()) + assert record["rejected_drafts"] and record["spent_ms"] > 0 + + +def test_training_rejects_unretained_search_labels(runtime): + control, adapter, snapshot, settings, root = runtime + records = collect_group( + control, "campaign", adapter, root, snapshot, settings, group_id="natural", size=2, seed=12 + ) + altered = (records[0] | {"gain": 999}, records[1]) + with pytest.raises(ValueError, match="retained"): + train_group( + control, "campaign", altered, snapshot, snapshot, cap_ms=5000, learning_rate=0.01 + ) + + +@pytest.mark.parametrize("mode", ["reset", "carry"]) +def test_outer_loop_resume_reuses_trained_checkpoints_and_frozen_roots(runtime, mode): + from inferrsi.harness.search_campaign import learn + + control, adapter, snapshot, settings, root = runtime + settings = replace(settings, search_ms=10000, max_steps=3) + args = dict(rounds=2, group_size=2, training_cap_ms=5000, learning_rate=0.01, seed=8, mode=mode) + trained = learn(control, "campaign", adapter, root, snapshot, settings, **args) + before = control.read("campaign") + resumed = learn(control, "campaign", adapter, root, snapshot, settings, **args) + assert resumed == trained + assert control.read("campaign") == before + with pytest.raises(ValueError, match="starting conditions"): + learn(control, "campaign", adapter, root, snapshot, settings, **(args | {"seed": 9})) diff --git a/tests/test_work_control.py b/tests/test_work_control.py new file mode 100644 index 00000000..4e768423 --- /dev/null +++ b/tests/test_work_control.py @@ -0,0 +1,152 @@ +import time +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from inferrsi.harness.run_store import RunStore +from inferrsi.harness.work_control import WorkControl + + +@pytest.fixture +def control(tmp_path): + result = WorkControl(RunStore(tmp_path / "store", "controls")) + result.create( + "campaign", + {"fixed": "contract"}, + {"incumbent": "root"}, + cap_ms=1000, + final_reserve_ms=100, + deadline=time.time() + 60, + ) + return result + + +def claim(control, op="work", **changes): + args = dict(category="verify", cap_ms=200, domains=("device",), binding={"candidate": "c"}) + return control.claim("campaign", control.read("campaign")["revision"], op, **(args | changes)) + + +def test_atomic_resource_and_budget_reservation_across_controllers(control): + other = WorkControl(control.store) + other.create("other", {}, {}, cap_ms=1000, final_reserve_ms=100, deadline=time.time() + 60) + with ThreadPoolExecutor(2) as pool: + + def reserve(name): + try: + control.claim( + name, 0, name, category="verify", cap_ms=200, domains=("device",), binding={} + ) + return True + except ValueError: + return False + + assert sum(pool.map(reserve, ("campaign", "other"))) == 1 + + +def test_unknown_cleanup_or_usage_retains_reservation_and_never_relaunches(control): + claim(control) + control.launch("campaign", "work") + control.settle( + "campaign", "work", {"result": "unknown"}, actual_ms=None, cleanup_verified=False + ) + assert control.available(control.read("campaign")) == 700 + with pytest.raises(ValueError, match="already claimed"): + claim(control) + with pytest.raises(ValueError, match="leased"): + claim(control, "another") + reopened = WorkControl(RunStore(control.store.root, control.store.namespace)) + assert reopened.read("campaign")["operations"]["work"]["status"] == "unknown" + + +def test_receipt_settlement_and_incumbent_are_atomic_and_idempotent(control): + claim(control) + control.launch("campaign", "work") + + def update(state): + return state | {"incumbent": "child"} + + result = control.settle( + "campaign", + "work", + {"proof": "bound"}, + actual_ms=50, + cleanup_verified=True, + transition=update, + ) + assert result["state"]["incumbent"] == "child" + control.settle("campaign", "work", {"proof": "bound"}, actual_ms=50, cleanup_verified=True) + assert control.read("campaign")["spent_ms"] == 50 + with pytest.raises(ValueError, match="Conflicting"): + control.settle("campaign", "work", {}, actual_ms=50, cleanup_verified=True) + claim(control, "next") + + +def test_expired_claim_cannot_start_and_does_not_charge_unstarted_work(control, monkeypatch): + claim(control) + monkeypatch.setattr("inferrsi.harness.work_control.time.time", lambda: 1e12) + with pytest.raises(ValueError, match="expired"): + control.launch("campaign", "work") + result = control.read("campaign") + assert result["operations"]["work"]["status"] == "settled" + assert result["spent_ms"] == 0 + + +def test_cas_and_failed_transition_roll_back_accounting(control): + before = control.read("campaign") + control.update("campaign", before["revision"], lambda state: state | {"x": 1}) + with pytest.raises(ValueError, match="Stale"): + control.update("campaign", before["revision"], lambda state: {}) + claim(control) + control.launch("campaign", "work") + + def fail(state): + raise ValueError("invalid receipt") + + with pytest.raises(ValueError, match="invalid receipt"): + control.settle("campaign", "work", {}, actual_ms=20, cleanup_verified=True, transition=fail) + assert control.read("campaign")["spent_ms"] == 0 + + +def test_holdout_is_once_across_campaigns_even_after_interruption(control): + before = control.read("campaign") + control.claim_final("campaign", before["revision"], "heldout-data-v1", {"candidate": "c"}) + control.create("second", {}, {}, cap_ms=1000, final_reserve_ms=100, deadline=time.time() + 60) + with pytest.raises(ValueError, match="consumed"): + control.claim_final("second", 0, "heldout-data-v1", {"candidate": "other"}) + with pytest.raises(ValueError, match="frozen"): + claim(control) + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "FAIL"}) + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "FAIL"}) + with pytest.raises(ValueError, match="immutable"): + control.finish_final("campaign", {"candidate": "c"}, {"verdict": "PASS"}) + + +def test_final_reserve_and_unknown_work_cannot_be_spent(control): + with pytest.raises(ValueError, match="budget"): + claim(control, cap_ms=901) + claim(control) + with pytest.raises(ValueError, match="unresolved"): + control.claim_final("campaign", control.read("campaign")["revision"], "holdout", {}) + + +def test_over_budget_final_is_accounted_but_never_promoted(control): + control.claim_final("campaign", 0, "holdout", {"candidate": "c"}) + claim(control, final=True, category="final") + control.launch("campaign", "work") + control.settle("campaign", "work", {"verdict": "PASS"}, actual_ms=201, cleanup_verified=True) + assert control.read("campaign")["spent_ms"] == 201 + assert control.read("campaign")["blocked"] + with pytest.raises(ValueError, match="cannot promote"): + control.finish_final( + "campaign", + {"candidate": "c"}, + {"verdict": "PASS"}, + transition=lambda state: state | {"incumbent": "c"}, + ) + + +def test_read_only_control_does_not_create_or_change_tables(control): + from inferrsi.harness.run_store import RunStoreReader + + reader = WorkControl(RunStoreReader(control.store.root, control.store.namespace)) + assert reader.read("campaign") == control.read("campaign") diff --git a/tools/speculative_skill.py b/tools/speculative_skill.py new file mode 100644 index 00000000..548d5573 --- /dev/null +++ b/tools/speculative_skill.py @@ -0,0 +1,165 @@ +"""Explicit bounded-provider Skill comparator for the speculative-search Python API. + +This checkout tool composes the existing comparison authority. It never selects +a model/client implicitly, and does not turn fixed-output fixtures into live runs. +""" + +import argparse +import copy +import json +import os +import sys +import time +from pathlib import Path + +from inferrsi.harness import candidates +from inferrsi.harness.artifact_primitives import digest, sha256 +from inferrsi.harness.atomic_json import atomic_write_json +from inferrsi.harness.optimization_session import runtime_environment +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.knowledge.knowledge_operations import KnowledgeReader +from tools import skill_eval_compare as compare +from tools.skill_eval_improve import materialize + + +class Comparator: + """Bind a repo, original baseline and two disjoint externally declared splits. + + config contains comparison (the existing comparator config, including client), + allowed_paths, and splits. Each split has path, sha256, scope and task_count. + Holdout bytes are only opened by run(split='holdout'), after the caller's + WorkControl final claim. The controller must use that declared scope. + """ + + def __init__(self, repo, config, baseline_source): + self.repo = Path(repo).resolve() + self.config = copy.deepcopy(config) + if set(config) != {"comparison", "allowed_paths", "splits"}: + raise ValueError("Unknown or missing Skill comparator configuration") + comparison = config["comparison"] + if "client" not in comparison or comparison["mode"] != "update": + raise ValueError("Skill search requires an explicit bounded provider update config") + if set(config["splits"]) != {"development", "holdout"}: + raise ValueError("Separate development and holdout descriptors are required") + dev, final = (config["splits"][key] for key in ("development", "holdout")) + if dev["scope"] == final["scope"] or dev["sha256"] == final["sha256"]: + raise ValueError("Development and holdout must have distinct declared identities") + for split in (dev, final): + if ( + set(split) != {"path", "sha256", "scope", "task_count"} + or type(split["task_count"]) is not int + or split["task_count"] < 1 + or not Path(split["path"]).is_absolute() + ): + raise ValueError("Split requires an absolute source and frozen identity/count") + self.baseline = comparison["snapshots"]["baseline"]["git_revision"] + root = Path(baseline_source) + library = compare.snapshot(self.repo, self.baseline, comparison["skill_root"], root) + bundle = KnowledgeReader(skill_root=library).read("checkout:" + comparison["skill"]) + self.original = dict(bundle.content) + if set(config["allowed_paths"]) - self.original.keys(): + raise ValueError("Allowed edits must belong to the original Skill bundle") + self.identity = digest( + { + "config": config, + "baseline": self.baseline, + "resources": {k: digest(v.hex()) for k, v in self.original.items()}, + "controllers": {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS}, + } + ) + self._config_digest = digest(self.config) + self._controllers = {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS} + self.holdout_scope = "skill-data:" + final["sha256"] + + def export_root(self, destination): + destination = Path(destination) + destination.mkdir(parents=True, exist_ok=False) + for name, value in self.original.items(): + path = destination / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + return destination + + def run(self, candidate, output, deadline, *, split): + if digest(self.config) != self._config_digest or any( + sha256(compare.ROOT / name) != expected for name, expected in self._controllers.items() + ): + raise ValueError("Frozen Skill configuration or controller changed") + files = candidates.members(candidate) + if set(files) != self.original.keys(): + raise ValueError("Skill candidate changed the frozen bundle membership") + edits = { + name: path.read_text(encoding="utf-8") + for name, path in files.items() + if path.read_bytes() != self.original[name] + } + if set(edits) - set(self.config["allowed_paths"]): + raise ValueError("Skill candidate changed a protected resource") + comparison = copy.deepcopy(self.config["comparison"]) + revisions, _ = materialize( + self.repo, + self.baseline, + comparison["skill_root"], + comparison["skill"], + {"inv": edits} if edits else {}, + self.config["allowed_paths"], + ) + descriptor = self.config["splits"][split] + tasks = Path(descriptor["path"]) + if sha256(tasks) != descriptor["sha256"]: + raise ValueError("Frozen Skill split changed") + cases = json.loads(tasks.read_text(encoding="utf-8")) + if len(cases) * comparison["limits"]["repeats"] != descriptor["task_count"]: + raise ValueError("Skill trial count differs from the frozen split") + if any(case.get("split") != split for case in cases): + raise ValueError("Skill task belongs to another declared split") + output.mkdir(parents=True, exist_ok=False) + comparison.update( + cases="tasks.json", + snapshots={ + "baseline": {"git_revision": self.baseline}, + "candidate": {"git_revision": revisions.get("inv", self.baseline)}, + }, + ) + atomic_write_json(output / "tasks.json", cases) + atomic_write_json(output / "config.json", comparison) + prepared = output / "comparison" + authorization = compare.prepare(output / "config.json", self.repo, prepared) + environment = runtime_environment() + environment["PYTHONPATH"] = str(compare.ROOT) + os.pathsep + environment["PYTHONPATH"] + command = ( + sys.executable, + "-m", + "tools.speculative_skill", + "--output", + str(prepared), + "--authorization", + authorization, + ) + record = execute_trial( + command, + compare.ROOT, + output / "execution", + time.monotonic() + max(0, deadline - time.time()), + None, + environment=environment, + include_cleanup=True, + ) + if sha256(prepared / "plan/freeze.json") != authorization: + raise ValueError("Skill comparison plan changed during execution") + report = compare.read_report(prepared) + atomic_write_json(output / "report.json", report) + return report, record["cleanup_verified"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--authorization", required=True) + args = parser.parse_args() + compare.execute(args.output, args.authorization, require_request_authorization=True) + compare.report(args.output) + + +if __name__ == "__main__": + main() From 5ee418121af783bcd62289e23cffa702b7f560c2 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:15:53 +0800 Subject: [PATCH 3/6] harness: retain evaluator identity across search execution Signed-off-by: amy-why-3459 --- src/inferrsi/apps/search_benchmarks.py | 6 ++++++ tests/test_search_benchmarks.py | 21 +++++++++++++++++++++ tools/speculative_skill.py | 11 +++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py index eb7a1172..95b77931 100644 --- a/src/inferrsi/apps/search_benchmarks.py +++ b/src/inferrsi/apps/search_benchmarks.py @@ -73,6 +73,12 @@ def evaluate(self, candidate, check, output, deadline): environment=environment, include_cleanup=True, ) + if { + p.relative_to(self.frozen).as_posix(): sha256(p) + for p in self.frozen.rglob("*") + if p.is_file() + } != self.plan["frozen_files"]: + raise ValueError("Frozen benchmark resources changed during execution") assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) if execution["cleanup_verified"] is not True: assessed = {"verdict": "INCONCLUSIVE", "score": None} diff --git a/tests/test_search_benchmarks.py b/tests/test_search_benchmarks.py index 5cc1ba6b..b2045f5d 100644 --- a/tests/test_search_benchmarks.py +++ b/tests/test_search_benchmarks.py @@ -78,6 +78,25 @@ def test_cli_requires_explicit_shared_store_and_read_commands_do_not_create_one( parser().parse_args(["search", "engine", "--session", str(tmp_path / "run")]) +def test_engine_rejects_frozen_resource_mutation_during_evaluation(tmp_path, monkeypatch): + evaluator = EngineEvaluator( + "qwen21-denoise-npu", + tmp_path / "frozen", + arguments=["--pairs", "1", "--samples", "1", "--warmups", "0"], + ) + source = tmp_path / "candidate" + source.mkdir() + (source / "candidate.py").write_text("unchanged candidate") + + def mutated_execution(*args, **kwargs): + (evaluator.frozen / "reference.py").write_text("changed reference") + return {"status": "completed", "returncode": 0, "cleanup_verified": True} + + monkeypatch.setattr("inferrsi.apps.search_benchmarks.execute_trial", mutated_execution) + with pytest.raises(ValueError, match="resources changed during"): + evaluator.evaluate(source, "evaluation", tmp_path / "output", time.time() + 1) + + def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): from tests.fixtures.harness_improvement_demo import configuration from tools.speculative_skill import Comparator @@ -89,6 +108,7 @@ def test_skill_comparator_preparation_does_not_access_holdout(tmp_path): client={"explicit": "preparation-only-test"}, snapshots={"baseline": {"git_revision": config["baseline"]}}, ) + def descriptor(split): return { "path": str(tmp_path / (split + ".json")), @@ -96,6 +116,7 @@ def descriptor(split): "scope": split, "task_count": 3, } + bound = Comparator( repo, { diff --git a/tools/speculative_skill.py b/tools/speculative_skill.py index 548d5573..09cb18ed 100644 --- a/tools/speculative_skill.py +++ b/tools/speculative_skill.py @@ -59,16 +59,23 @@ def __init__(self, repo, config, baseline_source): self.original = dict(bundle.content) if set(config["allowed_paths"]) - self.original.keys(): raise ValueError("Allowed edits must belong to the original Skill bundle") + self._controllers = { + name: sha256(compare.ROOT / name) + for name in ( + *compare.CONTROLLERS, + "tools/speculative_skill.py", + "tools/skill_eval_improve.py", + ) + } self.identity = digest( { "config": config, "baseline": self.baseline, "resources": {k: digest(v.hex()) for k, v in self.original.items()}, - "controllers": {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS}, + "controllers": self._controllers, } ) self._config_digest = digest(self.config) - self._controllers = {name: sha256(compare.ROOT / name) for name in compare.CONTROLLERS} self.holdout_scope = "skill-data:" + final["sha256"] def export_root(self, destination): From bd2bfc432d76b2af8ddcae8b19b7936ac2cc6b73 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:21:19 +0800 Subject: [PATCH 4/6] harness: bind GVO groups to identical initial observations Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 5 +++-- src/inferrsi/harness/gvo.py | 5 ++++- src/inferrsi/harness/vats.py | 19 +++++++++++++++++++ tests/test_vats_runtime.py | 2 ++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index d7639bb5..b99e2c97 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -148,8 +148,9 @@ External process-liveness reconciliation needs a qualified authority. `finalize` consumes a unique scope in the shared store before final data access, freezes selection and closes search/training. Engine compares again with the -original baseline. For Skill, supply a **holdout-bound adapter** and the declared -holdout scope, never the development adapter. Exact bounded radix utility keeps +original baseline. For Skill, supply a **holdout-bound adapter** and +`Comparator.holdout_scope` (derived from the frozen data digest), never the +development adapter or a replacement label. Exact bounded radix utility keeps quality before token use; reduced tokens cannot offset a quality regression. After interruption, final consumes existing receipts only. Even an unstarted diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py index 10e95096..f485040d 100644 --- a/src/inferrsi/harness/gvo.py +++ b/src/inferrsi/harness/gvo.py @@ -39,6 +39,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): set(), ) returns = [] + initial_conditions = set() for record in records: if record["status"] != "complete" or record["pending"] is not None: raise ValueError("Incomplete searches cannot supply GVO returns") @@ -53,6 +54,7 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): snapshots.add(record["snapshot"]) settings.add(record["settings"]) adapters.add(record["adapter"]) + initial_conditions.add(record["initial_conditions"]) returns.append( finite( record["gain"] @@ -62,7 +64,8 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): ) ) if len(set(identities)) != len(identities) or any( - len(values) != 1 for values in (roots, contracts, snapshots, settings, adapters) + len(values) != 1 + for values in (roots, contracts, snapshots, settings, adapters, initial_conditions) ): raise ValueError("Search group must have distinct IDs and identical starting conditions") if any(record["forced"] for record in records): diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index bf79022f..6ed33bf5 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -235,6 +235,24 @@ def start(self, initial, *, seed, collect=True, forced=False): tree.accept(initial) if not forced and len(initial.candidates) != 1: raise ValueError("Independent natural searches must start from one qualified root") + + def observation(attempt): + return { + key: value + for key, value in asdict(attempt).items() + if key not in ("attempt_id", "search_id") + } + + initial_conditions = digest( + { + "candidates": [asdict(node) for node in initial.candidates], + "dependencies": [asdict(item) for item in initial.dependencies], + "attempts": [observation(a) for a in initial.attempts], + "receipts": [ + {**asdict(r), "attempt": observation(r.attempt)} for r in initial.receipts + ], + } + ) doc = self.control.read(self.campaign) def transition(state): @@ -257,6 +275,7 @@ def transition(state): "draft_attempts": 0, "active_action": None, "initial_gain": tree.verified_gain(initial), + "initial_conditions": initial_conditions, "selected": initial.root_id, } return state diff --git a/tests/test_vats_runtime.py b/tests/test_vats_runtime.py index bd3a793f..1873789a 100644 --- a/tests/test_vats_runtime.py +++ b/tests/test_vats_runtime.py @@ -161,6 +161,8 @@ def test_gvo_rejects_shared_tree_leaves_forced_and_changed_start_conditions(runt examples((records[0] | {"forced": True}, records[1])) with pytest.raises(ValueError, match="identical"): examples((records[0] | {"settings": "other"}, records[1])) + with pytest.raises(ValueError, match="identical"): + examples((records[0] | {"initial_conditions": "different-root-proof"}, records[1])) def test_snapshot_change_on_resume_is_rejected(runtime): From 96e3c50c54a9c30581492b8dc86f916e5c5fec43 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:25:13 +0800 Subject: [PATCH 5/6] harness: inject search role operations across the model boundary Signed-off-by: amy-why-3459 --- docs/speculative-search.md | 3 +++ src/inferrsi/harness/gvo.py | 25 +++++++----------------- src/inferrsi/harness/search_adapter.py | 4 +--- src/inferrsi/harness/search_campaign.py | 3 +-- src/inferrsi/harness/vats.py | 9 +++------ src/inferrsi/model/search_roles.py | 26 +++++++++++++++++++++++++ 6 files changed, 41 insertions(+), 29 deletions(-) diff --git a/docs/speculative-search.md b/docs/speculative-search.md index b99e2c97..3c3e792b 100644 --- a/docs/speculative-search.md +++ b/docs/speculative-search.md @@ -123,6 +123,9 @@ Caller-authored unreceipted pair targets are rejected. These are real updates to a constrained CPU operator model. They are not LLM finetuning, prompt summarization or evidence of pretrained-model improvement. +Apps injects the role snapshot's `choose`, `predict`, `train`, `restore` and +`document` operations. Harness retains traces and controls when they may run; +it has no import dependency on Model. ## Accounting, recovery and final diff --git a/src/inferrsi/harness/gvo.py b/src/inferrsi/harness/gvo.py index f485040d..71490cc0 100644 --- a/src/inferrsi/harness/gvo.py +++ b/src/inferrsi/harness/gvo.py @@ -5,23 +5,14 @@ import uuid from dataclasses import replace -from inferrsi.model.search_roles import Choice, finite, update - from .artifact_primitives import digest from .search_records import decode -def choice_from_document(raw): - return Choice( - **( - raw - | { - "options": tuple(raw["options"]), - "features": tuple(tuple(row) for row in raw["features"]), - "probabilities": tuple(raw["probabilities"]), - } - ) - ) +def finite(value): + if type(value) not in (int, float) or not math.isfinite(value): + raise ValueError("Search returns require finite recorded observations") + return value def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): @@ -85,9 +76,8 @@ def examples(records, *, expense_weight=0.0, elapsed_weight=0.0): if step["forced"]: raise ValueError("Forced action leaked into a natural trajectory") for raw in (step["choice"], *step["draft_choices"]): - choice = choice_from_document(raw) - if choice.sampled: - actors.append((choice, advantage)) + if raw["sampled"]: + actors.append((raw, advantage)) return actors, values, {"search_ids": identities, "returns": returns, "advantages": advantages} @@ -183,8 +173,7 @@ def train_group( values.extend( suffix_examples(record, expense_weight=expense_weight, elapsed_weight=elapsed_weight) ) - trained = update( - snapshot, + trained = snapshot.train( reference, actors, values, diff --git a/src/inferrsi/harness/search_adapter.py b/src/inferrsi/harness/search_adapter.py index 4896f8a0..8375337f 100644 --- a/src/inferrsi/harness/search_adapter.py +++ b/src/inferrsi/harness/search_adapter.py @@ -12,7 +12,6 @@ from pathlib import Path from inferrsi.contracts.claim import Verdict -from inferrsi.model.search_roles import choose from . import candidates from . import search_tree as tree @@ -131,8 +130,7 @@ def draft(self, parent, state, snapshot, rng, operation, deadline): vectors = [ tuple(float(i == index) for i in range(len(self.edits))) for index, _ in legal ] - choice = choose( - snapshot, + choice = snapshot.choose( "drafter", [edit.token for _, edit in legal], vectors, diff --git a/src/inferrsi/harness/search_campaign.py b/src/inferrsi/harness/search_campaign.py index fea8bce9..96897c51 100644 --- a/src/inferrsi/harness/search_campaign.py +++ b/src/inferrsi/harness/search_campaign.py @@ -4,7 +4,6 @@ from dataclasses import asdict, replace from inferrsi.contracts.claim import Verdict -from inferrsi.model.search_roles import Snapshot from . import search_tree as tree from .artifact_primitives import digest @@ -215,7 +214,7 @@ def learn( or digest(op["binding"]["training"]) != digest(expected) ): raise ValueError("Frozen outer-loop training configuration changed") - snapshot = Snapshot(**op["receipt"]["checkpoint"]) + snapshot = snapshot.restore(op["receipt"]["checkpoint"]) else: snapshot = train_group( control, diff --git a/src/inferrsi/harness/vats.py b/src/inferrsi/harness/vats.py index 6ed33bf5..914ca415 100644 --- a/src/inferrsi/harness/vats.py +++ b/src/inferrsi/harness/vats.py @@ -12,8 +12,6 @@ from dataclasses import asdict, dataclass, replace from itertools import combinations -from inferrsi.model.search_roles import choose, dot - from . import search_tree as tree from .artifact_primitives import digest from .search_records import decode, encode @@ -140,8 +138,8 @@ def menu(state, jobs, available, settings, snapshot): continue trial = allowed[frozenset(union)] queries += 2 - if dot(snapshot.scorer, features(state, trial, available, settings)) > dot( - snapshot.scorer, features(state, left, available, settings) + if snapshot.predict(features(state, trial, available, settings)) > snapshot.predict( + features(state, left, available, settings) ): extra.append(trial) if not settings.bundled: @@ -526,8 +524,7 @@ def run(self, *, forced_action=None): features(current, action, remaining, self.settings) for action in actions ) rng = random.Random(f"{search['seed']}:policy:{len(search['trace'])}") - choice = choose( - self.snapshot, + choice = self.snapshot.choose( "policy", [action.key for action in actions], vectors, diff --git a/src/inferrsi/model/search_roles.py b/src/inferrsi/model/search_roles.py index 3e5df28c..e0a8e950 100644 --- a/src/inferrsi/model/search_roles.py +++ b/src/inferrsi/model/search_roles.py @@ -61,6 +61,32 @@ def document(self): "scorer": list(self.scorer), } + def choose(self, *args, **kwargs): + """Injected role operation; Harness does not import this Model module.""" + return choose(self, *args, **kwargs) + + def predict(self, features): + return dot(self.scorer, features) + + def restore(self, document): + return type(self)(**document) + + def train(self, reference, actors, values, pairs, **settings): + restored = [] + for raw, advantage in actors: + choice = Choice( + **( + raw + | { + "options": tuple(raw["options"]), + "features": tuple(tuple(row) for row in raw["features"]), + "probabilities": tuple(raw["probabilities"]), + } + ) + ) + restored.append((choice, advantage)) + return update(self, reference, restored, values, pairs, **settings) + @dataclass(frozen=True) class Choice: From bb6ecbd5a6888f43345ba5376f71f2c6e55f0c25 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Wed, 18 Nov 2026 21:42:11 +0800 Subject: [PATCH 6/6] apps: isolate benchmark loading and use bounded protocol processes Signed-off-by: amy-why-3459 --- src/inferrsi/apps/search_benchmarks.py | 21 +++++++++++++++++---- src/inferrsi/apps/speculative_search.py | 4 ++-- src/inferrsi/harness/task_protocol.py | 18 ++++++++++++++---- tests/test_task_protocol_child.py | 16 ++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/inferrsi/apps/search_benchmarks.py b/src/inferrsi/apps/search_benchmarks.py index 95b77931..b89442d1 100644 --- a/src/inferrsi/apps/search_benchmarks.py +++ b/src/inferrsi/apps/search_benchmarks.py @@ -49,9 +49,14 @@ def evaluate(self, candidate, check, output, deadline): raise ValueError("Frozen benchmark resources changed") output.mkdir(parents=True) candidates.retain(candidate / "candidate.py", output / "candidate.py") - protocol = task_protocol.protocol(self.frozen) - args = protocol.arguments( - output / "candidate.py", output / "measurement", self.plan["evaluation_settings"] + args = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "arguments", + deadline=time.monotonic() + max(0, deadline - time.time()), + candidate=output / "candidate.py", + output=output / "measurement", + settings=self.plan["evaluation_settings"], ) from inferrsi.harness import optimization_session @@ -79,7 +84,15 @@ def evaluate(self, candidate, check, output, deadline): if p.is_file() } != self.plan["frozen_files"]: raise ValueError("Frozen benchmark resources changed during execution") - assessed = protocol.assess(output, self.plan["evaluation_settings"], execution) + assessed = task_protocol.run( + self.frozen, + self.plan["frozen_files"]["optimization.py"], + "assess", + deadline=time.monotonic() + max(0, deadline - time.time()), + directory=output, + settings=self.plan["evaluation_settings"], + execution=execution, + ) if execution["cleanup_verified"] is not True: assessed = {"verdict": "INCONCLUSIVE", "score": None} atomic_write_json(output / "assessment.json", assessed) diff --git a/src/inferrsi/apps/speculative_search.py b/src/inferrsi/apps/speculative_search.py index 318939f0..01deb8b3 100644 --- a/src/inferrsi/apps/speculative_search.py +++ b/src/inferrsi/apps/speculative_search.py @@ -17,8 +17,6 @@ from inferrsi.harness.work_control import WorkControl from inferrsi.model.search_roles import Snapshot -from .search_benchmarks import EngineEvaluator - def add_arguments(commands): search = commands.add_parser("search", help="Bounded SpecRSI search and role training") @@ -80,6 +78,8 @@ def command(args): if args.search_action == "export": atomic_write_json(args.output, result) return result + from .search_benchmarks import EngineEvaluator + if root.exists(): raise ValueError("Use a new session directory; status/export never restart work") if args.groups < 2 or args.rounds < 1: diff --git a/src/inferrsi/harness/task_protocol.py b/src/inferrsi/harness/task_protocol.py index 340f1218..792b5564 100644 --- a/src/inferrsi/harness/task_protocol.py +++ b/src/inferrsi/harness/task_protocol.py @@ -7,6 +7,7 @@ Windows Job, and returns one strict JSON envelope (#412 R3). """ +import math import os import shutil import sys @@ -85,15 +86,23 @@ def _response_problem(response, operation: str) -> str | None: return None -def run(root: Path, protocol_sha256: str, operation: str, **arguments): +def run(root: Path, protocol_sha256: str, operation: str, *, deadline=None, **arguments): """Run one operation of ``root/optimization.py`` whose bytes have ``protocol_sha256``. Path arguments must be absolute. The child's stderr tail is copied to this process's stderr after every operation. A protocol ``SystemExit`` is raised again with its exit code; every other failure raises ``TaskProtocolError``. Descendants that the - protocol leaves running are stopped with the child. + protocol leaves running are stopped with the child. An optional absolute + monotonic deadline further bounds the shared protocol timeout. """ path = Path(root).resolve() / "optimization.py" + timeout = PROTOCOL_TIMEOUT_SECONDS + if deadline is not None: + if type(deadline) not in (int, float) or not math.isfinite(deadline): + raise TaskProtocolError("request", "deadline must be finite monotonic time") + timeout = min(timeout, deadline - time.monotonic()) + if timeout <= 0: + raise TaskProtocolError("timeout", "deadline expired before protocol execution") if sha256(path) != protocol_sha256: raise TaskProtocolError("identity", "protocol bytes differ from the frozen digest") request = {"schema_version": protocol_child.SCHEMA_VERSION, "operation": operation, @@ -113,7 +122,8 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): # cannot land among the frozen resources. record = local_process._supervise( argv, work, dict(os.environ), work / "stdout", work / "stderr", - time.monotonic() + PROTOCOL_TIMEOUT_SECONDS) + min(time.monotonic() + timeout, deadline) if deadline is not None + else time.monotonic() + timeout) stderr = _tail(work / "stderr") if (work / "stderr").is_file() else "" # Diagnostics contract: after every operation, whatever its result, the last # STDERR_TAIL_BYTES of the child's stderr (protocol prints included) are copied to @@ -125,7 +135,7 @@ def run(root: Path, protocol_sha256: str, operation: str, **arguments): raise KeyboardInterrupt if record["status"] == "timeout": raise TaskProtocolError( - "timeout", f"{operation} exceeded {PROTOCOL_TIMEOUT_SECONDS:g} s; {stderr}") + "timeout", f"{operation} exceeded {timeout:g} s; {stderr}") if record["status"] not in ("completed", "failed"): raise TaskProtocolError("launch", f"{record['status']}: {record['error']}") if record["returncode"] != 0: diff --git a/tests/test_task_protocol_child.py b/tests/test_task_protocol_child.py index 0446015c..973d1191 100644 --- a/tests/test_task_protocol_child.py +++ b/tests/test_task_protocol_child.py @@ -219,6 +219,22 @@ def test_deadline_stops_the_child_and_its_descendants(tmp_path, monkeypatch, cap assert not marker.exists() +def test_campaign_deadline_bounds_the_protocol_timeout(tmp_path): + digest = write_protocol( + tmp_path, "import time\ndef validate_arguments(a):\n time.sleep(60)\n") + started = time.monotonic() + error = failure(tmp_path, digest, deadline=started + 1.0, arguments=[]) + assert error.kind == "timeout" + assert time.monotonic() - started < 10 + + +def test_expired_campaign_deadline_never_starts_protocol(tmp_path, monkeypatch): + digest = write_protocol(tmp_path, "def validate_arguments(a):\n return {}\n") + monkeypatch.setattr(task_protocol.local_process, "_supervise", + lambda *a, **k: pytest.fail("expired protocol must not start")) + assert failure(tmp_path, digest, deadline=0, arguments=[]).kind == "timeout" + + @pytest.mark.parametrize("body, raises", [ (" return {}\n", False), (" raise ValueError('rejected')\n", True),