From e2e9d0876d587db0680e79889562911a1e5fadf2 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 17 Nov 2026 20:42:16 +0800 Subject: [PATCH 1/3] Add retained RSI result delivery bundles Project consistent experiment snapshots into Markdown, offline HTML, versioned results and a source artifact index. Keep historical verdicts separate from current verification and publish delivery after iterative final without changing execution outcomes. Agent-assisted-by: Codex (implementation) Signed-off-by: amy-why-3459 --- docs/operations.md | 1 + docs/optimization-benchmarks.md | 6 + docs/optimization-workflow-unification.md | 6 + docs/result-delivery.md | 190 +++++++ src/inferrsi/apps/cli/__init__.py | 12 +- src/inferrsi/apps/cli/arguments.py | 5 + src/inferrsi/apps/delivery/__init__.py | 272 ++++++++++ src/inferrsi/apps/delivery/adapters.py | 225 ++++++++ src/inferrsi/apps/delivery/projection.py | 515 ++++++++++++++++++ src/inferrsi/apps/delivery/render.py | 287 ++++++++++ src/inferrsi/apps/delivery/schema.py | 190 +++++++ src/inferrsi/apps/delivery/snapshot.py | 182 +++++++ tests/test_optimization_session.py | 30 ++ tests/test_project_optimization.py | 9 +- tests/test_result_delivery.py | 606 ++++++++++++++++++++++ 15 files changed, 2533 insertions(+), 3 deletions(-) create mode 100644 docs/result-delivery.md create mode 100644 src/inferrsi/apps/delivery/__init__.py create mode 100644 src/inferrsi/apps/delivery/adapters.py create mode 100644 src/inferrsi/apps/delivery/projection.py create mode 100644 src/inferrsi/apps/delivery/render.py create mode 100644 src/inferrsi/apps/delivery/schema.py create mode 100644 src/inferrsi/apps/delivery/snapshot.py create mode 100644 tests/test_result_delivery.py diff --git a/docs/operations.md b/docs/operations.md index 46eb49bc..b44582c0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -44,6 +44,7 @@ Effect classes used below: | Amend repository-owned knowledge | Scoped repository PR | — | Repository review and integration authority apply; this is not a runtime store Update | | Inspect/prepare experiments | CLI `doctor`; `plan` or `--dry-run` | Read-only inspection / preview | `doctor` may invoke bounded probes and does not certify target compatibility; `plan` previews a specification without persisting a campaign or authorizing execution | | Execute/inspect experiments | CLI `run`; `status`; `report` | Experiment-executing / read-only / report-generating | `run` executes caller-supplied commands and records evidence; `status` reads recorded progress; `report` generates HTML from recorded evidence | +| Deliver retained results | CLI `export --input --output`; Python `apps.delivery.export_bundle` / `read_bundle` | Report-generating | Publishes Markdown, offline HTML, versioned result JSON and artifact index from one snapshot without remeasurement. Iterative CLI final returns separate delivery status. See [result delivery](result-delivery.md) | | Inspect/verify archived evidence | `python tools/verify_evidence_archive.py`; [archive contract](evidence-archiving.md) | Read-only by default; explicit network/local scratch with `--download` | Default validates metadata offline; `--archive`/`--download` authenticates and bounds archive reading; `--recomputed` compares separately generated v2 results; never executes archived code or changes verdicts | | Rebuild scoped legacy evidence views | `python tools/rebuild_model_evidence.py --archive --output ` | CPU-only; creates a new output directory | Authenticates original bytes and pinned projection/code; no extraction, network, archived-code execution or campaign qualification | | Inspect evidence inventory / generate budget input | `python tools/validate_evidence.py [--budget-assets]` | Read-only | Offline layout/hash/schema checks; optional disposable #328 metadata view derived from campaign authority; no downloads | diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index ec0504eb..18c85ad6 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -1,5 +1,11 @@ # Optimize a task through RSI in its execution Agent session +The CLI `final` operation automatically publishes a [result delivery](result-delivery.md) +under `delivery//`: lead the response with the conclusion and `report.md`, +then `report.html` (**Open in a browser**), `campaign-result.json`, and retained code. +Execution and export outcomes are separate. Use `inferrsi export --input +--output ` for progress or a delivery retry without remeasurement. + An optimization benchmark covers the Agent's complete search under a frozen task and budget. The execution Agent proposes changes, uses permitted knowledge, submits candidates, reads independent feedback and completes the declared rounds. diff --git a/docs/optimization-workflow-unification.md b/docs/optimization-workflow-unification.md index 18f00606..c93ed921 100644 --- a/docs/optimization-workflow-unification.md +++ b/docs/optimization-workflow-unification.md @@ -1,5 +1,11 @@ # 普通任务与 Benchmark 任务共用的 RSI 优化流程 +CLI `final` 返回后自动生成 `delivery//report.md`、离线 `report.html`、 +`campaign-result.json` 和 `artifacts.json`。先阅读 Markdown;HTML 链接标注 +“Open in a browser”。执行结果与导出结果分别返回。中途快照或导出重试可使用 +`inferrsi export --input --output `, +不会重新测量。字段、证据缺口和历史格式边界见 [结果交付契约](result-delivery.md)。 + **Agent 理解需求并修改代码;Harness 固化明确提供的任务和验收协议,执行检查、判定留存结果,管理候选、轮次、预算和最终复测;Benchmark 提供题包及其专用规则;Knowledge 提供按需检索的资料。** 普通用户任务默认在当前 Agent 会话完成。题库 Benchmark 默认交给当前宿主中同一种 Agent 的全新会话;发起方只解析任务配置、派发并回收结果,新会话自行创建优化实验并执行到 final。用户明确要求当前会话时也可直接执行。新会话由 Agent 调用宿主工具创建,Benchmark 不自动启动 Agent。 diff --git a/docs/result-delivery.md b/docs/result-delivery.md new file mode 100644 index 00000000..c497a2b9 --- /dev/null +++ b/docs/result-delivery.md @@ -0,0 +1,190 @@ +# Result delivery + +Export retained records into a **new** directory: + +```bash +inferrsi export --input --output +``` + +Start with `report.md`. `report.html` is labelled **Open in a browser** and works +offline, including its search curve, tables and expandable structured snapshot. +`campaign-result.json` is the machine-readable result; `artifacts.json` indexes +the reports and source evidence. All four files describe one snapshot. +Code and evidence use digest-bound references to the original input directory; +they are **not copied** into this reference-based bundle. Offline reading does +not imply offline reproduction. Review records and paths before sharing; this +is not an automatic redaction tool. + +`campaign final` and `benchmarks session final` call the same Apps exporter after +Harness returns, including a failed final assessment. Output goes to +`/delivery//`. The existing execution result and exit code +remain unchanged; `delivery` separately reports publication success, failure or +uncertain durability. A report failure cannot undo a committed final. Retry +export, not measurement. Direct Harness calls remain execution-only. Python +callers can use `inferrsi.apps.delivery.export_bundle` and `read_bundle`. + +The export command exits 0 for successful publication, including FAIL and +incomplete reports, and 2 on export failure. It supports progress, blocked tasks +and retained records after termination. A busy writer returns an explicit retry +error. It never starts a missing final, executes a validator, recovers an attempt, +fetches a reference, or changes selection/control state. + +## Design decisions and authority + +This is the implementation contract for [RFC #468](https://github.com/InferMatrixLab/InferRSI/issues/468). +The RFC remains proposed until its decision owner records adoption. These +resolutions implement the discussion's constraints; implementation and RFC +acceptance remain separate. + +1. **Archive authority:** campaign `results/artifacts.json` owns external archive + metadata under [#464](https://github.com/InferMatrixLab/InferRSI/issues/464). + Delivery's separate bundle index points to that exact document and digest + through `archive_authority_ref`. It does not copy archive IDs, URLs, sizes, + hashes, owners, access rules, availability or integrity. Existing shared archive + readers own normalization and verification. Delivery's `verified` on this + entry means the **manifest bytes** were read, not that remote bytes verified. + No duplicate archive metadata needs reconciliation. +2. **Candidate binding:** selected and final retained code normally verify against + the same file/directory fingerprint. A no-fingerprint adapter can instead verify + an exact revision and fixed run/configuration documents, as specified below. + Missing, contradictory or unverifiable binding preserves the original verdict + as an audit fact and sets `delivery_accepted: false`. +3. **Degraded reads:** a separate read-only adapter never weakens Harness `_load` + or reassesses evidence. An intact digest-bound observation retains historical + PASS when measurement bytes are missing or modified. A missing, invalid or + changed observation cannot establish PASS from its unverified cached assessment; + the report says “Not assessed” and references preserved state for audit. +4. **Snapshot:** native exports share the controller's nonblocking OS lock. Since + writers hold it throughout an attempt, progress exports happen between committed + operations. A second digest pass checks every source read, negative lookup and + candidate member set. Uncoordinated writes trigger at most three complete retries, + then an explicit error. Native exports never consult `summary.json`. Live elapsed + time uses one observation cutoff throughout retries. + +## Version 1 result fields + +Identity: `schema_id: inferrsi.campaign-delivery`, `schema_version: 1`. +All fields below are required. `null` means unknown/unavailable, explained in +`gaps`; an empty collection means known empty. Numbers must be finite. Timestamps +are timezone-qualified ISO 8601. `read_bundle` rejects unsupported versions and +checks entry-file hashes. Optional additions retain the version; changes to +existing field types or semantics require a version increment. + +| Field | Meaning | +| --- | --- | +| `export` | ID, generation time, observation cutoff, `progress`/`final`, generator version | +| `campaign` | ID, recorded goal, task kind, frozen acceptance basis, plan reference | +| `sources` | Artifact IDs, source types and exact record digests | +| `execution` | Original state/namespace, budget, settled round count, round policy, stop reason, times | +| `completion` | `completed`/`incomplete` workflow and reason | +| `outcome` | `improved`, `baseline_retained`, `no_valid_candidate` or `undetermined`; selection rationale | +| `baseline` | Retained and observed fingerprints, configuration, source references | +| `selected_candidate` | Controller selection and retained code/configuration, or null | +| `final` | Independent final attempt, protocol, execution, assessment, binding; null if absent | +| `acceptance` | Original status/verdict, claim ID, scope, assessment reference; separate current delivery acceptance | +| `metrics` | Phase/scope/workload comparisons, units, statistic, configuration, sample counts, provenance | +| `checks` | Original assessment/check, requirement, execution, frozen gate, observation, verdict, evidence | +| `rounds` | Ordered attempts, including rejected and unresolved pending attempts | +| `usage` | Wall time, observed tokens, device time and cost, each with source and coverage | +| `gaps` | Missing steps, mapping limits, identity gaps and present evidence failures | +| `artifacts_ref` | `artifacts.json`, without a digest to avoid a hash cycle | + +`completed_rounds` counts settled search records, including rejected candidates, +as the controller does. Pending attempts retain their own IDs and do not increment +that count. Retries or independent finals are never invented from summary curves. + +Completion requires the frozen round policy (including a permitted early stop's +recorded reason) and a normally finished independent final assessment. A completed +assessment that violates a quality/performance gate can yield completed + FAIL. +A missing/interrupted final stays incomplete. A final snapshot means the source +stopped, not that it succeeded. Historical completion survives later evidence loss; +prominent current gaps prevent current delivery acceptance. + +The primary assessment is the final observation, never a favorable search result. +Missing claim identities stay null with `identity_incomplete`. Existing INCONCLUSIVE +is preserved; missing assessments are “Not assessed,” never fabricated INCONCLUSIVE. +`improved` also requires a non-baseline selection, passing verified final and a gain +under the frozen selection direction. Baseline retention and its final verdict are +separate. Baseline optimization speedup is 1.0 only after verified completed final; +its measured A/A ratio remains separate in `metrics`. A blocked state alone cannot +establish `no_valid_candidate`; unsupported selection outcomes stay `undetermined`. + +Project measurements use their retained paired median; task `case_metrics` preserve +separate measured cases. Missing workload IDs remain null. No concurrency tiers or +workloads are aggregated, and export establishes no statistical significance. +Positive comparable pairs yield baseline/candidate for minimize and candidate/baseline +for maximize, plus the corresponding improvement percentage. Missing units/statistics, +nonpositive values, inapplicable ratios, overflow and incomparable external records +retain raw values and null gains. Failed search scores leave holes in the HTML curve. + +Usage preserves the observed search subtotal separately from the caller session +export, never adds them together, and counts cached input as a subset of input. +Subtotal coverage concerns settled search rounds, not the whole campaign. A final +caller export alone cannot prove complete session coverage. Device time and cost +remain null without accounting observations; elapsed wall time is insufficient. + +## Adapter mapping + +| Input | Authoritative facts | Deliberate gaps | +| --- | --- | --- | +| Native project / Benchmark | Plan, campaign, state, digest-bound observations/execution/evidence and retained code | Missing identities stay unknown; summaries cannot substitute for final | +| Historical version 1 `summary.json` | Recorded final assessment and reported completion | No reconstructed rounds, independent final, binding or usage coverage | +| Legacy run directory | `run.lock.json`, `status.json`, `evaluation.json`; paired single-experiment median | No invented iterative rounds/final or generalized L4 claim | +| Apps `result.json` | Validated AppsResultV1, explicitly selected primary claim, fixed refs, all original claims | Operation success is not campaign completion; storage is not implicitly mounted/fetched | +| Explicit external record JSON | Version 1 `external_optimization_record`, source refs, performance/quality observations | No local revalidation or inferred comparability | + +External JSON uses `--input ` with references relative to its parent. +Arbitrary files named `result.json` or `summary.json` are not guessed into these +contracts. Campaign-specific historical formats need explicit adapters; unsupported +schemas fail instead of manufacturing facts. + +The no-fingerprint extension uses identical `best.binding` and +`final.candidate_binding` objects in retained native records: + +```json +{ + "revision": "", + "run_ref": {"path": "", "sha256": ""}, + "configuration_ref": {"path": "", "sha256": ""} +} +``` + +Revision is a full 40- or 64-character lowercase hexadecimal identity. Both fixed +documents must verify and record that `candidate_revision`. The run also records +the identical `configuration_ref`, final `attempt_id` and frozen `protocol_digest`. +Matching names alone are insufficient. This alternative cannot override contradictory +file fingerprints. It does not implement service-reuse execution in Harness, fetch +Git revisions, or claim local code-byte verification for a revision binding. + +## Artifact index and publication + +The index identity is `inferrsi.delivery-artifacts`, version 1, with `export_id`. +Entries include ID, role, location/kind (`bundle` or `source`), media type, size, +SHA-256, source revision, availability, integrity, check time and access explanation. +Source locations are relative to the original input; bundle locations are relative +to this export. Known digests survive missing/corrupt bytes; `observed_sha256` +separately records the current bytes. The index hashes JSON and both reports, not +itself. Both reports display the JSON digest and export ID. + +Reports escape imported text and use only fixed local navigation links. There is +no executable embedded data or arbitrary URL fetching. V1 does not bundle arbitrary +attachments, follow symlinks or traverse outside input paths. Documents have an +8 MiB read limit; larger raw evidence is streamed for hashing and indexed. Redacted +shared copies need separate identities; export never replaces frozen originals. + +Generation fixes the snapshot, creates JSON, renders reports, builds the index, +validates the bundle and uses the existing Apps no-replace directory publisher. +Existing destinations survive every failure. Post-rename directory-sync failure +returns `uncertain` with the same export ID and visible output path: inspect it with +`read_bundle` before retrying. Atomic visibility and durability retain the platform +limits of [Apps publication](apps-output.md). + +## Validation scope + +`tests/test_result_delivery.py` covers PASS/FAIL, missing final, baseline retention, +damaged evidence/code, both binding paths, usage, invalid ratios, archive authority, +source changes, writer locks, publication faults, hostile text/paths and historical +adapters. Project and Benchmark lifecycle tests exercise automatic export through +real CPU subprocess fixtures and retry without remeasurement. These verify software +behavior, not accelerator performance. RFC adoption, a future service-reuse producer +and real campaign performance acceptance remain separate. diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 7d84e23d..d3f59ee1 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -240,6 +240,12 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "export": + from ..delivery import export_bundle + + delivery = export_bundle(args.input, args.output, export_id=args.export_id) + _print(delivery) + return 0 if delivery["status"] == "published" else 2 if args.command == "mindie": from ..mindie.project import dispatch @@ -272,6 +278,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "benchmarks": if args.benchmark_action == "session": from ...harness.optimization_session import session_command + from ..delivery import after_final task_source = None if args.session_action == "start": @@ -284,7 +291,7 @@ def main(argv: list[str] | None = None) -> int: knowledge_reader=(KnowledgeReader() if args.session_action == "knowledge" else None), ) - _print(result) + _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 if args.benchmark_action == "run": from ..benchmark_operations import run_task @@ -521,6 +528,7 @@ def main(argv: list[str] | None = None) -> int: return 0 if getattr(args, "session_action", None) is not None: from ...harness.optimization_session import session_command + from ..delivery import after_final result = session_command( args, @@ -528,7 +536,7 @@ def main(argv: list[str] | None = None) -> int: knowledge_reader=(KnowledgeReader() if args.session_action == "knowledge" else None), ) - _print(result) + _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 from ...harness.records.campaign_records import ( assemble_trace, diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index b85c667d..e9ad7e59 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -237,6 +237,11 @@ def parser() -> argparse.ArgumentParser: trace = commands.add_parser("trace", help="Read recorded steps, checks and errors") _fixed_attempt_args(trace) + delivery = commands.add_parser("export", help="Export retained results without remeasurement") + delivery.add_argument("--input", type=Path, required=True, help="Campaign, run or record JSON") + delivery.add_argument("--output", type=Path, required=True, help="New delivery directory") + delivery.add_argument("--export-id", help="Optional identity for publication reconciliation") + dashboard = commands.add_parser( "dashboard", help="Export an offline dashboard for one run or recorded campaign" ) diff --git a/src/inferrsi/apps/delivery/__init__.py b/src/inferrsi/apps/delivery/__init__.py new file mode 100644 index 00000000..90d81a3e --- /dev/null +++ b/src/inferrsi/apps/delivery/__init__.py @@ -0,0 +1,272 @@ +"""Export retained experiment results; never execute, recover, select or assess.""" + +import errno +import hashlib +import time +import uuid +from contextlib import nullcontext +from pathlib import Path +from tempfile import TemporaryDirectory + +from ... import __version__ +from ...infra.artifacts import file_lock +from .. import archive +from . import adapters +from .projection import iterative +from .render import reports +from .schema import INDEX_ID, decode, empty_result, encode, gap, timestamp, validate_result +from .snapshot import Snapshot, SnapshotBusy, SnapshotChanged + + +def _safe_root(path): + path = Path(path).absolute() + if any(parent.is_symlink() for parent in (path, *path.parents)): + raise ValueError("Delivery paths cannot traverse symbolic links") + return path + + +def _project(source, export_id, observed_at): + root = source if source.is_dir() else source.parent + snapshot = Snapshot(root, timestamp(observed_at)) + result = empty_result(export_id, observed_at, __version__) + name = source.name if source.is_file() else None + if name is None and snapshot.path("state.json").exists(): + iterative(snapshot, result) + elif name is None and snapshot.path("run.lock.json").exists(): + adapters.legacy(snapshot, result) + else: + if name is None: + name = next( + ( + candidate + for candidate in ("result.json", "summary.json") + if snapshot.path(candidate).exists() + ), + None, + ) + if name is None: + raise ValueError("No supported retained records; supply a campaign, run or record JSON") + document, _ = snapshot.object(name, "source_record", required=True) + if document.get("kind") == "external_optimization_record": + adapters.external(snapshot, result, name) + elif "operation" in document: + adapters.operation(snapshot, result, name) + elif name == "summary.json": + adapters.summary(snapshot, result, name) + else: + raise ValueError("Unsupported source record schema") + # The archive authority stays opaque and canonical. The bundle points to it; + # it does not fork archive URLs, sizes, ownership or availability observations. + authority = snapshot.path("results/artifacts.json") + authority_ref = None + if authority.exists(): + authority_ref = snapshot.file("results/artifacts.json", "archive_authority") + result["sources"].append( + { + "artifact_ref": authority_ref, + "type": "archive_authority", + "revision": snapshot.entries[authority_ref]["source_revision"], + } + ) + # Include the negative lookup in the stability check, too. + else: + snapshot.file("results/artifacts.json", "optional_archive_authority") + snapshot.entries.pop("source:results/artifacts.json") + for entry in snapshot.entries.values(): + if entry["availability"] != "available" or entry["integrity"] != "verified": + gap( + result, + "EVIDENCE_UNVERIFIED", + entry["id"], + f"{entry['location']}: availability={entry['availability']}, " + f"integrity={entry['integrity']}; historical assessment is retained separately", + ) + for key in ("baseline", "selected_candidate", "metrics", "checks", "rounds"): + if result[key] is None: + gap(result, "NOT_RECORDED", key, f"{key} is not established by this source adapter") + if result["campaign"].get("goal") is None: + gap( + result, + "GOAL_NOT_RECORDED", + "campaign.goal", + "No structured goal was recorded; consult the frozen instructions when available", + ) + for key, usage in result["usage"].items(): + if usage["coverage"] != "complete": + gap( + result, + "USAGE_COVERAGE", + "usage." + key, + f"{key} coverage is {usage['coverage']}; unknown observations are not zero", + ) + validate_result(result) + return snapshot, result, authority_ref + + +def project(source: Path, *, export_id: str, observed_at=None): + """Read a committed snapshot or fail explicitly; no live summary is authoritative. + + Harness writers hold .session.lock across their operation. A busy writer + yields an explicit retryable error. Two digest passes additionally detect + uncoordinated edits to historical/external source files, with three attempts. + The observation cutoff is fixed across retries. + """ + source = _safe_root(source) + if not source.exists(): + raise ValueError("Delivery input does not exist") + observed_at = time.time() if observed_at is None else observed_at + root = source if source.is_dir() else source.parent + managed = source.is_dir() and (root / "state.json").exists() + lock = _safe_root(root / ".session.lock") + for _ in range(3): + try: + with file_lock(lock) if managed else nullcontext(): + snapshot, result, authority_ref = _project(source, export_id, observed_at) + if snapshot.stable(): + return snapshot, result, authority_ref + except SnapshotChanged: + continue + except BlockingIOError: + raise SnapshotBusy( + "Optimization writer is active; retry after its attempt commits" + ) from None + raise SnapshotChanged("Source changed during export; no consistent snapshot could be read") + + +def read_bundle(directory: Path): + """Validate published entry files without fetching or executing any source reference.""" + directory = _safe_root(directory) + index = decode(_safe_root(directory / "artifacts.json").read_bytes()) + if ( + index.get("schema_id") != INDEX_ID + or type(index.get("schema_version")) is not int + or index["schema_version"] != 1 + ): + raise ValueError("Unsupported delivery artifact index") + entries = index.get("artifacts", []) + if len({entry["id"] for entry in entries}) != len(entries): + raise ValueError("Duplicate delivery artifact ID") + bundled = {entry["location"]: entry for entry in entries if entry["location_kind"] == "bundle"} + if set(bundled) != {"report.md", "report.html", "campaign-result.json"}: + raise ValueError("Incomplete delivery bundle") + for name, entry in bundled.items(): + raw = _safe_root(directory / name).read_bytes() + if hashlib.sha256(raw).hexdigest() != entry["sha256"] or len(raw) != entry["size_bytes"]: + raise ValueError("Delivery file integrity mismatch") + result = decode((directory / "campaign-result.json").read_bytes()) + validate_result(result) + if result["export"]["id"] != index["export_id"]: + raise ValueError("Delivery export IDs disagree") + identities = {entry["id"] for entry in entries} + if any(source.get("artifact_ref") not in identities for source in result["sources"]): + raise ValueError("Delivery source does not resolve in its artifact index") + return result + + +def export_bundle(source: Path, output: Path, *, export_id=None): + """Publish a new four-file bundle atomically, preserving existing destinations.""" + export_id = export_id or uuid.uuid4().hex + archive.validate_output_id(export_id) + output = _safe_root(output) + published = False + result = None + try: + if output.exists(): + raise FileExistsError(errno.EEXIST, "Delivery destination already exists") + snapshot, result, authority_ref = project(source, export_id=export_id) + data = encode(result) + markdown, html = reports(result, hashlib.sha256(data).hexdigest()) + files = {"campaign-result.json": data, "report.md": markdown, "report.html": html} + entries = list(snapshot.entries.values()) + for name, raw in files.items(): + entries.append( + { + "id": "bundle:" + name, + "role": "report" if name.startswith("report") else "result", + "location": name, + "location_kind": "bundle", + "media_type": { + "report.md": "text/markdown", + "report.html": "text/html", + "campaign-result.json": "application/json", + }[name], + "size_bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + "source_revision": export_id, + "availability": "available", + "integrity": "verified", + "checked_at": result["export"]["generated_at"], + "access": "Bundled file", + } + ) + files["artifacts.json"] = encode( + { + "schema_id": INDEX_ID, + "schema_version": 1, + "export_id": export_id, + "archive_authority_ref": authority_ref, + "archive_metadata_policy": "Archive metadata is canonical only in the referenced " + "campaign results/artifacts.json; this index does not duplicate it " + "or check remote bytes", + "artifacts": entries, + } + ) + output.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix=".delivery-", dir=output.parent) as temporary: + staging = Path(temporary) + for name, raw in files.items(): + archive._write_file(staging / name, raw) + read_bundle(staging) + archive._sync_directory(staging) + archive._sync_directory(output.parent) + archive._publish_directory(staging, output) + published = True + archive._sync_directory(output.parent) + except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: + # In particular a post-rename sync error keeps the same export ID and + # visible destination so callers can reconcile instead of remeasuring. + return { + "status": "uncertain" if published else "error", + "export_id": export_id, + "output": str(output), + "error": str(error) + if isinstance(error, (SnapshotBusy, SnapshotChanged)) + else archive._publication_message(error, published=published), + "execution_preserved": True, + "reconcile": "Read this output with read_bundle before retrying" if published else None, + } + return { + "status": "published", + "export_id": export_id, + "output": str(output), + "conclusion": { + "completion": result["completion"], + "outcome": result["outcome"], + "acceptance": result["acceptance"], + }, + "report_md": str(output / "report.md"), + "report_html": {"path": str(output / "report.html"), "label": "Open in a browser"}, + "result": str(output / "campaign-result.json"), + "artifacts": str(output / "artifacts.json"), + "selected_code": result["selected_candidate"], + } + + +def after_final(args, result): + """Keep execution outcome and report publication outcome independent.""" + if args.session_action != "final": + return result + export_id = uuid.uuid4().hex + output = args.session / "delivery" / export_id + try: + delivery = export_bundle(args.session, output, export_id=export_id) + except Exception: + # Presentation failure must never change the already committed execution outcome. + delivery = { + "status": "error", + "export_id": export_id, + "output": str(output), + "execution_preserved": True, + "error": "Report generation failed; export retained records again", + } + return {**result, "delivery": delivery} diff --git a/src/inferrsi/apps/delivery/adapters.py b/src/inferrsi/apps/delivery/adapters.py new file mode 100644 index 00000000..72c75d09 --- /dev/null +++ b/src/inferrsi/apps/delivery/adapters.py @@ -0,0 +1,225 @@ +"""Historical adapters keep their original scope and explicitly disclose mapping gaps.""" + +from ..result import _selected_verdict +from ..result import validate_result as validate_apps_result +from .projection import assessment, metric +from .schema import gap, timestamp + + +def _source(result, snapshot, ref, kind): + result["sources"].append( + {"artifact_ref": ref, "type": kind, "revision": snapshot.entries[ref]["source_revision"]} + ) + + +def legacy(snapshot, result): + plan, plan_ref = snapshot.object("run.lock.json", "plan", required=True) + evaluation, ref = snapshot.object("evaluation.json", "assessment") + status, status_ref = snapshot.object("status.json", "execution") + _source(result, snapshot, plan_ref, "legacy_run") + _source(result, snapshot, ref, "legacy_evaluation") + _source(result, snapshot, status_ref, "legacy_status") + state = (status or {}).get("state") + result["campaign"].update( + id=plan.get("run_id"), + task_kind="single_experiment", + plan_ref=plan_ref, + acceptance_basis=plan, + ) + result["execution"] = { + "namespace": "inferrsi.legacy.run", + "state": state, + "completed_rounds": None, + "round_limit": None, + "started_at": timestamp((status or {}).get("started_at")), + "ended_at": timestamp((status or {}).get("ended_at")), + } + assessment(result, evaluation, ref, scope=plan.get("comparison_scope")) + if state == "completed": + result["completion"] = {"status": "completed", "reason": "Single experiment completed"} + result["export"]["kind"] = ( + "final" if state in ("completed", "failed", "cancelled") else "progress" + ) + if evaluation: + before, after = evaluation.get("baseline", {}), evaluation.get("candidate", {}) + result["metrics"] = [ + metric( + { + "name": evaluation.get("metric_key", plan.get("metric_key")), + "unit": evaluation.get("metric_unit", plan.get("metric_unit")), + "direction": evaluation.get("direction", plan.get("direction")), + "scope": plan.get("comparison_scope"), + "workload_id": plan.get("workload_id"), + "statistic": "median", + "baseline": before.get("median"), + "candidate": after.get("median"), + "configuration": plan, + "sample_counts": { + "baseline": len(before["values"]) if "values" in before else None, + "candidate": len(after["values"]) if "values" in after else None, + }, + "repetitions": plan.get("repetitions"), + }, + phase="single_experiment", + ref=ref, + protocol_ref=plan_ref, + comparable=bool(before and after), + ) + ] + gap( + result, + "LEGACY_SCOPE", + "final_acceptance", + "Single A/B experiment only; no iterative rounds or independent final were recorded", + ) + + +def summary(snapshot, result, name="summary.json"): + document, ref = snapshot.object(name, "historical_summary", required=True) + if type(document.get("schema_version")) is not int or document["schema_version"] != 1: + raise ValueError("Unsupported historical summary version") + _source(result, snapshot, ref, "historical_summary") + result["campaign"].update( + id=document.get("campaign_id"), + goal=document.get("goal"), + task_kind="historical_summary", + task=document.get("task"), + ) + result["execution"] = { + "namespace": "historical_summary", + "state": document.get("state"), + "completed_rounds": document.get("completed_rounds"), + "round_limit": document.get("planned_rounds"), + "stop_reason": document.get("stop_reason"), + } + result["export"]["kind"] = ( + "final" if document.get("state") in ("completed", "blocked") else "progress" + ) + assessment(result, document.get("final_assessment"), ref) + if document.get("state") == "completed": + result["completion"] = { + "status": "completed", + "reason": "Historical summary reports workflow completion", + } + result["selected_candidate"] = document.get("selected_candidate") + # The curve is retained verbatim as source material, never mistaken for attempt records. + gap( + result, + "SUMMARY_ONLY", + "verification", + "Only a rebuildable historical summary is available; original control, attempts, " + "usage coverage and final candidate binding cannot be verified", + ) + + +def operation(snapshot, result, name="result.json"): + document, ref = snapshot.object(name, "operation_result", required=True) + validate_apps_result(document) + _source(result, snapshot, ref, "apps_operation") + result["campaign"].update( + id=document.get("attempt_id"), + task_kind="operation", + acceptance_basis=document.get("prepared_ref"), + ) + result["execution"] = { + "namespace": "inferrsi.apps.result", + "operation": document["operation"], + "outcome": document["outcome"], + "record_ref": document["record_ref"], + "execution": document["execution"], + } + result["export"]["kind"] = ( + "final" if document["outcome"] not in ("in_progress", "recovery_required") else "progress" + ) + if document["assessment"] is not None: + verdict = _selected_verdict( + document["assessment"], (document["primary_claim_id"], document["primary_scope"]) + ) + assessment( + result, + {"verdict": verdict}, + ref, + scope=document["primary_scope"], + claim_id=document["primary_claim_id"], + ) + result["acceptance"]["original_assessment_ref"] = document["assessment_ref"] + result["acceptance"]["claims"] = document["assessment"]["claims"] + gap( + result, + "OPERATION_SCOPE", + "completion", + "Operation outcome is not completion or acceptance of an optimization campaign; " + "fixed storage references are preserved, not fetched", + ) + + +def external(snapshot, result, name): + document, ref = snapshot.object(name, "external_record", required=True) + if ( + document.get("kind") != "external_optimization_record" + or type(document.get("schema_version")) is not int + or document["schema_version"] != 1 + ): + raise ValueError("Unsupported external optimization record") + _source(result, snapshot, ref, "external_optimization_record") + refs = [] + for source in document.get("source_refs", []): + refs.append(snapshot.file(source["path"], "external_source", source.get("sha256"))) + result["campaign"].update( + id=document.get("candidate_id"), + task_kind="external_records", + goal=document.get("goal"), + acceptance_basis=document.get("protocol"), + ) + result["execution"] = { + "namespace": "external_optimization_record", + "state": document.get("state"), + } + result["export"]["kind"] = ( + "final" if document.get("state") in ("completed", "failed", "blocked") else "progress" + ) + if document.get("state") == "completed": + result["completion"] = { + "status": "completed", + "reason": "External source reports completion", + } + assessment( + result, + {"verdict": document.get("performance_verdict")}, + ref, + scope=document.get("scope"), + claim_id=document.get("claim_id"), + ) + result["acceptance"]["locally_revalidated"] = False + result["metrics"] = [ + metric( + { + **row, + "candidate": row.get("aggregate"), + "statistic": row.get("statistic", row.get("aggregation")), + }, + phase="external", + ref=ref, + protocol_ref=ref, + comparable=False, + ) + for row in document.get("metrics", []) + ] + result["checks"] = [ + { + "name": "external_quality", + "required": None, + "execution_state": None, + "gate_ref": None, + "observation": document.get("quality"), + "verdict": (document.get("quality") or {}).get("verdict"), + "evidence_refs": refs, + } + ] + gap( + result, + "EXTERNAL_SCOPE", + "final_acceptance", + "Externally reported facts have not been locally revalidated; " + "source hashes only verify bytes", + ) diff --git a/src/inferrsi/apps/delivery/projection.py b/src/inferrsi/apps/delivery/projection.py new file mode 100644 index 00000000..81d80ba2 --- /dev/null +++ b/src/inferrsi/apps/delivery/projection.py @@ -0,0 +1,515 @@ +"""Project retained iterative records without invoking execution or assessment.""" + +import re + +from ...harness.usage import normalize_usage +from ...infra.artifacts import digest +from .schema import gap, number, timestamp + + +def assessment(result, value, ref, *, scope=None, claim_id=None): + if not isinstance(value, dict) or value.get("verdict") not in ("PASS", "FAIL", "INCONCLUSIVE"): + gap(result, "ASSESSMENT_UNAVAILABLE", "acceptance", "No readable original assessment") + return + result["acceptance"].update( + status="assessed", + verdict=value["verdict"], + assessment_ref=ref, + claim_id=value.get("claim_id", claim_id), + scope=value.get("scope", scope), + ) + if any(result["acceptance"][key] is None for key in ("claim_id", "scope", "assessment_ref")): + gap( + result, + "identity_incomplete", + "acceptance", + "The source did not record all claim identities; none were invented", + ) + + +def metric(value, *, phase, ref, protocol_ref, comparable=False): + before, after = number(value.get("baseline")), number(value.get("candidate")) + direction = value.get("direction") + valid = ( + comparable + and bool(value.get("unit")) + and bool(value.get("statistic")) + and direction in ("minimize", "maximize") + and before is not None + and after is not None + and before > 0 + and after > 0 + and value.get("ratio_applicable", True) + ) + speedup = improvement = None + if valid: + speedup = number(before / after if direction == "minimize" else after / before) + improvement = number( + (after - before) / before * 100 * (-1 if direction == "minimize" else 1) + ) + reason = ( + None + if speedup is not None and improvement is not None + else ( + "Comparable protocol, workload, unit and statistic with positive finite values " + "and an applicable ratio are required" + ) + ) + return { + "name": value.get("name"), + "unit": value.get("unit"), + "direction": direction, + "scope": value.get("scope"), + "workload_id": value.get("workload_id"), + "configuration": value.get("configuration"), + "statistic": value.get("statistic"), + "baseline": before, + "candidate": after, + "sample_counts": value.get("sample_counts"), + "repetitions": value.get("repetitions"), + "phase": phase, + "evidence_refs": [ref], + "protocol_ref": protocol_ref, + "comparability": {"comparable": comparable, "reason": reason}, + "speedup": speedup if reason is None else None, + "improvement_pct": improvement if reason is None else None, + } + + +def _metrics(value, plan, phase, ref): + settings = plan.get("evaluation_settings", {}) + common = { + "scope": value.get("scope", settings.get("scope")), + "configuration": settings, + "repetitions": settings.get("repetitions"), + } + if plan.get("task_source") == "project" and isinstance(value.get("metrics"), dict): + raw = value["metrics"] + samples = raw.get("samples", {}) + return [ + metric( + { + **common, + **raw, + "statistic": "median", + "sample_counts": { + side: len(samples[side]) if side in samples else None + for side in ("baseline", "candidate") + }, + }, + phase=phase, + ref=ref, + protocol_ref="source:plan.json", + comparable=True, + ) + ] + rows = [] + for case in value.get("case_metrics", []): + samples = case.get("samples_ms", {}) + rows.append( + metric( + { + **common, + "name": "elapsed", + "unit": "ms", + "direction": "minimize", + "scope": "task_case", + "workload_id": case.get("id", case.get("case")), + "statistic": "median", + "baseline": case.get("baseline_median_ms"), + "candidate": case.get("candidate_median_ms"), + "sample_counts": { + side: len(samples[side]) if side in samples else None + for side in ("baseline", "candidate") + }, + }, + phase=phase, + ref=ref, + protocol_ref="source:plan.json", + comparable=True, + ) + ) + return rows + + +def _usage(records, source, scope): + observed = [ + normalize_usage((record.get("observations") or {}).get("usage")) for record in records + ] + totals = {} + counts = {} + for field in ("input_tokens", "output_tokens", "cached_input_tokens"): + values = [item[field] for item in observed if item and item.get(field) is not None] + totals[field] = sum(values) if values else None + counts[field] = len(values) + # Only paired observations contribute to total_tokens; cache is an input subset. + paired = [ + item["input_tokens"] + item["output_tokens"] + for item in observed + if item and item.get("input_tokens") is not None and item.get("output_tokens") is not None + ] + totals["total_tokens"] = sum(paired) if paired else None + counts["total_tokens"] = len(paired) + return { + "value": totals if any(counts.values()) else None, + "source": source, + "coverage": "complete" + if observed + and all(n == len(records) for n in (counts["input_tokens"], counts["output_tokens"])) + else "partial" + if any(counts.values()) + else "unknown", + "scope": scope, + "observed_counts": counts, + "record_count": len(records), + } + + +def _binding(snapshot, selected, final, *, attempt_id, protocol_digest): + """A fixed run/config pair can bind an exact revision without file fingerprints. + + The run must itself record the exact revision and the same configuration ref; + matching two caller-supplied strings alone is insufficient. + """ + if not selected or not final or selected != final: + return False + revision = selected.get("revision") + if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", revision): + return False + refs = [] + documents = [] + for field in ("run_ref", "configuration_ref"): + ref = selected.get(field) + if not isinstance(ref, dict) or not re.fullmatch(r"[0-9a-f]{64}", ref.get("sha256", "")): + return False + document, identity = snapshot.object(ref["path"], field, ref["sha256"]) + documents.append(document) + refs.append(identity) + run, config = documents + return bool( + snapshot.verified(refs) + and run + and config + and run.get("candidate_revision") == revision + and config.get("candidate_revision") == revision + and run.get("configuration_ref") == selected["configuration_ref"] + and run.get("attempt_id") == attempt_id + and run.get("protocol_digest") == protocol_digest + ) + + +def iterative(snapshot, result): + plan, plan_ref = snapshot.object("plan.json", "plan", required=True) + state, state_ref = snapshot.object("state.json", "control", required=True) + campaign, campaign_ref = snapshot.object("campaign.json", "campaign", required=True) + if ( + type(plan.get("schema_version")) is not int + or plan["schema_version"] not in (1, 2) + or type(state.get("schema_version")) is not int + or state["schema_version"] != 1 + ): + raise ValueError("Unsupported optimization record version") + if digest(plan) != state.get("plan_digest") or campaign.get("plan_digest") != digest(plan): + raise ValueError("Frozen plan and committed state disagree") + result["campaign"].update( + id=campaign.get("campaign_id"), + goal=plan.get("goal"), + task_kind=plan["task_source"], + acceptance_basis=plan.get("evaluation_settings"), + plan_ref=plan_ref, + task=plan.get("task"), + instructions_ref="source:frozen/TASK.md", + ) + result["sources"] = [ + {"artifact_ref": ref, "type": kind, "revision": snapshot.entries[ref]["source_revision"]} + for ref, kind in ( + (plan_ref, "plan"), + (state_ref, "optimization_state"), + (campaign_ref, "campaign"), + ) + ] + frozen_refs = [ + snapshot.file("frozen/" + name, "protocol", expected) + for name, expected in plan["frozen_files"].items() + ] + # A changed member set also invalidates the frozen task, even if known files match. + frozen = snapshot.path("frozen") + if frozen.is_dir(): + snapshot.watch_tree("frozen") + actual = {p.relative_to(frozen).as_posix() for p in frozen.rglob("*") if p.is_file()} + if actual != set(plan["frozen_files"]): + gap(result, "FROZEN_INPUTS_CHANGED", "final_acceptance", "Frozen file set changed") + for context in state.get("contexts", []): + snapshot.file(context["path"], "knowledge", context.get("sha256")) + best = state.get("best") + baseline = snapshot.candidate(plan["baseline_path"], plan.get("baseline_sha256")) + result["baseline"] = {**baseline, "configuration": plan.get("evaluation_settings")} + selected = None + if best: + selected = ( + snapshot.candidate(best["path"], best.get("sha256")) + if best.get("path") + else {"fingerprint": None, "verified": False} + ) + result["selected_candidate"] = { + **best, + **selected, + "configuration": plan.get("evaluation_settings"), + } + rounds = state.get("rounds", []) + final = state.get("final") + parsed = [] + final_value = None + final_refs = [] + final_candidate = None + final_observation = None + result["rounds"], result["metrics"], result["checks"] = [], [], [] + records = [*rounds, *([final] if final else [])] + for index, record in enumerate(records): + is_final = index == len(rounds) + phase = "final" if is_final else "search" + directory = record["path"] + observation, ref = snapshot.object( + directory + "/observation.json", "assessment", record.get("observation_sha256") + ) + # Compare cached assessment with its independently retained original. + if observation and observation != { + k: v for k, v in record.items() if k != "observation_sha256" + }: + observation = None + gap(result, "ASSESSMENT_CONFLICT", ref, "Cached and retained observations disagree") + value = observation.get("assessment") if observation else None + if ( + not isinstance(value, dict) + or value.get("status") + not in ("completed", "candidate_failure", "infrastructure_failure") + or value.get("verdict") not in ("PASS", "FAIL", "INCONCLUSIVE") + ): + value = None + gap( + result, + "ASSESSMENT_UNAVAILABLE", + ref, + "The original assessment is absent or malformed", + ) + candidate = ( + snapshot.candidate( + directory + "/" + plan["candidate_name"], record.get("candidate_sha256") + ) + if record.get("candidate_sha256") + else None + ) + evidence_refs = [ + snapshot.file( + directory + "/execution/execution.json", "execution", record.get("execution_sha256") + ) + ] + for name, expected in (value or {}).get("evidence_files", {}).items(): + evidence_refs.append(snapshot.file(directory + "/" + name, "measurement", expected)) + usage_source = (record.get("observations") or {}).get("usage_source") + usage_record = observation or {} + if usage_source: + usage_ref = snapshot.file( + directory + "/agent-usage.json", "usage", usage_source.get("sha256") + ) + if not snapshot.verified([usage_ref]): + usage_record = {} + parsed.append(usage_record) + metric_start = len(result["metrics"]) + if value: + result["metrics"].extend(_metrics(value, plan, phase, ref)) + result["checks"].append( + { + "name": "frozen_assessment", + "required": True, + "phase": phase, + "execution_state": value.get("status"), + "gate_ref": plan_ref, + "observation": value, + "verdict": value.get("verdict"), + "evidence_refs": [ref, *evidence_refs], + } + ) + if is_final: + final_value, final_refs = value, [ref, *evidence_refs] + final_candidate, final_observation = candidate, observation + result["final"] = { + "attempt_id": directory.rsplit("/", 1)[-1], + "fingerprint": record.get("candidate_sha256"), + "protocol_ref": plan_ref, + "assessment_ref": ref, + "execution": observation.get("execution") if observation else None, + "status": (value or {}).get("status"), + "candidate_binding": record.get("candidate_binding"), + } + assessment(result, value, ref, scope=plan.get("evaluation_settings", {}).get("scope")) + else: + best_after = record.get("best_after", {}) + result["rounds"].append( + { + "round": index + 1, + "attempt_id": directory.rsplit("/", 1)[-1], + "candidate_fingerprint": record.get("candidate_sha256"), + "hypothesis": record.get("hypothesis"), + "change_summary": record.get("change_summary"), + "execution_state": (observation or {}).get("execution", {}).get("status"), + "assessment_ref": ref, + "verdict": (value or {}).get("verdict"), + "reason": (value or {}).get("reason"), + "score": (value or {}).get("score"), + "metric_refs": list(range(metric_start, len(result["metrics"]))), + "selection_decision": "retained" + if best_after.get("candidate_id") == (record.get("candidate_id")) + else "not_retained", + "best_candidate": best_after, + "best_score": best_after.get("score"), + } + ) + if pending := state.get("pending"): + result["rounds"].append( + { + "round": None if pending["kind"] == "final" else len(rounds) + 1, + "attempt_id": pending["path"].rsplit("/", 1)[-1], + "phase": pending["kind"], + "candidate_fingerprint": pending.get("candidate_sha256"), + "hypothesis": pending.get("hypothesis"), + "change_summary": None, + "execution_state": "pending", + "assessment_ref": None, + "metric_refs": [], + "selection_decision": None, + "best_candidate": best, + } + ) + terminal = state.get("state") in ("completed", "blocked") + result["export"]["kind"] = "final" if terminal else "progress" + result["execution"] = { + "namespace": "inferrsi.harness.optimization", + "state": state.get("state"), + "budget": { + "max_seconds": plan.get("max_seconds"), + "final_reserve_seconds": plan.get("final_reserve_seconds"), + }, + "completed_rounds": len(rounds), + "round_limit": plan.get("round_limit"), + "round_mode": plan.get("round_mode"), + "stop_reason": state.get("stop_reason"), + "started_at": timestamp(state.get("started_at")), + "ended_at": timestamp(state.get("settled_at")), + } + rounds_complete = ( + len(rounds) == plan["round_limit"] + if plan["round_mode"] == "exact" + else 0 < len(rounds) <= plan["round_limit"] and bool(state.get("stop_reason")) + ) + finished = ( + rounds_complete + and final_value is not None + and final_value.get("status") in ("completed", "candidate_failure") + and final_observation.get("execution", {}).get("status") in ("completed", "failed") + and state.get("pending") is None + ) + result["completion"] = { + "status": "completed" if finished else "incomplete", + "reason": "Required search and independent final assessment completed" + if finished + else "Required rounds or independent final assessment are incomplete", + } + if not final: + gap(result, "FINAL_NOT_RUN", "final_acceptance", "Independent final evaluation is absent") + fingerprint_bound = bool( + selected + and final_candidate + and selected["verified"] + and final_candidate["verified"] + and selected["fingerprint"] == final_candidate["fingerprint"] + ) + # Substitute metadata cannot override a contradictory file fingerprint. + substitute_bound = ( + not (best or {}).get("sha256") + and not (final or {}).get("candidate_sha256") + and _binding( + snapshot, + (best or {}).get("binding"), + (final or {}).get("candidate_binding"), + attempt_id=(final or {}).get("path", "").rsplit("/", 1)[-1], + protocol_digest=state["plan_digest"], + ) + ) + bound = fingerprint_bound or substitute_bound + if result["final"] is not None: + result["final"]["binding"] = { + "verified": bound, + "kind": "fingerprint" + if fingerprint_bound + else "exact_revision" + if substitute_bound + else None, + } + if not bound: + gap( + result, + "CANDIDATE_BINDING_UNVERIFIED", + "final_acceptance", + "Neither file fingerprint nor exact revision/run/config binding verifies", + ) + current = ( + bound + and snapshot.verified(frozen_refs) + and snapshot.verified(final_refs) + and baseline["verified"] + and not state.get("integrity_error") + and not any(g["code"] == "FROZEN_INPUTS_CHANGED" for g in result["gaps"]) + ) + result["acceptance"]["delivery_accepted"] = bool( + finished and current and result["acceptance"]["verdict"] == "PASS" + ) + optimization_speedup = None + if best and best.get("kind") == "baseline": + result["outcome"] = { + "kind": "baseline_retained", + "reason": "Controller retained the baseline", + } + if finished and current and final_value.get("status") == "completed": + optimization_speedup = 1.0 + elif result["acceptance"]["delivery_accepted"]: + selection = plan.get("selection", {}) + score = number(final_value.get("score")) + baseline_score = number(final_value.get("baseline_score", selection.get("initial_score"))) + improved = ( + score is not None + and baseline_score is not None + and ( + score < baseline_score + if selection.get("direction") == "minimize" + else score > baseline_score + ) + ) + if improved and final_value.get("eligible", True): + result["outcome"] = { + "kind": "improved", + "reason": "Selected code passed independent final", + } + optimization_speedup = number(final_value.get("speedup")) + if optimization_speedup is None and selection.get("metric") == "speedup": + optimization_speedup = score + result["outcome"]["optimization_speedup"] = optimization_speedup + result["usage"]["tokens"] = _usage(parsed[: len(rounds)], state_ref, "settled_search_rounds") + if final: + # A final caller export observes the whole session; never add it to round usage. + result["usage"]["session_tokens"] = _usage( + parsed[-1:], final_refs[0], "caller_session_export" + ) + result["usage"]["session_tokens"]["coverage"] = "unknown" + start = number(state.get("started_at")) + end = number(state.get("settled_at")) + if end is None: + from datetime import datetime + + end = datetime.fromisoformat(result["export"]["snapshot_at"]).timestamp() + if start is not None and end >= start: + result["usage"]["wall_seconds"] = { + "value": end - start, + "source": state_ref, + "coverage": "complete" if terminal else "partial", + } diff --git a/src/inferrsi/apps/delivery/render.py b/src/inferrsi/apps/delivery/render.py new file mode 100644 index 00000000..46e7a623 --- /dev/null +++ b/src/inferrsi/apps/delivery/render.py @@ -0,0 +1,287 @@ +"""Both offline reports render one projection; imported text is never markup.""" + +import html +import json + +from .schema import number + + +def _text(value): + if value is None: + return "Unknown" + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False) + return str(value) + + +def _markdown(value): + # Numeric entities prevent caller text from becoming HTML, Markdown links, + # headings, fences or table rows, including javascript: links and . + return "".join(f"&#{ord(c)};" if c in '\\`*_{}[]()#+-.!|<>"&\n\r' else c for c in _text(value)) + + +def _sections(result): + acceptance = result["acceptance"] + selected = result["selected_candidate"] + verdict = acceptance["verdict"] if acceptance["status"] == "assessed" else "Not assessed" + metrics = result["metrics"] or [] + sections = [ + ( + "Result summary", + ["Item", "Result"], + [ + ["Goal", result["campaign"].get("goal")], + ["Workflow", result["completion"]["status"]], + ["Completion reason", result["completion"]["reason"]], + ["Outcome", result["outcome"]["kind"]], + ["Original assessment", verdict], + ["Acceptance scope", acceptance["scope"]], + ["Delivery acceptance verified", acceptance["delivery_accepted"]], + [ + "Selected code", + selected.get("path", selected.get("revision")) if selected else None, + ], + ["Optimization speedup", result["outcome"].get("optimization_speedup")], + ], + ), + ( + "Experiment conditions", + ["Item", "Recorded value"], + [ + ["Task kind", result["campaign"]["task_kind"]], + ["Task", result["campaign"].get("task")], + ["Frozen plan", result["campaign"].get("plan_ref")], + ["Acceptance basis", result["campaign"].get("acceptance_basis")], + ["Baseline", result["baseline"]], + ], + ), + ( + "Final comparison", + [ + "Metric", + "Scope / workload", + "Phase", + "Statistic", + "Unit", + "Baseline", + "Candidate", + "Speedup", + "Improvement %", + "Sample counts", + "Repetitions", + "Comparison limits", + "Evidence", + ], + [ + [ + m["name"], + [m["scope"], m["workload_id"]], + m["phase"], + m["statistic"], + m["unit"], + m["baseline"], + m["candidate"], + m["speedup"], + m["improvement_pct"], + m["sample_counts"], + m["repetitions"], + m["comparability"]["reason"], + m["evidence_refs"], + ] + for m in metrics + if m["phase"] != "search" + ], + ), + ( + "Optimization history", + [ + "Round", + "Attempt", + "Hypothesis", + "Change", + "State", + "Verdict", + "Score", + "Best score", + "Selection / reason", + ], + [ + [ + r.get("round"), + r.get("attempt_id"), + r.get("hypothesis"), + r.get("change_summary"), + r.get("execution_state"), + r.get("verdict"), + r.get("score"), + r.get("best_score"), + [r.get("selection_decision"), r.get("reason")], + ] + for r in result["rounds"] or [] + ], + ), + ( + "Validation and limitations", + ["Item", "Recorded value"], + [ + ["Independent final", result["final"]], + ["Checks", result["checks"]], + *[[g["code"] + " (" + g["affects"] + ")", g["reason"]] for g in result["gaps"]], + ["Statistical significance", "Not established by export"], + ], + ), + ( + "Code and reproduction", + ["Item", "Reference"], + [ + ["Selected retained code", selected], + ["Frozen commands / configuration", result["campaign"].get("acceptance_basis")], + ["Source records", result["sources"]], + [ + "Evidence access", + "Source paths in artifacts.json require the original input " + "directory. External archives retain their own access requirements. " + "No evidence, models or dependencies were downloaded or executed.", + ], + ], + ), + ( + "Time and resources", + ["Measure", "Value / source / coverage"], + [ + ["Execution", result["execution"]], + *[[key, value] for key, value in result["usage"].items()], + ], + ), + ] + return sections + + +def _curve(result): + rounds = [row for row in result["rounds"] or [] if row.get("round") is not None] + values = [number(row.get(key)) for row in rounds for key in ("score", "best_score")] + positive = [value for value in values if value is not None and value > 0] + if not positive: + return "" + scale = max(positive) + nodes = [ + '
', + "Search only; independent final acceptance is reported separately", + '', + ] + for key, color, label in ( + ("score", "#176fa6", "Round score"), + ("best_score", "#bd6e0a", "Best retained score"), + ): + previous = None + for index, row in enumerate(rounds): + value = number(row.get(key)) + if value is None or value <= 0: + previous = None # Missing/failed observations leave gaps, never zeros. + continue + x, y = 60 + 690 * index / max(1, len(rounds) - 1), 205 - 180 * (value / scale) + if previous is not None: + nodes.append( + f'' + ) + nodes.append( + f'' + f"{label}, round {index + 1}: {value}" + ) + previous = x, y + nodes += [ + 'Round score (blue) · best retained score (orange)', + "
Search rounds in submission order; missing scores leave gaps. " + "Hover a point for its recorded value. Final acceptance remains separate.
" + "
", + ] + return "".join(nodes) + + +def reports(result, result_digest): + metadata = ( + f"Export: {result['export']['id']} · Snapshot: {result['export']['snapshot_at']} · " + f"Kind: {result['export']['kind']} · Structured result SHA-256: {result_digest}" + ) + conclusion = ( + f"{result['completion']['status']}: {result['outcome']['kind']}. " + f"Original assessment: {result['acceptance']['verdict'] or 'Not assessed'}. " + f"{result['completion']['reason']}." + ) + md = [ + "# RSI result report", + "", + _markdown(conclusion), + "", + _markdown(metadata), + "", + "[Open in a browser](report.html) · [Structured result](campaign-result.json) · " + "[Artifact index](artifacts.json)", + "", + "Evidence remains in the referenced source locations; it is not bundled. " + "Review source records and references before sharing this local export.", + "", + ] + body = [ + "

RSI result report

", + f"

{html.escape(conclusion)}

", + f"

{html.escape(metadata)}

", + '', + "

Evidence remains in the referenced source locations; it is not bundled. " + "Core content is available offline without scripts. Review before sharing.

", + ] + # Verification failures must be on the first screen even for a historical PASS. + if result["gaps"]: + warning = "; ".join(g["reason"] for g in result["gaps"]) + md += ["**Current gaps:** " + _markdown(warning), ""] + body += [f'

Current gaps: {html.escape(warning)}

'] + for title, headers, rows in _sections(result): + md += [ + "## " + title, + "", + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + md.extend("| " + " | ".join(_markdown(v) for v in row) + " |" for row in rows) + if not rows: + md.append("| Not recorded | " + " | " * (len(headers) - 2) + "|") + md.append("") + body += [ + f"

{title}

", + _curve(result) if title == "Optimization history" else "", + "
", + "".join(f"" for h in headers), + "", + ] + body.extend( + "" + "".join(f"" for v in row) + "" + for row in rows + ) + body += ["
{h}
{html.escape(_text(v))}
"] + # Native details supplies offline interaction without executable imported data. + body += [ + "
Full structured snapshot
",
+        html.escape(json.dumps(result, ensure_ascii=False, indent=2)),
+        "
", + ] + document = ( + """ + + +RSI result report""" + + "\n".join(body) + + "\n" + ) + return "\n".join(md).encode("utf-8"), document.encode("utf-8") diff --git a/src/inferrsi/apps/delivery/schema.py b/src/inferrsi/apps/delivery/schema.py new file mode 100644 index 00000000..f35c067d --- /dev/null +++ b/src/inferrsi/apps/delivery/schema.py @@ -0,0 +1,190 @@ +"""Versioned delivery projection, independent of execution and AppsResultV1.""" + +import json +import math +from datetime import datetime, timezone + +SCHEMA_ID = "inferrsi.campaign-delivery" +INDEX_ID = "inferrsi.delivery-artifacts" +FIELDS = { + "schema_id", + "schema_version", + "export", + "campaign", + "sources", + "execution", + "completion", + "outcome", + "baseline", + "selected_candidate", + "final", + "acceptance", + "metrics", + "checks", + "rounds", + "usage", + "gaps", + "artifacts_ref", +} + + +def encode(value): + return ( + json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False) + "\n" + ).encode("utf-8") + + +def decode(raw): + def constant(_): + raise ValueError("Nonfinite JSON number") + + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("Duplicate JSON key") + result[key] = value + return result + + value = json.loads(raw, parse_constant=constant, object_pairs_hook=pairs) + encode(value) # Also rejects overflowing float tokens and invalid Unicode. + return value + + +def number(value): + try: + return value if type(value) in (int, float) and math.isfinite(value) else None + except OverflowError: + return None + + +def timestamp(value): + if number(value) is not None: + try: + return datetime.fromtimestamp(value, timezone.utc).isoformat() + except (OverflowError, OSError, ValueError): + return None + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is not None: + return parsed.isoformat() + except ValueError: + pass + return None + + +def gap(result, code, affects, reason): + entry = {"code": code, "affects": affects, "reason": reason} + if entry not in result["gaps"]: + result["gaps"].append(entry) + + +def empty_result(export_id, observed_at, version): + return { + "schema_id": SCHEMA_ID, + "schema_version": 1, + "export": { + "id": export_id, + "generated_at": timestamp(observed_at), + "snapshot_at": timestamp(observed_at), + "kind": "progress", + "generator_version": version, + }, + "campaign": { + "id": None, + "goal": None, + "task_kind": None, + "acceptance_basis": None, + "plan_ref": None, + }, + "sources": [], + "execution": {}, + "completion": {"status": "incomplete", "reason": "Workflow facts are unavailable"}, + "outcome": {"kind": "undetermined", "reason": "Final improvement is not established"}, + "baseline": None, + "selected_candidate": None, + "final": None, + "acceptance": { + "status": "not_assessed", + "verdict": None, + "claim_id": None, + "scope": None, + "assessment_ref": None, + "delivery_accepted": False, + }, + "metrics": None, + "checks": None, + "rounds": None, + "usage": { + key: {"value": None, "source": None, "coverage": "unknown"} + for key in ("wall_seconds", "tokens", "device_seconds", "cost") + }, + "gaps": [], + "artifacts_ref": "artifacts.json", + } + + +def validate_result(value): + if not isinstance(value, dict) or not FIELDS <= value.keys(): + raise ValueError("Missing delivery contract fields") + if ( + value["schema_id"] != SCHEMA_ID + or type(value["schema_version"]) is not int + or value["schema_version"] != 1 + ): + raise ValueError("Unsupported delivery schema or version") + for key in ("export", "campaign", "execution", "completion", "outcome", "acceptance", "usage"): + if not isinstance(value[key], dict): + raise ValueError("Invalid delivery object field") + for key in ("baseline", "selected_candidate", "final"): + if value[key] is not None and not isinstance(value[key], dict): + raise ValueError("Invalid optional delivery object") + for key in ("sources", "gaps", "metrics", "checks", "rounds"): + if value[key] is None and key not in ("sources", "gaps"): + continue + if not isinstance(value[key], list) or any(not isinstance(row, dict) for row in value[key]): + raise ValueError("Invalid delivery collection") + if value["completion"].get("status") not in ("completed", "incomplete"): + raise ValueError("Invalid workflow completion") + if value["outcome"].get("kind") not in ( + "improved", + "baseline_retained", + "no_valid_candidate", + "undetermined", + ): + raise ValueError("Invalid selection outcome") + acceptance = value["acceptance"] + if ( + (acceptance.get("status") == "not_assessed" and acceptance.get("verdict") is not None) + or ( + acceptance.get("status") == "assessed" + and acceptance.get("verdict") not in ("PASS", "FAIL", "INCONCLUSIVE") + ) + or acceptance.get("status") not in ("assessed", "not_assessed") + ): + raise ValueError("Invalid assessment semantics") + if type(acceptance.get("delivery_accepted")) is not bool: + raise ValueError("Invalid delivery acceptance flag") + if acceptance["delivery_accepted"] and ( + acceptance["verdict"] != "PASS" + or value["completion"]["status"] != "completed" + or not value["final"] + or value["final"].get("binding", {}).get("verified") is not True + ): + raise ValueError("Delivery acceptance requires a completed final with verified binding") + for measure in value["usage"].values(): + if ( + not isinstance(measure, dict) + or measure.get("coverage") not in ("complete", "partial", "unknown") + or not {"value", "source"} <= measure.keys() + ): + raise ValueError("Invalid usage coverage") + if value["export"].get("kind") not in ("progress", "final"): + raise ValueError("Invalid snapshot kind") + for key in ("generated_at", "snapshot_at"): + if timestamp(value["export"].get(key)) is None: + raise ValueError("Invalid export timestamp") + if value["artifacts_ref"] != "artifacts.json": + raise ValueError("Invalid artifact index reference") + encode(value) diff --git a/src/inferrsi/apps/delivery/snapshot.py b/src/inferrsi/apps/delivery/snapshot.py new file mode 100644 index 00000000..9d8ef19a --- /dev/null +++ b/src/inferrsi/apps/delivery/snapshot.py @@ -0,0 +1,182 @@ +"""Read-only, bounded snapshots with independent availability and integrity facts.""" + +import hashlib +import mimetypes +import stat +from pathlib import Path, PurePosixPath + +from ...harness import candidates +from ...infra.artifacts import sha256 +from .schema import decode + +MAX_DOCUMENT_BYTES = 8 * 1024 * 1024 + + +def relative_path(name): + if ( + not isinstance(name, str) + or not name + or "\\" in name + or ":" in name + or "\x00" in name + or name.startswith("/") + ): + raise ValueError("Unsafe delivery source path") + path = PurePosixPath(name) + if ".." in path.parts or path.as_posix() != name or name == ".": + raise ValueError("Unsafe delivery source path") + return path + + +class Snapshot: + def __init__(self, root: Path, observed_at: str): + self.root = root + self.observed_at = observed_at + self.entries = {} + self.documents = {} + self.observations = {} + self.trees = {} + + def path(self, name): + relative_path(name) + path = self.root / name + for parent in ( + self.root, + *[self.root / Path(*Path(name).parts[:i]) for i in range(1, len(Path(name).parts) + 1)], + ): + if parent.is_symlink(): + raise ValueError("Symbolic delivery source path") + return path + + def file(self, name, role, expected=None, *, document=False): + path = self.path(name) + identity = "source:" + name + previous = self.entries.get(identity) + expected = expected or (previous or {}).get("sha256") + raw = None + observed = None + size = None + try: + mode = path.stat().st_mode + if not stat.S_ISREG(mode): + raise ValueError("Delivery source must be a regular file") + size = path.stat().st_size + if document: + if size > MAX_DOCUMENT_BYTES: + raise ValueError("Delivery source document exceeds the read limit") + raw = path.read_bytes() + size = len(raw) + observed = hashlib.sha256(raw).hexdigest() + else: + observed = sha256(path) + availability = "available" + except FileNotFoundError: + availability = "missing" + except PermissionError: + availability = "restricted" + entry = { + "id": identity, + "role": role, + "location": name, + "location_kind": "source", + "media_type": mimetypes.guess_type(name)[0] or "application/octet-stream", + "size_bytes": size, + "sha256": expected or observed, + "observed_sha256": observed, + "source_revision": expected or observed, + "availability": availability, + "integrity": ("mismatch" if expected and observed != expected else "verified") + if observed + else "not_checked", + "checked_at": self.observed_at, + "access": "Requires the original input directory; no remote fetch was performed", + } + if previous and previous["observed_sha256"] != observed: + raise SnapshotChanged("Source changed during snapshot") + if previous and expected and previous["sha256"] != expected: + # Different frozen references to the same path are a conflict, not last-write-wins. + entry["integrity"] = "mismatch" + self.entries[identity] = entry + self.observations[name] = (availability, observed, size) + if raw is not None: + self.documents[name] = raw + return identity + + def object(self, name, role, expected=None, *, required=False): + ref = self.file(name, role, expected, document=True) + raw = self.documents.get(name) + if raw is not None and self.entries[ref]["integrity"] != "mismatch": + try: + value = decode(raw) + if isinstance(value, dict): + return value, ref + except (ValueError, UnicodeError): + pass + self.entries[ref]["integrity"] = "mismatch" + if required: + raise ValueError("Required delivery control document is missing or invalid") + return None, ref + + def candidate(self, name, expected): + path = self.path(name) + try: + members = candidates.members(path) + actual = candidates.identity(path) + names = [p.relative_to(self.root).as_posix() for p in members.values()] + refs = [ + self.file(n, "source_code", expected if path.is_file() else None) for n in names + ] + self.trees[name] = actual + return { + "path": name, + "fingerprint": expected, + "observed_fingerprint": actual, + "artifact_refs": refs, + "verified": bool(expected) and actual == expected, + } + except (FileNotFoundError, PermissionError): + pass + except ValueError: + # Missing code is a verification gap. Links/unsafe paths are rejected. + if path.exists() or path.is_symlink(): + raise + self.trees[name] = None + ref = self.file(name, "source_code", expected) + return { + "path": name, + "fingerprint": expected, + "observed_fingerprint": None, + "artifact_refs": [ref], + "verified": False, + } + + def verified(self, refs): + return bool(refs) and all(self.entries[ref]["integrity"] == "verified" for ref in refs) + + def watch_tree(self, name): + self.trees[name] = candidates.identity(self.path(name)) + + def stable(self): + """Detect uncoordinated writes, including candidate member-set changes.""" + probe = Snapshot(self.root, self.observed_at) + for name in self.observations: + probe.file(name, "snapshot_check") + if probe.observations != self.observations: + return False + for name, expected in self.trees.items(): + path = probe.path(name) + try: + actual = candidates.identity(path) + except (FileNotFoundError, PermissionError, ValueError): + actual = None + if actual != expected: + return False + return True + + +class SnapshotChanged(ValueError): + pass + + +class SnapshotBusy(ValueError): + pass diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index beded41f..88374a13 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -151,6 +151,12 @@ def test_real_loop_rejects_bad_round_keeps_best_and_retests_independently(sessio assert main(["benchmarks", "session", "final", "--session", str(session)]) == 0 final = json.loads(capsys.readouterr().out) assert final["state"] == "completed" + from inferrsi.apps.delivery import read_bundle + + assert final["delivery"]["status"] == "published" + delivery = read_bundle(session / "delivery" / final["delivery"]["export_id"]) + assert delivery["acceptance"]["verdict"] == "PASS" + assert delivery["outcome"]["kind"] == "improved" assert (session / "calls.log").read_text().splitlines() == [*map(str, factors), "2"] assert failed_record.read_bytes() == failed_bytes state = json.loads((session / "state.json").read_text()) @@ -203,6 +209,30 @@ def test_final_cannot_skip_optimization_rounds(session): assert not (session / "final").exists() +def test_final_report_error_preserves_execution_outcome_and_can_be_exported_later( + session, capsys, monkeypatch, +): + from inferrsi.apps import delivery + + for factor in (2, 3, 4): + submit(session, factor, capsys) + original = delivery.export_bundle + + def fail(*args, **kwargs): + raise OSError("Synthetic publication failure") + + monkeypatch.setattr(delivery, "export_bundle", fail) + assert main(["benchmarks", "session", "final", "--session", str(session)]) == 0 + result = json.loads(capsys.readouterr().out) + assert result["state"] == "completed" and result["delivery"]["status"] == "error" + calls = (session / "calls.log").read_bytes() + monkeypatch.setattr(delivery, "export_bundle", original) + assert main(["export", "--input", str(session), + "--output", str(session / "delivery" / "retry")]) == 0 + assert json.loads(capsys.readouterr().out)["status"] == "published" + assert (session / "calls.log").read_bytes() == calls + + def test_missing_usage_remains_unknown_and_baseline_noise_is_not_a_gain(session, capsys): for _ in range(3): submit(session, 1, capsys) diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 7444ef67..9e08239c 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -128,7 +128,14 @@ def test_project_runs_shared_rounds_and_final_from_retained_multifile_candidate( (repo / "engine.py").write_text("raise RuntimeError('unsubmitted edit')", encoding="utf-8") assert main(["campaign", "final", "--directory", str(root), "--reason", "Caller ended the up-to-budget search"]) == 0 - capsys.readouterr() + receipt = json.loads(capsys.readouterr().out) + from inferrsi.apps.delivery import read_bundle + + assert receipt["delivery"]["status"] == "published" + delivery = read_bundle(root / "delivery" / receipt["delivery"]["export_id"]) + assert delivery["completion"]["status"] == "completed" + assert delivery["outcome"]["kind"] == "improved" + assert delivery["acceptance"]["delivery_accepted"] is True plan, state, summary = read_records(root) assert plan["round_mode"] == "up_to" and summary["completed_rounds"] == 1 assert state["final"]["assessment"]["verdict"] == "PASS" diff --git a/tests/test_result_delivery.py b/tests/test_result_delivery.py new file mode 100644 index 00000000..ad7d1821 --- /dev/null +++ b/tests/test_result_delivery.py @@ -0,0 +1,606 @@ +"""Delivery acceptance fixtures use synthetic measurements, never a performance claim.""" + +import json + +import pytest + +from inferrsi.apps import archive, delivery +from inferrsi.apps.cli import main +from inferrsi.apps.delivery.projection import metric +from inferrsi.apps.delivery.schema import decode, validate_result +from inferrsi.apps.delivery.snapshot import Snapshot +from inferrsi.apps.result import preparation_result +from inferrsi.harness import optimization +from inferrsi.infra.artifacts import atomic_write_json, sha256 + + +def make_campaign(tmp_path, *, scores=(5,), final=True, final_verdict="PASS", usage=False): + root = tmp_path / "campaign" + frozen = root / "frozen" + frozen.mkdir(parents=True) + (frozen / "baseline.py").write_text("10", encoding="utf-8") + (frozen / "TASK.md").write_text("Synthetic test; no accelerator claim", encoding="utf-8") + plan = { + "schema_version": 2, + "task": "synthetic", + "task_source": "project", + "condition": "test", + "candidate_kind": "file", + "candidate_name": "candidate.py", + "baseline_path": "frozen/baseline.py", + "working_candidate": "candidate.py", + "round_mode": "exact", + "round_limit": len(scores), + "max_seconds": 600, + "final_reserve_seconds": 60, + "selection": {"direction": "minimize", "metric": "latency", "initial_score": 10}, + "evaluation_settings": { + "metric_key": "latency", + "metric_unit": "ms", + "scope": "synthetic", + "direction": "minimize", + "repetitions": 2, + }, + "frozen_files": {p.name: sha256(p) for p in frozen.iterdir()}, + } + optimization.initialize(root, plan) + + def execute(root, directory, plan, deadline): + (directory / "execution").mkdir() + execution = {"status": "completed", "returncode": 0, "elapsed_seconds": 1} + atomic_write_json(directory / "execution/execution.json", execution) + value = float((directory / "candidate.py").read_text()) + atomic_write_json(directory / "measurement.json", {"synthetic_value": value}) + return execution + + def assess(root, directory, plan, execution): + value = json.loads((directory / "measurement.json").read_text())["synthetic_value"] + verdict = final_verdict if "final" in directory.parts else "PASS" + if value >= 10: + verdict = "FAIL" + return { + "status": "completed", + "score": value, + "baseline_score": 10, + "verdict": verdict, + "eligible": verdict == "PASS", + "quality_passed": True, + "speedup": 10 / value, + "scope": "synthetic", + "claim_id": "synthetic-primary", + "reason": "Synthetic fixture assessment", + "metrics": { + "name": "latency", + "unit": "ms", + "direction": "minimize", + "baseline": 10, + "candidate": value, + "samples": {"baseline": [10, 10], "candidate": [value, value]}, + }, + "evidence_files": {"measurement.json": sha256(directory / "measurement.json")}, + } + + for index, score in enumerate(scores): + candidate = root / "candidate.py" + candidate.write_text(str(score), encoding="utf-8") + usage_path = None + if usage and index == 0: + usage_path = root / "tokens.json" + atomic_write_json( + usage_path, {"input_tokens": 100, "output_tokens": 20, "cached_input_tokens": 50} + ) + optimization.evaluate_round( + root, + candidate=candidate, + hypothesis=f"Synthetic score {score}", + usage=usage_path, + execute=execute, + assess=assess, + ) + if final: + usage_path = root / "session-tokens.json" if usage else None + if usage_path: + atomic_write_json(usage_path, {"input_tokens": 1000, "output_tokens": 200}) + optimization.evaluate_round( + root, + candidate=None, + hypothesis="Independent final", + usage=usage_path, + execute=execute, + assess=assess, + final=True, + ) + return root + + +def export(root, tmp_path, name="delivery"): + output = tmp_path / name + receipt = delivery.export_bundle(root, output, export_id=name) + assert receipt["status"] == "published", receipt + return delivery.read_bundle(output), output + + +def state(root): + return json.loads((root / "state.json").read_text()) + + +def rewrite_final(root, value): + final = value["final"] + path = root / final["path"] / "observation.json" + atomic_write_json( + path, {key: item for key, item in final.items() if key != "observation_sha256"} + ) + final["observation_sha256"] = sha256(path) + atomic_write_json(root / "state.json", value) + + +def test_complete_bundle_agrees_across_views_and_preserves_sources(tmp_path): + root = make_campaign(tmp_path) + before = {p.relative_to(root): p.read_bytes() for p in root.rglob("*") if p.is_file()} + result, output = export(root, tmp_path) + assert result["completion"]["status"] == "completed" + assert result["outcome"]["kind"] == "improved" + assert result["acceptance"]["verdict"] == "PASS" + assert result["acceptance"]["delivery_accepted"] is True + assert result["final"]["binding"] == {"verified": True, "kind": "fingerprint"} + assert len(result["rounds"]) == 1 + final_metric = next(m for m in result["metrics"] if m["phase"] == "final") + assert ( + final_metric["baseline"], + final_metric["candidate"], + final_metric["speedup"], + final_metric["improvement_pct"], + ) == (10, 5, 2, 50) + fingerprint = sha256(output / "campaign-result.json") + for filename in ("report.md", "report.html"): + text = (output / filename).read_text() + assert fingerprint in text and "improved" in text and "PASS" in text + assert "latency" in text and "ms" in text + assert {p.name for p in output.iterdir()} == { + "report.md", + "report.html", + "campaign-result.json", + "artifacts.json", + } + assert all((root / name).read_bytes() == raw for name, raw in before.items()) + second, _ = export(root, tmp_path, "second") + assert second["metrics"] == result["metrics"] + assert second["outcome"] == result["outcome"] + + +@pytest.mark.parametrize("final,verdict", [(False, "PASS"), (True, "FAIL"), (True, "INCONCLUSIVE")]) +def test_search_pass_is_never_substituted_for_missing_or_failed_final(tmp_path, final, verdict): + root = make_campaign(tmp_path, final=final, final_verdict=verdict) + result, _ = export(root, tmp_path) + assert result["rounds"][0]["verdict"] == "PASS" + assert result["outcome"]["kind"] == "undetermined" + assert result["acceptance"]["delivery_accepted"] is False + assert result["acceptance"]["verdict"] == (verdict if final else None) + assert result["final"] is not None if final else result["final"] is None + assert result["completion"]["status"] == ("completed" if final else "incomplete") + + +@pytest.mark.parametrize("final", [False, True]) +def test_baseline_gain_only_after_completed_final(tmp_path, final): + root = make_campaign(tmp_path, scores=(10,), final=final) + result, _ = export(root, tmp_path) + assert result["outcome"]["kind"] == "baseline_retained" + assert result["outcome"]["optimization_speedup"] == (1 if final else None) + assert result["acceptance"]["verdict"] == ("FAIL" if final else None) + + +@pytest.mark.parametrize( + "damage", + [ + "missing_measurement", + "changed_measurement", + "missing_assessment", + "invalid_assessment", + "changed_candidate", + "changed_final", + ], +) +def test_damaged_evidence_does_not_rewrite_history_or_accept_different_code(tmp_path, damage): + root = make_campaign(tmp_path) + value = state(root) + directory = root / value["final"]["path"] + if damage == "missing_measurement": + (directory / "measurement.json").unlink() + elif damage == "changed_measurement": + (directory / "measurement.json").write_text('{"synthetic_value": 1}') + elif damage == "missing_assessment": + (directory / "observation.json").unlink() + elif damage == "invalid_assessment": + (directory / "observation.json").write_text('{"assessment": "PASS"') + elif damage == "changed_candidate": + (root / value["best"]["path"]).write_text("1000") + else: + (directory / "candidate.py").write_text("1000") + result, output = export(root, tmp_path) + assert result["acceptance"]["delivery_accepted"] is False + assert result["outcome"]["kind"] == "undetermined" + recoverable = "assessment" not in damage + assert result["acceptance"]["verdict"] == ("PASS" if recoverable else None) + assert result["completion"]["status"] == ("completed" if recoverable else "incomplete") + assert "Current gaps" in (output / "report.md").read_text() + with pytest.raises((ValueError, FileNotFoundError)): + optimization.read_records(root) # Normal execution reader remains strict. + + +@pytest.mark.parametrize("binding_state", ["verified", "missing", "mismatch", "bad_revision"]) +def test_no_fingerprint_requires_verified_exact_revision_run_and_config(tmp_path, binding_state): + root = make_campaign(tmp_path) + value = state(root) + revision = "a" * 40 + config = root / "fixed-config.json" + atomic_write_json(config, {"candidate_revision": revision, "parameters": {"batch": 1}}) + config_ref = {"path": config.name, "sha256": sha256(config)} + run = root / "fixed-run.json" + atomic_write_json( + run, + { + "candidate_revision": revision, + "configuration_ref": config_ref, + "attempt_id": value["final"]["path"].rsplit("/", 1)[-1], + "protocol_digest": value["plan_digest"], + }, + ) + binding = { + "revision": revision, + "run_ref": {"path": run.name, "sha256": sha256(run)}, + "configuration_ref": config_ref, + } + value["best"]["sha256"] = None + value["best"]["binding"] = binding + value["final"]["candidate_sha256"] = None + value["final"]["candidate_binding"] = binding + if binding_state == "missing": + run.unlink() + elif binding_state == "mismatch": + run.write_text("{}") + elif binding_state == "bad_revision": + binding["revision"] = "main" + rewrite_final(root, value) + result, _ = export(root, tmp_path) + assert result["acceptance"]["verdict"] == "PASS" + assert result["acceptance"]["delivery_accepted"] is (binding_state == "verified") + assert result["final"]["binding"]["kind"] == ( + "exact_revision" if binding_state == "verified" else None + ) + + +def test_usage_subtotal_and_session_observation_are_not_added(tmp_path): + root = make_campaign(tmp_path, scores=(5, 4), usage=True) + result, _ = export(root, tmp_path) + rounds, session = result["usage"]["tokens"], result["usage"]["session_tokens"] + assert rounds["coverage"] == "partial" and rounds["value"]["total_tokens"] == 120 + assert rounds["value"]["cached_input_tokens"] == 50 + assert session["value"]["total_tokens"] == 1200 and session["coverage"] == "unknown" + assert result["usage"]["device_seconds"]["value"] is None + assert result["usage"]["cost"]["value"] is None + + +@pytest.mark.parametrize( + "direction,before,after,comparable,expected", + [ + ("minimize", 10, 5, True, (2, 50)), + ("maximize", 10, 20, True, (2, 100)), + ("minimize", 0, 5, True, (None, None)), + ("minimize", 10, 0, True, (None, None)), + ("maximize", 10, 20, False, (None, None)), + (None, 10, 20, True, (None, None)), + ("minimize", 1e308, 1e-308, True, (None, None)), + ], +) +def test_ratio_semantics(direction, before, after, comparable, expected): + row = metric( + { + "baseline": before, + "candidate": after, + "direction": direction, + "unit": "test", + "statistic": "median", + }, + phase="test", + ref="source:test", + protocol_ref=None, + comparable=comparable, + ) + assert (row["speedup"], row["improvement_pct"]) == expected + assert row["baseline"] == before and row["candidate"] == after + + +def test_opaque_archive_authority_does_not_fork_metadata_or_fetch(tmp_path, monkeypatch): + root = make_campaign(tmp_path) + manifest = { + "schema_version": 2, + "archive": {"url": "https://example.org/archive.tar.gz", "sha256": "b" * 64, "bytes": 123}, + } + (root / "results").mkdir() + atomic_write_json(root / "results/artifacts.json", manifest) + import urllib.request + + monkeypatch.setattr(urllib.request, "urlopen", lambda *a, **kw: pytest.fail("network access")) + _, output = export(root, tmp_path) + index = json.loads((output / "artifacts.json").read_text()) + assert index["archive_authority_ref"] == "source:results/artifacts.json" + assert "https://example.org" not in (output / "artifacts.json").read_text() + assert json.loads((root / "results/artifacts.json").read_text()) == manifest + + +def test_progress_uses_committed_control_and_a_fixed_clock(tmp_path): + root = make_campaign(tmp_path, final=False) + (root / "summary.json").write_text( + '{"state":"completed","final_assessment":{"verdict":"PASS"}}' + ) + observed = state(root)["started_at"] + 10 + _, result, _ = delivery.project(root, export_id="progress", observed_at=observed) + assert result["export"]["kind"] == "progress" + assert result["completion"]["status"] == "incomplete" + assert result["usage"]["wall_seconds"]["value"] == pytest.approx(10, abs=1e-6) + with optimization._locked(root), pytest.raises(ValueError, match="writer is active"): + delivery.project(root, export_id="busy") + + +def test_snapshot_retries_whole_projection_on_interleaved_write(tmp_path, monkeypatch): + root = make_campaign(tmp_path) + complete = (root / "state.json").read_bytes() + progress = state(root) + progress.update(final=None, state="active") + progress.pop("settled_at") + atomic_write_json(root / "state.json", progress) + original = Snapshot.stable + calls = [] + + def interleaved(snapshot): + calls.append(True) + if len(calls) == 1: + (root / "state.json").write_bytes(complete) + return original(snapshot) + + monkeypatch.setattr(Snapshot, "stable", interleaved) + result, _ = export(root, tmp_path) + assert len(calls) == 2 + assert result["completion"]["status"] == "completed" + assert result["acceptance"]["verdict"] == "PASS" + monkeypatch.setattr(Snapshot, "stable", lambda self: False) + with pytest.raises(ValueError, match="consistent snapshot"): + delivery.project(root, export_id="changing") + + +def test_publication_conflict_and_write_failure_preserve_committed_records(tmp_path, monkeypatch): + root = make_campaign(tmp_path) + _, output = export(root, tmp_path) + before = (root / "state.json").read_bytes() + original = (output / "campaign-result.json").read_bytes() + assert delivery.export_bundle(root, output)["status"] == "error" + assert (output / "campaign-result.json").read_bytes() == original + + def broken(*args): + raise OSError("synthetic full disk") + + monkeypatch.setattr(archive, "_write_file", broken) + failed = tmp_path / "failed" + assert delivery.export_bundle(root, failed)["status"] == "error" + assert not failed.exists() and not list(tmp_path.glob(".delivery-*")) + assert (root / "state.json").read_bytes() == before + + +def test_post_rename_failure_keeps_identity_for_reconciliation(tmp_path, monkeypatch): + root = make_campaign(tmp_path) + output = tmp_path / "uncertain" + original = archive._sync_directory + + def fail_after_publish(path): + if output.exists() and path == output.parent: + raise OSError("synthetic fsync failure") + original(path) + + monkeypatch.setattr(archive, "_sync_directory", fail_after_publish) + receipt = delivery.export_bundle(root, output, export_id="same-export") + assert receipt["status"] == "uncertain" and receipt["export_id"] == "same-export" + assert delivery.read_bundle(output)["export"]["id"] == "same-export" + + +@pytest.mark.parametrize( + "bad_path", ["../outside.py", "/outside.py", "x/../../outside.py", "C:\\outside.py", "link.py"] +) +def test_source_traversal_and_symlinks_are_rejected(tmp_path, bad_path): + root = make_campaign(tmp_path) + value = state(root) + outside = tmp_path / "outside.py" + outside.write_text("private content") + if bad_path == "link.py": + (root / bad_path).symlink_to(outside) + value["best"]["path"] = bad_path + atomic_write_json(root / "state.json", value) + output = tmp_path / "rejected" + assert delivery.export_bundle(root, output)["status"] == "error" + assert not output.exists() and outside.read_text() == "private content" + + +def test_untrusted_text_cannot_inject_html_or_markdown_links(tmp_path): + root = make_campaign(tmp_path) + value = state(root) + payload = " [click](javascript:alert(2))\n# heading |" + record = value["rounds"][0] + record["hypothesis"] = payload + path = root / record["path"] / "observation.json" + atomic_write_json(path, {k: v for k, v in record.items() if k != "observation_sha256"}) + record["observation_sha256"] = sha256(path) + atomic_write_json(root / "state.json", value) + result, output = export(root, tmp_path) + assert result["rounds"][0]["hypothesis"] == payload + html = (output / "report.html").read_text() + markdown = (output / "report.md").read_text() + assert "" not in html + assert "[click](javascript:" not in markdown and " p").first.inner_text() + assert page.get_by_role("img").count() == 1 + page.locator("details summary").click() + assert ( + '"schema_id": "inferrsi.campaign-delivery"' in page.locator("details pre").inner_text() + ) + assert not [url for url in requests if url.startswith(("http:", "https:"))] + browser.close() From a5818e64f2edbb8bf4ac415c5785c5844fd083ae Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 17 Nov 2026 21:27:14 +0800 Subject: [PATCH 2/3] Fix delivery history preservation and browser CI coverage Preserve cached final and search assessments with explicit unverified provenance when retained observations are unavailable or conflicting. Keep current acceptance fail-closed and run both offline delivery report cases in browser CI. Agent-assisted-by: Codex (implementation) Signed-off-by: amy-why-3459 --- .github/workflows/ci-ready.yml | 3 +- docs/result-delivery.md | 30 +++++- src/inferrsi/apps/delivery/projection.py | 63 ++++++++++--- src/inferrsi/apps/delivery/render.py | 20 +++- src/inferrsi/apps/delivery/schema.py | 5 + tests/test_result_delivery.py | 114 +++++++++++++++++++++-- 6 files changed, 209 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 0dcd349b..bec17ed5 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -70,12 +70,13 @@ jobs: run: | python -m pip install -e ".[dev]" playwright python -m playwright install --with-deps chromium - - name: Test dashboard navigation and live refresh in Chromium + - name: Test dashboards and offline delivery reports in Chromium run: | python -m pytest -vv -o addopts='' \ tests/test_profile_story_checks.py::test_profile_and_trace_shortcuts_open_recorded_evidence \ tests/test_knowledge_navigation_browser.py \ tests/test_dashboard_experiments_browser.py \ + tests/test_result_delivery.py::test_offline_html_remains_readable_without_scripts_or_network \ --junitxml="$RUNNER_TEMP/dashboard-browser-results.xml" - name: Retain browser test results if: always() diff --git a/docs/result-delivery.md b/docs/result-delivery.md index c497a2b9..d321e9ce 100644 --- a/docs/result-delivery.md +++ b/docs/result-delivery.md @@ -51,9 +51,12 @@ acceptance remain separate. as an audit fact and sets `delivery_accepted: false`. 3. **Degraded reads:** a separate read-only adapter never weakens Harness `_load` or reassesses evidence. An intact digest-bound observation retains historical - PASS when measurement bytes are missing or modified. A missing, invalid or - changed observation cannot establish PASS from its unverified cached assessment; - the report says “Not assessed” and references preserved state for audit. + PASS when measurement bytes are missing or modified. If the observation is + missing, invalid or conflicts with committed state, a readable cached assessment + in `state.json` retains the historical verdict and completion facts, explicitly + labelled **cached state; unverified**. It cannot establish current delivery + acceptance. If neither source supplies a readable assessment, the report says + “Not assessed”; it never invents a verdict or substitutes a search PASS for final. 4. **Snapshot:** native exports share the controller's nonblocking OS lock. Since writers hold it throughout an attempt, progress exports happen between committed operations. A second digest pass checks every source read, negative lookup and @@ -100,7 +103,24 @@ A missing/interrupted final stays incomplete. A final snapshot means the source stopped, not that it succeeded. Historical completion survives later evidence loss; prominent current gaps prevent current delivery acceptance. -The primary assessment is the final observation, never a favorable search result. +The primary assessment is the final record, never a favorable search result. +For native records, `assessment_provenance` on acceptance, final, checks and rounds +identifies `kind: retained_observation / cached_state`, an indexed `artifact_ref`, +the assessment's `json_pointer` inside that document, and +`verification: verified / unverified`. A retained observation must agree with the +committed record and its recorded digest. A readable cached assessment is used +only when that original cannot be read consistently; its reference resolves to +`state.json`, with `/final/assessment` or `/rounds//assessment`. Conflicting +values retain an explicit gap instead of choosing the more favorable verdict. +The index still records the missing or damaged observation and raw evidence. +Observation verification does not certify all evidence: `delivery_accepted` +separately requires current evidence and candidate binding to verify. Cached +assessments always remain unverified, even if the other evidence is intact. +Their raw measurements remain in the historical check for audit, but do not +produce comparison metrics, derived gains or usage totals. Cached search scores +remain labelled in the history and curve. Malformed or absent cached assessments +cannot establish historical completion; cached timeouts stay incomplete. + Missing claim identities stay null with `identity_incomplete`. Existing INCONCLUSIVE is preserved; missing assessments are “Not assessed,” never fabricated INCONCLUSIVE. `improved` also requires a non-baseline selection, passing verified final and a gain @@ -127,7 +147,7 @@ remain null without accounting observations; elapsed wall time is insufficient. | Input | Authoritative facts | Deliberate gaps | | --- | --- | --- | -| Native project / Benchmark | Plan, campaign, state, digest-bound observations/execution/evidence and retained code | Missing identities stay unknown; summaries cannot substitute for final | +| Native project / Benchmark | Plan, campaign, state, digest-bound observations/execution/evidence and retained code; labelled cached history if observations are lost | Cached history cannot verify delivery; summaries cannot substitute for final | | Historical version 1 `summary.json` | Recorded final assessment and reported completion | No reconstructed rounds, independent final, binding or usage coverage | | Legacy run directory | `run.lock.json`, `status.json`, `evaluation.json`; paired single-experiment median | No invented iterative rounds/final or generalized L4 claim | | Apps `result.json` | Validated AppsResultV1, explicitly selected primary claim, fixed refs, all original claims | Operation success is not campaign completion; storage is not implicitly mounted/fetched | diff --git a/src/inferrsi/apps/delivery/projection.py b/src/inferrsi/apps/delivery/projection.py index 81d80ba2..2774c89b 100644 --- a/src/inferrsi/apps/delivery/projection.py +++ b/src/inferrsi/apps/delivery/projection.py @@ -263,6 +263,7 @@ def iterative(snapshot, result): final_refs = [] final_candidate = None final_observation = None + final_assessment_verified = False result["rounds"], result["metrics"], result["checks"] = [], [], [] records = [*rounds, *([final] if final else [])] for index, record in enumerate(records): @@ -273,12 +274,25 @@ def iterative(snapshot, result): directory + "/observation.json", "assessment", record.get("observation_sha256") ) # Compare cached assessment with its independently retained original. - if observation and observation != { + if observation is not None and observation != { k: v for k, v in record.items() if k != "observation_sha256" }: observation = None gap(result, "ASSESSMENT_CONFLICT", ref, "Cached and retained observations disagree") - value = observation.get("assessment") if observation else None + # Committed state retains history when the original is lost or conflicts; + # this fallback is never current verification. + historical = observation if observation is not None else record + assessment_ref = ref if observation is not None else state_ref + provenance = { + "kind": "retained_observation" if observation is not None else "cached_state", + "artifact_ref": assessment_ref, + "json_pointer": ( + "" if observation is not None else "/final" if is_final else f"/rounds/{index}" + ) + + "/assessment", + "verification": "verified" if observation is not None else "unverified", + } + value = historical.get("assessment") if ( not isinstance(value, dict) or value.get("status") @@ -290,7 +304,16 @@ def iterative(snapshot, result): result, "ASSESSMENT_UNAVAILABLE", ref, - "The original assessment is absent or malformed", + "No readable assessment remains in the retained observation or committed state", + ) + elif observation is None: + gap( + result, + "CACHED_ASSESSMENT_UNVERIFIED", + ref, + "Cached historical assessment from state.json is retained for audit; " + "the original observation is unavailable or conflicts. Current verification " + "is not established by this cached record.", ) candidate = ( snapshot.candidate( @@ -317,7 +340,8 @@ def iterative(snapshot, result): parsed.append(usage_record) metric_start = len(result["metrics"]) if value: - result["metrics"].extend(_metrics(value, plan, phase, ref)) + if observation is not None: + result["metrics"].extend(_metrics(value, plan, phase, ref)) result["checks"].append( { "name": "frozen_assessment", @@ -327,22 +351,32 @@ def iterative(snapshot, result): "gate_ref": plan_ref, "observation": value, "verdict": value.get("verdict"), - "evidence_refs": [ref, *evidence_refs], + "assessment_provenance": provenance, + "evidence_refs": list(dict.fromkeys([assessment_ref, ref, *evidence_refs])), } ) if is_final: final_value, final_refs = value, [ref, *evidence_refs] - final_candidate, final_observation = candidate, observation + final_candidate, final_observation = candidate, historical + final_assessment_verified = observation is not None and value is not None result["final"] = { "attempt_id": directory.rsplit("/", 1)[-1], "fingerprint": record.get("candidate_sha256"), "protocol_ref": plan_ref, - "assessment_ref": ref, - "execution": observation.get("execution") if observation else None, + "assessment_ref": assessment_ref, + "assessment_provenance": provenance if value else None, + "execution": historical.get("execution"), "status": (value or {}).get("status"), "candidate_binding": record.get("candidate_binding"), } - assessment(result, value, ref, scope=plan.get("evaluation_settings", {}).get("scope")) + assessment( + result, + value, + assessment_ref, + scope=plan.get("evaluation_settings", {}).get("scope"), + ) + if value: + result["acceptance"]["assessment_provenance"] = provenance else: best_after = record.get("best_after", {}) result["rounds"].append( @@ -352,8 +386,9 @@ def iterative(snapshot, result): "candidate_fingerprint": record.get("candidate_sha256"), "hypothesis": record.get("hypothesis"), "change_summary": record.get("change_summary"), - "execution_state": (observation or {}).get("execution", {}).get("status"), - "assessment_ref": ref, + "execution_state": historical.get("execution", {}).get("status"), + "assessment_ref": assessment_ref, + "assessment_provenance": provenance if value else None, "verdict": (value or {}).get("verdict"), "reason": (value or {}).get("reason"), "score": (value or {}).get("score"), @@ -415,6 +450,11 @@ def iterative(snapshot, result): if finished else "Required rounds or independent final assessment are incomplete", } + if finished and not final_assessment_verified: + result["completion"]["reason"] = ( + "Required search and independent final assessment completed according to " + "cached state; unverified against the retained final observation" + ) if not final: gap(result, "FINAL_NOT_RUN", "final_acceptance", "Independent final evaluation is absent") fingerprint_bound = bool( @@ -455,6 +495,7 @@ def iterative(snapshot, result): ) current = ( bound + and final_assessment_verified and snapshot.verified(frozen_refs) and snapshot.verified(final_refs) and baseline["verified"] diff --git a/src/inferrsi/apps/delivery/render.py b/src/inferrsi/apps/delivery/render.py index 46e7a623..8caeeb6f 100644 --- a/src/inferrsi/apps/delivery/render.py +++ b/src/inferrsi/apps/delivery/render.py @@ -20,6 +20,12 @@ def _markdown(value): return "".join(f"&#{ord(c)};" if c in '\\`*_{}[]()#+-.!|<>"&\n\r' else c for c in _text(value)) +def _assessment_source(value): + if not value: + return None + return "cached state; unverified" if value["kind"] == "cached_state" else "retained observation" + + def _sections(result): acceptance = result["acceptance"] selected = result["selected_candidate"] @@ -35,6 +41,8 @@ def _sections(result): ["Completion reason", result["completion"]["reason"]], ["Outcome", result["outcome"]["kind"]], ["Original assessment", verdict], + ["Assessment source", _assessment_source(acceptance.get("assessment_provenance"))], + ["Assessment reference", acceptance["assessment_ref"]], ["Acceptance scope", acceptance["scope"]], ["Delivery acceptance verified", acceptance["delivery_accepted"]], [ @@ -101,6 +109,7 @@ def _sections(result): "Change", "State", "Verdict", + "Assessment source", "Score", "Best score", "Selection / reason", @@ -113,6 +122,7 @@ def _sections(result): r.get("change_summary"), r.get("execution_state"), r.get("verdict"), + _assessment_source(r.get("assessment_provenance")), r.get("score"), r.get("best_score"), [r.get("selection_decision"), r.get("reason")], @@ -197,6 +207,11 @@ def _curve(result): "Hover a point for its recorded value. Final acceptance remains separate." "", ] + if any((r.get("assessment_provenance") or {}).get("kind") == "cached_state" for r in rounds): + nodes.append( + "

Some search scores are from cached state; unverified. " + "See each round's assessment source below.

" + ) return "".join(nodes) @@ -205,9 +220,12 @@ def reports(result, result_digest): f"Export: {result['export']['id']} · Snapshot: {result['export']['snapshot_at']} · " f"Kind: {result['export']['kind']} · Structured result SHA-256: {result_digest}" ) + verdict = result["acceptance"]["verdict"] or "Not assessed" + if (result["acceptance"].get("assessment_provenance") or {}).get("kind") == "cached_state": + verdict += " (cached state; unverified)" conclusion = ( f"{result['completion']['status']}: {result['outcome']['kind']}. " - f"Original assessment: {result['acceptance']['verdict'] or 'Not assessed'}. " + f"Original assessment: {verdict}. " f"{result['completion']['reason']}." ) md = [ diff --git a/src/inferrsi/apps/delivery/schema.py b/src/inferrsi/apps/delivery/schema.py index f35c067d..d172c2fe 100644 --- a/src/inferrsi/apps/delivery/schema.py +++ b/src/inferrsi/apps/delivery/schema.py @@ -111,6 +111,7 @@ def empty_result(export_id, observed_at, version): "claim_id": None, "scope": None, "assessment_ref": None, + "assessment_provenance": None, "delivery_accepted": False, }, "metrics": None, @@ -171,6 +172,10 @@ def validate_result(value): or value["completion"]["status"] != "completed" or not value["final"] or value["final"].get("binding", {}).get("verified") is not True + or ( + acceptance.get("assessment_provenance") is not None + and acceptance["assessment_provenance"].get("verification") != "verified" + ) ): raise ValueError("Delivery acceptance requires a completed final with verified binding") for measure in value["usage"].values(): diff --git a/tests/test_result_delivery.py b/tests/test_result_delivery.py index ad7d1821..688e0ee8 100644 --- a/tests/test_result_delivery.py +++ b/tests/test_result_delivery.py @@ -219,14 +219,93 @@ def test_damaged_evidence_does_not_rewrite_history_or_accept_different_code(tmp_ result, output = export(root, tmp_path) assert result["acceptance"]["delivery_accepted"] is False assert result["outcome"]["kind"] == "undetermined" - recoverable = "assessment" not in damage - assert result["acceptance"]["verdict"] == ("PASS" if recoverable else None) - assert result["completion"]["status"] == ("completed" if recoverable else "incomplete") + assert result["acceptance"]["verdict"] == "PASS" + assert result["completion"]["status"] == "completed" assert "Current gaps" in (output / "report.md").read_text() with pytest.raises((ValueError, FileNotFoundError)): optimization.read_records(root) # Normal execution reader remains strict. +@pytest.mark.parametrize("verdict", ["PASS", "FAIL", "INCONCLUSIVE"]) +@pytest.mark.parametrize("damage", ["missing", "invalid", "changed"]) +def test_cached_final_preserves_history_without_verifying_delivery(tmp_path, verdict, damage): + root = make_campaign(tmp_path, final_verdict=verdict) + committed = (root / "state.json").read_bytes() + record = state(root)["final"] + observation = root / record["path"] / "observation.json" + if damage == "missing": + observation.unlink() + elif damage == "invalid": + observation.write_text('{"assessment":') + else: + changed = json.loads(observation.read_text()) + changed["assessment"]["verdict"] = "FAIL" if verdict == "PASS" else "PASS" + atomic_write_json(observation, changed) + result, output = export(root, tmp_path) + assert (root / "state.json").read_bytes() == committed + assert result["completion"]["status"] == "completed" + assert result["acceptance"]["status"] == "assessed" + assert result["acceptance"]["verdict"] == verdict + assert result["acceptance"]["delivery_accepted"] is False + assert result["outcome"]["kind"] == "undetermined" + assert result["outcome"]["optimization_speedup"] is None + provenance = { + "kind": "cached_state", + "artifact_ref": "source:state.json", + "json_pointer": "/final/assessment", + "verification": "unverified", + } + assert result["acceptance"]["assessment_ref"] == "source:state.json" + assert result["acceptance"]["assessment_provenance"] == provenance + assert result["final"]["assessment_provenance"] == provenance + check = next(c for c in result["checks"] if c["phase"] == "final") + assert check["observation"] == record["assessment"] + assert check["assessment_provenance"] == provenance + assert "source:state.json" in check["evidence_refs"] + assert "source:" + record["path"] + "/observation.json" in check["evidence_refs"] + assert not [m for m in result["metrics"] if m["phase"] == "final"] + for filename in ("report.md", "report.html"): + text = (output / filename).read_text() + assert "cached state; unverified" in text + assert "Current gaps" in text + + +def test_conflicting_retained_observation_cannot_verify_cached_pass(tmp_path): + root = make_campaign(tmp_path) + value = state(root) + record = value["final"] + observation = root / record["path"] / "observation.json" + changed = json.loads(observation.read_text()) + changed["assessment"]["verdict"] = "FAIL" + atomic_write_json(observation, changed) + record["observation_sha256"] = sha256(observation) + atomic_write_json(root / "state.json", value) + result, _ = export(root, tmp_path) + assert result["acceptance"]["verdict"] == "PASS" + assert result["acceptance"]["assessment_provenance"]["kind"] == "cached_state" + assert result["acceptance"]["delivery_accepted"] is False + assert any(g["code"] == "ASSESSMENT_CONFLICT" for g in result["gaps"]) + result["acceptance"]["delivery_accepted"] = True + with pytest.raises(ValueError, match="Delivery acceptance requires"): + validate_result(result) + + +def test_missing_search_observation_retains_labelled_history(tmp_path): + root = make_campaign(tmp_path) + record = state(root)["rounds"][0] + (root / record["path"] / "observation.json").unlink() + result, _ = export(root, tmp_path) + search = result["rounds"][0] + assert search["verdict"] == "PASS" + assert search["score"] == record["assessment"]["score"] + assert search["assessment_ref"] == "source:state.json" + assert search["assessment_provenance"]["json_pointer"] == "/rounds/0/assessment" + assert search["assessment_provenance"]["verification"] == "unverified" + # The independently retained final still supplies its own verified assessment. + assert result["acceptance"]["assessment_provenance"]["kind"] == "retained_observation" + assert result["acceptance"]["delivery_accepted"] is True + + @pytest.mark.parametrize("binding_state", ["verified", "missing", "mismatch", "bad_revision"]) def test_no_fingerprint_requires_verified_exact_revision_run_and_config(tmp_path, binding_state): root = make_campaign(tmp_path) @@ -539,7 +618,10 @@ def test_export_cli_success_describes_export_not_experiment_verdict(tmp_path, ca assert receipt["report_html"]["label"] == "Open in a browser" -def test_interrupted_final_is_incomplete_and_keeps_original_inconclusive(tmp_path): +@pytest.mark.parametrize("missing_observation", [False, True]) +def test_interrupted_final_is_incomplete_and_keeps_original_inconclusive( + tmp_path, missing_observation +): root = make_campaign(tmp_path) value = state(root) record = value["final"] @@ -550,6 +632,8 @@ def test_interrupted_final_is_incomplete_and_keeps_original_inconclusive(tmp_pat atomic_write_json(execution, record["execution"]) record["execution_sha256"] = sha256(execution) rewrite_final(root, value) + if missing_observation: + (root / record["path"] / "observation.json").unlink() result, _ = export(root, tmp_path) assert result["execution"]["state"] == "blocked" assert result["completion"]["status"] == "incomplete" @@ -557,11 +641,17 @@ def test_interrupted_final_is_incomplete_and_keeps_original_inconclusive(tmp_pat assert result["outcome"]["optimization_speedup"] is None -def test_malformed_assessment_cannot_become_historical_pass(tmp_path): +@pytest.mark.parametrize("missing_observation", [False, True]) +@pytest.mark.parametrize("assessment", [None, "PASS", {}]) +def test_malformed_assessment_cannot_become_historical_pass( + tmp_path, missing_observation, assessment +): root = make_campaign(tmp_path) value = state(root) - value["final"]["assessment"] = "PASS" + value["final"]["assessment"] = assessment rewrite_final(root, value) + if missing_observation: + (root / value["final"]["path"] / "observation.json").unlink() result, _ = export(root, tmp_path) assert result["acceptance"]["status"] == "not_assessed" assert result["completion"]["status"] == "incomplete" @@ -585,9 +675,12 @@ def changed(snapshot): assert not (tmp_path / "changing").exists() -def test_offline_html_remains_readable_without_scripts_or_network(tmp_path): +@pytest.mark.parametrize("missing_observation", [False, True]) +def test_offline_html_remains_readable_without_scripts_or_network(tmp_path, missing_observation): playwright = pytest.importorskip("playwright.sync_api") root = make_campaign(tmp_path) + if missing_observation: + (root / state(root)["final"]["path"] / "observation.json").unlink() _, output = export(root, tmp_path) with playwright.sync_playwright() as driver: browser = driver.chromium.launch() @@ -596,7 +689,12 @@ def test_offline_html_remains_readable_without_scripts_or_network(tmp_path): page.on("request", lambda request: requests.append(request.url)) page.goto((output / "report.html").as_uri()) assert page.locator("h1").inner_text() == "RSI result report" - assert "completed: improved" in page.locator("body > p").first.inner_text() + conclusion = page.locator("body > p").first.inner_text() + expected = "completed: undetermined" if missing_observation else "completed: improved" + assert expected in conclusion + if missing_observation: + assert "Original assessment: PASS (cached state; unverified)" in conclusion + assert "Cached historical assessment" in page.locator(".gaps").inner_text() assert page.get_by_role("img").count() == 1 page.locator("details summary").click() assert ( From 9b55c12de4fb8dd1c9389f6f18551c0a206aba49 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 17 Nov 2026 21:35:40 +0800 Subject: [PATCH 3/3] Harden delivery readers for duplicate paths and oversized observations Reject duplicate bundled locations before checksum reconciliation. Keep oversized optional observations unreadable without hashing them and preserve labelled cached history, while required control documents still prevent export. Agent-assisted-by: Codex (implementation) Signed-off-by: amy-why-3459 --- docs/result-delivery.md | 8 +++- src/inferrsi/apps/delivery/__init__.py | 9 +++- src/inferrsi/apps/delivery/snapshot.py | 19 +++++--- tests/test_result_delivery.py | 65 +++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/docs/result-delivery.md b/docs/result-delivery.md index d321e9ce..2f2c41db 100644 --- a/docs/result-delivery.md +++ b/docs/result-delivery.md @@ -189,11 +189,17 @@ itself. Both reports display the JSON digest and export ID. Reports escape imported text and use only fixed local navigation links. There is no executable embedded data or arbitrary URL fetching. V1 does not bundle arbitrary attachments, follow symlinks or traverse outside input paths. Documents have an -8 MiB read limit; larger raw evidence is streamed for hashing and indexed. Redacted +8 MiB read limit. Oversized optional documents are indexed as `unreadable` with +integrity `not_checked`, retaining the expected digest without reading or hashing +their bytes, including during snapshot rechecks. They allow labelled cached history; +oversized required control documents still prevent export. Larger raw evidence is +streamed for hashing and indexed. Redacted shared copies need separate identities; export never replaces frozen originals. Generation fixes the snapshot, creates JSON, renders reports, builds the index, validates the bundle and uses the existing Apps no-replace directory publisher. +The reader rejects duplicate artifact IDs and duplicate bundled locations before +checking file digests, so a later entry cannot hide an earlier invalid checksum. Existing destinations survive every failure. Post-rename directory-sync failure returns `uncertain` with the same export ID and visible output path: inspect it with `read_bundle` before retrying. Atomic visibility and durability retain the platform diff --git a/src/inferrsi/apps/delivery/__init__.py b/src/inferrsi/apps/delivery/__init__.py index 90d81a3e..cc93f3d4 100644 --- a/src/inferrsi/apps/delivery/__init__.py +++ b/src/inferrsi/apps/delivery/__init__.py @@ -146,7 +146,14 @@ def read_bundle(directory: Path): entries = index.get("artifacts", []) if len({entry["id"] for entry in entries}) != len(entries): raise ValueError("Duplicate delivery artifact ID") - bundled = {entry["location"]: entry for entry in entries if entry["location_kind"] == "bundle"} + bundled = {} + for entry in entries: + if entry["location_kind"] != "bundle": + continue + name = entry["location"] + if name in bundled: + raise ValueError("Duplicate delivery bundle location") + bundled[name] = entry if set(bundled) != {"report.md", "report.html", "campaign-result.json"}: raise ValueError("Incomplete delivery bundle") for name, entry in bundled.items(): diff --git a/src/inferrsi/apps/delivery/snapshot.py b/src/inferrsi/apps/delivery/snapshot.py index 9d8ef19a..ea4ce422 100644 --- a/src/inferrsi/apps/delivery/snapshot.py +++ b/src/inferrsi/apps/delivery/snapshot.py @@ -34,6 +34,7 @@ def __init__(self, root: Path, observed_at: str): self.observed_at = observed_at self.entries = {} self.documents = {} + self.document_names = set() self.observations = {} self.trees = {} @@ -50,6 +51,8 @@ def path(self, name): def file(self, name, role, expected=None, *, document=False): path = self.path(name) + if document: + self.document_names.add(name) identity = "source:" + name previous = self.entries.get(identity) expected = expected or (previous or {}).get("sha256") @@ -61,15 +64,15 @@ def file(self, name, role, expected=None, *, document=False): if not stat.S_ISREG(mode): raise ValueError("Delivery source must be a regular file") size = path.stat().st_size - if document: - if size > MAX_DOCUMENT_BYTES: - raise ValueError("Delivery source document exceeds the read limit") + availability = "available" + if document and size > MAX_DOCUMENT_BYTES: + availability = "unreadable" + elif document: raw = path.read_bytes() size = len(raw) observed = hashlib.sha256(raw).hexdigest() else: observed = sha256(path) - availability = "available" except FileNotFoundError: availability = "missing" except PermissionError: @@ -89,7 +92,11 @@ def file(self, name, role, expected=None, *, document=False): if observed else "not_checked", "checked_at": self.observed_at, - "access": "Requires the original input directory; no remote fetch was performed", + "access": ( + "Document exceeds the 8 MiB read limit; bytes were not read" + if availability == "unreadable" + else "Requires the original input directory; no remote fetch was performed" + ), } if previous and previous["observed_sha256"] != observed: raise SnapshotChanged("Source changed during snapshot") @@ -160,7 +167,7 @@ def stable(self): """Detect uncoordinated writes, including candidate member-set changes.""" probe = Snapshot(self.root, self.observed_at) for name in self.observations: - probe.file(name, "snapshot_check") + probe.file(name, "snapshot_check", document=name in self.document_names) if probe.observations != self.observations: return False for name, expected in self.trees.items(): diff --git a/tests/test_result_delivery.py b/tests/test_result_delivery.py index 688e0ee8..03e8878a 100644 --- a/tests/test_result_delivery.py +++ b/tests/test_result_delivery.py @@ -8,7 +8,7 @@ from inferrsi.apps.cli import main from inferrsi.apps.delivery.projection import metric from inferrsi.apps.delivery.schema import decode, validate_result -from inferrsi.apps.delivery.snapshot import Snapshot +from inferrsi.apps.delivery.snapshot import MAX_DOCUMENT_BYTES, Snapshot from inferrsi.apps.result import preparation_result from inferrsi.harness import optimization from inferrsi.infra.artifacts import atomic_write_json, sha256 @@ -306,6 +306,51 @@ def test_missing_search_observation_retains_labelled_history(tmp_path): assert result["acceptance"]["delivery_accepted"] is True +@pytest.mark.parametrize("phase", ["search", "final"]) +def test_oversized_optional_observation_keeps_cached_history_without_reading_it( + tmp_path, monkeypatch, phase +): + root = make_campaign(tmp_path) + value = state(root) + record = value["final"] if phase == "final" else value["rounds"][0] + path = root / record["path"] / "observation.json" + path.write_bytes(b" " * (MAX_DOCUMENT_BYTES + 1)) + original_open = type(path).open + + def bounded_open(self, *args, **kwargs): + assert self != path, "An oversized observation must not be read, including during recheck" + return original_open(self, *args, **kwargs) + + monkeypatch.setattr(type(path), "open", bounded_open) + result, output = export(root, tmp_path) + assert result["completion"]["status"] == "completed" + historical = result["acceptance"] if phase == "final" else result["rounds"][0] + assert historical["verdict"] == "PASS" + assert historical["assessment_provenance"]["kind"] == "cached_state" + assert result["acceptance"]["delivery_accepted"] is (phase == "search") + index = json.loads((output / "artifacts.json").read_text()) + entry = next( + e for e in index["artifacts"] if e["id"] == "source:" + record["path"] + "/observation.json" + ) + assert entry["availability"] == "unreadable" + assert entry["integrity"] == "not_checked" + assert entry["observed_sha256"] is None + assert entry["sha256"] == record["observation_sha256"] + assert "read limit" in entry["access"] + + +@pytest.mark.parametrize("name", ["plan.json", "state.json", "campaign.json"]) +def test_oversized_required_control_document_prevents_publication(tmp_path, name): + root = make_campaign(tmp_path) + (root / name).write_bytes(b" " * (MAX_DOCUMENT_BYTES + 1)) + with pytest.raises(ValueError, match="Required delivery control document"): + delivery.project(root, export_id="oversized-control") + output = tmp_path / "delivery" + receipt = delivery.export_bundle(root, output) + assert receipt["status"] == "error" + assert not output.exists() + + @pytest.mark.parametrize("binding_state", ["verified", "missing", "mismatch", "bad_revision"]) def test_no_fingerprint_requires_verified_exact_revision_run_and_config(tmp_path, binding_state): root = make_campaign(tmp_path) @@ -610,6 +655,24 @@ def test_reader_rejects_versions_nonfinite_and_modified_reports(tmp_path): delivery.read_bundle(output) +@pytest.mark.parametrize("name", ["report.md", "report.html", "campaign-result.json"]) +@pytest.mark.parametrize("position", [0, -1]) +def test_reader_rejects_unique_id_entries_with_duplicate_bundle_locations(tmp_path, name, position): + root = make_campaign(tmp_path) + _, output = export(root, tmp_path) + index_path = output / "artifacts.json" + index = json.loads(index_path.read_text()) + entry = next(e for e in index["artifacts"] if e["id"] == "bundle:" + name) + duplicate = {**entry, "id": "duplicate:" + name, "sha256": "0" * 64} + if position == 0: + index["artifacts"].insert(0, duplicate) + else: + index["artifacts"].append(duplicate) + atomic_write_json(index_path, index) + with pytest.raises(ValueError, match="Duplicate delivery bundle location"): + delivery.read_bundle(output) + + def test_export_cli_success_describes_export_not_experiment_verdict(tmp_path, capsys): root = make_campaign(tmp_path, final_verdict="FAIL") assert main(["export", "--input", str(root), "--output", str(tmp_path / "cli")]) == 0