diff --git a/docs/apps-module-migrations.json b/docs/apps-module-migrations.json index 4040cca2..28429c51 100644 --- a/docs/apps-module-migrations.json +++ b/docs/apps-module-migrations.json @@ -21,6 +21,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/archive.py", "src/inferrsi/apps/delivery/__init__.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", @@ -50,6 +51,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/artifact_reader.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/views/attempt.py", "tests/check_installed.py", @@ -80,6 +82,7 @@ "src/inferrsi/apps/delivery/adapters.py", "src/inferrsi/apps/handoff/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", + "src/inferrsi/apps/result.py", "src/inferrsi/apps/storage/archive.py", "src/inferrsi/apps/storage/artifact_reader.py", "src/inferrsi/apps/views/attempt.py", @@ -100,6 +103,7 @@ "docs/contracts/current-design.md", "docs/module-layout.md", "docs/operations.md", + "src/inferrsi/apps/evidence_layout.py", "src/inferrsi/contracts/evidence.py", "tests/_gate_policy_support.py", "tests/check_installed.py", @@ -125,6 +129,7 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/bridge.py", "src/inferrsi/apps/handoff/candidate_provider.py", "src/inferrsi/apps/views/attempt.py", "tests/check_installed.py", @@ -146,6 +151,7 @@ "docs/architecture.md", "docs/module-layout.md", "docs/operations.md", + "src/inferrsi/apps/context_provider.py", "tests/check_installed.py", "tests/test_apps_context_provider.py", "tests/test_apps_module_migrations.py", @@ -164,8 +170,129 @@ "docs/module-layout.md", "docs/operations.md", "src/inferrsi/apps/api.py", + "src/inferrsi/apps/candidate_provider.py", "tests/check_installed.py", "tests/test_apps_module_migrations.py" ] + }, + { + "old": "inferrsi.apps.campaign", + "canonical": "inferrsi.apps.workflows.campaign", + "exports": [ + "create_campaign", + "capture_context", + "save_retrieval", + "verify_capture" + ], + "consumers": [ + "docs/architecture.md", + "docs/knowledge-retrieval-trace.md", + "docs/module-layout.md", + "docs/operations.md", + "docs/optimization-workflow-unification.md", + "src/inferrsi/apps/cli/__init__.py", + "src/inferrsi/apps/views/capture_delivery.py", + "src/inferrsi/apps/views/optimization_story.py", + "src/inferrsi/harness/optimization_session.py", + "tests/check_installed.py", + "tests/fixtures/dashboard_profile_review.py", + "tests/fixtures/knowledge_retrieval.py", + "tests/test_apps_module_migrations.py", + "tests/test_campaign_record_injection.py", + "tests/test_campaign_task_selection.py", + "tests/test_knowledge_navigation_browser.py", + "tests/test_optimization_session.py", + "tests/test_optimization_story.py", + "tests/test_search_archive.py", + "tests/test_search_attribution.py" + ] + }, + { + "old": "inferrsi.apps.benchmark_operations", + "canonical": "inferrsi.apps.workflows.benchmark_operations", + "exports": [ + "run_task" + ], + "consumers": [ + "docs/architecture.md", + "docs/module-layout.md", + "src/inferrsi/apps/cli/__init__.py", + "tests/_gate_policy_support.py", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_benchmark_operations.py", + "tests/test_optimization_session.py", + "tests/test_package_root_inventory.py", + "tests/test_project_optimization.py", + "tests/test_target_module_import_gate.py" + ] + }, + { + "old": "inferrsi.apps.improvement", + "canonical": "inferrsi.apps.workflows.improvement", + "exports": [ + "assessment", + "choose", + "holdout_accepts", + "rerun_assessment" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/harness-improvement.md", + "docs/module-layout.md", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_harness_improvement.py", + "tools/skill_eval_improve.py" + ] + }, + { + "old": "inferrsi.apps.skill_evaluation", + "canonical": "inferrsi.apps.workflows.skill_evaluation", + "exports": [ + "export_bundle", + "summarize", + "run_trial" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/module-layout.md", + "docs/skill-evaluation.md", + "tests/check_installed.py", + "tests/test_anthropic_agent.py", + "tests/test_apps_module_migrations.py", + "tests/test_skill_eval_compare.py", + "tests/test_skill_eval_generation.py", + "tests/test_skill_eval_pipeline.py", + "tests/test_skill_evaluation.py", + "tools/skill_eval_compare.py", + "tools/skill_eval_pilot.py" + ] + }, + { + "old": "inferrsi.apps.skill_eval_engineering", + "canonical": "inferrsi.apps.workflows.skill_eval_engineering", + "exports": [ + "verify", + "main" + ], + "consumers": [ + "docs/contracts/function-owners.md", + "docs/local-skill-evaluation.md", + "docs/module-layout.md", + "src/inferrsi/apps/workflows/skill_evaluation.py", + "tests/check_installed.py", + "tests/test_apps_module_migrations.py", + "tests/test_skill_eval_compare.py", + "tests/test_skill_eval_engineering.py", + "tests/test_skill_eval_generation.py", + "tests/test_skill_eval_local.py", + "tests/test_skill_eval_local_cli.py", + "tools/skill_eval_cloud_replay.py", + "tools/skill_eval_compare.py", + "tools/skill_eval_local.py", + "tools/skill_eval_qualify.py" + ], + "script_entry": "main" } ] diff --git a/docs/architecture.md b/docs/architecture.md index 87d68ac9..578b8d66 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,7 +7,7 @@ packages. `inferrsi benchmarks run ...` launches the package's `benchmark.py`. Task-specific orchestration, sampling and assessment live inside each package. The task library owns its `index.json` registration. The existing `benchmark/registry.py` discovers the resources. The CLI package `apps/cli/` -and `apps/benchmark_operations.py` provide application assembly and launch. +and `apps/workflows/benchmark_operations.py` provide application assembly and launch. Each task retains its own protocol and task-specific code. The [current-Agent optimization session](optimization-benchmarks.md) assembles @@ -16,7 +16,7 @@ budget, attempts, feedback records, selection and independent final execution. `harness/task_protocol.py` freezes a supplied resource directory and loads its task-owned assessment protocol. `harness/optimization_session.py` assembles execution and assessment; the CLI resolves task resources and injects knowledge -access and the capture callback from `apps/campaign.py`. +access and the capture callback from `apps/workflows/campaign.py`. The external Agent chooses and edits candidates between round submissions. The same controller serves ordinary projects through `campaign start/round/final` and packaged tasks through `benchmarks session`. The @@ -158,7 +158,6 @@ the comparator; the engine AB/BA path retains its independent acceptance contrac ```text src/inferrsi/ - apps/cli/ CLI entry `main` and its argument parser benchmark/ registry.py Generic task resource discovery tasks/ Task-specific definitions, workloads and assessment @@ -168,19 +167,6 @@ src/inferrsi/ skill_eval_diagnostic.py Synthetic KV admission diagnosis task and grader skill_eval_engineering.py Serving-parser task, public vectors and vector judgment improvement.py Improvement-step thresholds and acceptance rules - apps/ - api.py Python entry points for assembling and publishing Apps operations - benchmark_operations.py Resolve a task entry and launch it through Harness - archive.py Publish and read the standardized Apps output in a workspace - result.py Assemble and check the AppsResultV1 output envelope - skill_evaluation.py Composed bundle export, trial run and summary - skill_eval_engineering.py Composed artifact check: task rules around sandbox replay - improvement.py Composed assessment, selection, rerun and holdout acceptance - inspection.py Environment inspection bound to the host probe and Git identity - legacy/ - run.py Old run composed from Harness, Benchmark and the report - evaluation.py Old evaluate entry: RunSpec to the Benchmark gates - reporting.py HTML evidence report contracts/ runspec.py Immutable experiment contract; the initializer re-exports only RunSpec json_v1.py Canonical inferrsi-json-v1 encoder and decoder @@ -190,6 +176,13 @@ src/inferrsi/ __init__.py Import-free initializer repository_packs.py Neutral REPO_TYPES and pack.json reader; imports no inferrsi module apps/ + api.py Python entry points for assembling and publishing Apps operations + cli/ CLI entry `main` and its argument parser + inspection.py Environment inspection bound to the host probe and Git identity + legacy/ + run.py Old run composed from Harness, Benchmark and the report + evaluation.py Old evaluate entry: RunSpec to the Benchmark gates + reporting.py HTML evidence report handoff/ bridge.py Bind preparations to specs and fixed records to evidence context_provider.py Deliver Knowledge bytes and pins for Harness sealing @@ -199,7 +192,12 @@ src/inferrsi/ artifact_reader.py Read verified artifacts through authorized mounts result.py AppsResultV1 construction and validation evidence_layout.py Create plan/results/patch/validator evidence directories - campaign.py Knowledge values connected to Harness campaign records + workflows/ + campaign.py Knowledge values connected to Harness campaign records + benchmark_operations.py Resolve a task entry and launch it through Harness + skill_evaluation.py Composed bundle export, trial run and summary + skill_eval_engineering.py Composed artifact check: task rules around sandbox replay + improvement.py Composed assessment, selection, rerun and holdout acceptance mindie/ project.py Opt-in per-checkout grant, the worker launch and the recorded read context.py Export supplied MindIE reads into a caller-reviewed campaign trace @@ -289,7 +287,7 @@ The package root holds exactly two tracked files: `__init__.py` and `__main__.py `mcp.py` and the standalone `read_worker.py`) and deleted root `mindie_project.py`, `mindie_context.py`, `mindie_mcp.py` and `_mindie_read_worker.py` with no forwarding shell or alias. #408 A2 moved the chart renderer to -`apps/views/summary_chart.py` and the task launcher to `apps/benchmark_operations.py` +`apps/views/summary_chart.py` and the task launcher to `apps/workflows/benchmark_operations.py` and deleted root `summary_chart.py` and `benchmark_operations.py` the same way. #408 A1 moved the CLI and its parser to `apps/cli/` and deleted root `cli.py` and `arguments.py`; `__main__.py` and the console script call `inferrsi.apps.cli:main`. #412 R1 deleted eleven root modules with no diff --git a/docs/contracts/function-owners.md b/docs/contracts/function-owners.md index 6f147b30..5d9f173e 100644 --- a/docs/contracts/function-owners.md +++ b/docs/contracts/function-owners.md @@ -31,7 +31,7 @@ Any later split of a mixed function is accepted by real pytest/host/wheel/record ## Implemented rows -The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py` and `apps/improvement.py`. Two points follow the tree as it stands after #338, not the row text: +The skill-evaluation split (#410) implements the `skill_evaluation`, `skill_eval_diagnostic`, `skill_eval_engineering` and `improvement` rows. H keeps `harness/skill_evaluation.py`, `harness/artifact_sandbox.py` and `harness/improvement.py`; B holds `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py` and `benchmark/improvement.py`; Apps composes them in `apps/workflows/skill_evaluation.py`, `apps/workflows/skill_eval_engineering.py` and `apps/workflows/improvement.py`. Two points follow the tree as it stands after #338, not the row text: - `usage_totals` stays in `harness/usage.py`, because `harness/optimization.py` consumes it for budget accounting. B `assessment` receives the baseline and candidate totals from Apps and does not import H. - `SelectionPolicy` holds only the two thresholds, so it lives whole in B. H `choose` receives the assessments, the validity block and the composition flag; it holds no threshold and no policy class. diff --git a/docs/harness-improvement.md b/docs/harness-improvement.md index df136e83..d7d0abdb 100644 --- a/docs/harness-improvement.md +++ b/docs/harness-improvement.md @@ -46,7 +46,7 @@ flowchart TB `src/inferrsi/benchmark/improvement.py` owns the thresholds and acceptance rules, `src/inferrsi/harness/improvement.py` the search progression and -`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/improvement.py` +`src/inferrsi/harness/usage.py` usage accounting; `src/inferrsi/apps/workflows/improvement.py` composes them. `tools/skill_eval_improve.py` owns freezing, declarative snapshot materialization and orchestration through `tools/skill_eval_compare.py`. There is no second diff --git a/docs/knowledge-retrieval-trace.md b/docs/knowledge-retrieval-trace.md index a5958347..816b4420 100644 --- a/docs/knowledge-retrieval-trace.md +++ b/docs/knowledge-retrieval-trace.md @@ -107,7 +107,7 @@ same immutable capture envelope used by `capture-context`: ```python from pathlib import Path from inferrsi.knowledge.catalog import search_knowledge -from inferrsi.apps.campaign import save_retrieval +from inferrsi.apps.workflows.campaign import save_retrieval result = search_knowledge("decode launch", explain=True) path, sha256 = save_retrieval( diff --git a/docs/local-skill-evaluation.md b/docs/local-skill-evaluation.md index 81e3cf44..fae59406 100644 --- a/docs/local-skill-evaluation.md +++ b/docs/local-skill-evaluation.md @@ -23,7 +23,7 @@ snapshots. `tools/skill_eval_local.py` adds manual stage/capture, without launch an agent. `src/inferrsi/benchmark/skill_eval_engineering.py` owns the one task, its 19 public behavior probes and their judgment. `src/inferrsi/harness/artifact_sandbox.py` replays the retained artifact under bubblewrap and separates the single writable -artifact from frozen inputs; `src/inferrsi/apps/skill_eval_engineering.py` composes the two. +artifact from frozen inputs; `src/inferrsi/apps/workflows/skill_eval_engineering.py` composes the two. The existing cloud read/submit route is unchanged; it cannot perform this artifact-editing task and is rejected for this adapter. diff --git a/docs/module-layout.md b/docs/module-layout.md index 1c93b831..4ebd3f17 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -6,14 +6,14 @@ moved Knowledge bodies. R1 (#412) deleted the seven static root compatibility shells that K01 left, so those names import only from `inferrsi.knowledge`. The engineering task contribution now implements `benchmark/registry.py` and `benchmark/tasks/`. The CLI package `apps/cli/` and -`apps/benchmark_operations.py` assemble this slice. The former `observability/` +`apps/workflows/benchmark_operations.py` assemble this slice. The former `observability/` package is dissolved. Its campaign records, traces, logs and profile captures are in `harness/records/`, its dashboard views and templates are in `apps/views/`, and the collected-context projection is in `knowledge/context_projection.py`. #371 owns the Apps composition. Its accepted directory follow-up -[#490](https://github.com/InferMatrixLab/InferRSI/issues/490) starts with the -`apps/storage/` slice described below. +[#490](https://github.com/InferMatrixLab/InferRSI/issues/490) groups the implemented +storage, handoff and workflows slices described below. The old `run` chain is split by owner. Its gates are in `benchmark/legacy_run_v0.py`, its execution facts stay in `harness/`, and @@ -41,8 +41,8 @@ through these exports; new pickles use the canonical definition. The retired root `inferrsi.evidence_paths` has no alias and remains unavailable. Shim removal requires a separate reviewed PR recording consumer migration, -wheel and plugin validation, and the first removal version. Workflows, -integrations and integrated acceptance remain subsequent #490 slices. +wheel and plugin validation, and the first removal version. Integrations and +integrated acceptance remain subsequent #490 slices. ### Apps handoff migration @@ -61,6 +61,28 @@ path performs no Knowledge reads, device probes, process launches or service registration. Harness owns the sealed context, original attempt and request digest; these ownership boundaries and DTOs remain unchanged. +### Apps workflows migration + +`apps/workflows/` owns `campaign.py`, `benchmark_operations.py`, `improvement.py`, +`skill_evaluation.py` and `skill_eval_engineering.py`. It joins Knowledge reads, +Harness execution and Benchmark judgments. CLI commands, views and evaluation +tools import these canonical implementations. Its initializer imports nothing; +imports of either path perform no data reads, probes, process launches or service +registration. Signatures, return values and retained-record schemas are unchanged. + +The five former Apps paths keep explicit exports from the same +[migration inventory](apps-module-migrations.json). The engineering verifier also +keeps its former file and `python -m` entry points through a static absolute import +and a `__main__` guard, recorded as `script_entry` in the inventory. Other shims +contain only their docstring, explicit import and literal `__all__`. New pickles +use the canonical module; patches to helpers or imported dependencies target it. + +New optimization plans and evaluation controller snapshots fingerprint the +canonical source files, not the compatibility shims. A frozen pre-move runtime +map remains incompatible with a new runtime, including for `status` and `recover`. +Do not edit a frozen plan to bypass this check. Use the original pinned runtime +to resume, following [release-boundary recovery](optimization-workflow-unification.md#release-boundary-recovery). + ### Core and technical packages The four target modules are peers: `knowledge`, `harness`, `benchmark`, and @@ -89,7 +111,8 @@ The two are `__init__.py` and `__main__.py`. M1 moved root `mindie_project.py`, the four root files with no forwarding shell or alias. A2 moved root `summary_chart.py` to `apps/views/summary_chart.py` and root `benchmark_operations.py` to -`apps/benchmark_operations.py` and deleted both root files the same way. A1 moved root `cli.py` and +`apps/benchmark_operations.py` and deleted both root files the same way. The workflows +slice later moved that Apps implementation into `apps/workflows/`. A1 moved root `cli.py` and `arguments.py` to `apps/cli/__init__.py` and `apps/cli/arguments.py` and deleted both root files. A new root file now fails the guard, so a later module belongs in a subpackage. The build-generated `_version.py` is ignored wheel @@ -148,17 +171,17 @@ Several flat modules remain shared seams until their slice starts. | Current path | Tentative reading | This slice | | --- | --- | --- | | `apps/cli/` (`__init__.py`, `arguments.py`), root `__main__.py` | Apps CLI entry and its parser | A1 (#408) moved root `cli.py` and `arguments.py` here; the entry is `inferrsi.apps.cli:main`, and `__main__.py` only delegates to it | -| `apps/benchmark_operations.py` | Single-task application launcher | A2 (#408) moved root `benchmark_operations.py` here; resolves a task entry and invokes the Harness runner, and ordinary projects do not use it | +| `apps/workflows/benchmark_operations.py` | Single-task application launcher | A2 (#408) moved the root launcher into Apps; #490 groups its implementation under workflows. Resolves a task entry and invokes the Harness runner; ordinary projects do not use it | | `benchmark/registry.py`, `benchmark/tasks/` | Task discovery, definitions and specialized rules | Task-specific quality and measurement code stays with each task | | `harness/optimization_session.py`, `harness/project.py`, `harness/task_protocol.py` | Shared preparation, execution and retained-result assessment | CLI supplies task resources and knowledge access; ordinary project lifecycle does not import Benchmark | | `harness/optimization.py`, `harness/candidates.py`, `harness/usage.py` | Shared project and packaged-task lifecycle | Injected execution/assessment; declared file/directory candidates, selection direction and exact/up-to rounds | | `harness/agent_sessions.py` | Native host handoff records | Freezes requests and retains native IDs and exported observations; the initiating Agent calls host tools | | `harness/skill_evaluation.py`, `harness/artifact_sandbox.py`, `harness/improvement.py`, `harness/atomic_json.py` | Skill-evaluation host adapter, trial execution and retention, sandbox replay, search progression | Verdict rules and task definitions arrive as injected values; no Model, Infra, Knowledge, Benchmark or Apps import | | `benchmark/skill_evaluation.py`, `benchmark/skill_eval_diagnostic.py`, `benchmark/skill_eval_engineering.py`, `benchmark/improvement.py` | Skill-evaluation criteria, task definitions and acceptance rules | Pure judgments over retained facts; no Harness import and no process start | -| `apps/skill_evaluation.py`, `apps/skill_eval_engineering.py`, `apps/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | +| `apps/workflows/skill_evaluation.py`, `apps/workflows/skill_eval_engineering.py`, `apps/workflows/improvement.py` | Composed skill-evaluation entries | Join Knowledge reads, Harness execution and Benchmark rules behind the signatures that `harness/` held before the split | | `knowledge/knowledge_operations.py`, `knowledge/knowledge_store.py`, `knowledge/catalog.py`, `knowledge/research.py`, `knowledge/skill_catalog.py`, `knowledge/_skill_selection.py`, `knowledge/_source_snapshot.py` | The seven Knowledge implementations, moved by K01 | Interfaces unchanged; the same-named root shells were deleted by R1 | -| `harness/records/` | Campaign manifests, traces, logs, profile captures and delivered-context captures | Imports no Knowledge or Apps module; `apps/campaign.py` supplies the backend names, the selection default and the context projection | -| `apps/views/`, `apps/campaign.py` | Dashboard views, templates and the Knowledge-to-records connection | Apps composition under #371; campaign organization follows in #490 | +| `harness/records/` | Campaign manifests, traces, logs, profile captures and delivered-context captures | Imports no Knowledge or Apps module; `apps/workflows/campaign.py` supplies the backend names, the selection default and the context projection | +| `apps/views/`, `apps/workflows/campaign.py` | Dashboard views, templates and the Knowledge-to-records connection | Apps composition under #371; campaign composition is grouped under workflows by #490 | | `apps/mindie/` (`project.py`, `context.py`, `mcp.py`, `read_worker.py`) | The local MindIE adapter: project grant, stdio MCP, context export and the read worker | M1 (#452) moved the four root MindIE files here; no Harness, Knowledge or Infra module names MindIE, and the initializer imports nothing | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `harness/runner.py`, `harness/scenarios/external.py`, `harness/atomic_json.py` | Old run execution facts and Harness-owned JSON writes | Ports injected; no Model edge, and no Infra edge except the local process executor | @@ -561,7 +584,7 @@ A2 (#408) deleted two more root modules the same way: | Retired root module | Canonical module | | --- | --- | | `inferrsi.summary_chart` | `inferrsi.apps.views.summary_chart` | -| `inferrsi.benchmark_operations` | `inferrsi.apps.benchmark_operations` | +| `inferrsi.benchmark_operations` | `inferrsi.apps.workflows.benchmark_operations` | `tools/campaign_summary_chart.py` keeps its command line and calls `inferrsi.apps.views.summary_chart:main`, so its arguments, exit codes, output and diff --git a/docs/operations.md b/docs/operations.md index 283ce6a6..de95f7a6 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -27,7 +27,7 @@ Effect classes used below: | Intent | Entry | Effect class | Boundary | | --- | --- | --- | --- | -| Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution | +| Complete a task optimization in the current Agent | CLI `benchmarks session start/knowledge/round/status/final/recover` | Experiment-executing / local records | Frozen budget, actual candidates and independent task outcomes. The current Agent drives edits. Real usage is imported explicitly. Recovery collects an existing execution using its [pinned runtime](optimization-workflow-unification.md#release-boundary-recovery) | | Optimize an ordinary project in the current Agent | CLI `campaign start/knowledge/round/status/final/recover` | Experiment-executing / local records | Harness freezes explicit caller-provided checks, manages multi-file candidates, executes and assesses retained results. This path does not depend on Benchmark or infer acceptance from the goal. See [shared workflow](optimization-workflow-unification.md) | | Prepare and record a fresh native Agent | CLI `agents prepare/dispatch/bind/record/status` | Local records / explicit host handoff | The initiating Agent calls native host tools. Records prevent repeated dispatch; exported host observations do not establish task acceptance. See [native sessions](native-agent-sessions.md) | | Discover packaged tasks | CLI `benchmarks list` / `benchmarks show`; Python `benchmark.registry.builtin_tasks` / `describe_task` | Read-only | Reads the task library's registry and opaque manifests from installed resources | diff --git a/docs/optimization-workflow-unification.md b/docs/optimization-workflow-unification.md index c93ed921..23d0a3b2 100644 --- a/docs/optimization-workflow-unification.md +++ b/docs/optimization-workflow-unification.md @@ -32,7 +32,7 @@ CLI `final` 返回后自动生成 `delivery//report.md`、离线 `rep ## 时序图 -以下是普通项目的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 +以下是普通项目的完整流程。CLI 接入既有入口,调用 Harness 的 `optimization_session.session_command()`。初始化、执行、判定和结算均在 Harness 内;CLI 注入 KnowledgeReader 和已有知识记录回调。知识记录回调在 `apps/workflows/campaign.py`,展示在 `apps/views`,记录由 `harness/records` 写入;Harness 仍负责优化状态与预算。 先准备任务条件,再调用 start。start 内复制任务、脚本和基线并写入计划/状态,不执行测量。Agent 逐次调用 round,最后调用 final 独立重测已保存的最佳版本。 @@ -151,7 +151,7 @@ Harness 没有常驻 Agent 驱动。Agent 逐次调用这些操作,代码在 两个优化入口都调用 `harness.optimization_session.session_command`,再调用 **同一个** `harness.optimization.evaluate_round`。执行与评测适配也在 Harness 内。普通项目由 `harness.project` 固化调用方协议并判定留存结果,整个生命周期不导入 Benchmark。 -题库 `start` 由 CLI 经 `benchmark.registry` 定位资源目录,再交给 `harness.task_protocol.prepare` 固化;后续从冻结目录加载题包自己的 `optimization.py`,无需再次查询注册表。专用协议检查仍属于题包。两类 `assess` 都只读留存结果,不重新测量。KnowledgeReader 与 `apps/campaign.py` 的记录回调由 CLI 注入,Harness 不直接导入 Benchmark 或目标 Knowledge 模块,也不经记录模块间接引入知识实现。 +题库 `start` 由 CLI 经 `benchmark.registry` 定位资源目录,再交给 `harness.task_protocol.prepare` 固化;后续从冻结目录加载题包自己的 `optimization.py`,无需再次查询注册表。专用协议检查仍属于题包。两类 `assess` 都只读留存结果,不重新测量。KnowledgeReader 与 `apps/workflows/campaign.py` 的记录回调由 CLI 注入,Harness 不直接导入 Benchmark 或目标 Knowledge 模块,也不经记录模块间接引入知识实现。 本次交付可以分成三块:**Benchmark 的优化题目、Harness 的优化管理与验收、Harness 与宿主协作的新会话派发和回收**。第三块不改变第二块的生命周期;新会话仍自行 start 到 final。 @@ -215,6 +215,40 @@ inferrsi dashboard --input --output 执行器抛出异常时,若已有完成记录则收集该记录,不重复启动;没有记录则标记 blocked,清理状态保持未知,不自动重试。进程被中断后的 `recover` 只接收已有结果,不重新启动命令;没有完成记录时,需确认原进程及资源清理。运行中的旧 session 应继续使用其冻结 runtime,不能直接换新代码恢复。 +### Release-boundary recovery + +Apps workflows 目录迁移会改变运行时摘要的路径和控制器源码。迁移前的 +`plan.json` 即使只是执行 `status` 或 `recover`,在新版本下也会被拒绝。 +保留原始计划、冻结输入和所有已留存结果,不要替换摘要、重新格式化计划或 +把旧运行当作新运行初始化。评测工具的冻结 controller 快照也遵循相同原则。 + +恢复时,从运行准备记录取得创建该计划时的精确软件版本和 Python 环境。 +`runtime_identity` 是校验表,不是可安装的软件包;单靠摘要不能重建原始环境。 +优先使用先前留存的虚拟环境或 wheel。若只有精确源码版本,应在独立目录恢复 +该版本及原依赖,构建 wheel 并安装到独立环境;不要覆盖新版本环境。 + +```text + -I -m inferrsi benchmarks session status --session + -I -m inferrsi benchmarks session recover --session +``` + +普通项目对应 `campaign status/recover --directory `。 +原环境的所有文件摘要与计划匹配后,Harness 才会接收已有 execution 记录; +恢复不会再次执行已完成的候选。无法取得匹配的原始环境时保持拒绝,保留证据。 +需要在新版本继续实验,应另开新运行,明确新的版本和冻结输入。 + +跨版本回归使用两个独立安装环境:迁移前 runtime 创建计划并在执行完成后 +中断;新 runtime 的 status/recover 必须拒绝且不改动计划和结果;原 runtime +恢复同一运行并验证候选只执行一次。设置 `INFERRSI_PRE_MOVE_PYTHON` 指向 +固定旧 wheel 的解释器,`INFERRSI_POST_MOVE_PYTHON` 指向新 wheel 的解释器后运行: + +```text +python -m pytest tests/test_optimization_session.py::test_pinned_pre_move_wheel_recovers_its_original_plan +``` + +未提供旧 runtime 时该安装集成用例明确跳过;默认回归仍检查旧路径的摘要表 +被拒绝、计划字节不变,以及新计划摘要读取 `apps/workflows/campaign.py` 的实际源码。 + 判定器异常按基础设施失败结算,`assessment.reason` 保留异常类型和消息,避免普通编程错误被隐藏成无类型的失败描述。执行器异常的原始类型和消息保存在该次尝试的 `coordinator-error.json` 中。此处捕获异常用于保留失败状态,不代表候选通过;进程中断仍沿用上述恢复流程。 每轮 token 只接受实际增量导出,未知不补零。整会话用量另存,不重复叠加到曲线。终止后的总耗时固定,之后打开 dashboard 不增加。HTML 只投影已清洗字段;原始本地记录可能含命令、路径等私有信息,公开前需要检查。 diff --git a/docs/skill-evaluation.md b/docs/skill-evaluation.md index 739d1ea9..1f42f440 100644 --- a/docs/skill-evaluation.md +++ b/docs/skill-evaluation.md @@ -22,7 +22,7 @@ platform source is copied. Existing knowledge retains its source/license limits. Ownership follows the module split. `inferrsi.harness.skill_evaluation` adapts the client, executes each trial and retains its facts. `inferrsi.benchmark.skill_evaluation` holds check validity, the task verdict and condition statistics. -`inferrsi.apps.skill_evaluation` composes both with the bundle export behind the +`inferrsi.apps.workflows.skill_evaluation` composes both with the bundle export behind the `run_trial`, `summarize` and `export_bundle` signatures; retained trial records are unchanged. diff --git a/src/inferrsi/apps/benchmark_operations.py b/src/inferrsi/apps/benchmark_operations.py index 7770c3f5..c425b618 100644 --- a/src/inferrsi/apps/benchmark_operations.py +++ b/src/inferrsi/apps/benchmark_operations.py @@ -1,19 +1,5 @@ -"""Resolve packaged tasks at the application boundary and launch their entries.""" +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.benchmark_operations`.""" -import subprocess -import sys -from pathlib import Path +from .workflows.benchmark_operations import run_task -from inferrsi.benchmark.registry import task_resources -from inferrsi.harness import optimization_session - - -def run_task(name: str, arguments: list[str]) -> int: - entry = task_resources(name).joinpath("benchmark.py") - if not entry.is_file(): - raise ValueError(f"Missing benchmark.py entry for task: {name}") - return subprocess.call( - [sys.executable, str(Path(optimization_session.__file__).resolve()), - str(entry), *arguments], - env=optimization_session.runtime_environment(), - ) +__all__ = ['run_task'] diff --git a/src/inferrsi/apps/campaign.py b/src/inferrsi/apps/campaign.py index b30c2aef..4ac66483 100644 --- a/src/inferrsi/apps/campaign.py +++ b/src/inferrsi/apps/campaign.py @@ -1,156 +1,5 @@ -"""Connect Knowledge values to the campaign records that Harness keeps. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.campaign`.""" -Harness records never import Knowledge. This module supplies the backend -vocabulary, the selection default, the context projection and the capture -orchestration they need: it calls the reader, keeps the retrieval envelope -the reader returned and hands Harness only the bytes to store. -""" +from .workflows.campaign import capture_context, create_campaign, save_retrieval, verify_capture -from __future__ import annotations - -from pathlib import Path - -from inferrsi import __version__ -from inferrsi.harness.records import campaign_records, knowledge_capture -from inferrsi.knowledge._skill_selection import TASK_RELEVANCE -from inferrsi.knowledge.catalog import BACKENDS -from inferrsi.knowledge.context_projection import ( - PROVENANCE_FIELDS, - assemble_context_text, - context_item, -) - - -def create_campaign(root: Path, campaign_id: str, *, target: float | None = None, - noise: float | None = None, synthetic: bool = False, - backend: str | None = None, profile_first: bool = False) -> Path: - """Create a campaign manifest that accepts the Knowledge backend names.""" - return campaign_records.create_campaign( - root, campaign_id, target=target, noise=noise, synthetic=synthetic, backend=backend, - profile_first=profile_first, backends=BACKENDS, - ) - - -def capture_context(root: Path, *, context_id: str, reader, query: str = "", - relevance: str | None = None, - relevance_method: str | None = None, - ids: list[str] | None = None, kind: str | None = None, - repo: str | None = None, tier: str | None = None, - backend: str | None = None, scenario: str | None = None, - limit: int = 5, context_bytes: int = 65536, - sources: tuple[str, ...] = ("package",)) -> tuple[Path, str]: - """Run ``collect`` with a context budget and keep its complete envelope. - - The capture is produced by the same ``collect`` operation the agent uses, - not by a caller narrative. Returns the new campaign-relative file and its - SHA-256. - """ - campaign = knowledge_capture.capture_campaign(root, context_id) - if relevance_method is None: - relevance_method = TASK_RELEVANCE if relevance and ids is None else "weighted-lexical-v1" - if relevance_method == TASK_RELEVANCE and ids is None and backend is None: - backend = campaign.get("backend") - result = reader.collect( - query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, - scenario=scenario, limit=limit, relevance=relevance, - relevance_method=relevance_method, context_bytes=context_bytes, - ) - request = { - "query": query, "relevance": relevance, - "relevance_method": relevance_method if relevance else None, - "ids": ids, "kind": kind, "repo": repo, "tier": tier, - "backend": backend, "scenario": scenario, "limit": limit, - "context_bytes": context_bytes, "sources": list(sources), - } - if _supplies_no_observation(result): - # Such a reader gets the record this entry wrote before observations - # existed: the same fields, order and bytes, with no empty provenance. - contexts = _contexts(result) - return knowledge_capture.store_capture( - root, campaign_id=campaign["campaign_id"], context_id=context_id, - # Only the source categories are kept; store and checkout paths can be private. - request={**request, "ids": ids or []}, - selection=result.get("selection"), - items=[context_item(view) for view in result.get("items", [])], - omissions=result.get("omissions", []), - context_omissions=result.get("context_omissions", []), - context_budget=result.get("context_budget"), - context_bytes_used=result.get("context_bytes"), - contexts=contexts, - assembled_text=assemble_context_text(contexts), - ) - return save_retrieval(root, context_id=context_id, result=result, request=request, - wrapper="campaign capture-context") - - -def _supplies_no_observation(result: dict) -> bool: - """Whether a result has neither a retrieval observation nor any item provenance.""" - return not isinstance(result.get("retrieval"), dict) and not any( - key in view for view in result.get("items", []) for key in PROVENANCE_FIELDS) - - -def _contexts(result: dict) -> list[dict]: - return [{ - "ref": context["ref"], "revision": context["revision"], - "content_bytes": context["content_bytes"], - "resources": [{key: resource[key] for key in ("path", "role", "sha256", "content")} - for resource in context["resources"]], - "omissions": context["omissions"], - "sources": context.get("sources", []), - } for context in result.get("contexts", [])] - - -def save_retrieval(root: Path, *, context_id: str, result: dict, - request: dict | None = None, wrapper: str | None = None) -> tuple[Path, str]: - """Explicitly retain a search/collect result without executing another search. - - The caller owns this local capture. Review/redact the bundle before sharing; - hashes attest bytes, not publication permission or actual model use. - """ - campaign = knowledge_capture.capture_campaign(root, context_id) - retrieval = result.get("retrieval") - if isinstance(retrieval, dict): - retrieval = {**retrieval} - if wrapper: - retrieval.update(entrypoint=wrapper, calls=[wrapper, *retrieval["calls"]]) - request = request if request is not None else (retrieval or {}).get("request", {}) - observations = {item["ref"]: item for item in (retrieval or {}).get("items", [])} - items = [] - for view in result.get("items", []): - ref = view.get("ref") or "package:" + view["id"] - items.append({**context_item({**view, "ref": ref}, provenance=True), - "revision": view.get("revision") - or observations.get(ref, {}).get("revision")}) - contexts = _contexts(result) - return knowledge_capture.store_capture( - root, campaign_id=campaign["campaign_id"], context_id=context_id, - # Only the source categories are kept; store and checkout paths can be private. - request=request, - selection=result.get("selection"), - items=items, - omissions=result.get("omissions", []), - context_omissions=result.get("context_omissions", []), - context_budget=result.get("context_budget"), - context_bytes_used=result.get("context_bytes"), - contexts=contexts, - assembled_text=assemble_context_text(contexts), - retrieval=retrieval, - # A result without an observation has no recorded entry point. It is - # labelled unknown; a plausible current default is not evidence. - captured_by=f"inferrsi {(retrieval or {}).get('entrypoint', 'unknown')} {__version__}", - ) - - -def verify_capture(raw: bytes) -> tuple[dict | None, str]: - """Return the capture and ``verified``, or ``None`` and a fixed reason code. - - Harness checks the stored bytes. This adds that the assembled text is the - Knowledge projection of the listed resources, so a reader sees exactly the - text those resources produce. - """ - document, state = knowledge_capture.verify_capture(raw) - if document is None: - return None, state - if assemble_context_text(document["contexts"]) != document["assembled_text"]: - return None, "assembly_mismatch" - return document, "verified" +__all__ = ['create_campaign', 'capture_context', 'save_retrieval', 'verify_capture'] diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 974f3d9a..a5889c8d 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -11,11 +11,11 @@ from ...knowledge.catalog import get_knowledge, search_knowledge from ...knowledge.knowledge_operations import KnowledgeReader from ...knowledge.research import research_knowledge -from ..campaign import capture_context, create_campaign from ..inspection import inspect_environment from ..knowledge import knowledge_reader from ..knowledge import read as read_knowledge from ..legacy.reporting import write_report +from ..workflows.campaign import capture_context, create_campaign from .arguments import device_ids, make_spec, parser @@ -368,7 +368,7 @@ def main(argv: list[str] | None = None) -> int: _print(after_final(args, result)) return 2 if result.get("state") == "blocked" else 0 if args.benchmark_action == "run": - from ..benchmark_operations import run_task + from ..workflows.benchmark_operations import run_task return run_task(args.task, args.arguments) from ...benchmark.registry import builtin_tasks, describe_task diff --git a/src/inferrsi/apps/improvement.py b/src/inferrsi/apps/improvement.py index 187329e7..d3a4d6ef 100644 --- a/src/inferrsi/apps/improvement.py +++ b/src/inferrsi/apps/improvement.py @@ -1,29 +1,5 @@ -# Composed improvement-step entries: Harness usage totals and search progression -# joined with Benchmark acceptance rules. Signatures and results match the -# functions that harness.improvement held before the ownership split. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.improvement`.""" -from inferrsi.benchmark import improvement as rules -from inferrsi.benchmark.improvement import SelectionPolicy -from inferrsi.harness import improvement as progression -from inferrsi.harness.usage import usage_totals +from .workflows.improvement import assessment, choose, holdout_accepts, rerun_assessment - -def assessment(report: dict, policy: SelectionPolicy) -> dict: - pairs = report["pairs"] - usage = {role: usage_totals([p[role] for p in pairs]) for role in ("baseline", "candidate")} - return rules.assessment(report, policy, usage) - - -def choose(probes: dict[str, dict], policy: SelectionPolicy, *, composable: bool) -> dict: - scores = {name: assessment(report, policy) for name, report in probes.items()} - return progression.choose( - scores, blocked=rules.search_block(scores, probes), composable=composable) - - -def holdout_accepts(report: dict, policy: SelectionPolicy) -> bool: - return rules.holdout_accepts(assessment(report, policy)) - - -def rerun_assessment(report: dict, probes: dict[str, dict], policy: SelectionPolicy) -> dict: - scores = {name: assessment(probe, policy) for name, probe in probes.items()} - return rules.rerun_assessment(assessment(report, policy), report, probes, scores) +__all__ = ['assessment', 'choose', 'holdout_accepts', 'rerun_assessment'] diff --git a/src/inferrsi/apps/skill_eval_engineering.py b/src/inferrsi/apps/skill_eval_engineering.py index b907e9a5..bbe7de38 100644 --- a/src/inferrsi/apps/skill_eval_engineering.py +++ b/src/inferrsi/apps/skill_eval_engineering.py @@ -1,28 +1,8 @@ -# Composed serving-parser artifact check: Benchmark task rules and vector judgment -# around the Harness sandbox replay. No expected result enters the sandbox. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.skill_eval_engineering`.""" -import json -import os -import sys -from pathlib import Path +from inferrsi.apps.workflows.skill_eval_engineering import main, verify -from inferrsi.benchmark import skill_eval_engineering as task_rules -from inferrsi.harness import artifact_sandbox - - -def verify(task, output): - source, rejected = task_rules.submission(task, output) - if rejected is not None: - return rejected - return task_rules.judge(artifact_sandbox.replay(source, task_rules.vector_inputs())) - - -def main(): - task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) - output = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) - result = verify(task, output) - Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") - raise SystemExit(0 if result["passed"] else 1) +__all__ = ['verify', 'main'] if __name__ == "__main__": diff --git a/src/inferrsi/apps/skill_evaluation.py b/src/inferrsi/apps/skill_evaluation.py index 06482510..0fd84045 100644 --- a/src/inferrsi/apps/skill_evaluation.py +++ b/src/inferrsi/apps/skill_evaluation.py @@ -1,61 +1,5 @@ -# Composed skill-evaluation entries: Knowledge bundle export, and Harness trial -# execution joined with Benchmark verdict rules. Signatures and retained records -# match the functions that harness.skill_evaluation held before the ownership split. +"""Compatibility exports; use :mod:`inferrsi.apps.workflows.skill_evaluation`.""" -from pathlib import Path +from .workflows.skill_evaluation import export_bundle, run_trial, summarize -from inferrsi.benchmark import skill_eval_engineering as engineering_task -from inferrsi.benchmark import skill_evaluation as rules -from inferrsi.harness import skill_evaluation as host -from inferrsi.harness.usage import normalize_usage -from inferrsi.knowledge.knowledge_operations import KnowledgeReader - -from . import skill_eval_engineering - - -def export_bundle(reader: KnowledgeReader, reference: str, destination: Path) -> dict: - # Knowledge resolves the bundle; Harness receives and retains bytes only. - bundle = reader.read(reference) - host.write_files(destination, bundle.content) - return bundle.view() - - -def summarize(trials: list[dict]) -> dict: - normalized = [ - {**trial, "observations": {**trial["observations"], - "usage": normalize_usage(trial["observations"]["usage"])}} - for trial in trials - ] - return rules.summarize(normalized) - - -def run_trial( - client, - task: dict, - condition: str, - repeat: int, - workspace: Path, - trial_dir: Path, - schema: Path, - verifier: Path, - prompt: str, - deadline: float, - *, - timeout_seconds: float | None = None, - engineering_artifact: bool = False, - generation_only: bool = False, -) -> dict: - artifact = None - if engineering_artifact: - artifact = host.ArtifactPort( - kind=engineering_task.ADAPTER, - generation_mode=engineering_task.GENERATION_MODE, - starter=engineering_task.STARTER.encode("utf-8"), - max_source_bytes=engineering_task.MAX_SOURCE_BYTES, - verify=skill_eval_engineering.verify, - ) - return host.run_trial( - client, task, condition, repeat, workspace, trial_dir, schema, verifier, prompt, - deadline, valid_checks=rules.valid_checks, task_verdict=rules.task_verdict, - artifact=artifact, timeout_seconds=timeout_seconds, generation_only=generation_only, - ) +__all__ = ['export_bundle', 'summarize', 'run_trial'] diff --git a/src/inferrsi/apps/views/capture_delivery.py b/src/inferrsi/apps/views/capture_delivery.py index 5bae55f4..0a1d0121 100644 --- a/src/inferrsi/apps/views/capture_delivery.py +++ b/src/inferrsi/apps/views/capture_delivery.py @@ -3,7 +3,7 @@ import hashlib import json -from inferrsi.apps.campaign import verify_capture +from inferrsi.apps.workflows.campaign import verify_capture from inferrsi.harness.records.campaign_records import _identifier from inferrsi.harness.records.evidence import ( inside as _inside, diff --git a/src/inferrsi/apps/views/optimization_story.py b/src/inferrsi/apps/views/optimization_story.py index f3bb9548..b8761620 100644 --- a/src/inferrsi/apps/views/optimization_story.py +++ b/src/inferrsi/apps/views/optimization_story.py @@ -14,7 +14,7 @@ from inferrsi.harness.records.knowledge_capture import MAX_CONTEXT_CAPTURE_BYTES from inferrsi.harness.records.optimization_log import read_latest_log -from ..campaign import verify_capture +from ..workflows.campaign import verify_capture _IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") _GOOD = {"improved"} diff --git a/src/inferrsi/apps/workflows/__init__.py b/src/inferrsi/apps/workflows/__init__.py new file mode 100644 index 00000000..79aac851 --- /dev/null +++ b/src/inferrsi/apps/workflows/__init__.py @@ -0,0 +1 @@ +"""Application workflows joining Knowledge, Harness and Benchmark ports.""" diff --git a/src/inferrsi/apps/workflows/benchmark_operations.py b/src/inferrsi/apps/workflows/benchmark_operations.py new file mode 100644 index 00000000..7770c3f5 --- /dev/null +++ b/src/inferrsi/apps/workflows/benchmark_operations.py @@ -0,0 +1,19 @@ +"""Resolve packaged tasks at the application boundary and launch their entries.""" + +import subprocess +import sys +from pathlib import Path + +from inferrsi.benchmark.registry import task_resources +from inferrsi.harness import optimization_session + + +def run_task(name: str, arguments: list[str]) -> int: + entry = task_resources(name).joinpath("benchmark.py") + if not entry.is_file(): + raise ValueError(f"Missing benchmark.py entry for task: {name}") + return subprocess.call( + [sys.executable, str(Path(optimization_session.__file__).resolve()), + str(entry), *arguments], + env=optimization_session.runtime_environment(), + ) diff --git a/src/inferrsi/apps/workflows/campaign.py b/src/inferrsi/apps/workflows/campaign.py new file mode 100644 index 00000000..b30c2aef --- /dev/null +++ b/src/inferrsi/apps/workflows/campaign.py @@ -0,0 +1,156 @@ +"""Connect Knowledge values to the campaign records that Harness keeps. + +Harness records never import Knowledge. This module supplies the backend +vocabulary, the selection default, the context projection and the capture +orchestration they need: it calls the reader, keeps the retrieval envelope +the reader returned and hands Harness only the bytes to store. +""" + +from __future__ import annotations + +from pathlib import Path + +from inferrsi import __version__ +from inferrsi.harness.records import campaign_records, knowledge_capture +from inferrsi.knowledge._skill_selection import TASK_RELEVANCE +from inferrsi.knowledge.catalog import BACKENDS +from inferrsi.knowledge.context_projection import ( + PROVENANCE_FIELDS, + assemble_context_text, + context_item, +) + + +def create_campaign(root: Path, campaign_id: str, *, target: float | None = None, + noise: float | None = None, synthetic: bool = False, + backend: str | None = None, profile_first: bool = False) -> Path: + """Create a campaign manifest that accepts the Knowledge backend names.""" + return campaign_records.create_campaign( + root, campaign_id, target=target, noise=noise, synthetic=synthetic, backend=backend, + profile_first=profile_first, backends=BACKENDS, + ) + + +def capture_context(root: Path, *, context_id: str, reader, query: str = "", + relevance: str | None = None, + relevance_method: str | None = None, + ids: list[str] | None = None, kind: str | None = None, + repo: str | None = None, tier: str | None = None, + backend: str | None = None, scenario: str | None = None, + limit: int = 5, context_bytes: int = 65536, + sources: tuple[str, ...] = ("package",)) -> tuple[Path, str]: + """Run ``collect`` with a context budget and keep its complete envelope. + + The capture is produced by the same ``collect`` operation the agent uses, + not by a caller narrative. Returns the new campaign-relative file and its + SHA-256. + """ + campaign = knowledge_capture.capture_campaign(root, context_id) + if relevance_method is None: + relevance_method = TASK_RELEVANCE if relevance and ids is None else "weighted-lexical-v1" + if relevance_method == TASK_RELEVANCE and ids is None and backend is None: + backend = campaign.get("backend") + result = reader.collect( + query, ids=ids, kind=kind, repo=repo, tier=tier, backend=backend, + scenario=scenario, limit=limit, relevance=relevance, + relevance_method=relevance_method, context_bytes=context_bytes, + ) + request = { + "query": query, "relevance": relevance, + "relevance_method": relevance_method if relevance else None, + "ids": ids, "kind": kind, "repo": repo, "tier": tier, + "backend": backend, "scenario": scenario, "limit": limit, + "context_bytes": context_bytes, "sources": list(sources), + } + if _supplies_no_observation(result): + # Such a reader gets the record this entry wrote before observations + # existed: the same fields, order and bytes, with no empty provenance. + contexts = _contexts(result) + return knowledge_capture.store_capture( + root, campaign_id=campaign["campaign_id"], context_id=context_id, + # Only the source categories are kept; store and checkout paths can be private. + request={**request, "ids": ids or []}, + selection=result.get("selection"), + items=[context_item(view) for view in result.get("items", [])], + omissions=result.get("omissions", []), + context_omissions=result.get("context_omissions", []), + context_budget=result.get("context_budget"), + context_bytes_used=result.get("context_bytes"), + contexts=contexts, + assembled_text=assemble_context_text(contexts), + ) + return save_retrieval(root, context_id=context_id, result=result, request=request, + wrapper="campaign capture-context") + + +def _supplies_no_observation(result: dict) -> bool: + """Whether a result has neither a retrieval observation nor any item provenance.""" + return not isinstance(result.get("retrieval"), dict) and not any( + key in view for view in result.get("items", []) for key in PROVENANCE_FIELDS) + + +def _contexts(result: dict) -> list[dict]: + return [{ + "ref": context["ref"], "revision": context["revision"], + "content_bytes": context["content_bytes"], + "resources": [{key: resource[key] for key in ("path", "role", "sha256", "content")} + for resource in context["resources"]], + "omissions": context["omissions"], + "sources": context.get("sources", []), + } for context in result.get("contexts", [])] + + +def save_retrieval(root: Path, *, context_id: str, result: dict, + request: dict | None = None, wrapper: str | None = None) -> tuple[Path, str]: + """Explicitly retain a search/collect result without executing another search. + + The caller owns this local capture. Review/redact the bundle before sharing; + hashes attest bytes, not publication permission or actual model use. + """ + campaign = knowledge_capture.capture_campaign(root, context_id) + retrieval = result.get("retrieval") + if isinstance(retrieval, dict): + retrieval = {**retrieval} + if wrapper: + retrieval.update(entrypoint=wrapper, calls=[wrapper, *retrieval["calls"]]) + request = request if request is not None else (retrieval or {}).get("request", {}) + observations = {item["ref"]: item for item in (retrieval or {}).get("items", [])} + items = [] + for view in result.get("items", []): + ref = view.get("ref") or "package:" + view["id"] + items.append({**context_item({**view, "ref": ref}, provenance=True), + "revision": view.get("revision") + or observations.get(ref, {}).get("revision")}) + contexts = _contexts(result) + return knowledge_capture.store_capture( + root, campaign_id=campaign["campaign_id"], context_id=context_id, + # Only the source categories are kept; store and checkout paths can be private. + request=request, + selection=result.get("selection"), + items=items, + omissions=result.get("omissions", []), + context_omissions=result.get("context_omissions", []), + context_budget=result.get("context_budget"), + context_bytes_used=result.get("context_bytes"), + contexts=contexts, + assembled_text=assemble_context_text(contexts), + retrieval=retrieval, + # A result without an observation has no recorded entry point. It is + # labelled unknown; a plausible current default is not evidence. + captured_by=f"inferrsi {(retrieval or {}).get('entrypoint', 'unknown')} {__version__}", + ) + + +def verify_capture(raw: bytes) -> tuple[dict | None, str]: + """Return the capture and ``verified``, or ``None`` and a fixed reason code. + + Harness checks the stored bytes. This adds that the assembled text is the + Knowledge projection of the listed resources, so a reader sees exactly the + text those resources produce. + """ + document, state = knowledge_capture.verify_capture(raw) + if document is None: + return None, state + if assemble_context_text(document["contexts"]) != document["assembled_text"]: + return None, "assembly_mismatch" + return document, "verified" diff --git a/src/inferrsi/apps/workflows/improvement.py b/src/inferrsi/apps/workflows/improvement.py new file mode 100644 index 00000000..187329e7 --- /dev/null +++ b/src/inferrsi/apps/workflows/improvement.py @@ -0,0 +1,29 @@ +# Composed improvement-step entries: Harness usage totals and search progression +# joined with Benchmark acceptance rules. Signatures and results match the +# functions that harness.improvement held before the ownership split. + +from inferrsi.benchmark import improvement as rules +from inferrsi.benchmark.improvement import SelectionPolicy +from inferrsi.harness import improvement as progression +from inferrsi.harness.usage import usage_totals + + +def assessment(report: dict, policy: SelectionPolicy) -> dict: + pairs = report["pairs"] + usage = {role: usage_totals([p[role] for p in pairs]) for role in ("baseline", "candidate")} + return rules.assessment(report, policy, usage) + + +def choose(probes: dict[str, dict], policy: SelectionPolicy, *, composable: bool) -> dict: + scores = {name: assessment(report, policy) for name, report in probes.items()} + return progression.choose( + scores, blocked=rules.search_block(scores, probes), composable=composable) + + +def holdout_accepts(report: dict, policy: SelectionPolicy) -> bool: + return rules.holdout_accepts(assessment(report, policy)) + + +def rerun_assessment(report: dict, probes: dict[str, dict], policy: SelectionPolicy) -> dict: + scores = {name: assessment(probe, policy) for name, probe in probes.items()} + return rules.rerun_assessment(assessment(report, policy), report, probes, scores) diff --git a/src/inferrsi/apps/workflows/skill_eval_engineering.py b/src/inferrsi/apps/workflows/skill_eval_engineering.py new file mode 100644 index 00000000..b907e9a5 --- /dev/null +++ b/src/inferrsi/apps/workflows/skill_eval_engineering.py @@ -0,0 +1,29 @@ +# Composed serving-parser artifact check: Benchmark task rules and vector judgment +# around the Harness sandbox replay. No expected result enters the sandbox. + +import json +import os +import sys +from pathlib import Path + +from inferrsi.benchmark import skill_eval_engineering as task_rules +from inferrsi.harness import artifact_sandbox + + +def verify(task, output): + source, rejected = task_rules.submission(task, output) + if rejected is not None: + return rejected + return task_rules.judge(artifact_sandbox.replay(source, task_rules.vector_inputs())) + + +def main(): + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + output = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, output) + Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(json.dumps(result), encoding="utf-8") + raise SystemExit(0 if result["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/src/inferrsi/apps/workflows/skill_evaluation.py b/src/inferrsi/apps/workflows/skill_evaluation.py new file mode 100644 index 00000000..06482510 --- /dev/null +++ b/src/inferrsi/apps/workflows/skill_evaluation.py @@ -0,0 +1,61 @@ +# Composed skill-evaluation entries: Knowledge bundle export, and Harness trial +# execution joined with Benchmark verdict rules. Signatures and retained records +# match the functions that harness.skill_evaluation held before the ownership split. + +from pathlib import Path + +from inferrsi.benchmark import skill_eval_engineering as engineering_task +from inferrsi.benchmark import skill_evaluation as rules +from inferrsi.harness import skill_evaluation as host +from inferrsi.harness.usage import normalize_usage +from inferrsi.knowledge.knowledge_operations import KnowledgeReader + +from . import skill_eval_engineering + + +def export_bundle(reader: KnowledgeReader, reference: str, destination: Path) -> dict: + # Knowledge resolves the bundle; Harness receives and retains bytes only. + bundle = reader.read(reference) + host.write_files(destination, bundle.content) + return bundle.view() + + +def summarize(trials: list[dict]) -> dict: + normalized = [ + {**trial, "observations": {**trial["observations"], + "usage": normalize_usage(trial["observations"]["usage"])}} + for trial in trials + ] + return rules.summarize(normalized) + + +def run_trial( + client, + task: dict, + condition: str, + repeat: int, + workspace: Path, + trial_dir: Path, + schema: Path, + verifier: Path, + prompt: str, + deadline: float, + *, + timeout_seconds: float | None = None, + engineering_artifact: bool = False, + generation_only: bool = False, +) -> dict: + artifact = None + if engineering_artifact: + artifact = host.ArtifactPort( + kind=engineering_task.ADAPTER, + generation_mode=engineering_task.GENERATION_MODE, + starter=engineering_task.STARTER.encode("utf-8"), + max_source_bytes=engineering_task.MAX_SOURCE_BYTES, + verify=skill_eval_engineering.verify, + ) + return host.run_trial( + client, task, condition, repeat, workspace, trial_dir, schema, verifier, prompt, + deadline, valid_checks=rules.valid_checks, task_verdict=rules.task_verdict, + artifact=artifact, timeout_seconds=timeout_seconds, generation_only=generation_only, + ) diff --git a/src/inferrsi/harness/optimization_session.py b/src/inferrsi/harness/optimization_session.py index b4e8d277..6718697b 100644 --- a/src/inferrsi/harness/optimization_session.py +++ b/src/inferrsi/harness/optimization_session.py @@ -121,7 +121,7 @@ def _runtime_identity(): "harness/usage.py", "harness/agent_sessions.py", "harness/task_protocol.py", "harness/protocol_child.py", "harness/candidates.py", "harness/project.py", - "harness/records/knowledge_capture.py", "apps/campaign.py", + "harness/records/knowledge_capture.py", "apps/workflows/campaign.py", "knowledge/context_projection.py") return {name: sha256(root / name) for name in names} diff --git a/tests/_gate_policy_support.py b/tests/_gate_policy_support.py index 990e6d9f..3c1ae728 100644 --- a/tests/_gate_policy_support.py +++ b/tests/_gate_policy_support.py @@ -18,7 +18,8 @@ NEW_CORES = ("knowledge", "benchmark", "apps") CORE_NAMES = ("knowledge", "harness", "benchmark", "apps") # No root file is application assembly. A1 (#408) moved the CLI and its parser into -# ``apps/cli/``, and A2 (#408) moved the task launcher to ``apps/benchmark_operations.py``. +# ``apps/cli/``, and A2 (#408) moved the task launcher into Apps. The #490 workflows +# slice groups it at ``apps/workflows/benchmark_operations.py``. # With that last move the ``APPLICATION_ENTRIES`` listing is gone: Apps is the # ``inferrsi.apps`` package by its first path segment and nothing else, so a root # ``cli.py``, ``arguments.py`` or ``benchmark_operations.py`` that returns is an unlisted @@ -76,7 +77,7 @@ RETIRED_BY_A1 = frozenset({"inferrsi.cli", "inferrsi.arguments"}) # #408 A2 deleted the root chart renderer and task launcher the same way. The # implementations are ``inferrsi.apps.views.summary_chart`` and -# ``inferrsi.apps.benchmark_operations``. +# ``inferrsi.apps.workflows.benchmark_operations``. RETIRED_BY_A2 = frozenset({"inferrsi.summary_chart", "inferrsi.benchmark_operations"}) # #452 M1 deleted the four root MindIE files the same way. The adapter is the # ``inferrsi.apps.mindie`` package: ``project``, ``context``, ``mcp`` and ``read_worker``. diff --git a/tests/check_installed.py b/tests/check_installed.py index 04de8547..945ae7c4 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -17,6 +17,7 @@ from inferrsi.apps.api import CandidateProvider, EvaluationPort from inferrsi.apps.cli import main from inferrsi.benchmark.registry import builtin_tasks, describe_task, task_resources +from inferrsi.benchmark.skill_eval_engineering import REQUEST from inferrsi.contracts import RunSpec, claim, evidence, json_v1 from inferrsi.knowledge._source_snapshot import BUNDLE_LIBRARY, LIBRARY from inferrsi.knowledge.catalog import knowledge_records, search_knowledge @@ -621,6 +622,9 @@ def _inventory(root, prefix): ("result", "storage"), ("evidence_layout", "storage"), ("bridge", "handoff"), ("context_provider", "handoff"), ("candidate_provider", "handoff"), + ("campaign", "workflows"), ("benchmark_operations", "workflows"), + ("improvement", "workflows"), ("skill_evaluation", "workflows"), + ("skill_eval_engineering", "workflows"), ): old_name = f"inferrsi.apps.{name}" canonical_name = f"inferrsi.apps.{group}.{name}" @@ -637,6 +641,32 @@ def _inventory(root, prefix): ) assert CandidateProvider is import_module("inferrsi.apps.candidate_provider").CandidateProvider assert EvaluationPort.__module__ == "inferrsi.apps.api" +# Both the former engineering verifier script and its canonical entry survive installation. +with tempfile.TemporaryDirectory() as script_directory: + script_root = Path(script_directory) + script_task, script_output, script_result = ( + script_root / name for name in ("task.json", "output.json", "result.json") + ) + script_task.write_text(json.dumps({ + "id": "synthetic-parser", "split": "development", "request": REQUEST, + "provenance": "Synthetic installed-entry control", + }), encoding="utf-8") + script_output.write_text("{}", encoding="utf-8") + for script_module in ("inferrsi.apps.skill_eval_engineering", + "inferrsi.apps.workflows.skill_eval_engineering"): + for script_entry in (["-I", "-m", script_module], + ["-I", import_module(script_module).__file__]): + script_result.unlink(missing_ok=True) + script_run = subprocess.run( + [sys.executable, *script_entry, str(script_task), str(script_output)], + env={**os.environ, "INFERRSI_RESULT_PATH": str(script_result)}, + cwd=script_root, capture_output=True, text=True, timeout=30, + ) + assert script_run.returncode == 1, script_run.stderr + assert script_run.stdout == script_run.stderr == "" + assert script_result.read_bytes() == ( + b'{"passed": false, "assertions": {"artifact_schema": false}}' + ) for retired in ("_repo_pack_data", "evidence_paths"): assert find_spec(f"inferrsi.{retired}") is None, retired assert not installed_root & {"_repo_pack_data.py", "evidence_paths.py"} @@ -686,7 +716,8 @@ def _inventory(root, prefix): else: raise AssertionError(f"retired module inferrsi.{a1_retired} must not import") assert main.__module__ == "inferrsi.apps.cli" -for canonical in ("inferrsi.apps.views.summary_chart", "inferrsi.apps.benchmark_operations", +for canonical in ("inferrsi.apps.views.summary_chart", + "inferrsi.apps.workflows.benchmark_operations", "inferrsi.apps.mindie.project", "inferrsi.apps.mindie.context", "inferrsi.apps.mindie.mcp", "inferrsi.apps.mindie.read_worker"): assert find_spec(canonical) is not None, canonical diff --git a/tests/fixtures/dashboard_profile_review.py b/tests/fixtures/dashboard_profile_review.py index 519c0769..61ca7b30 100644 --- a/tests/fixtures/dashboard_profile_review.py +++ b/tests/fixtures/dashboard_profile_review.py @@ -24,7 +24,7 @@ def build(root: Path): start = datetime(2026, 1, 1, 12, tzinfo=timezone.utc) with ( patch("inferrsi.__version__", "0+synthetic"), - patch("inferrsi.apps.campaign.__version__", "0+synthetic"), + patch("inferrsi.apps.workflows.campaign.__version__", "0+synthetic"), patch("inferrsi.harness.records.knowledge_capture.datetime", wraps=datetime) as clock, patch("inferrsi.harness.records.optimization_log.datetime", wraps=datetime) as log_clock, ): diff --git a/tests/fixtures/knowledge_retrieval.py b/tests/fixtures/knowledge_retrieval.py index 454fc30c..e53ef2e3 100644 --- a/tests/fixtures/knowledge_retrieval.py +++ b/tests/fixtures/knowledge_retrieval.py @@ -10,9 +10,9 @@ from pathlib import Path from unittest.mock import patch -from inferrsi.apps.campaign import capture_context, create_campaign, save_retrieval from inferrsi.apps.views.dashboard import write_dashboard from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import capture_context, create_campaign, save_retrieval from inferrsi.harness.records.campaign_records import record_candidate from inferrsi.harness.records.optimization_log import record_log from inferrsi.knowledge._source_snapshot import source_entry diff --git a/tests/test_anthropic_agent.py b/tests/test_anthropic_agent.py index f00c5f97..be63f356 100644 --- a/tests/test_anthropic_agent.py +++ b/tests/test_anthropic_agent.py @@ -6,7 +6,7 @@ import pytest -from inferrsi.apps import skill_evaluation as evaluation +from inferrsi.apps.workflows import skill_evaluation as evaluation from inferrsi.harness import skill_evaluation as harness from inferrsi.infra.artifacts import sha256, write_json from inferrsi.model import anthropic_agent as agent diff --git a/tests/test_apps_module_migrations.py b/tests/test_apps_module_migrations.py index 606297e0..4582c3ca 100644 --- a/tests/test_apps_module_migrations.py +++ b/tests/test_apps_module_migrations.py @@ -22,10 +22,21 @@ def test_shim_is_an_explicit_export_of_the_same_objects(entry): path = REPO / "src" / (entry["old"].replace(".", "/") + ".py") tree = ast.parse(path.read_text(encoding="utf-8")) - docstring, imported, declared = tree.body + if entry.get("script_entry"): + docstring, imported, declared, guard = tree.body + assert ast.dump(guard) == ast.dump(ast.parse( + f'if __name__ == "__main__":\n {entry["script_entry"]}()\n' + ).body[0]) + else: + docstring, imported, declared = tree.body assert isinstance(docstring, ast.Expr) and isinstance(docstring.value, ast.Constant) - assert isinstance(imported, ast.ImportFrom) and imported.level == 1 - assert "inferrsi.apps." + imported.module == entry["canonical"] + assert isinstance(imported, ast.ImportFrom) + if entry.get("script_entry"): + # This former file entry must also work without a package context. + assert imported.level == 0 and imported.module == entry["canonical"] + else: + assert imported.level == 1 + assert "inferrsi.apps." + imported.module == entry["canonical"] assert sorted((name.name, name.asname) for name in imported.names) == sorted( (name, None) for name in entry["exports"] ) @@ -148,7 +159,7 @@ def test_storage_dependency_check_rejects_a_handoff_back_edge(tmp_path, indirect def test_migrated_initializers_import_nothing_and_module_bodies_do_no_io(): - packages = ["inferrsi.apps.storage", "inferrsi.apps.handoff"] + packages = ["inferrsi.apps.storage", "inferrsi.apps.handoff", "inferrsi.apps.workflows"] for package in packages: path = REPO / "src" / package.replace(".", "/") / "__init__.py" tree = ast.parse(path.read_text(encoding="utf-8")) @@ -212,13 +223,15 @@ def test_delivered_context_draft_loads_a_legacy_instance_pickle(): assert type(restored) is DeliveredContextDraft and restored == value -def test_cold_handoff_imports_do_not_read_data_probe_or_launch_services(): +@pytest.mark.parametrize("group", ["handoff", "workflows"]) +def test_cold_handoff_and_workflow_imports_do_not_read_data_probe_or_launch_services(group): script = """ import builtins, importlib, importlib.abc, io, json, socket, sqlite3, subprocess, sys, warnings from unittest.mock import patch -blocked = ('inferrsi.apps.api', 'inferrsi.infra', 'inferrsi.benchmark', 'inferrsi.model', - 'inferrsi.harness.api', 'inferrsi.harness.run_store', 'inferrsi.harness.generation', - 'torch', 'numpy', 'PIL', 'vllm') +blocked = ('inferrsi.apps.api', 'inferrsi.harness.api', 'inferrsi.harness.run_store', + 'inferrsi.harness.generation', 'torch', 'numpy', 'PIL', 'vllm') +if sys.argv[2] == 'handoff': + blocked += ('inferrsi.infra', 'inferrsi.benchmark', 'inferrsi.model') class Block(importlib.abc.MetaPathFinder): def find_spec(self, name, path=None, target=None): if any(name == item or name.startswith(item + '.') for item in blocked): @@ -228,7 +241,7 @@ def deny(*args, **kwargs): raise AssertionError('unexpected data read, device probe, process or service') with warnings.catch_warnings(record=True) as caught: warnings.simplefilter('always') - importlib.import_module('inferrsi.apps.handoff') + importlib.import_module('inferrsi.apps.' + sys.argv[2]) parent_only = sorted(name for name in sys.modules if name.startswith('inferrsi.apps')) with patch.object(builtins, 'open', deny), patch.object(io, 'open', deny), \\ patch.object(subprocess, 'Popen', deny), patch.object(sqlite3, 'connect', deny), \\ @@ -242,8 +255,8 @@ def deny(*args, **kwargs): print(json.dumps({'parent_only': parent_only, 'warnings': [str(w.message) for w in caught], 'control': control})) """ - modules = [entry[key] for entry in MIGRATIONS if ".handoff." in entry["canonical"] + modules = [entry[key] for entry in MIGRATIONS if f".{group}." in entry["canonical"] for key in ("old", "canonical")] - result = _fresh(script, json.dumps(modules)) - assert result == {"parent_only": ["inferrsi.apps", "inferrsi.apps.handoff"], + result = _fresh(script, json.dumps(modules), group) + assert result == {"parent_only": ["inferrsi.apps", f"inferrsi.apps.{group}"], "warnings": [], "control": True} diff --git a/tests/test_benchmark_operations.py b/tests/test_benchmark_operations.py index 091dd462..0f23e917 100644 --- a/tests/test_benchmark_operations.py +++ b/tests/test_benchmark_operations.py @@ -4,8 +4,8 @@ import pytest -from inferrsi.apps import benchmark_operations from inferrsi.apps.cli import main +from inferrsi.apps.workflows import benchmark_operations from inferrsi.benchmark import registry diff --git a/tests/test_campaign_record_injection.py b/tests/test_campaign_record_injection.py index 1c94519b..6664b07e 100644 --- a/tests/test_campaign_record_injection.py +++ b/tests/test_campaign_record_injection.py @@ -10,7 +10,7 @@ import pytest from inferrsi import __version__ -from inferrsi.apps import campaign +from inferrsi.apps.workflows import campaign from inferrsi.harness.records import campaign_records, knowledge_capture from inferrsi.knowledge.context_projection import context_item from tests._gate_policy_support import ( diff --git a/tests/test_campaign_task_selection.py b/tests/test_campaign_task_selection.py index 5d1a2da1..575d9d70 100644 --- a/tests/test_campaign_task_selection.py +++ b/tests/test_campaign_task_selection.py @@ -1,6 +1,6 @@ import json -from inferrsi.apps.campaign import capture_context, create_campaign, verify_capture +from inferrsi.apps.workflows.campaign import capture_context, create_campaign, verify_capture from inferrsi.knowledge.knowledge_operations import KnowledgeReader BASELINE_CUE = "reproducible diffusion baseline correctness fixed seed end to end benchmark" diff --git a/tests/test_harness_improvement.py b/tests/test_harness_improvement.py index d2890fff..149d261c 100644 --- a/tests/test_harness_improvement.py +++ b/tests/test_harness_improvement.py @@ -5,7 +5,7 @@ import pytest -from inferrsi.apps.improvement import assessment, choose, holdout_accepts +from inferrsi.apps.workflows.improvement import assessment, choose, holdout_accepts from inferrsi.benchmark.improvement import SelectionPolicy from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import usage_totals @@ -265,7 +265,7 @@ def factory(phase, candidate): def test_rerun_with_unstable_baseline_is_rejected(tmp_path): - from inferrsi.apps.improvement import rerun_assessment + from inferrsi.apps.workflows.improvement import rerun_assessment _, _, output, _, _ = run(tmp_path) probe = compare.read_report(output / "results/inv") diff --git a/tests/test_knowledge_navigation_browser.py b/tests/test_knowledge_navigation_browser.py index 0742394e..1005e0bf 100644 --- a/tests/test_knowledge_navigation_browser.py +++ b/tests/test_knowledge_navigation_browser.py @@ -8,9 +8,9 @@ from test_optimization_story import campaign as make_campaign from test_retrieval_projection import FIXTURE -from inferrsi.apps.campaign import capture_context from inferrsi.apps.views.dashboard import dashboard_snapshot, render_dashboard from inferrsi.apps.views.dashboard_live import DashboardServer +from inferrsi.apps.workflows.campaign import capture_context playwright = pytest.importorskip("playwright.sync_api") diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index a37836ff..c671ec2d 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -1,6 +1,10 @@ """Run the current-Agent protocol using real child processes and synthetic work.""" import json +import os +import subprocess +import sys +from pathlib import Path import pytest @@ -410,7 +414,7 @@ def test_child_argument_error_keeps_its_cli_message_at_start(task, tmp_path, cap def test_task_runner_does_not_expose_harness_as_top_level_modules(task, tmp_path): - from inferrsi.apps.benchmark_operations import run_task + from inferrsi.apps.workflows.benchmark_operations import run_task output = tmp_path / "imports.json" (task / "benchmark.py").write_text( @@ -460,6 +464,122 @@ def test_runtime_identity_covers_migrated_process_domain_and_not_removed_executo assert "harness/artifact_primitives.py" in identity # #412 R3: the protocol child entry runs every task protocol operation. assert "harness/protocol_child.py" in identity + assert "apps/workflows/campaign.py" in identity + assert "apps/campaign.py" not in identity assert "infra/executors/local.py" not in identity for name in identity: assert (root / name).is_file(), name + from inferrsi.harness.artifact_primitives import sha256 + + assert identity["apps/workflows/campaign.py"] == sha256(root / "apps/workflows/campaign.py") + + +@pytest.mark.parametrize("action", ["status", "recover"]) +def test_pre_move_runtime_identity_is_rejected_without_rewriting_plan(session, action): + path = session / "plan.json" + plan = json.loads(path.read_text(encoding="utf-8")) + plan["runtime_identity"]["apps/campaign.py"] = plan["runtime_identity"].pop( + "apps/workflows/campaign.py" + ) + # A prior release used the old key. Even equal bytes cannot admit that runtime map. + frozen = json.dumps(plan, indent=2).encode("utf-8") + path.write_bytes(frozen) + with pytest.raises(SystemExit): + main(["benchmarks", "session", action, "--session", str(session)]) + assert path.read_bytes() == frozen + assert not (session / "calls.log").exists() + + +@pytest.fixture +def pre_move_python(): + executable = os.environ.get("INFERRSI_PRE_MOVE_PYTHON") + if not executable: + pytest.skip("Set INFERRSI_PRE_MOVE_PYTHON to an isolated, pinned pre-move wheel runtime") + assert Path(executable).is_file() + return executable + + +def test_pinned_pre_move_wheel_recovers_its_original_plan( + tmp_path, task, pre_move_python, +): + """Two independent runtimes; no patched identity map or rewritten frozen plan.""" + driver = """ +import json, sys +from pathlib import Path +if sys.argv[1]: + sys.path.insert(0, sys.argv[1]) +from inferrsi.apps.cli import main +from inferrsi.benchmark import registry +from inferrsi.harness import optimization_session +registry._library_root = lambda: Path(sys.argv[2]) +arguments = sys.argv[3:] +if arguments[0] == 'interrupt': + arguments.pop(0) + execute = optimization_session._session_execute + def interrupt(*args): + execute(*args) + raise KeyboardInterrupt('synthetic interruption after the retained execution') + optimization_session._session_execute = interrupt + try: + main(arguments) + except KeyboardInterrupt: + print(json.dumps({'interrupted': True})) + else: + raise AssertionError('execution was not interrupted') +else: + raise SystemExit(main(arguments)) +""" + post_move_python = os.environ.get("INFERRSI_POST_MOVE_PYTHON", sys.executable) + current_source = "" if "INFERRSI_POST_MOVE_PYTHON" in os.environ else str( + Path(__file__).resolve().parents[1] / "src" + ) + + def run(python, source, arguments, expected=0): + result = subprocess.run( + [python, "-I", "-c", driver, source, str(task.parent), *arguments], + cwd=tmp_path, capture_output=True, text=True, timeout=60, + ) + assert result.returncode == expected, result.stderr + return result + + def start(root): + return ["benchmarks", "session", "start", task.name, "--session", str(root), + "--rounds", "3", "--max-minutes", "5", "--evaluation-timeout-seconds", "15", + "--condition", "synthetic-release-boundary", "--evaluation-args", ""] + + old = tmp_path / "old-session" + run(pre_move_python, "", start(old)) + frozen = (old / "plan.json").read_bytes() + old_identity = json.loads(frozen)["runtime_identity"] + assert "apps/campaign.py" in old_identity + assert "apps/workflows/campaign.py" not in old_identity + candidate = old / "work/candidate.py" + candidate.write_text("FACTOR = 2\n", encoding="utf-8") + interrupted = run(pre_move_python, "", [ + "interrupt", "benchmarks", "session", "round", "--session", str(old), + "--candidate", str(candidate), "--hypothesis", "Synthetic release recovery control", + ]) + assert json.loads(interrupted.stdout)["interrupted"] + retained = {p.relative_to(old): p.read_bytes() for p in old.rglob("*") if p.is_file()} + for action in ("status", "recover"): + rejected = run(post_move_python, current_source, + ["benchmarks", "session", action, "--session", str(old)], expected=2) + assert "Optimization runtime changed" in rejected.stderr + after = {p.relative_to(old): p.read_bytes() for p in old.rglob("*") if p.is_file()} + assert after == retained + recovered = json.loads(run(pre_move_python, "", [ + "benchmarks", "session", "recover", "--session", str(old), + ]).stdout) + assert recovered["completed_rounds"] == 1 and recovered["best"]["score"] == 2 + run(pre_move_python, "", ["benchmarks", "session", "status", "--session", str(old)]) + assert (old / "plan.json").read_bytes() == frozen + assert (old / "calls.log").read_text().splitlines() == ["2"] + + new = tmp_path / "new-session" + run(post_move_python, current_source, start(new)) + new_identity = json.loads((new / "plan.json").read_bytes())["runtime_identity"] + assert "apps/workflows/campaign.py" in new_identity and "apps/campaign.py" not in new_identity + from inferrsi.harness.artifact_primitives import sha256 + + canonical = Path(__file__).resolve().parents[1] / "src/inferrsi/apps/workflows/campaign.py" + assert new_identity["apps/workflows/campaign.py"] == sha256(canonical) diff --git a/tests/test_optimization_story.py b/tests/test_optimization_story.py index c22048a8..725a1ce3 100644 --- a/tests/test_optimization_story.py +++ b/tests/test_optimization_story.py @@ -9,10 +9,10 @@ import pytest -from inferrsi.apps.campaign import capture_context, verify_capture from inferrsi.apps.cli import main from inferrsi.apps.views.dashboard import dashboard_snapshot, render_dashboard, write_dashboard from inferrsi.apps.views.dashboard_data import load_dashboard +from inferrsi.apps.workflows.campaign import capture_context, verify_capture from inferrsi.harness.records.campaign_records import ( create_campaign, record_candidate, diff --git a/tests/test_package_root_inventory.py b/tests/test_package_root_inventory.py index 8f800205..a2bc065d 100644 --- a/tests/test_package_root_inventory.py +++ b/tests/test_package_root_inventory.py @@ -120,6 +120,7 @@ def test_inventory_is_the_exact_final_two_file_root() -> None: if isinstance(node, (ast.Import, ast.ImportFrom))] # A2 moved its two files into Apps. assert (REPO / PACKAGE / "apps" / "benchmark_operations.py").is_file() + assert (REPO / PACKAGE / "apps" / "workflows" / "benchmark_operations.py").is_file() assert (REPO / PACKAGE / "apps" / "views" / "summary_chart.py").is_file() # A1 moved its two files; the implementation is the apps/cli package. for name in ("__init__.py", "arguments.py"): diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index 9e08239c..32ae27ce 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -77,7 +77,7 @@ def test_project_lifecycle_never_imports_benchmark(project): from pathlib import Path class NoBenchmark(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path=None, target=None): - if fullname in ('inferrsi.benchmark', 'inferrsi.apps.benchmark_operations') or ( + if fullname in ('inferrsi.benchmark', 'inferrsi.apps.workflows.benchmark_operations') or ( fullname.startswith('inferrsi.benchmark.') ): raise AssertionError('Ordinary optimization imported ' + fullname) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index e0c29138..a47a9ba1 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -9,8 +9,8 @@ import pytest -from inferrsi.apps.campaign import capture_context, create_campaign from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import capture_context, create_campaign from inferrsi.knowledge.context_projection import assemble_context_text REVISION = "f6d200" + "a" * 34 diff --git a/tests/test_search_attribution.py b/tests/test_search_attribution.py index 127d87c4..6c45c371 100644 --- a/tests/test_search_attribution.py +++ b/tests/test_search_attribution.py @@ -4,9 +4,9 @@ import pytest -from inferrsi.apps.campaign import create_campaign from inferrsi.apps.cli import main from inferrsi.apps.views.search_attribution import import_search +from inferrsi.apps.workflows.campaign import create_campaign def command(corpus, cmd, output, event_id, exit_code=0): diff --git a/tests/test_skill_eval_compare.py b/tests/test_skill_eval_compare.py index 4a64ea08..7f98a7d9 100644 --- a/tests/test_skill_eval_compare.py +++ b/tests/test_skill_eval_compare.py @@ -1013,9 +1013,9 @@ def test_changed_runtime_controllers_reject_execution_and_report_without_calls(t assert "src/inferrsi/infra/artifacts.py" in module.CONTROLLERS assert "tools/skill_eval_artifact_guidance.py" in module.CONTROLLERS # Each owner of the split trial path is frozen: composed entry, rules and host. - for owner in ("apps", "benchmark", "harness"): + for owner in ("apps/workflows", "benchmark", "harness"): assert f"src/inferrsi/{owner}/skill_evaluation.py" in module.CONTROLLERS - for name in ("apps/skill_eval_engineering.py", "benchmark/skill_eval_engineering.py", + for name in ("apps/workflows/skill_eval_engineering.py", "benchmark/skill_eval_engineering.py", "benchmark/skill_eval_diagnostic.py", "harness/artifact_sandbox.py", "harness/atomic_json.py", "harness/input_binding.py"): assert "src/inferrsi/" + name in module.CONTROLLERS diff --git a/tests/test_skill_eval_engineering.py b/tests/test_skill_eval_engineering.py index c0b0434f..c5cab497 100644 --- a/tests/test_skill_eval_engineering.py +++ b/tests/test_skill_eval_engineering.py @@ -1,5 +1,6 @@ import copy import json +import os import signal import subprocess import sys @@ -8,7 +9,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox @@ -17,6 +18,26 @@ ANSWER = {"artifact": "parser.py", "summary": "Repair", "artifact_source": engineering.STARTER} +@pytest.mark.parametrize("module", [ + "inferrsi.apps.skill_eval_engineering", "inferrsi.apps.workflows.skill_eval_engineering", +]) +@pytest.mark.parametrize("by_file", [False, True]) +def test_old_and_canonical_script_entries_write_the_same_result(tmp_path, module, by_file): + task, output, result = (tmp_path / name for name in ("task.json", "output.json", "result.json")) + task.write_text(json.dumps(TASK), encoding="utf-8") + output.write_text("{}", encoding="utf-8") + source = Path(__file__).resolve().parents[1] / "src" + arguments = [str(source / (module.replace(".", "/") + ".py"))] if by_file else ["-m", module] + environment = {**os.environ, "PYTHONPATH": str(source), "INFERRSI_RESULT_PATH": str(result)} + executed = subprocess.run( + [sys.executable, *arguments, str(task), str(output)], cwd=tmp_path, + env=environment, capture_output=True, text=True, timeout=30, + ) + assert executed.returncode == 1, executed.stderr + assert executed.stdout == executed.stderr == "" + assert result.read_bytes() == b'{"passed": false, "assertions": {"artifact_schema": false}}' + + def test_starter_matches_frozen_fixture(): fixture = Path(__file__).parent / "fixtures/skill_eval_engineering/parser.py" assert fixture.read_text() == engineering.STARTER diff --git a/tests/test_skill_eval_generation.py b/tests/test_skill_eval_generation.py index 79e312ae..872e840c 100644 --- a/tests/test_skill_eval_generation.py +++ b/tests/test_skill_eval_generation.py @@ -6,8 +6,8 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check -from inferrsi.apps import skill_evaluation as evaluation +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_evaluation as evaluation from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.harness import skill_evaluation as harness diff --git a/tests/test_skill_eval_local.py b/tests/test_skill_eval_local.py index d0a68e56..2afbf54b 100644 --- a/tests/test_skill_eval_local.py +++ b/tests/test_skill_eval_local.py @@ -3,7 +3,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.infra.artifacts import sha256, write_json diff --git a/tests/test_skill_eval_local_cli.py b/tests/test_skill_eval_local_cli.py index 3f08e1d5..0eb98df3 100644 --- a/tests/test_skill_eval_local_cli.py +++ b/tests/test_skill_eval_local_cli.py @@ -11,7 +11,7 @@ import pytest -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox as sandbox from inferrsi.infra.artifacts import sha256, write_json diff --git a/tests/test_skill_eval_pipeline.py b/tests/test_skill_eval_pipeline.py index 15aa483e..3ebb4a18 100644 --- a/tests/test_skill_eval_pipeline.py +++ b/tests/test_skill_eval_pipeline.py @@ -8,7 +8,7 @@ import pytest -from inferrsi.apps.skill_evaluation import run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import run_trial, summarize from inferrsi.infra.artifacts import atomic_write_json, sha256 from tools import skill_eval_pilot diff --git a/tests/test_skill_evaluation.py b/tests/test_skill_evaluation.py index ae398a38..5944038a 100644 --- a/tests/test_skill_evaluation.py +++ b/tests/test_skill_evaluation.py @@ -6,7 +6,7 @@ import pytest -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.benchmark.skill_evaluation import task_verdict from inferrsi.harness.skill_evaluation import CodexClient, toml_value, trace_metrics from inferrsi.knowledge.knowledge_operations import KnowledgeReader diff --git a/tests/test_target_module_import_gate.py b/tests/test_target_module_import_gate.py index d05d352b..9d4ef97f 100644 --- a/tests/test_target_module_import_gate.py +++ b/tests/test_target_module_import_gate.py @@ -146,6 +146,7 @@ def test_only_files_directly_in_a_non_package_bundle_are_bundle_programs(tmp_pat "harness/task_protocol.py", "harness/other_protocol.py", "apps/benchmark_operations.py", + "apps/workflows/benchmark_operations.py", "knowledge/data/skills/scripts/loader.py", ]) def test_by_path_loading_is_admitted_nowhere_else(tmp_path, relative): @@ -182,6 +183,7 @@ def test_no_new_cross_core_imports(self) -> None: "_mindie_read_worker.py"): self.assertFalse((ROOT / retired).exists(), retired) for moved in ("cli/__init__.py", "cli/arguments.py", "benchmark_operations.py", + "workflows/benchmark_operations.py", "views/summary_chart.py", "mindie/project.py", "mindie/context.py", "mindie/mcp.py", "mindie/read_worker.py"): self.assertTrue((ROOT / "apps" / moved).is_file(), moved) @@ -308,7 +310,7 @@ def test_real_tree_allows_apps_and_same_core_only(self) -> None: "arguments_extra.py": "from inferrsi.knowledge import catalog\n", # #408 A2: the task launcher is Apps by path too. The root file that held # it is no longer an application entry and fails like the others. - "apps/benchmark_operations.py": "import inferrsi.benchmark\n", + "apps/workflows/benchmark_operations.py": "import inferrsi.benchmark\n", "benchmark_operations.py": "import inferrsi.benchmark\n", "unowned.py": "import inferrsi.benchmark\n", } diff --git a/tools/skill_eval_cloud_replay.py b/tools/skill_eval_cloud_replay.py index 876879ac..230a7553 100644 --- a/tools/skill_eval_cloud_replay.py +++ b/tools/skill_eval_cloud_replay.py @@ -6,7 +6,7 @@ import urllib.request from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox from tools import skill_eval_artifact_guidance as artifact_guidance diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 5615d61c..43b312f6 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -12,8 +12,8 @@ from dataclasses import replace from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.benchmark import skill_eval_diagnostic as diagnostic from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox @@ -39,8 +39,8 @@ GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "tools/skill_eval_artifact_guidance.py", - "src/inferrsi/apps/skill_evaluation.py", - "src/inferrsi/apps/skill_eval_engineering.py", + "src/inferrsi/apps/workflows/skill_evaluation.py", + "src/inferrsi/apps/workflows/skill_eval_engineering.py", "src/inferrsi/benchmark/skill_evaluation.py", "src/inferrsi/benchmark/skill_eval_diagnostic.py", "src/inferrsi/benchmark/skill_eval_engineering.py", diff --git a/tools/skill_eval_improve.py b/tools/skill_eval_improve.py index 46bc6d70..b1a6098e 100644 --- a/tools/skill_eval_improve.py +++ b/tools/skill_eval_improve.py @@ -12,7 +12,12 @@ import time from pathlib import Path -from inferrsi.apps.improvement import assessment, choose, holdout_accepts, rerun_assessment +from inferrsi.apps.workflows.improvement import ( + assessment, + choose, + holdout_accepts, + rerun_assessment, +) from inferrsi.benchmark.improvement import SelectionPolicy from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import usage_totals @@ -21,7 +26,8 @@ from tools import skill_eval_compare as compare CONTROLLERS = (*compare.CONTROLLERS, "tools/skill_eval_improve.py", - "src/inferrsi/apps/improvement.py", "src/inferrsi/benchmark/improvement.py", + "src/inferrsi/apps/workflows/improvement.py", + "src/inferrsi/benchmark/improvement.py", "src/inferrsi/harness/improvement.py") diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py index 95f7253d..46967643 100644 --- a/tools/skill_eval_local.py +++ b/tools/skill_eval_local.py @@ -12,7 +12,7 @@ import stat from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox from inferrsi.harness.usage import normalize_usage diff --git a/tools/skill_eval_pilot.py b/tools/skill_eval_pilot.py index 7d17f853..579b121c 100644 --- a/tools/skill_eval_pilot.py +++ b/tools/skill_eval_pilot.py @@ -7,7 +7,7 @@ import time from pathlib import Path -from inferrsi.apps.skill_evaluation import export_bundle, run_trial, summarize +from inferrsi.apps.workflows.skill_evaluation import export_bundle, run_trial, summarize from inferrsi.harness.skill_evaluation import CodexClient from inferrsi.harness.usage import normalize_usage from inferrsi.infra.artifacts import sha256, write_json @@ -205,7 +205,7 @@ def prepare( bundles = {name: reader.read("checkout:" + name).view() for name in SKILLS} schedule = build_schedule(tasks, final_only) source_files = [ - ROOT / "src/inferrsi/apps/skill_evaluation.py", + ROOT / "src/inferrsi/apps/workflows/skill_evaluation.py", ROOT / "src/inferrsi/benchmark/skill_evaluation.py", ROOT / "src/inferrsi/harness/skill_evaluation.py", Path(__file__), diff --git a/tools/skill_eval_qualify.py b/tools/skill_eval_qualify.py index 2ff26714..fc775de1 100644 --- a/tools/skill_eval_qualify.py +++ b/tools/skill_eval_qualify.py @@ -12,7 +12,7 @@ import subprocess from pathlib import Path -from inferrsi.apps import skill_eval_engineering as artifact_check +from inferrsi.apps.workflows import skill_eval_engineering as artifact_check from inferrsi.benchmark import skill_eval_engineering as engineering from inferrsi.harness import artifact_sandbox