diff --git a/agentdescent/__init__.py b/agentdescent/__init__.py index 5ff00d23..8380feba 100644 --- a/agentdescent/__init__.py +++ b/agentdescent/__init__.py @@ -476,6 +476,14 @@ tree_summary, ) from .treestrategy import EDIT_PROTOCOL, FileTree, parse_edits, parse_proposal, tree_reflector +from .artifact_snapshot import ( + ArtifactSnapshot, + SnapshotEvalReport, + evaluate_snapshot, + evaluate_snapshot_spec, + load_snapshot_source, +) +from .ledger import read_artifact_lineage from .runners import ( LAYOUTS, PLUGIN_FROZEN, PLUGIN_HOSTS, TEST_FAILURE_MARKER, PluginHost, code_runner, gated_reward, plugin_runner, tree_runner, @@ -1035,6 +1043,12 @@ "tree_summary", "EDIT_PROTOCOL", "FileTree", + "ArtifactSnapshot", + "SnapshotEvalReport", + "evaluate_snapshot", + "evaluate_snapshot_spec", + "load_snapshot_source", + "read_artifact_lineage", "parse_edits", "parse_proposal", "tree_reflector", diff --git a/agentdescent/artifact_snapshot.py b/agentdescent/artifact_snapshot.py new file mode 100644 index 00000000..56296efe --- /dev/null +++ b/agentdescent/artifact_snapshot.py @@ -0,0 +1,428 @@ +"""A portable snapshot of one candidate: full context plus ledger lineage. + +A snapshot is not a second ledger. It is the FileTree state already stored +on the ledger (or in a run's ``result.json``) plus the id / parent / root +that history already implies, written as a JSON file so the same candidate +can be restored or scored elsewhere. + +Algorithms still edit only the declared editable scope; evaluation and +restore use the complete tree, including frozen files. Taking a selected +snapshot and scoring it on another compatible dataset does not search, +commit, or apply anything. + +This is not "run this snapshot inside Claude Code". Plugin kind still +means "evolve the plugin itself". Host selection stays the existing +agent / runner path on the destination spec. + + snap = ArtifactSnapshot.from_filetree(tree, artifact_id="skill") + snap.save("skill.json") + report = evaluate_snapshot(snap, tasks, run=run, reward=reward) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import dataclass, field, replace +from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple + +from .evolution import Task +from .filetree import canonical, materialize, match_any, safe_relpath +from .ledger import Ledger, read_artifact_lineage +from .treestrategy import FileTree + +__all__ = [ + "FORMAT", + "FORMAT_VERSION", + "ArtifactSnapshot", + "SnapshotEvalReport", + "evaluate_snapshot", + "evaluate_snapshot_spec", + "read_artifact_lineage", + "load_snapshot_source", +] + +FORMAT = "agentdescent.artifact_snapshot" +FORMAT_VERSION = 1 + + +def _as_patterns(value: Optional[Sequence[str]], default: Tuple[str, ...]) -> Tuple[str, ...]: + if value is None: + return default + return tuple(str(item) for item in value) + + +def _files(value: Mapping[str, str]) -> Dict[str, str]: + if not isinstance(value, Mapping): + raise TypeError("snapshot files must be a mapping of path to text") + files: Dict[str, str] = {} + for path, content in value.items(): + if not isinstance(path, str) or not isinstance(content, str): + raise TypeError("snapshot files must be str paths to str content") + files[safe_relpath(path) if path not in (".", "") else path] = content + return files + + +def _ident(artifact_id: str, version: Optional[int]) -> Optional[str]: + if not artifact_id: + return None + if version is None: + return artifact_id + return f"{artifact_id}@v{version}" + + +@dataclass(frozen=True) +class ArtifactSnapshot: + """One candidate's complete context and the lineage already on the ledger.""" + + artifact_id: str + files: Mapping[str, str] + editable: Tuple[str, ...] = ("**",) + frozen: Tuple[str, ...] = () + kind: str = "" + version: Optional[int] = None + id: Optional[str] = None + parent: Optional[str] = None + root: Optional[str] = None + commit: Optional[str] = None + parent_commit: Optional[str] = None + root_commit: Optional[str] = None + branch: Optional[str] = None + rendered: Optional[str] = None + format: str = FORMAT + format_version: int = FORMAT_VERSION + extra: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not isinstance(self.artifact_id, str) or not self.artifact_id.strip(): + raise ValueError("artifact_id must be a non-empty string") + object.__setattr__(self, "files", _files(self.files)) + object.__setattr__(self, "editable", _as_patterns(self.editable, ("**",))) + object.__setattr__(self, "frozen", _as_patterns(self.frozen, ())) + if self.id is None: + object.__setattr__(self, "id", _ident(self.artifact_id, self.version)) + + @property + def snapshot_id(self) -> str: + return self.id or self.artifact_id + + def role(self, path: str) -> str: + """``frozen``, ``editable``, or ``context`` (loaded, but not in scope).""" + rel = safe_relpath(path) + if match_any(rel, self.frozen): + return "frozen" + if match_any(rel, self.editable): + return "editable" + return "context" + + def roles(self) -> Dict[str, str]: + return {path: self.role(path) for path in self.files} + + def to_filetree(self) -> FileTree: + return FileTree(self.files, editable=self.editable, frozen=self.frozen) + + def render(self) -> str: + if self.rendered is not None: + return self.rendered + return self.to_filetree().render(self.files) + + def materialize(self, dest: str) -> None: + materialize(dict(self.files), dest) + + def digest(self) -> str: + body = canonical(self.files) + return hashlib.sha256(body.encode()).hexdigest() + + def to_dict(self) -> Dict[str, Any]: + payload = { + "format": self.format, + "format_version": self.format_version, + "id": self.snapshot_id, + "parent": self.parent, + "root": self.root, + "artifact_id": self.artifact_id, + "kind": self.kind, + "version": self.version, + "commit": self.commit, + "parent_commit": self.parent_commit, + "root_commit": self.root_commit, + "branch": self.branch, + "editable": list(self.editable), + "frozen": list(self.frozen), + "files": dict(self.files), + "roles": self.roles(), + "digest": self.digest(), + } + if self.rendered is not None: + payload["rendered"] = self.rendered + if self.extra: + payload["extra"] = dict(self.extra) + return payload + + @classmethod + def from_dict(cls, payload: Mapping[str, Any]) -> "ArtifactSnapshot": + if not isinstance(payload, Mapping): + raise TypeError("snapshot payload must be an object") + fmt = payload.get("format") + if fmt not in (None, FORMAT): + raise ValueError(f"unsupported snapshot format: {fmt!r}") + version = payload.get("format_version", FORMAT_VERSION) + if version != FORMAT_VERSION: + raise ValueError(f"unsupported snapshot format_version: {version!r}") + files = payload.get("files") + if files is None: + raise ValueError("snapshot is missing files") + return cls( + artifact_id=str(payload.get("artifact_id") or payload.get("id") or "artifact"), + files=files, + editable=tuple(payload.get("editable") or ("**",)), + frozen=tuple(payload.get("frozen") or ()), + kind=str(payload.get("kind") or ""), + version=payload.get("version"), + id=payload.get("id"), + parent=payload.get("parent"), + root=payload.get("root"), + commit=payload.get("commit"), + parent_commit=payload.get("parent_commit"), + root_commit=payload.get("root_commit"), + branch=payload.get("branch"), + rendered=payload.get("rendered"), + extra=dict(payload.get("extra") or {}), + ) + + def save(self, path: str) -> None: + os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + json.dump(self.to_dict(), fh, indent=2, sort_keys=True) + fh.write("\n") + + @classmethod + def load(cls, path: str) -> "ArtifactSnapshot": + with open(path, encoding="utf-8") as fh: + return cls.from_dict(json.load(fh)) + + @classmethod + def from_filetree( + cls, tree: FileTree, *, + artifact_id: str, + state: Optional[Mapping[str, str]] = None, + kind: str = "", + lineage: Optional[Mapping[str, Any]] = None, + rendered: Optional[str] = None, + ) -> "ArtifactSnapshot": + files = dict(state if state is not None else tree.initial()) + # Evaluation/restore must see frozen context even if a caller passed + # only the editable slice; fill any missing frozen keys from the tree. + for path, content in tree.frozen_files(tree.initial_files).items(): + files.setdefault(path, content) + line = dict(lineage or {}) + return cls( + artifact_id=artifact_id, + files=files, + editable=tuple(tree.editable), + frozen=tuple(tree.frozen), + kind=kind, + version=line.get("version"), + id=line.get("id"), + parent=line.get("parent"), + root=line.get("root"), + commit=line.get("commit"), + parent_commit=line.get("parent_commit"), + root_commit=line.get("root_commit"), + branch=line.get("branch"), + rendered=rendered, + ) + + @classmethod + def from_result( + cls, result: Any, *, + artifact_id: str, + kind: str = "", + editable: Sequence[str] = ("**",), + frozen: Sequence[str] = (), + lineage: Optional[Mapping[str, Any]] = None, + ) -> "ArtifactSnapshot": + tree = FileTree(dict(result.state), editable=editable, frozen=frozen) + return cls.from_filetree( + tree, artifact_id=artifact_id, state=result.state, kind=kind, + lineage=lineage, rendered=getattr(result, "rendered", None)) + + @classmethod + def from_ledger( + cls, ledger: Ledger, artifact_id: str, *, + branch: str = Ledger.DEV, + strategy: Optional[FileTree] = None, + kind: str = "", + ) -> "ArtifactSnapshot": + view = ledger.snapshot(branch) + artifact = view.get(artifact_id) + if artifact is None: + raise KeyError(f"ledger has no artifact {artifact_id!r} on {branch}") + state = dict(getattr(artifact, "state", None) or {}) + if not state and hasattr(artifact, "table"): + # Non-FileTree Evolvable used in tests; keep a portable mapping. + state = {"state.json": json.dumps(getattr(artifact, "table"), + sort_keys=True)} + tree = strategy or FileTree(state) + lineage = read_artifact_lineage(ledger.repo_path, artifact_id, branch=branch) or {} + version = getattr(artifact, "version", None) + if version is not None: + lineage.setdefault("version", version) + return cls.from_filetree( + tree, artifact_id=artifact_id, state=state or dict(tree.initial()), + kind=kind, lineage=lineage, + rendered=artifact.render() if hasattr(artifact, "render") else None) + + @classmethod + def from_run(cls, run_id: str, *, store: Optional[str] = None) -> "ArtifactSnapshot": + from . import runstore + from .evolvespec import EvolveSpec + + rd = runstore.get(run_id, store=store) + result = rd.result() + if result is None: + raise runstore.RunStoreError(f"run {run_id} has no result yet") + spec = EvolveSpec.from_dict(rd.spec_dict()) + aid = spec.artifact_id() + lineage = None + if os.path.isdir(rd.ledger_path): + lineage = read_artifact_lineage( + rd.ledger_path, aid, branch=Ledger.DEV) + return cls.from_result( + result, artifact_id=aid, kind=spec.kind, + editable=spec.editable, frozen=spec.frozen, lineage=lineage) + + +@dataclass(frozen=True) +class SnapshotEvalReport: + """A read-only score of one snapshot; no search happened.""" + + snapshot_id: str + n: int + reward: Optional[float] + scores: Mapping[str, float] + failed: Tuple[str, ...] = () + kind: str = "" + digest: str = "" + searched: bool = False + + def to_dict(self) -> Dict[str, Any]: + return { + "snapshot_id": self.snapshot_id, + "n": self.n, + "reward": self.reward, + "scores": dict(self.scores), + "failed": list(self.failed), + "kind": self.kind, + "digest": self.digest, + "searched": self.searched, + } + + +def _task_id(task: Any) -> str: + value = getattr(task, "id", None) + if value is None and isinstance(task, Mapping): + value = task.get("id") + if value is None: + raise ValueError("every snapshot eval task needs an id") + return str(value) + + + +def load_snapshot_source(source: str, *, store: Optional[str] = None) -> ArtifactSnapshot: + """Load a snapshot JSON file, or capture one from a finished run id.""" + path = os.path.abspath(os.path.expanduser(source)) + if os.path.isfile(path): + return ArtifactSnapshot.load(path) + return ArtifactSnapshot.from_run(source, store=store) + + +def evaluate_snapshot( + snapshot: ArtifactSnapshot, + tasks: Sequence[Task], + *, + run: Callable[[str, Task], str], + reward: Callable[[Task, str], float], +) -> SnapshotEvalReport: + """Score a frozen snapshot on ``tasks``. Does not search or write a ledger.""" + rendered = snapshot.render() + scores: Dict[str, float] = {} + failed = [] + for task in tasks: + task_id = _task_id(task) + try: + output = run(rendered, task) + value = float(reward(task, output)) + if value != value or value in (float("inf"), float("-inf")): + raise ValueError(f"reward for {task_id!r} must be finite") + except Exception: # noqa: BLE001 - a scorer/harness failure, not a zero + failed.append(task_id) + continue + scores[task_id] = value + n = len(scores) + return SnapshotEvalReport( + snapshot_id=snapshot.snapshot_id, + n=n, + reward=(sum(scores.values()) / n if n else None), + scores=scores, + failed=tuple(failed), + kind=snapshot.kind, + digest=snapshot.digest(), + searched=False, + ) + + +def _write_target(snapshot: ArtifactSnapshot, kind: str, tmp: str) -> str: + if kind == "text": + path = os.path.join(tmp, snapshot.artifact_id) + text = snapshot.render() + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return path + dest = os.path.join(tmp, snapshot.artifact_id) + os.makedirs(dest, exist_ok=True) + snapshot.materialize(dest) + return dest + + +def evaluate_snapshot_spec( + snapshot: ArtifactSnapshot, + spec: Any, + **compose_hooks: Any, +) -> SnapshotEvalReport: + """Score ``snapshot`` with a destination spec's tasks and scorer. + + The destination supplies the dataset and the run/reward (including the + already-selectable host agent). The candidate is the snapshot, not the + spec's ``target``. Nothing is evolved. + """ + from .evolvespec import EvolveSpec, compose + + if not isinstance(spec, EvolveSpec): + raise TypeError("evaluate_snapshot_spec expects an EvolveSpec") + if snapshot.kind and spec.kind and snapshot.kind != spec.kind: + raise ValueError( + f"snapshot kind {snapshot.kind!r} is not compatible with " + f"destination kind {spec.kind!r}") + kind = snapshot.kind or spec.kind + with tempfile.TemporaryDirectory(prefix="ad-snapshot-eval-") as tmp: + target = _write_target(snapshot, kind, tmp) + dest = replace( + spec, target=target, + editable=tuple(snapshot.editable), + frozen=tuple(snapshot.frozen), + ) + # Search knobs on the dest spec must not start a search: we only compose. + dest = replace(dest, evolve={}) + comp = compose(dest, **compose_hooks) + run = comp.kwargs.get("run") + if run is None: + agent = comp.kwargs.get("agent") + if agent is not None and callable(getattr(agent, "solve", None)): + run = agent.solve + if run is None: + raise ValueError("destination spec produced no run callable") + return evaluate_snapshot( + snapshot, comp.tasks, run=run, reward=comp.reward) diff --git a/agentdescent/cli.py b/agentdescent/cli.py index 71e61e63..6a7bc022 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -16,6 +16,7 @@ agentdescent watch follow rounds agentdescent show [--diff] the evolved tree agentdescent apply [--to PATH] [--dry-run] install it, backed up + agentdescent snapshot export|show|eval|apply portable candidate, readonly re-eval agentdescent cancel / resume agentdescent algorithms list|show|run … named Algorithm cards agentdescent demo one offline run, no key @@ -3684,6 +3685,80 @@ def cmd_apply(a: argparse.Namespace) -> int: return 0 +def _snapshot_from_args(a: argparse.Namespace): + from .artifact_snapshot import load_snapshot_source + source = a.source + path = os.path.abspath(os.path.expanduser(source)) + if not os.path.isfile(path): + source = resolve_run_id(source, store=a.store) or source + return load_snapshot_source(source, store=a.store) + + +def _print_snapshot(snap, *, as_json: bool) -> None: + payload = snap.to_dict() + if as_json: + _out(payload, as_json=True) + return + lines = [ + f"snapshot {payload.get('id')}", + f"kind: {payload.get('kind') or '(none)'} version: {payload.get('version')}", + f"parent: {payload.get('parent') or '(none)'} root: {payload.get('root') or '(none)'}", + f"files: {len(payload.get('files') or {})}", + ] + roles = payload.get("roles") or {} + for path, role in sorted(roles.items()): + lines.append(f" [{role}] {path}") + _out('\n'.join(lines), as_json=False) + + +def cmd_snapshot_export(a: argparse.Namespace) -> int: + snap = _snapshot_from_args(a) + if a.out: + try: + snap.save(a.out) + except OSError as e: + raise _cannot_write(a.out, e) from None + if a.json: + _out(snap.to_dict(), as_json=True) + else: + print(f"wrote {a.out}") + return 0 + _out(snap.to_dict(), as_json=True) + return 0 + + +def cmd_snapshot_show(a: argparse.Namespace) -> int: + _print_snapshot(_snapshot_from_args(a), as_json=a.json) + return 0 + + +def cmd_snapshot_eval(a: argparse.Namespace) -> int: + from .artifact_snapshot import evaluate_snapshot_spec + snap = _snapshot_from_args(a) + spec = load_spec(a.spec) + report = evaluate_snapshot_spec(snap, spec) + payload = report.to_dict() + payload["ok"] = True + _out(payload, as_json=True) + return 0 if not report.failed else 1 + + +def cmd_snapshot_apply(a: argparse.Namespace) -> int: + import tempfile + snap = _snapshot_from_args(a) + if not a.to: + raise runstore.RunStoreError("snapshot apply needs --to PATH") + with tempfile.TemporaryDirectory(prefix="ad-snapshot-apply-") as tmp: + src = os.path.join(tmp, snap.artifact_id or "artifact") + os.makedirs(src, exist_ok=True) + snap.materialize(src) + payload = apply_from_payload( + src, to=a.to, dry_run=a.dry_run, backup=not a.no_backup, + check_write=_operator_write_guard()) + _out(payload, as_json=True) + return 0 + + def cmd_cancel(a: argparse.Namespace) -> int: st = runstore.cancel(a.run_id, store=a.store) print(_status_line(st)) @@ -5959,6 +6034,33 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--no-backup", action="store_true", dest="no_backup") s.set_defaults(fn=cmd_apply) + s = sub.add_parser( + "snapshot", + help="export, show, re-evaluate or apply a portable candidate snapshot") + snap_sub = s.add_subparsers(dest="snapshot_cmd", required=True) + sp = snap_sub.add_parser( + "export", help="write a snapshot JSON from a run or an existing snapshot") + sp.add_argument("source", help="a run id, `latest`, or a snapshot JSON file") + sp.add_argument("-o", "--out", help="write JSON to this path (default: stdout)") + sp.set_defaults(fn=cmd_snapshot_export) + sp = snap_sub.add_parser( + "show", help="lineage, file roles and the complete candidate") + sp.add_argument("source", help="a run id, `latest`, or a snapshot JSON file") + sp.set_defaults(fn=cmd_snapshot_show) + sp = snap_sub.add_parser( + "eval", help="score a frozen snapshot on a destination spec; does not search") + sp.add_argument("source", help="a run id, `latest`, or a snapshot JSON file") + sp.add_argument("--spec", required=True, + help="destination EvolveSpec (tasks and scorer only; target is ignored)") + sp.set_defaults(fn=cmd_snapshot_eval) + sp = snap_sub.add_parser( + "apply", help="materialise a snapshot to a directory, with the usual backup") + sp.add_argument("source", help="a run id, `latest`, or a snapshot JSON file") + sp.add_argument("--to", required=True, help="destination directory") + sp.add_argument("--dry-run", action="store_true", dest="dry_run") + sp.add_argument("--no-backup", action="store_true", dest="no_backup") + sp.set_defaults(fn=cmd_snapshot_apply) + s = sub.add_parser("cancel", help="stop a run and every worker it started") s.add_argument("run_id") s.set_defaults(fn=cmd_cancel) diff --git a/agentdescent/ledger.py b/agentdescent/ledger.py index c18de80b..48debc2c 100644 --- a/agentdescent/ledger.py +++ b/agentdescent/ledger.py @@ -731,3 +731,45 @@ def discard_head(self, name: str) -> None: if self._current_branch == name: self._checkout(self.DEV) _git(self.repo_path, "branch", "-D", name) + +def read_artifact_lineage( + repo_path: str, artifact_id: str, *, + branch: str = Ledger.DEV, +) -> Optional[Dict[str, Any]]: + """id / parent / root already implied by a ledger repo, without opening it. + + ``id`` is ``{artifact_id}@v{version}``, ``parent`` is the previous version + (or ``None`` at v1), and ``root`` is ``{artifact_id}@v1``. Commit hashes + come from ``git log`` of ``artifacts/.json``. Returns ``None`` when + the repo or artifact is missing rather than inventing a second identifier. + """ + if not repo_path or not os.path.isdir(os.path.join(repo_path, ".git")): + return None + if not isinstance(artifact_id, str) or not artifact_id.strip(): + raise ValueError("artifact_id must be a non-empty string") + rel = os.path.join("artifacts", f"{artifact_id}.json") + try: + raw = _git(repo_path, "show", f"{branch}:versions.json") + versions = json.loads(raw) if raw else {} + log = _git(repo_path, "log", branch, "--pretty=format:%H", "--", rel) + except (GitError, json.JSONDecodeError, OSError): + return None + if not isinstance(versions, dict) or artifact_id not in versions: + return None + try: + version = int(versions[artifact_id]) + except (TypeError, ValueError): + return None + commits = [line.strip() for line in (log or "").splitlines() if line.strip()] + ident = f"{artifact_id}@v{version}" + return { + "id": ident, + "parent": (f"{artifact_id}@v{version - 1}" if version > 1 else None), + "root": f"{artifact_id}@v1", + "artifact_id": artifact_id, + "version": version, + "commit": commits[0] if commits else None, + "parent_commit": commits[1] if len(commits) > 1 else None, + "root_commit": commits[-1] if commits else None, + "branch": branch, + } diff --git a/changelog.d/1138-artifact-snapshot.md b/changelog.d/1138-artifact-snapshot.md new file mode 100644 index 00000000..f1ac855f --- /dev/null +++ b/changelog.d/1138-artifact-snapshot.md @@ -0,0 +1,6 @@ +Added +Portable artifact snapshots capture a complete FileTree candidate +(editable files plus frozen context) and the ledger's existing +id/parent/root lineage, and can be re-scored on another compatible +dataset without further search (`agentdescent snapshot +export|show|eval|apply`, #1138). diff --git a/docs/api.md b/docs/api.md index 352b2ee3..99ec966d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -13,7 +13,7 @@ means the parameter has none. Each section links to the page that explains *why* the module is shaped the way it is; this page is the *what*. -631 public names across 125 modules. +637 public names across 126 modules. --- @@ -1516,6 +1516,19 @@ Ledger( An immutable view of one branch at one point in time. +### `read_artifact_lineage(...)` + +id / parent / root already implied by a ledger repo, without opening it. + +```python +read_artifact_lineage( + repo_path: str, + artifact_id: str, + *, + branch: str = 'dev' +) -> Optional[Dict[str, Any]] +``` + --- ## The verifier @@ -5485,6 +5498,10 @@ One high-risk candidate's approval request and its latest decision. Append-only JSONL decisions for high-risk candidates. +### `ArtifactSnapshot` + +One candidate's complete context and the lineage already on the ledger. + ### `BootstrapResult` A paired bootstrap of per-task gains. @@ -6001,6 +6018,10 @@ What the archive says about one decision slot's evolution. One decision slot's adoption history and whether it is still rising. +### `SnapshotEvalReport` + +A read-only score of one snapshot; no search happened. + ### `SocietyGraph` A directed graph of `AgentNode` with delegation edges. @@ -6165,6 +6186,14 @@ The per-arm plan estimate without running anything. Compare `verifier` with `oracle` on a deterministic sample. +### `evaluate_snapshot` + +Score a frozen snapshot on `tasks`. Does not search or write a ledger. + +### `evaluate_snapshot_spec` + +Score `snapshot` with a destination spec's tasks and scorer. + ### `evaluate_source` Run the candidate against the tests and the benchmark. @@ -6237,6 +6266,10 @@ An inner `evolve()` whose candidate is the **whole bundle**. One layer per decision slot, aggregated from the archive's rule records. +### `load_snapshot_source` + +Load a snapshot JSON file, or capture one from a finished run id. + ### `meta_population_run` Run a population of meta sessions from the archive's best rules, promote the winner. diff --git a/docs/directory-evolution.md b/docs/directory-evolution.md index ce3d9a0a..2a7d2553 100644 --- a/docs/directory-evolution.md +++ b/docs/directory-evolution.md @@ -396,3 +396,11 @@ claude_code(extra_args=["--permission-mode", "acceptEdits"], timeout=300) And check that the skill is actually being *used*: run one round against an empty skill directory as a control. If the score does not move, what you are measuring is the model's prior knowledge, not your skill. + +## Taking the candidate with you + +The tree the engine scores is the complete directory: editable files plus the +frozen overlay. A [portable snapshot](snapshot.md) writes that whole candidate +and its ledger lineage (`id` / `parent` / `root`) to JSON, so you can restore +it or re-score it on another compatible dataset without running search again. +`kind: plugin` is still the plugin tree, not a host to run inside. diff --git a/docs/docs.md b/docs/docs.md index aa378278..c7e571bc 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -90,7 +90,7 @@ never read a card run as paper numbers. | path | you have | the product surface | |---|---|---| -| **A — evolve one artifact** | tasks, a reward, an actor | `agentdescent plan/evolve/status/show/apply` | +| **A — evolve one artifact** | tasks, a reward, an actor | `agentdescent plan/evolve/status/show/apply/snapshot` | | **B — governed release** | paired held-out scores | `agentdescent release` + `release verify` | | **C — continuous platform** | a run database and callbacks | `agentdescent platform` + `platform status` | @@ -105,6 +105,7 @@ own page under **Product surfaces** in the navigation. **The engine.** [Architecture](architecture.md) · [Concepts](concepts.md) · [The `evolve()` method](evolution.md) · [Strategies](strategies.md) · +[Portable snapshots](snapshot.md) · [Policies](policies.md) · [API reference](api.md). **Run and operate it.** diff --git a/docs/ledger.md b/docs/ledger.md index 058234e0..a0ff4010 100644 --- a/docs/ledger.md +++ b/docs/ledger.md @@ -197,3 +197,10 @@ b912f19 genesis Or open the repo with any git tool you already know — it is an ordinary repository. + +## Lineage on a portable snapshot + +The ledger already has the identifiers a snapshot needs: artifact id, integer +version, and the git parent of `artifacts/.json`. +[Portable snapshots](snapshot.md) read those as `id` / `parent` / `root` +(`{artifact_id}@v{version}`) rather than inventing a second store. diff --git a/docs/main-path.md b/docs/main-path.md index cc824be2..cbb389b1 100644 --- a/docs/main-path.md +++ b/docs/main-path.md @@ -26,9 +26,9 @@ tasks + reward + agent + Strategy | loop | `evolution`, `async_evolve`, `aggregator`, `policies`, `sampling`, `selection`, `fusion`, `staleness`, `governance` | | execution | `agents`, `runners`, `executor`, `parallel`, `scheduler`, `sandbox`, `sandbox_container`, `sandbox_shared` | | evidence | `verifier`, `verifier_ensemble`, `audit/`, `stats`, `stats_eval`, `conformal`, `sequential` | -| state | `ledger`, `runstore`, `checkpoint`, `manifest`, `trajectory`, `decisions` | +| state | `ledger`, `runstore`, `snapshot`, `checkpoint`, `manifest`, `trajectory`, `decisions` | -This is the path every CLI/MCP `plan/evolve/status/show/apply` verb uses. +This is the path every CLI/MCP `plan/evolve/status/show/apply` verb uses. A [portable snapshot](snapshot.md) is that same candidate taken off the run: full FileTree context plus ledger lineage, scored or restored without a new search. ## Path B — governed release diff --git a/docs/snapshot.md b/docs/snapshot.md new file mode 100644 index 00000000..e76b6fec --- /dev/null +++ b/docs/snapshot.md @@ -0,0 +1,82 @@ +# Portable snapshots + +A snapshot is the **complete candidate** the ledger already stores — every +editable file plus the frozen context around it — together with the lineage +that history already implies (`id`, `parent`, `root`). It is a portable JSON +file, not a second ledger. + +Algorithms still edit only the declared scope (`FileTree.editable`, with +`frozen` winning). Evaluation and restore use the whole tree. You can take a +selected snapshot and score it, read-only, on another compatible dataset; +that pass does not search, commit, or apply anything. + +This is not “run this snapshot inside Claude Code.” Host selection stays the +existing agent / runner path on the destination spec. `kind: plugin` still +means “evolve the plugin itself.” + +## Capture + +From a finished run (the run’s `result.json` supplies the files; the run +ledger, when still on disk, supplies lineage): + +```bash +agentdescent snapshot export -o skill.json +agentdescent snapshot show skill.json +``` + +From a `FileTree` or a live ledger in Python: + +```python +from agentdescent import ArtifactSnapshot, FileTree +from agentdescent.ledger import Ledger, read_artifact_lineage + +snap = ArtifactSnapshot.from_filetree( + FileTree(files, frozen=["tests/**"]), + artifact_id="skill", kind="skill_dir") +# or ArtifactSnapshot.from_ledger(ledger, "skill", strategy=tree, kind="skill_dir") +snap.save("skill.json") +``` + +`id` is `{artifact_id}@v{version}`, `parent` is the previous version, and +`root` is `{artifact_id}@v1`. Those are the identifiers the ledger already +uses. Existing `status` / `show` / `apply` on a run are unchanged; a snapshot +is what you take *with you* when the run directory is not the destination. + +## Re-evaluate without search + +```bash +agentdescent snapshot eval skill.json --spec dest.yaml +``` + +The destination spec supplies the **tasks and the scorer** (including whatever +host agent that spec already selects). Its `target` is ignored; the candidate +is the snapshot. Kind must match. In Python: + +```python +from agentdescent import evaluate_snapshot, evaluate_snapshot_spec, load_spec + +report = evaluate_snapshot(snap, tasks, run=run, reward=reward) +# or evaluate_snapshot_spec(snap, load_spec("dest.yaml")) +report.reward, report.scores, report.searched # searched is always False +``` + +This does **not** change `evolve(test_frac=…)` defaults or add a new +train/test product verb. It only scores a frozen candidate. + +## Restore + +```bash +agentdescent snapshot apply skill.json --to ./out --dry-run +agentdescent snapshot apply skill.json --to ./out +``` + +That materialises the complete tree (frozen files included) with the same +backup rules as `apply --from`. `show` and `apply` on the original run_id +still work as they do today. + +## What a file contains + +The JSON names its format (`agentdescent.artifact_snapshot`, version 1) and +carries `files`, `editable`, `frozen`, derived `roles`, a content `digest`, +and the lineage fields above. Missing lineage (a run whose ledger was +removed) is left empty rather than invented. diff --git a/mkdocs.yml b/mkdocs.yml index 40a01374..cf37fce8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -124,6 +124,7 @@ nav: - The evolve method: evolution.md - Strategies — what evolves: strategies.md - Evolving a directory: directory-evolution.md + - Portable snapshots: snapshot.md - Document backends: backends.md - The aggregator (the optimizer): aggregator.md - Population engine: population.md diff --git a/tests/test_self_improve.py b/tests/test_self_improve.py index 6e9f5bf3..e41c2d86 100644 --- a/tests/test_self_improve.py +++ b/tests/test_self_improve.py @@ -45,6 +45,21 @@ def _archive(store, run_id, slot, reward, rounds, *, artifact_id="meta-rule"): # -- evidence ------------------------------------------------------------------ +def test_package_level_snapshot_is_the_evidence_callable(): + """The evidence snapshot() must keep the package name ``snapshot``. + + ``artifact_snapshot`` is a sibling module; importing it (or the package) + must not replace the callable with that module. + """ + import agentdescent + from agentdescent.artifact_snapshot import ArtifactSnapshot + from agentdescent.evidence import snapshot as evidence_snapshot + + assert agentdescent.snapshot is evidence_snapshot + assert callable(agentdescent.snapshot) + assert agentdescent.ArtifactSnapshot is ArtifactSnapshot + + def test_snapshot_composes_slots_ports_and_exam_counts(tmp_path): store = ConsolidationStore(str(tmp_path / "archive")) # slot A: three rules, mean reward 0.8 over 5 rounds -> rpc 0.16 diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py new file mode 100644 index 00000000..79ab5a58 --- /dev/null +++ b/tests/test_snapshot.py @@ -0,0 +1,192 @@ +"""Portable FileTree snapshots reuse the ledger; re-eval does not search.""" + +import json +from dataclasses import replace + +import pytest + +from agentdescent.evolution import EvolvingArtifact, Task +from agentdescent.evolvespec import EvolveSpec +from agentdescent.ledger import Ledger, read_artifact_lineage +from agentdescent.artifact_snapshot import ( + ArtifactSnapshot, + evaluate_snapshot, + evaluate_snapshot_spec, +) +from agentdescent.treestrategy import FileTree + + +def _tree(): + files = { + "SKILL.md": "name: skill\nUse R0 when asked.\n", + "notes.md": "editable note\n", + "tests/gold.md": "do not change this fixture\n", + } + return FileTree(files, editable=["**"], frozen=["tests/**"]) + + +def _ser(artifact): + return {"state": dict(artifact.state), "blast_radius": artifact.blast_radius} + + +def _deser(aid, version, state): + return EvolvingArtifact(aid, state.get("state", {}), version, + state.get("blast_radius", 0.2)) + + +def _ledger(tmp_path): + return Ledger(str(tmp_path / "repo"), _ser, _deser) + + +def _tasks(): + return [ + Task(id="t0", prompt="need R0", meta={"rule": "R0"}), + Task(id="t1", prompt="need R1", meta={"rule": "R1"}), + ] + + +def _run(rendered, task): + return "good" if task.meta["rule"] in rendered else "bad" + + +def _reward(task, output): + return 1.0 if output == "good" else 0.0 + + +def test_snapshot_includes_frozen_context_and_roles(): + snap = ArtifactSnapshot.from_filetree(_tree(), artifact_id="skill", + kind="skill_dir") + assert "tests/gold.md" in snap.files + assert "SKILL.md" in snap.files + assert snap.role("tests/gold.md") == "frozen" + assert snap.role("SKILL.md") == "editable" + assert snap.to_filetree().writable("tests/gold.md") is False + assert snap.to_filetree().writable("notes.md") is True + + +def test_from_filetree_fills_frozen_keys_omitted_from_editable_state(): + tree = _tree() + editable_only = {p: c for p, c in tree.initial().items() + if tree.writable(p)} + snap = ArtifactSnapshot.from_filetree( + tree, artifact_id="skill", state=editable_only, kind="skill_dir") + assert snap.files["tests/gold.md"] == "do not change this fixture\n" + assert snap.role("tests/gold.md") == "frozen" + + +def test_json_roundtrip_preserves_lineage_and_files(tmp_path): + snap = ArtifactSnapshot.from_filetree( + _tree(), artifact_id="skill", kind="skill_dir", + lineage={"version": 3, "id": "skill@v3", "parent": "skill@v2", + "root": "skill@v1", "commit": "abc", "branch": "dev"}) + path = tmp_path / "skill.json" + snap.save(path) + loaded = ArtifactSnapshot.load(path) + assert loaded.files == snap.files + assert loaded.id == "skill@v3" + assert loaded.parent == "skill@v2" + assert loaded.root == "skill@v1" + assert loaded.commit == "abc" + assert loaded.roles() == snap.roles() + assert loaded.digest() == snap.digest() + + +def test_lineage_uses_ledger_versions_not_a_second_store(tmp_path): + led = _ledger(tmp_path) + first = EvolvingArtifact("skill", _tree().initial()) + led.register(first) + base = led.head_version(Ledger.DEV) + second = EvolvingArtifact("skill", {**first.state, "notes.md": "v2\n"}) + led.commit(second, base, branch=Ledger.DEV) + base = led.head_version(Ledger.DEV) + third = EvolvingArtifact("skill", {**second.state, "notes.md": "v3\n"}) + led.commit(third, base, branch=Ledger.DEV) + + line = read_artifact_lineage(led.repo_path, "skill", branch=Ledger.DEV) + assert line["id"] == "skill@v3" + assert line["parent"] == "skill@v2" + assert line["root"] == "skill@v1" + assert line["version"] == 3 + assert line["commit"] and line["parent_commit"] and line["root_commit"] + assert line["commit"] != line["parent_commit"] + assert line["root_commit"] != line["commit"] + + snap = ArtifactSnapshot.from_ledger( + led, "skill", strategy=_tree(), kind="skill_dir") + assert snap.id == "skill@v3" + assert snap.parent == "skill@v2" + assert snap.root == "skill@v1" + assert snap.files["notes.md"] == "v3\n" + assert snap.files["tests/gold.md"] == "do not change this fixture\n" + assert list((tmp_path / "repo").iterdir()) # still one ledger repo + + +def test_evaluate_snapshot_scores_readonly_and_does_not_search(monkeypatch): + snap = ArtifactSnapshot.from_filetree( + _tree(), artifact_id="skill", kind="skill_dir") + # The frozen tests file is in the candidate even though it is not editable. + assert "do not change this fixture" in snap.render() + + def boom(*args, **kwargs): + raise AssertionError("evaluate_snapshot must not call evolve") + + monkeypatch.setattr("agentdescent.artifact_snapshot.evolve", boom, raising=False) + # Also patch the real engine entry point the scorer might import. + monkeypatch.setattr("agentdescent.evolution.evolve", boom) + + report = evaluate_snapshot(snap, _tasks(), run=_run, reward=_reward) + assert report.searched is False + assert report.n == 2 + assert report.scores["t0"] == 1.0 + assert report.scores["t1"] == 0.0 + assert report.reward == 0.5 + assert report.failed == () + + +def test_evaluate_snapshot_spec_uses_dest_tasks_and_rejects_kind_mismatch(): + snap = ArtifactSnapshot( + artifact_id="skill", kind="text", + files={"skill": "answer with yes"}, + rendered="answer with yes", + ) + dest = EvolveSpec( + kind="text", target="ignored-target", + data={"inline": [ + {"id": "a", "prompt": "say yes", "gold": "yes"}, + {"id": "b", "prompt": "say yes again", "gold": "yes"}, + ]}, + score="contains", agent="echo", + evolve={"rounds": 8, "n_workers": 4}, + ) + + def boom(*args, **kwargs): + raise AssertionError("spec re-eval must not search") + + import agentdescent.evolvespec as es + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(es, "run_spec", boom) + monkeypatch.setattr("agentdescent.evolution.evolve", boom) + try: + report = evaluate_snapshot_spec(snap, dest) + finally: + monkeypatch.undo() + + assert report.searched is False + assert report.n == 2 + assert report.reward == 1.0 + + other = replace(dest, kind="skill_dir") + with pytest.raises(ValueError, match="not compatible"): + evaluate_snapshot_spec(snap, other) + + +def test_plugin_kind_snapshot_is_the_plugin_tree_not_a_host(): + files = {"plugin.py": "print('plugin')\n", "README.md": "a plugin\n"} + snap = ArtifactSnapshot.from_filetree( + FileTree(files, frozen=["README.md"]), + artifact_id="my-plugin", kind="plugin") + assert snap.kind == "plugin" + assert snap.files["plugin.py"].startswith("print") + assert snap.role("README.md") == "frozen" + # Plugin kind is the evolved artifact, not a host execution environment. + assert "claude" not in json.dumps(snap.to_dict()) diff --git a/tests/test_snapshot_cli.py b/tests/test_snapshot_cli.py new file mode 100644 index 00000000..0ea4bfec --- /dev/null +++ b/tests/test_snapshot_cli.py @@ -0,0 +1,70 @@ +"""CLI surface for portable snapshots: show, eval, apply — no search.""" + +import json +from pathlib import Path + +from agentdescent.cli import main +from agentdescent.artifact_snapshot import ArtifactSnapshot +from agentdescent.treestrategy import FileTree + + +def _snap(path: Path) -> ArtifactSnapshot: + snap = ArtifactSnapshot.from_filetree( + FileTree({"SKILL.md": "always say yes\n", "tests/n.txt": "frozen\n"}, + frozen=["tests/**"]), + artifact_id="skill", kind="text", + lineage={"version": 2, "id": "skill@v2", "parent": "skill@v1", + "root": "skill@v1"}) + # text re-eval uses render(); keep the instruction body. + return ArtifactSnapshot( + artifact_id=snap.artifact_id, files=snap.files, editable=snap.editable, + frozen=snap.frozen, kind="text", version=2, id="skill@v2", + parent="skill@v1", root="skill@v1", rendered="always say yes") + + +def test_snapshot_show_export_eval_and_apply(tmp_path, capsys, monkeypatch): + snap = _snap(tmp_path) + source = tmp_path / "skill.json" + snap.save(source) + + assert main(["--json", "snapshot", "show", str(source)]) == 0 + shown = json.loads(capsys.readouterr().out) + assert shown["id"] == "skill@v2" + assert shown["parent"] == "skill@v1" + assert shown["roles"]["tests/n.txt"] == "frozen" + + out = tmp_path / "exported.json" + assert main(["snapshot", "export", str(source), "-o", str(out)]) == 0 + capsys.readouterr() # "wrote …" on stdout + assert ArtifactSnapshot.load(out).id == "skill@v2" + + dest = tmp_path / "dest.json" + dest.write_text(json.dumps({ + "kind": "text", "target": "ignored", + "agent": "echo", "score": "contains", + "data": {"inline": [ + {"id": "a", "prompt": "say yes", "gold": "yes"}, + ]}, + "evolve": {"rounds": 9}, + })) + + def boom(*args, **kwargs): + raise AssertionError("snapshot eval must not call evolve") + + monkeypatch.setattr("agentdescent.evolution.evolve", boom) + assert main(["--json", "snapshot", "eval", str(source), "--spec", str(dest)]) == 0 + report = json.loads(capsys.readouterr().out) + assert report["ok"] is True + assert report["searched"] is False + assert report["n"] == 1 + assert report["reward"] == 1.0 + + target = tmp_path / "installed" + assert main(["--json", "snapshot", "apply", str(source), "--to", str(target), + "--dry-run"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload.get("ok") is not False + assert not target.exists() or not any(target.iterdir()) + assert main(["--json", "snapshot", "apply", str(source), "--to", str(target)]) == 0 + assert (target / "SKILL.md").read_text() == "always say yes\n" + assert (target / "tests/n.txt").read_text() == "frozen\n"