diff --git a/agentdescent/cli.py b/agentdescent/cli.py index 71e61e63..fc8d5a1d 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -2224,16 +2224,17 @@ def _merge_policies_replaced(comp) -> Optional[str]: "#807). evolve() may also RuntimeWarning.") -def _selection_not_merging(comp) -> Optional[str]: - """Say so when `n_workers` buys selection rather than the merge it looks like. - - The reflective pair is the default now, so this fires on the one case that - is left: a spec whose only model is a file-editing CLI agent, where paying a - whole agent session per merge is not something to switch on unasked. Without - a fusion policy a one-key artifact -- `SingleSlot`, every prompt -- has its - worker proposals contradict by construction and collapse to one candidate, - so no fusion is built. Asked against the *composed* bundle rather than the - spec's `policies` block, because the default never appears in the latter. +def _one_key_artifact(comp) -> Optional[str]: + """Say so when the artifact shape leaves the merge nothing to fuse (#1113). + + Without a fusion policy a one-key artifact -- `SingleSlot`, one prompt, a + one-file tree -- has its worker proposals contradict by construction and + collapse to one candidate, so no union is ever built; `n_workers` is + per-round best-of-N selection and the note says what to change. Asked against + the *composed* bundle rather than the spec's `policies` block, because the + default never appears in the latter: when the reflective pair is installed a + one-key artifact *is* merged by the reflector, so the advice would be wrong + and stays quiet. An ``aggregator_factory`` (gepa's Pareto pool) replaces that conflict pipeline: each surviving proposal enters its pool on its own, so nothing @@ -2245,32 +2246,53 @@ def _selection_not_merging(comp) -> Optional[str]: if installed is not None and (installed.fusion is not None or installed.conflict is not None): return None - strategy = comp.kwargs.get("strategy") - workers = comp.kwargs.get("n_workers") or 1 - try: - keys = list(strategy.keys()) - except Exception: # noqa: BLE001 - a strategy that cannot enumerate: say nothing - return None - if workers > 1 and len(keys) == 1: - return (f"n_workers={workers} on a one-key artifact ({type(strategy).__name__}) " - "is best-of-N selection, not merging: worker proposals contradict by " - "construction and collapse to one candidate, so no fusion is built. " - "Name a cheap model in `reflect` and it merges them instead.") - return None + from .strategies import _one_key_artifact_note + + return _one_key_artifact_note(comp.kwargs.get("strategy"), + workers=comp.kwargs.get("n_workers") or 1) #: Same rule as ``evolve()`` / ``_build_engine`` — keep the strings aligned (#343). _MIN_TASKS_FOR_SPLIT = 4 _MIN_TASKS_MSG = "need at least 4 tasks to split train/held-out" +#: `plan --baseline-sample`: a seed at or above this score has no headroom left. +#: `docs/results.md` records a 0.95 baseline as no headroom and the bench probe +#: refuses above 0.90; the plain number keeps the CLI off the `bench/` package. +DEFAULT_SATURATION = 0.90 + + +def _seed_baseline_score(comp, n: int) -> Tuple[float, int]: + """Score the **seed** artifact on the first ``n`` tasks (#1113). + + One agent call per task: the seed is what every round starts from, so its + score is the ceiling a run has to climb. Only an explicit + ``--baseline-sample`` gets here, and the caller quotes the calls before the + first one is spent -- plain `plan` still sends nothing. + """ + strategy = comp.kwargs.get("strategy") + run = comp.kwargs.get("run") + tasks = list(comp.tasks)[:n] + if strategy is None or run is None or not tasks: + raise ValueError("this spec has no seed artifact and tasks to sample") + rendered = strategy.render(strategy.initial()) + scores = [float(comp.reward(task, run(rendered, task))) for task in tasks] + return sum(scores) / len(scores), len(tasks) + def plan_payload(spec: EvolveSpec, *, usd_per_call: Optional[float] = None, budget_usd: Optional[float] = None) -> Dict[str, Any]: - comp = compose(spec) + return _plan_payload(compose(spec), spec, usd_per_call=usd_per_call, + budget_usd=budget_usd) + + +def _plan_payload(comp, spec: EvolveSpec, *, + usd_per_call: Optional[float] = None, + budget_usd: Optional[float] = None) -> Dict[str, Any]: warnings = _unusable_refs(spec) - selection = _selection_not_merging(comp) - if selection: - warnings.append(selection) + shape = _one_key_artifact(comp) + if shape: + warnings.append(shape) replaced = _merge_policies_replaced(comp) if replaced: warnings.append(replaced) @@ -2393,13 +2415,19 @@ def _print_plan(payload: Dict[str, Any], *, spec_path: str, next_line: Optional[str] = None, printed: Optional[set] = None, probe_lines: Optional[Sequence[str]] = None, - then_line: Optional[str] = None) -> None: + then_line: Optional[str] = None, + baseline_lines: Optional[Sequence[str]] = None, + refused_headroom: bool = False) -> None: """Print the human `plan` summary; warnings and the held-out note go to stderr. `payload` is exactly the object `--json` prints, so both surfaces carry the same facts and `--json` stays byte-identical (#919). `printed` holds lines an earlier report already showed (`algorithms run` prints init's report first), so a warning is never repeated. + + `refused_headroom` withholds the paid quote -- the call estimate and the + `next:` step -- because `--baseline-sample` found nothing to learn (#1113); + the refusal, with its fix, takes its place. """ already = set(printed or ()) for line in payload.get("warnings") or []: @@ -2427,10 +2455,13 @@ def _print_plan(payload: Dict[str, Any], *, spec_path: str, usd = est.get("usd_upper_bound") summary += (f" (${usd:.2f} upper bound)" if usd is not None else " (no $ estimate: pass --usd-per-call)") - lines = [_plan_target_line(payload), summary] + lines = [_plan_target_line(payload)] + if not refused_headroom: + lines.append(summary) agent = _plan_agent_line(payload.get("spec") or {}) if agent: lines.append(agent) + lines.extend(baseline_lines or ()) other_notes = [n for n in notes if not n.startswith(_HELD_OUT_NOTE)] for line in other_notes[:3]: lines.append(f"note: {line}") @@ -2443,7 +2474,11 @@ def _print_plan(payload: Dict[str, Any], *, spec_path: str, lines.extend(probe_lines or ()) if then_line: lines.append(f"then: {then_line}") - lines.append(f"next: {next_line or f'agentdescent evolve {spec_path}'}") + if refused_headroom: + lines.append("refused: no headroom -- the paid run is not quoted; " + "raise the difficulty or pass --force") + else: + lines.append(f"next: {next_line or f'agentdescent evolve {spec_path}'}") print("\n".join(lines)) @@ -2488,13 +2523,29 @@ def _probe_if_asked(a: argparse.Namespace, def cmd_plan(a: argparse.Namespace) -> int: + sample = getattr(a, "baseline_sample", None) + if sample is not None and sample < 1: + print("error: --baseline-sample must be >= 1", file=sys.stderr) + return 2 + saturation = getattr(a, "saturation", None) + if saturation is None: + saturation = DEFAULT_SATURATION + # Both flags are only read by the sample: refuse a nonsense saturation where + # it would be read, and leave `plan --saturation 0` inert rather than failing + # a plain plan over a flag that changes nothing (#1113 review). + if sample is not None and not 0.0 < saturation <= 1.0: + print("error: --saturation must be in (0, 1]", file=sys.stderr) + return 2 try: # Loading sits inside the guard so an unreadable or malformed spec file # answers `--json` with the same `{"ok": false, "error": ...}` object a # spec that fails to compile already gets (#648). spec = load_spec(a.spec) _apply_algorithm_flag(spec, a) - payload = plan_payload(spec, usd_per_call=a.usd_per_call, + # The composition is kept: `--baseline-sample` scores the seed on it, + # and composing twice would load the data twice for one plan (#1113). + comp = compose(spec) + payload = _plan_payload(comp, spec, usd_per_call=a.usd_per_call, budget_usd=getattr(a, 'budget', None)) except (SpecError, ValueError) as e: # JSON only on request; otherwise the same `error: ...` line on stderr @@ -2528,6 +2579,49 @@ def cmd_plan(a: argparse.Namespace) -> int: next_line = _first_failed_probe_next(probe) payload["next"] = next_line then_line = f"agentdescent plan {a.spec} --probe" + # `--baseline-sample` spends real calls, so it runs last: a broken spec or a + # refused probe must not first pay for N rollouts (#1113). The sample's own + # call count is printed before the first one; it is opt-in, and plain `plan` + # still sends nothing. + baseline_lines: List[str] = [] + refused_headroom = False + if (sample is not None and payload.get("ok", True) + and (probe is None or probe.get("ok", True))): + quote = f"{sample} agent call(s)" + usd_per_call = getattr(a, "usd_per_call", None) + if usd_per_call: + quote += f" (~${sample * usd_per_call:.2f})" + print(f"baseline sample: scoring the seed on up to {sample} task(s) " + f"costs {quote}", file=sys.stderr) + try: + score, used = _seed_baseline_score(comp, sample) + except Exception as e: # noqa: BLE001 - a backend or contract failure + print(f"error: baseline sample failed: {type(e).__name__}: {e}", + file=sys.stderr) + return 2 + headroom = score < saturation + payload["baseline"] = {"sample": used, "score": score, + "saturation": saturation, "headroom": headroom} + baseline_lines.append( + f"baseline: the seed scores {score:.3f} on {used} task(s); " + f"saturation {saturation:.3f} -> " + f"{'headroom' if headroom else 'no headroom'}") + if not headroom: + forced = bool(getattr(a, "force", False)) + payload["warnings"].append( + f"no headroom: the seed already scores {score:.3f} on {used} " + f"sampled task(s), at or above saturation {saturation:.3f}, so a " + "run can only report below-threshold. Raise the task difficulty " + "(docs/results.md) or lower the model's baseline" + + ("; the run is quoted anyway because --force was passed." + if forced else "; pass --force to quote the run anyway.")) + # `--force` buys the quote, not a different verdict: `ready` stays + # false either way, so a host following it cannot auto-start a run + # the plan says has nothing to learn. The human refusal (exit 1, + # withheld quote) is what force skips, nothing else (#1113). + payload["ready"] = False + if not forced: + refused_headroom = True if a.json: _out(payload, as_json=True) else: @@ -2538,9 +2632,14 @@ def cmd_plan(a: argparse.Namespace) -> int: _print_plan(payload, spec_path=a.spec, next_line=next_line, probe_lines=(_probe_lines(probe) if probe is not None else None), - then_line=then_line) + then_line=then_line, baseline_lines=baseline_lines, + refused_headroom=refused_headroom) if not payload.get("ok", True): return 2 + # A saturated seed is a config answer, like a refused probe: plan found the + # run cannot learn anything, before the run store gained an entry (#1113). + if refused_headroom: + return 1 # A refused probe is a config answer, not a spec error: the same non-zero # exit `evolve` would have taken on its first call, taken before the run # store gained an entry (#1048). @@ -5894,6 +5993,20 @@ def build_parser() -> argparse.ArgumentParser: "refused key/endpoint/model/subscription is answered " "before any run starts (default: runs nothing, spends " "nothing)") + s.add_argument("--baseline-sample", type=int, default=None, metavar="N", + dest="baseline_sample", + help="score the seed artifact on the first N tasks before " + "quoting the run, so a baseline with nothing left to " + "learn is caught before paying; one agent call per task, " + "quoted first (default: runs nothing)") + s.add_argument("--saturation", type=float, default=DEFAULT_SATURATION, + metavar="SCORE", + help="with --baseline-sample: a seed at or above this score " + "has no headroom and the paid run is not quoted " + "(default: %(default).2f)") + s.add_argument("--force", action="store_true", + help="with --baseline-sample: quote the paid run even when " + "the seed has no headroom") s.set_defaults(fn=cmd_plan) s = sub.add_parser("evolve", help="run a spec") diff --git a/agentdescent/pilot.py b/agentdescent/pilot.py index d5b9ec51..7538b05b 100644 --- a/agentdescent/pilot.py +++ b/agentdescent/pilot.py @@ -450,7 +450,8 @@ def _shared_artifact( def _notes(config: PilotConfig, evidence: Optional[TrustEvidence], *, reported_seed: Optional[int] = None, - artifacts_agree: Optional[bool] = None) -> Tuple[str, ...]: + artifacts_agree: Optional[bool] = None, + strategy: Any = None) -> Tuple[str, ...]: notes = [] if len(config.seeds) == 1: notes.append("one seed; rerun with at least two before quoting the " @@ -465,6 +466,14 @@ def _notes(config: PilotConfig, evidence: Optional[TrustEvidence], notes.append( f"every seed produced the same artifact (seed {reported_seed}); " "the scores pool every seed") + # Same sentence `plan` prints, from the same helper: a one-key artifact has + # no union to merge into, so the card says so once (#1113). + if strategy is not None: + from .strategies import _one_key_artifact_note + + shape = _one_key_artifact_note(strategy, workers=config.n_workers) + if shape: + notes.append(shape) notes.append("token counts are real only when the model adapter shares " "the run's Usage") if config.contaminated is None: @@ -539,6 +548,9 @@ def pilot(policy: PilotPolicy, notes=("refused before any model call",)) if config.dry_run: + from .strategies import _one_key_artifact_note + + shape = _one_key_artifact_note(policy.strategy, workers=config.n_workers) return PilotReport( artifact_id=artifact_id, status="planned", manifest=manifest, run={"tasks": len(train_ids), "test_tasks": len(test_ids), @@ -546,7 +558,8 @@ def pilot(policy: PilotPolicy, "n_workers": config.n_workers, "budget": dict(config.budget or {})}, next=("rerun without --dry-run to spend the quoted budget",), - notes=("dry run: no model call was made",)) + notes=tuple(["dry run: no model call was made"] + + ([shape] if shape else []))) if config.contaminated is True: # Refuse before spending: a contaminated dataset cannot yield a @@ -659,7 +672,7 @@ def pilot(policy: PilotPolicy, } cost = _cost(usage) notes = _notes(config, None, reported_seed=reported_seed, - artifacts_agree=artifacts_agree) + artifacts_agree=artifacts_agree, strategy=policy.strategy) failed_all = list(dict.fromkeys(failed_all)) if errors or not before_all or not after_all: @@ -688,4 +701,5 @@ def pilot(policy: PilotPolicy, before=before_all, after=after_all, evidence=evidence, manifest=manifest, run=run_report, cost=cost, next=_next_actions(decision), notes=_notes(config, evidence, reported_seed=reported_seed, - artifacts_agree=artifacts_agree), alpha=config.alpha) + artifacts_agree=artifacts_agree, strategy=policy.strategy), + alpha=config.alpha) diff --git a/agentdescent/strategies.py b/agentdescent/strategies.py index 0b7eae20..f11d5418 100644 --- a/agentdescent/strategies.py +++ b/agentdescent/strategies.py @@ -24,7 +24,7 @@ import hashlib import re from dataclasses import dataclass -from typing import Dict, Optional, Protocol, Sequence, runtime_checkable +from typing import Any, Dict, Optional, Protocol, Sequence, runtime_checkable from .evolvable import Diff @@ -214,3 +214,37 @@ def to_diff(self, state, proposal, author, base_version, target) -> Optional[Dif ops={key: value}, author=author) +def _one_key_artifact_note(strategy: Any, *, workers: int = 1) -> Optional[str]: + """One line: a declared one-key artifact cannot be merged by union (#1113). + + ``plan`` and the pilot report both print this, from here, so the two + surfaces cannot drift. Only a strategy that **declares** its key space can be + one-key: :class:`SingleSlot`, :class:`KeyedRules` with one category, a + ``FileTree`` with one writable file. ``AppendRules``/``SeededAppendRules`` + content-address every new rule, so a second key is always possible and no + note is due. + + Measured, not assumed: on a one-key artifact every pair of proposals + contradicts on the same key, conflict resolution collapses them to one, and + ``docs/results.md`` records ``fusion.contested == 0`` for exactly that shape + -- so the parallelism is per-round best-of-N *selection*. + """ + try: + keys = list(strategy.keys()) # type: ignore[attr-defined] + except Exception: # noqa: BLE001 - no declared key space: nothing to say + return None + if len(keys) != 1: + return None + name = type(strategy).__name__ + body = (f"one-key artifact ({name}): every worker proposal targets " + f"{keys[0]!r}, so they contradict and collapse to one candidate -- " + "merge cannot fuse") + if workers > 1: + body += (f", and n_workers={workers} is best-of-N selection, not " + "merging; parallelism buys throughput only.") + else: + body += ", and parallelism buys throughput only." + return body + (" Split the artifact into several keys (several files, " + "AppendRules) if you want merging.") + + diff --git a/changelog.d/1113-plan-headroom-artifact-shape.md b/changelog.d/1113-plan-headroom-artifact-shape.md new file mode 100644 index 00000000..a9264f16 --- /dev/null +++ b/changelog.d/1113-plan-headroom-artifact-shape.md @@ -0,0 +1,10 @@ +Added +`plan` can now catch two wasteful runs before they are paid for. The opt-in +`--baseline-sample N` scores the seed artifact on the first N tasks (one agent +call per task, quoted before the first call) and a seed at or above +`--saturation` (default `0.90`) prints `no headroom` and withholds the paid +run's quote unless `--force` is passed; `--json` carries the result as +`baseline` and sets `ready: false`. A plain `plan` still sends nothing. +`plan` and the pilot report also say when the artifact holds one key (a +`SingleSlot` prompt, a one-file tree): worker proposals contradict on that key, +so extra workers buy best-of-N selection, not merging (#1113). diff --git a/docs/start.md b/docs/start.md index dfbf0384..996b1fab 100644 --- a/docs/start.md +++ b/docs/start.md @@ -95,6 +95,26 @@ has. `plan` quotes an estimated call count and starts nothing. On a model-backed spec, `plan --probe` makes one minimal call so an expired subscription or a bad endpoint is refused before a run is written. +Two more checks are opt-in and answer "is this run worth paying for?". Nothing +about them is sent by a plain `plan`: + +```bash +agentdescent plan spec.json --baseline-sample 8 # one call per task, quoted first +``` + +`--baseline-sample N` scores the **seed** artifact on the first N tasks before +the run is quoted: the seed is what every round starts from, so its score is the +ceiling the run has to climb. A seed at or above `--saturation` (default `0.90`) +prints `no headroom` and the paid run is **not** quoted — it could only report +`below-threshold` — unless `--force` asks for the estimate anyway. The sample +itself is priced before its first call (one agent call per task). + +`plan` also says when the artifact holds **one key** (a `SingleSlot` prompt, a +one-file tree): every pair of worker proposals contradicts on that key and +collapses to one candidate, so `n_workers` buys best-of-N selection, not +merging. Several keys — several files, `AppendRules` — give the merge something +to fuse. + A dollar cap needs a per-call price, because the engine counts calls and cannot know a tool-using agent's bill in advance: diff --git a/tests/test_plan_headroom_shape.py b/tests/test_plan_headroom_shape.py new file mode 100644 index 00000000..d6a5d906 --- /dev/null +++ b/tests/test_plan_headroom_shape.py @@ -0,0 +1,310 @@ +"""`plan` says whether a paid run can learn anything, before it is paid for (#1113). + +Two avoidable ways a user spends money and sees no effect: + +* the seed artifact already scores 0.95+, so every round is ``below-threshold`` + and the user only learns that after paying; +* the artifact holds **one key** (``SingleSlot``, a single prompt, a one-file + tree), so every pair of worker proposals contradicts and the fusion + tournament never builds a fused candidate -- ``docs/results.md`` records + ``contested == 0`` for exactly that shape, but ``plan`` did not say so. + +The headroom sample is a real model call per task, against a local endpoint: +the flag is opt-in, the call count is quoted before the first call, and a +saturated seed refuses to quote the run unless ``--force`` is passed. +""" +from __future__ import annotations + +import io +import json +import threading +from contextlib import redirect_stderr, redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from agentdescent import cli +from agentdescent.evolution import Task +from agentdescent.pilot import PilotConfig, PilotPolicy, pilot +from agentdescent.strategies import AppendRules, SingleSlot + +OK = {"choices": [{"message": {"content": "ok"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1}} + + +class _Handler(BaseHTTPRequestHandler): + hits = 0 + lock = threading.Lock() + + def do_POST(self): # noqa: N802 + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + with type(self).lock: + type(self).hits += 1 + payload = json.dumps(OK).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): # noqa: D401 + pass + + +@pytest.fixture +def endpoint(monkeypatch): + _Handler.hits = 0 + server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setenv("FAKE_KEY", "sk-not-real") + monkeypatch.setenv("FAKE_BASE_URL", f"http://127.0.0.1:{server.server_address[1]}") + yield + server.shutdown() + server.server_close() + + +def _model(): + return {"ref": "openai_compatible", "model": "m", "api_key_env": "FAKE_KEY", + "base_url_env": "FAKE_BASE_URL", "retries": 0} + + +def _spec(tmp_path, *, gold="zzz", n=8, name="spec.json"): + spec = { + "version": 1, + "kind": "text", + "target": "You are a helpful assistant.", + "data": {"inline": [{"prompt": f"q{i}", "gold": gold} for i in range(n)]}, + "score": "contains", + "agent": _model(), + "reflect": _model(), + "evolve": {"rounds": 1, "n_workers": 1, "seed": 0}, + } + path = tmp_path / name + path.write_text(json.dumps(spec), encoding="utf-8") + return str(path) + + +def _run(args): + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = cli.main(args) + return code, out.getvalue(), err.getvalue() + + +# -- the headroom sample ----------------------------------------------------- + + +def test_plain_plan_still_samples_nothing(tmp_path, endpoint): + """The dry run stays model-free: no flag, no call, no sample line.""" + code, out, err = _run(["plan", _spec(tmp_path)]) + + assert code == 0, (out, err) + assert _Handler.hits == 0, "plain plan must stay free" + assert "baseline" not in (out + err).lower() + + +def test_the_sample_costs_one_call_per_task_and_quotes_it_first(tmp_path, endpoint): + """`--baseline-sample N` is opt-in and priced before the first call.""" + code, out, err = _run(["plan", _spec(tmp_path), "--baseline-sample", "4", + "--usd-per-call", "0.01"]) + + # Its own cost is named before anything is scored. + assert "baseline sample:" in err + assert "4 agent call(s)" in err + assert "$0.04" in err + assert _Handler.hits == 4, "exactly one call per sampled task" + + # And it does not change the shape of the run's own quote. + assert code == 0, (out, err) + assert "up to 11 agent calls" in out + assert "baseline: the seed scores 0.000 on 4 task(s)" in out + assert "headroom" in out + + +def test_a_saturated_seed_refuses_the_paid_quote(tmp_path, endpoint): + """Baseline 1.0: say "no headroom" and do not quote the run.""" + code, out, err = _run(["plan", _spec(tmp_path, gold="ok"), + "--baseline-sample", "4"]) + + assert code == 1, (out, err) + assert "no headroom" in err + assert "baseline: the seed scores 1.000 on 4 task(s)" in out + assert "1.000" in err and "saturation 0.900" in err + # The paid quote -- the call count and the next step -- is withheld. + assert "up to " not in out and "agent calls" not in out + assert "next: agentdescent evolve" not in out + assert "refused: no headroom" in out + assert _Handler.hits == 4 + + +def test_force_quotes_the_run_anyway(tmp_path, endpoint): + """`--force` is the only way to see the estimate on a saturated seed.""" + code, out, err = _run(["plan", _spec(tmp_path, gold="ok"), + "--baseline-sample", "4", "--force"]) + + assert code == 0, (out, err) + assert "no headroom" in err and "--force" in err + assert "up to 11 agent calls" in out + assert "next: agentdescent evolve" in out + assert _Handler.hits == 4 + + +def test_a_forced_json_plan_still_says_not_ready(tmp_path, endpoint): + """`--force` buys the quote, not a verdict change. + + `ready` is what a host follows, so it stays false on a saturated seed even + when the estimate was forced out: auto-starting that run is exactly what + the flag exists to question. + """ + code, out, err = _run(["plan", _spec(tmp_path, gold="ok"), "--json", + "--baseline-sample", "4", "--force"]) + + payload = json.loads(out) + assert code == 0 + assert payload["estimate"]["agent_calls_upper_bound"] == 11 + assert payload["baseline"]["headroom"] is False + assert payload["ready"] is False + assert any("because --force was passed" in w for w in payload["warnings"]) + assert _Handler.hits == 4 + + +def test_a_saturated_sample_withholds_the_estimate_from_json_too(tmp_path, endpoint): + code, out, err = _run(["plan", _spec(tmp_path, gold="ok"), "--json", + "--baseline-sample", "4"]) + + payload = json.loads(out) + assert code == 1 + assert payload["ok"] is True + assert payload["baseline"] == {"sample": 4, "score": 1.0, + "saturation": 0.9, "headroom": False} + assert payload["ready"] is False + assert _Handler.hits == 4 + + +def test_a_baseline_sample_below_one_is_an_input_error(tmp_path, endpoint): + code, out, err = _run(["plan", _spec(tmp_path), "--baseline-sample", "0"]) + + assert code == 2 + assert "--baseline-sample must be >= 1" in err + assert _Handler.hits == 0 + + +def test_an_out_of_range_saturation_is_an_input_error(tmp_path, endpoint): + code, out, err = _run(["plan", _spec(tmp_path), "--baseline-sample", "4", + "--saturation", "1.5"]) + + assert code == 2 + assert "--saturation must be in (0, 1]" in err + assert _Handler.hits == 0, "the input error must be caught before any call" + + +def test_a_saturation_without_a_sample_changes_nothing(tmp_path, endpoint): + """`--saturation` is only read by the sample; a plain plan stays free.""" + code, out, err = _run(["plan", _spec(tmp_path), "--saturation", "0"]) + + assert code == 0, (out, err) + assert _Handler.hits == 0 + assert "baseline" not in (out + err).lower() + + +# -- the artifact-shape note ------------------------------------------------- + + +def _tree_spec(tmp_path, files, *, name="skill-spec.json"): + tree = tmp_path / name.replace(".json", "") + tree.mkdir() + for rel, text in files.items(): + path = tree / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + cli_ref = {"ref": "claude_code", "isolate": False} + spec = { + "version": 1, + "kind": "skill_dir", + "target": str(tree), + "data": {"inline": [{"prompt": f"q{i}", "gold": "q"} for i in range(8)]}, + "score": "contains", + "agent": dict(cli_ref), + "reflect": dict(cli_ref), + "prompt_template": "{prompt}", + "evolve": {"rounds": 1, "n_workers": 4, "seed": 0}, + } + path = tmp_path / name + path.write_text(json.dumps(spec), encoding="utf-8") + return str(path) + + +@pytest.fixture +def fake_claude(monkeypatch): + real = cli.doctor_report + + def fake_doctor(): + report = real() + clis = dict(report["agent_clis"]) + clis["claude"] = clis.get("claude") or "/fake/claude" + return {**report, "agent_clis": clis} + + monkeypatch.setattr(cli, "doctor_report", fake_doctor) + + +def test_a_one_key_artifact_warns_that_merging_cannot_fuse(tmp_path, fake_claude): + """A one-file tree is one key: proposals contradict, fusion never runs.""" + spec = _tree_spec(tmp_path, {"rules.md": "MODE: forward\n"}) + + code, out, err = _run(["plan", spec]) + + assert code == 0, (out, err) + assert "one-key artifact (FileTree)" in err + assert "best-of-N" in err + assert "several keys" in err + + +def test_a_multi_key_artifact_says_nothing_about_merging(tmp_path, fake_claude): + spec = _tree_spec(tmp_path, {"rules.md": "MODE: forward\n", + "extra.md": "MODE: back\n"}) + + code, out, err = _run(["plan", spec]) + + assert code == 0, (out, err) + assert "one-key artifact" not in err + assert "best-of-N" not in err + + +# -- the same note on the pilot card ---------------------------------------- + + +def _pilot_policy(strategy): + train = [Task(id=f"train-{i}", prompt="q", meta={"rule": "R0"}) + for i in range(30)] + test = [Task(id=f"test-{i}", prompt="q", + meta={"rule": "R0", "split": "test"}) for i in range(12)] + + def run(rendered, task): + return "good" if "R0" in rendered else "bad" + + def reward(task, output): + return 1.0 if output == "good" else 0.0 + + def propose(rendered, task, output, reward): + return None if "R0" in rendered else "R0" + + return PilotPolicy(tasks=train, test_tasks=test, reward=reward, run=run, + propose=propose, strategy=strategy, artifact_id="skill") + + +def _pilot_config(): + return PilotConfig(rounds=1, n_workers=1, max_concurrency=1, + contaminated=False) + + +def test_the_pilot_card_carries_the_note_once(): + report = pilot(_pilot_policy(SingleSlot()), _pilot_config()) + + assert sum("one-key artifact" in note for note in report.notes) == 1 + assert report.to_markdown().count("one-key artifact") == 1 + + +def test_the_pilot_card_is_quiet_on_a_multi_key_artifact(): + report = pilot(_pilot_policy(AppendRules()), _pilot_config()) + + assert not any("one-key artifact" in note for note in report.notes)