From b7b4e03f713b25c09c8ae1517fefceff2f464bc9 Mon Sep 17 00:00:00 2001 From: AgentDescent Dev-1 Date: Tue, 17 Nov 2026 01:36:50 +0000 Subject: [PATCH] fix(pilot): name the per-seed artifacts when a multi-seed promote report names none (#1141) The #1137 fix stopped pairing seed 0's artifact with pooled scores and says in the note that the numbers describe the method, not one artifact; the promote steps still said 'review the artifact diff' / 'apply it'. _next_actions now takes the per-seed agreement and, when the seeds disagree, points at run.per_seed and asks for one seed's artifact to be chosen and applied explicitly. Single-seed and agreeing-seed reports keep their text. --- agentdescent/pilot.py | 16 +++++++++-- changelog.d/1141-pilot-next-artifact.md | 6 ++++ tests/test_pilot.py | 38 +++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 changelog.d/1141-pilot-next-artifact.md diff --git a/agentdescent/pilot.py b/agentdescent/pilot.py index d5b9ec51..182f7580 100644 --- a/agentdescent/pilot.py +++ b/agentdescent/pilot.py @@ -407,10 +407,18 @@ def _cost(usage: Usage) -> Dict[str, Any]: "failures": usage.failures} -def _next_actions(decision: Optional[PromotionDecision]) -> Tuple[str, ...]: +def _next_actions(decision: Optional[PromotionDecision], *, + artifacts_agree: Optional[bool] = None) -> Tuple[str, ...]: if decision is None: return ("fix the report's error and rerun",) if decision.action == PromotionAction.PROMOTE: + if artifacts_agree is False: + # The report names no artifact, so the steps must not name one + # either (#1141): point at the per-seed rows it does publish. + return ("pick one seed's artifact from `run.per_seed` and review " + "that diff", + "apply the chosen artifact as a separate, explicit step", + "keep the manifest and this report for audit") return ("review the artifact diff", "apply it as a separate, explicit step", "keep the manifest and this report for audit") if decision.action == PromotionAction.DEFER: @@ -669,7 +677,8 @@ def pilot(policy: PilotPolicy, return PilotReport( artifact_id=artifact_id, status="error", decision=decision, before=before_all, after=after_all, manifest=manifest, - run=run_report, cost=cost, next=_next_actions(decision), + run=run_report, cost=cost, + next=_next_actions(decision, artifacts_agree=artifacts_agree), notes=notes, alpha=config.alpha, error=reason) health = RunHealth( @@ -686,6 +695,7 @@ def pilot(policy: PilotPolicy, return PilotReport( artifact_id=artifact_id, status="decided", decision=decision, before=before_all, after=after_all, evidence=evidence, manifest=manifest, - run=run_report, cost=cost, next=_next_actions(decision), + run=run_report, cost=cost, + next=_next_actions(decision, artifacts_agree=artifacts_agree), notes=_notes(config, evidence, reported_seed=reported_seed, artifacts_agree=artifacts_agree), alpha=config.alpha) diff --git a/changelog.d/1141-pilot-next-artifact.md b/changelog.d/1141-pilot-next-artifact.md new file mode 100644 index 00000000..73b15173 --- /dev/null +++ b/changelog.d/1141-pilot-next-artifact.md @@ -0,0 +1,6 @@ +Fixed +A promoting multi-seed pilot report no longer tells the reader to "review the +artifact diff" or "apply it" when the seeds produced different artifacts and the +report names none. The steps point at `run.per_seed` and ask for one seed's +artifact to be chosen, reviewed and applied as an explicit step; the single-seed +and agreeing-seed reports keep their current text (#1141). diff --git a/tests/test_pilot.py b/tests/test_pilot.py index 7ae793ec..8b21b6a8 100644 --- a/tests/test_pilot.py +++ b/tests/test_pilot.py @@ -260,6 +260,44 @@ def row_gains_differ(report) -> bool: return len(set(gains)) > 1 +def test_a_promoting_multi_seed_report_never_points_at_one_artifact(monkeypatch): + """Different-but-equally-good artifacts promote, so ``next`` must name the + per-seed rows instead of the one diff this report refuses to name (#1141).""" + + def fake_evolve(*args, **kwargs): + seed = kwargs.get("seed", 0) + rendered = "R0 R1 R2" if seed == 0 else "R0 R1 R2 plus seed1" + return _StubResult(rendered=rendered, final_reward=0.9) + + monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve) + report = pilot(_policy(), _config(seeds=(0, 1))) + + assert report.action == PromotionAction.PROMOTE + assert report.run["artifacts_agree"] is False + assert report.run["artifact_hash"] is None + assert report.run["reported_seed"] is None + steps = " ".join(report.next) + assert "review the artifact diff" not in steps + assert "apply it" not in steps + assert "run.per_seed" in steps + assert "keep the manifest and this report for audit" in report.next + + +def test_an_agreeing_multi_seed_report_keeps_its_promote_steps(monkeypatch): + """One shared artifact is nameable, so the single-diff steps stay.""" + + def fake_evolve(*args, **kwargs): + return _StubResult(rendered="R0 R1 R2", final_reward=0.9) + + monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve) + report = pilot(_policy(), _config(seeds=(0, 1))) + + assert report.action == PromotionAction.PROMOTE + assert report.run["artifacts_agree"] is True + assert "review the artifact diff" in report.next + assert "apply it as a separate, explicit step" in report.next + + def test_single_seed_report_is_unchanged(): report = pilot(_policy(), _config())