Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions agentdescent/pilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,10 +407,18 @@ def _cost(usage: Usage) -> Dict[str, Any]:
"failures": usage.failures}


def _next_actions(decision: Optional[PromotionDecision]) -> Tuple[str, ...]:
def _next_actions(decision: Optional[PromotionDecision], *,
artifacts_agree: Optional[bool] = None) -> Tuple[str, ...]:
if decision is None:
return ("fix the report's error and rerun",)
if decision.action == PromotionAction.PROMOTE:
if artifacts_agree is False:
# The report names no artifact, so the steps must not name one
# either (#1141): point at the per-seed rows it does publish.
return ("pick one seed's artifact from `run.per_seed` and review "
"that diff",
"apply the chosen artifact as a separate, explicit step",
"keep the manifest and this report for audit")
return ("review the artifact diff", "apply it as a separate, explicit step",
"keep the manifest and this report for audit")
if decision.action == PromotionAction.DEFER:
Expand Down Expand Up @@ -682,7 +690,8 @@ def pilot(policy: PilotPolicy,
return PilotReport(
artifact_id=artifact_id, status="error", decision=decision,
before=before_all, after=after_all, manifest=manifest,
run=run_report, cost=cost, next=_next_actions(decision),
run=run_report, cost=cost,
next=_next_actions(decision, artifacts_agree=artifacts_agree),
notes=notes, alpha=config.alpha, error=reason)

health = RunHealth(
Expand All @@ -699,7 +708,8 @@ def pilot(policy: PilotPolicy,
return PilotReport(
artifact_id=artifact_id, status="decided", decision=decision,
before=before_all, after=after_all, evidence=evidence, manifest=manifest,
run=run_report, cost=cost, next=_next_actions(decision),
run=run_report, cost=cost,
next=_next_actions(decision, artifacts_agree=artifacts_agree),
notes=_notes(config, evidence, reported_seed=reported_seed,
artifacts_agree=artifacts_agree, strategy=policy.strategy),
alpha=config.alpha)
6 changes: 6 additions & 0 deletions changelog.d/1141-pilot-next-artifact.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fixed
A promoting multi-seed pilot report no longer tells the reader to "review the
artifact diff" or "apply it" when the seeds produced different artifacts and the
report names none. The steps point at `run.per_seed` and ask for one seed's
artifact to be chosen, reviewed and applied as an explicit step; the single-seed
and agreeing-seed reports keep their current text (#1141).
38 changes: 38 additions & 0 deletions tests/test_pilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,44 @@ def row_gains_differ(report) -> bool:
return len(set(gains)) > 1


def test_a_promoting_multi_seed_report_never_points_at_one_artifact(monkeypatch):
"""Different-but-equally-good artifacts promote, so ``next`` must name the
per-seed rows instead of the one diff this report refuses to name (#1141)."""

def fake_evolve(*args, **kwargs):
seed = kwargs.get("seed", 0)
rendered = "R0 R1 R2" if seed == 0 else "R0 R1 R2 plus seed1"
return _StubResult(rendered=rendered, final_reward=0.9)

monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve)
report = pilot(_policy(), _config(seeds=(0, 1)))

assert report.action == PromotionAction.PROMOTE
assert report.run["artifacts_agree"] is False
assert report.run["artifact_hash"] is None
assert report.run["reported_seed"] is None
steps = " ".join(report.next)
assert "review the artifact diff" not in steps
assert "apply it" not in steps
assert "run.per_seed" in steps
assert "keep the manifest and this report for audit" in report.next


def test_an_agreeing_multi_seed_report_keeps_its_promote_steps(monkeypatch):
"""One shared artifact is nameable, so the single-diff steps stay."""

def fake_evolve(*args, **kwargs):
return _StubResult(rendered="R0 R1 R2", final_reward=0.9)

monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve)
report = pilot(_policy(), _config(seeds=(0, 1)))

assert report.action == PromotionAction.PROMOTE
assert report.run["artifacts_agree"] is True
assert "review the artifact diff" in report.next
assert "apply it as a separate, explicit step" in report.next


def test_single_seed_report_is_unchanged():
report = pilot(_policy(), _config())

Expand Down
Loading