diff --git a/agentdescent/pilot.py b/agentdescent/pilot.py index 6c159f8b..d5b9ec51 100644 --- a/agentdescent/pilot.py +++ b/agentdescent/pilot.py @@ -24,7 +24,11 @@ * a missing contamination check defers, a contaminated dataset rejects before any model call; * the test split must be disjoint from the evolve pool by task id. Overlap is - refused before spending, not discovered in the report. + refused before spending, not discovered in the report; +* a multi-seed run never reports one seed's artifact next to scores pooled + across every seed. A single artifact is reported only when every seed + produced it; otherwise the report summarizes per seed and the pooled + scores describe the method, not one candidate. The module does not apply anything. ``apply``/release stays an explicit, separate step. @@ -418,7 +422,35 @@ def _next_actions(decision: Optional[PromotionDecision]) -> Tuple[str, ...]: return ("do not ship this change", "keep the run for the next proposal") -def _notes(config: PilotConfig, evidence: Optional[TrustEvidence]) -> Tuple[str, ...]: +def _artifact_hash(rendered: str) -> str: + return hashlib.sha256(rendered.encode()).hexdigest() + + +def _shared_artifact( + per_seed: Sequence[Mapping[str, Any]], +) -> Optional[Tuple[int, str]]: + """Return ``(index, hash)`` only when every produced artifact agrees. + + Seeds that never recorded an artifact hash are skipped. If two hashes + differ, return ``None`` so pooled scores cannot be pinned on one seed. + """ + seen: list[Tuple[int, str]] = [] + for index, row in enumerate(per_seed): + digest = row.get("artifact_hash") + if not digest: + continue + seen.append((index, str(digest))) + if not seen: + return None + first = seen[0][1] + if any(digest != first for _, digest in seen): + return None + return seen[0][0], first + + +def _notes(config: PilotConfig, evidence: Optional[TrustEvidence], + *, reported_seed: Optional[int] = None, + artifacts_agree: Optional[bool] = None) -> Tuple[str, ...]: notes = [] if len(config.seeds) == 1: notes.append("one seed; rerun with at least two before quoting the " @@ -426,8 +458,13 @@ def _notes(config: PilotConfig, evidence: Optional[TrustEvidence]) -> Tuple[str, else: notes.append("seeds reuse the same test tasks; the interval is a paired " "bootstrap over seed-task pairs, not a cluster-robust one") - notes.append(f"the reported artifact is seed {config.seeds[0]}; the " - "scores pool every seed") + if artifacts_agree is False: + notes.append("seeds produced different artifacts; the pooled scores " + "describe the method, not a single artifact") + elif artifacts_agree is True and reported_seed is not None: + notes.append( + f"every seed produced the same artifact (seed {reported_seed}); " + "the scores pool every seed") notes.append("token counts are real only when the model adapter shares " "the run's Usage") if config.contaminated is None: @@ -534,7 +571,7 @@ def pilot(policy: PilotPolicy, after_all: Dict[str, float] = {} failed_all: list[str] = [] per_seed = [] - chosen = None + results = [] errors = [] for seed in config.seeds: failed_all.extend(f"{seed}:{task}" for task in baseline_failures) @@ -584,19 +621,45 @@ def pilot(policy: PilotPolicy, "final_reward": result.final_reward, "test_gain": ((sum(after[t] - before[t] for t in common) / len(common)) if common else None), + "artifact_hash": _artifact_hash(result.rendered), }) - if chosen is None: - chosen = result - - run_report = { - "rounds": config.rounds, "n_workers": config.n_workers, - "seeds": list(config.seeds), "stop_reason": chosen.stop_reason, - "error": chosen.error, "accepted": chosen.outcomes().get("committed", 0), - "outcomes": dict(chosen.outcomes()), "per_seed": per_seed, - "initial_reward": chosen.initial_reward, - "final_reward": chosen.final_reward, - } + results.append(result) + + shared = _shared_artifact(per_seed) + if shared is not None: + chosen = results[shared[0]] + artifact_hash = shared[1] + reported_seed = per_seed[shared[0]]["seed"] + artifacts_agree = True + artifact_changed = initial != chosen.rendered + run_report = { + "rounds": config.rounds, "n_workers": config.n_workers, + "seeds": list(config.seeds), "stop_reason": chosen.stop_reason, + "error": chosen.error, + "accepted": chosen.outcomes().get("committed", 0), + "outcomes": dict(chosen.outcomes()), "per_seed": per_seed, + "initial_reward": chosen.initial_reward, + "final_reward": chosen.final_reward, + "reported_seed": reported_seed, "artifact_hash": artifact_hash, + "artifacts_agree": True, + } + else: + chosen = None + artifact_hash = None + reported_seed = None + artifacts_agree = False + artifact_changed = any(initial != result.rendered for result in results) + run_report = { + "rounds": config.rounds, "n_workers": config.n_workers, + "seeds": list(config.seeds), "stop_reason": None, "error": None, + "accepted": None, "outcomes": {}, "per_seed": per_seed, + "initial_reward": None, "final_reward": None, + "reported_seed": None, "artifact_hash": None, + "artifacts_agree": False, + } cost = _cost(usage) + notes = _notes(config, None, reported_seed=reported_seed, + artifacts_agree=artifacts_agree) failed_all = list(dict.fromkeys(failed_all)) if errors or not before_all or not after_all: @@ -607,7 +670,7 @@ def pilot(policy: PilotPolicy, artifact_id=artifact_id, status="error", decision=decision, before=before_all, after=after_all, manifest=manifest, run=run_report, cost=cost, next=_next_actions(decision), - notes=_notes(config, None), alpha=config.alpha, error=reason) + notes=notes, alpha=config.alpha, error=reason) health = RunHealth( attempted=len(policy.test_tasks) * len(config.seeds), @@ -615,14 +678,14 @@ def pilot(policy: PilotPolicy, threshold=config.max_failure_rate) evidence = build_trust_evidence( before_all, after_all, contaminated=config.contaminated, - risk=config.risk, artifact_changed=(initial != chosen.rendered), + risk=config.risk, artifact_changed=artifact_changed, alpha=config.alpha, min_effect=config.min_effect) evidence = replace(evidence, run_health=health) - artifact_hash = hashlib.sha256(chosen.rendered.encode()).hexdigest() evidence, decision = _decide(evidence, artifact_id, config, artifact_hash=artifact_hash) return PilotReport( artifact_id=artifact_id, status="decided", decision=decision, before=before_all, after=after_all, evidence=evidence, manifest=manifest, run=run_report, cost=cost, next=_next_actions(decision), - notes=_notes(config, evidence), alpha=config.alpha) + notes=_notes(config, evidence, reported_seed=reported_seed, + artifacts_agree=artifacts_agree), alpha=config.alpha) diff --git a/changelog.d/1137-pilot-multi-seed-artifact.md b/changelog.d/1137-pilot-multi-seed-artifact.md new file mode 100644 index 00000000..0d4eda88 --- /dev/null +++ b/changelog.d/1137-pilot-multi-seed-artifact.md @@ -0,0 +1,4 @@ +Fixed +`pilot` no longer reports the seed-0 artifact next to scores pooled +across every seed. A single artifact is reported only when every seed +produced it; otherwise the report summarizes per seed (#1137). diff --git a/tests/test_pilot.py b/tests/test_pilot.py index de31d5f3..7ae793ec 100644 --- a/tests/test_pilot.py +++ b/tests/test_pilot.py @@ -9,8 +9,8 @@ import json from agentdescent.evolution import Task -from agentdescent.pilot import (PilotConfig, PilotPolicy, pilot, - pilot_scores) +from agentdescent.pilot import (PilotConfig, PilotPolicy, _shared_artifact, + pilot, pilot_scores) from agentdescent.promotion import PromotionAction from agentdescent.strategies import AppendRules @@ -173,7 +173,105 @@ def test_two_seeds_pool_the_same_test_tasks(): assert report.action == PromotionAction.PROMOTE assert report.evidence.eval_result.n == 24 assert [row["seed"] for row in report.run["per_seed"]] == [0, 1] + assert report.run["artifacts_agree"] is True + assert report.run["reported_seed"] == 0 assert any("pool every seed" in note for note in report.notes) + hashes = {row["artifact_hash"] for row in report.run["per_seed"]} + assert len(hashes) == 1 + assert report.run["artifact_hash"] in hashes + + +def test_shared_artifact_is_none_when_hashes_differ(): + assert _shared_artifact([ + {"artifact_hash": "aaa"}, + {"artifact_hash": "bbb"}, + ]) is None + assert _shared_artifact([ + {"artifact_hash": "aaa"}, + {"artifact_hash": "aaa"}, + ]) == (0, "aaa") + assert _shared_artifact([{"seed": 0}, {"seed": 1}]) is None + + +def _pairs_pooled_scores_with_one_seed_artifact(report) -> bool: + """True when a report pins pooled scores on a single seed's artifact.""" + rows = list(report.run.get("per_seed") or []) + if len(rows) <= 1: + return False + hashes = [row.get("artifact_hash") for row in rows] + if any(not digest for digest in hashes): + return False + if len(set(hashes)) <= 1: + return False + if report.run.get("final_reward") is not None: + return True + if report.run.get("artifact_hash") in hashes: + return True + if report.run.get("reported_seed") is not None: + return True + text = " ".join(report.notes) + if "the reported artifact is seed" in text: + return True + return False + + +class _StubResult: + def __init__(self, rendered, final_reward): + self.rendered = rendered + self.final_reward = final_reward + self.initial_reward = 0.0 + self.error = None + self.stop_reason = "rounds" + + def outcomes(self): + return {"committed": 1} + + +def test_pooled_scores_are_not_paired_with_the_first_seed_artifact(monkeypatch): + """Different per-seed artifacts must not be hidden behind seed-0 + a pool.""" + + def fake_evolve(*args, **kwargs): + seed = kwargs.get("seed", 0) + if seed == 0: + return _StubResult(rendered="", final_reward=0.1) + return _StubResult(rendered="R0 R1 R2", final_reward=0.9) + + monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve) + report = pilot(_policy(), _config(seeds=(0, 1))) + + hashes = [row["artifact_hash"] for row in report.run["per_seed"]] + assert hashes[0] != hashes[1] + assert row_gains_differ(report) + assert report.evidence.eval_result.n == 24 + assert report.run["artifacts_agree"] is False + assert report.run["reported_seed"] is None + assert report.run["final_reward"] is None + assert report.run["artifact_hash"] is None + assert report.run["per_seed"][0]["final_reward"] == 0.1 + assert report.run["per_seed"][1]["final_reward"] == 0.9 + text = " ".join(report.notes) + assert "the reported artifact is seed 0" not in text + assert "not a single artifact" in text + assert not _pairs_pooled_scores_with_one_seed_artifact(report) + + +def row_gains_differ(report) -> bool: + gains = [row["test_gain"] for row in report.run["per_seed"]] + return len(set(gains)) > 1 + + +def test_single_seed_report_is_unchanged(): + report = pilot(_policy(), _config()) + + assert report.action == PromotionAction.PROMOTE + assert report.run["seeds"] == [0] + assert report.run["reported_seed"] == 0 + assert report.run["artifacts_agree"] is True + assert report.run["final_reward"] is not None + assert report.run["artifact_hash"] + assert any("one seed" in note for note in report.notes) + assert not any("the reported artifact is seed" in note + for note in report.notes) def test_same_inputs_produce_the_same_manifest_id():