Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 83 additions & 20 deletions agentdescent/pilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@
* a missing contamination check defers, a contaminated dataset rejects before
any model call;
* the test split must be disjoint from the evolve pool by task id. Overlap is
refused before spending, not discovered in the report.
refused before spending, not discovered in the report;
* a multi-seed run never reports one seed's artifact next to scores pooled
across every seed. A single artifact is reported only when every seed
produced it; otherwise the report summarizes per seed and the pooled
scores describe the method, not one candidate.

The module does not apply anything. ``apply``/release stays an explicit,
separate step.
Expand Down Expand Up @@ -418,16 +422,49 @@ def _next_actions(decision: Optional[PromotionDecision]) -> Tuple[str, ...]:
return ("do not ship this change", "keep the run for the next proposal")


def _notes(config: PilotConfig, evidence: Optional[TrustEvidence]) -> Tuple[str, ...]:
def _artifact_hash(rendered: str) -> str:
return hashlib.sha256(rendered.encode()).hexdigest()


def _shared_artifact(
per_seed: Sequence[Mapping[str, Any]],
) -> Optional[Tuple[int, str]]:
"""Return ``(index, hash)`` only when every produced artifact agrees.

Seeds that never recorded an artifact hash are skipped. If two hashes
differ, return ``None`` so pooled scores cannot be pinned on one seed.
"""
seen: list[Tuple[int, str]] = []
for index, row in enumerate(per_seed):
digest = row.get("artifact_hash")
if not digest:
continue
seen.append((index, str(digest)))
if not seen:
return None
first = seen[0][1]
if any(digest != first for _, digest in seen):
return None
return seen[0][0], first


def _notes(config: PilotConfig, evidence: Optional[TrustEvidence],
*, reported_seed: Optional[int] = None,
artifacts_agree: Optional[bool] = None) -> Tuple[str, ...]:
notes = []
if len(config.seeds) == 1:
notes.append("one seed; rerun with at least two before quoting the "
"magnitude")
else:
notes.append("seeds reuse the same test tasks; the interval is a paired "
"bootstrap over seed-task pairs, not a cluster-robust one")
notes.append(f"the reported artifact is seed {config.seeds[0]}; the "
"scores pool every seed")
if artifacts_agree is False:
notes.append("seeds produced different artifacts; the pooled scores "
"describe the method, not a single artifact")
elif artifacts_agree is True and reported_seed is not None:
notes.append(
f"every seed produced the same artifact (seed {reported_seed}); "
"the scores pool every seed")
notes.append("token counts are real only when the model adapter shares "
"the run's Usage")
if config.contaminated is None:
Expand Down Expand Up @@ -534,7 +571,7 @@ def pilot(policy: PilotPolicy,
after_all: Dict[str, float] = {}
failed_all: list[str] = []
per_seed = []
chosen = None
results = []
errors = []
for seed in config.seeds:
failed_all.extend(f"{seed}:{task}" for task in baseline_failures)
Expand Down Expand Up @@ -584,19 +621,45 @@ def pilot(policy: PilotPolicy,
"final_reward": result.final_reward,
"test_gain": ((sum(after[t] - before[t] for t in common) /
len(common)) if common else None),
"artifact_hash": _artifact_hash(result.rendered),
})
if chosen is None:
chosen = result

run_report = {
"rounds": config.rounds, "n_workers": config.n_workers,
"seeds": list(config.seeds), "stop_reason": chosen.stop_reason,
"error": chosen.error, "accepted": chosen.outcomes().get("committed", 0),
"outcomes": dict(chosen.outcomes()), "per_seed": per_seed,
"initial_reward": chosen.initial_reward,
"final_reward": chosen.final_reward,
}
results.append(result)

shared = _shared_artifact(per_seed)
if shared is not None:
chosen = results[shared[0]]
artifact_hash = shared[1]
reported_seed = per_seed[shared[0]]["seed"]
artifacts_agree = True
artifact_changed = initial != chosen.rendered
run_report = {
"rounds": config.rounds, "n_workers": config.n_workers,
"seeds": list(config.seeds), "stop_reason": chosen.stop_reason,
"error": chosen.error,
"accepted": chosen.outcomes().get("committed", 0),
"outcomes": dict(chosen.outcomes()), "per_seed": per_seed,
"initial_reward": chosen.initial_reward,
"final_reward": chosen.final_reward,
"reported_seed": reported_seed, "artifact_hash": artifact_hash,
"artifacts_agree": True,
}
else:
chosen = None
artifact_hash = None
reported_seed = None
artifacts_agree = False
artifact_changed = any(initial != result.rendered for result in results)
run_report = {
"rounds": config.rounds, "n_workers": config.n_workers,
"seeds": list(config.seeds), "stop_reason": None, "error": None,
"accepted": None, "outcomes": {}, "per_seed": per_seed,
"initial_reward": None, "final_reward": None,
"reported_seed": None, "artifact_hash": None,
"artifacts_agree": False,
}
cost = _cost(usage)
notes = _notes(config, None, reported_seed=reported_seed,
artifacts_agree=artifacts_agree)

failed_all = list(dict.fromkeys(failed_all))
if errors or not before_all or not after_all:
Expand All @@ -607,22 +670,22 @@ def pilot(policy: PilotPolicy,
artifact_id=artifact_id, status="error", decision=decision,
before=before_all, after=after_all, manifest=manifest,
run=run_report, cost=cost, next=_next_actions(decision),
notes=_notes(config, None), alpha=config.alpha, error=reason)
notes=notes, alpha=config.alpha, error=reason)

health = RunHealth(
attempted=len(policy.test_tasks) * len(config.seeds),
failed_tasks=tuple(failed_all),
threshold=config.max_failure_rate)
evidence = build_trust_evidence(
before_all, after_all, contaminated=config.contaminated,
risk=config.risk, artifact_changed=(initial != chosen.rendered),
risk=config.risk, artifact_changed=artifact_changed,
alpha=config.alpha, min_effect=config.min_effect)
evidence = replace(evidence, run_health=health)
artifact_hash = hashlib.sha256(chosen.rendered.encode()).hexdigest()
evidence, decision = _decide(evidence, artifact_id, config,
artifact_hash=artifact_hash)
return PilotReport(
artifact_id=artifact_id, status="decided", decision=decision,
before=before_all, after=after_all, evidence=evidence, manifest=manifest,
run=run_report, cost=cost, next=_next_actions(decision),
notes=_notes(config, evidence), alpha=config.alpha)
notes=_notes(config, evidence, reported_seed=reported_seed,
artifacts_agree=artifacts_agree), alpha=config.alpha)
4 changes: 4 additions & 0 deletions changelog.d/1137-pilot-multi-seed-artifact.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fixed
`pilot` no longer reports the seed-0 artifact next to scores pooled
across every seed. A single artifact is reported only when every seed
produced it; otherwise the report summarizes per seed (#1137).
102 changes: 100 additions & 2 deletions tests/test_pilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import json

from agentdescent.evolution import Task
from agentdescent.pilot import (PilotConfig, PilotPolicy, pilot,
pilot_scores)
from agentdescent.pilot import (PilotConfig, PilotPolicy, _shared_artifact,
pilot, pilot_scores)
from agentdescent.promotion import PromotionAction
from agentdescent.strategies import AppendRules

Expand Down Expand Up @@ -173,7 +173,105 @@ def test_two_seeds_pool_the_same_test_tasks():
assert report.action == PromotionAction.PROMOTE
assert report.evidence.eval_result.n == 24
assert [row["seed"] for row in report.run["per_seed"]] == [0, 1]
assert report.run["artifacts_agree"] is True
assert report.run["reported_seed"] == 0
assert any("pool every seed" in note for note in report.notes)
hashes = {row["artifact_hash"] for row in report.run["per_seed"]}
assert len(hashes) == 1
assert report.run["artifact_hash"] in hashes


def test_shared_artifact_is_none_when_hashes_differ():
assert _shared_artifact([
{"artifact_hash": "aaa"},
{"artifact_hash": "bbb"},
]) is None
assert _shared_artifact([
{"artifact_hash": "aaa"},
{"artifact_hash": "aaa"},
]) == (0, "aaa")
assert _shared_artifact([{"seed": 0}, {"seed": 1}]) is None


def _pairs_pooled_scores_with_one_seed_artifact(report) -> bool:
"""True when a report pins pooled scores on a single seed's artifact."""
rows = list(report.run.get("per_seed") or [])
if len(rows) <= 1:
return False
hashes = [row.get("artifact_hash") for row in rows]
if any(not digest for digest in hashes):
return False
if len(set(hashes)) <= 1:
return False
if report.run.get("final_reward") is not None:
return True
if report.run.get("artifact_hash") in hashes:
return True
if report.run.get("reported_seed") is not None:
return True
text = " ".join(report.notes)
if "the reported artifact is seed" in text:
return True
return False


class _StubResult:
def __init__(self, rendered, final_reward):
self.rendered = rendered
self.final_reward = final_reward
self.initial_reward = 0.0
self.error = None
self.stop_reason = "rounds"

def outcomes(self):
return {"committed": 1}


def test_pooled_scores_are_not_paired_with_the_first_seed_artifact(monkeypatch):
"""Different per-seed artifacts must not be hidden behind seed-0 + a pool."""

def fake_evolve(*args, **kwargs):
seed = kwargs.get("seed", 0)
if seed == 0:
return _StubResult(rendered="", final_reward=0.1)
return _StubResult(rendered="R0 R1 R2", final_reward=0.9)

monkeypatch.setattr("agentdescent.pilot.evolve", fake_evolve)
report = pilot(_policy(), _config(seeds=(0, 1)))

hashes = [row["artifact_hash"] for row in report.run["per_seed"]]
assert hashes[0] != hashes[1]
assert row_gains_differ(report)
assert report.evidence.eval_result.n == 24
assert report.run["artifacts_agree"] is False
assert report.run["reported_seed"] is None
assert report.run["final_reward"] is None
assert report.run["artifact_hash"] is None
assert report.run["per_seed"][0]["final_reward"] == 0.1
assert report.run["per_seed"][1]["final_reward"] == 0.9
text = " ".join(report.notes)
assert "the reported artifact is seed 0" not in text
assert "not a single artifact" in text
assert not _pairs_pooled_scores_with_one_seed_artifact(report)


def row_gains_differ(report) -> bool:
gains = [row["test_gain"] for row in report.run["per_seed"]]
return len(set(gains)) > 1


def test_single_seed_report_is_unchanged():
report = pilot(_policy(), _config())

assert report.action == PromotionAction.PROMOTE
assert report.run["seeds"] == [0]
assert report.run["reported_seed"] == 0
assert report.run["artifacts_agree"] is True
assert report.run["final_reward"] is not None
assert report.run["artifact_hash"]
assert any("one seed" in note for note in report.notes)
assert not any("the reported artifact is seed" in note
for note in report.notes)


def test_same_inputs_produce_the_same_manifest_id():
Expand Down
Loading