Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f6d8a4b
feat(runs): read a task's repeated attempts together, by pass rate, w…
aitianshi Nov 16, 2026
e87c4c1
Merge remote-tracking branch 'origin/main' into feat/compare-pass-cou…
aitianshi Nov 16, 2026
24382d2
refactor(runs): say repeat for one run of a task, not attempt
aitianshi Nov 16, 2026
5267cea
feat(standard): add --attempts, trying a repeat again when the infras…
aitianshi Nov 16, 2026
61d7f49
fix(standard): count every try in the progress, and word the live boa…
aitianshi Nov 16, 2026
8ec9680
fix(runs): say attempt for one run of a task, and number attempts as …
aitianshi Nov 16, 2026
ff59dfd
feat(standard): retry an attempt that fails on the infrastructure or …
aitianshi Nov 16, 2026
831b821
feat(standard): show each try of a retried attempt, and retry only wh…
aitianshi Nov 17, 2026
800803d
Merge remote-tracking branch 'origin/main' into feat/compare-pass-cou…
aitianshi Nov 17, 2026
0b47a74
docs(runs): cut the field guide's attempts and compare text to what c…
aitianshi Nov 17, 2026
a818938
Merge branch 'feat/compare-pass-count-sorting' into feat/standard-att…
aitianshi Nov 17, 2026
608ef2a
fix(standard): wait out a retry's pause before moving the try before,…
aitianshi Nov 17, 2026
dc795aa
docs(standard): say an agent that does not start in time is retried
aitianshi Nov 17, 2026
30ce4ff
feat(standard): run retries as many at once as the run, and count tas…
aitianshi Nov 17, 2026
845f705
fix(runs): count the tries a retry replaced in the run's cost
aitianshi Nov 17, 2026
5544e1d
perf(traces): read an attempt's tries once a request
aitianshi Nov 17, 2026
5fb96be
refactor(standard): say --attempts wherever --repeat was said, confor…
aitianshi Nov 17, 2026
7a70d3b
feat(standard): before a run, refuse a model its provider does not li…
aitianshi Nov 17, 2026
8f86d32
docs(standard): cut the retries text to one line a row, the model che…
aitianshi Nov 17, 2026
e3dd0ec
Merge remote-tracking branch 'origin/main' into feat/standard-attempts
aitianshi Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude/skills/import-agent/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ run only if your instructions already say yes.
## 4. Prove

1. `cli conform KEY --model <cheap model> --unsandboxed` (or `--backend
docker`): every kit it can take; `--port PORT` for one kit, `--repeat 3`
docker`): every kit it can take; `--port PORT` for one kit, `--attempts 3`
to hold a flaky check by the majority.
2. **The relay saw its calls:**
`cat jobs_standard/*__kit__*__<key>*/*/agent/calls.jsonl | head -3`. Empty
Expand Down
5 changes: 5 additions & 0 deletions .claude/skills/run-analysis/scripts/runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,11 @@ def find_jobs(root, max_depth=6):
if isinstance(data, dict) and "task_name" in data:
jobs[os.path.dirname(cur)].append(cur)
dirs[:] = [] # nothing of interest below a trial
continue
if "config.json" in files and "retried" in dirs:
# a Nullius standard job keeps an attempt's tries before its last in retried/<trial>.try-N:
# none is a trial of the job, nor a job of its own
dirs.remove("retried")
return dict(jobs)


Expand Down
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ and its resources. The source layout stays under `app/`.
`feed`: the live feed any Harbor writes, from its own trial hooks),
`importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`),
`standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`,
`jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its
`jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `retries` (what an attempt is tried again after, and how many times), `pace` (how soon the next unit starts after a rate limit), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its
`payload` snapshots the running backend and effective catalogue for a remote host;
conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the
tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue:
Expand All @@ -72,7 +72,7 @@ and its resources. The source layout stays under `app/`.
(the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities/<key>/` (the keyword-memory `capability`); `workspace` (and
`workspace_cli`, its shell command),
`cli` and `__main__` (the unified `nullius` and `python -m nullius` commands),
`paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request, never by a run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine
`paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request and before a capped run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine
says is done, and where each step is done), `gitrefs` (a repository's branches and tags,
by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (Docker's state, and the Docker Hub login Data providers uses) and `redact` at the top.
- `app/templates/`, `app/static/` — the interface, grouped by the same capability
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
"setting": "--k",
"values": "1,10,100",
"default": "1,10,100",
"nullius": "fixed:pass@1 to pass@n with n the run's attempts (cli run --repeat n, the builder's Attempts per problem) - the job's result.json reports it, 1 - C(n-c, k)/C(n, k) as estimate_pass_at_k reads it; one attempt by default, pass@1"
"nullius": "fixed:pass@1 to pass@n with n the run's attempts (cli run --attempts n, the builder's Attempts per problem) - the job's result.json reports it, 1 - C(n-c, k)/C(n, k) as estimate_pass_at_k reads it; one attempt by default, pass@1"
},
{
"entry": "README",
"setting": "num_samples_per_task",
"values": "200",
"default": "200",
"nullius": "fixed:the run's attempts (--repeat), 1 by default - 200 agent runs per problem are offered, never assumed"
"nullius": "fixed:the run's attempts (--attempts), 1 by default - 200 agent runs per problem are offered, never assumed"
},
{
"entry": "evaluate_functional_correctness",
Expand Down
2 changes: 1 addition & 1 deletion app/nullius/catalogue/benchmarks/tau3-bench/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
retries, its user simulator), its max steps and max errors the run's
``max_steps`` and ``max_errors`` options (``bounds``). It draws the trial's seed from the
base seed as it does for every run: the unit's first attempt is
trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1
trial 0 of a one-trial run, its attempt N (``--attempts``) trial N-1
at the seed run_tasks draws for that trial (``trial``), and each
attempt is its ``run_single_task``: an ``Orchestrator`` built by its
``build_text_orchestrator`` at that seed, run by its
Expand Down
2 changes: 1 addition & 1 deletion app/nullius/catalogue/benchmarks/tau3-bench/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@
"limits": {
"turn_seconds": 7200,
"turns": 100,
"attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it."
"attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --attempts 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it."
},
"setups": {
"original": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
"setting": "--num-trials",
"values": "an integer from 1",
"default": "1 (DEFAULT_NUM_TRIALS)",
"nullius": "fixed:1 trial per attempt - a run's --repeat K (the builder's Attempts per unit) runs each unit K times, attempt N being the upstream's trial N-1 at the seed run_tasks draws for it (limits.attempts); the job's result.json carries pass^k"
"nullius": "fixed:1 trial per attempt - a run's --attempts K (the builder's Attempts per unit) runs each unit K times, attempt N being the upstream's trial N-1 at the seed run_tasks draws for it (limits.attempts); the job's result.json carries pass^k"
},
{
"entry": "tau2 run (src/tau2/cli.py)",
Expand Down Expand Up @@ -606,7 +606,7 @@
"setting": "trials (4 or more)",
"values": "an integer",
"default": "4",
"nullius": "fixed:1 a unit's attempt - --repeat 4 runs the leaderboard's four trials (limits.attempts)"
"nullius": "fixed:1 a unit's attempt - --attempts 4 runs the leaderboard's four trials (limits.attempts)"
},
{
"entry": "docs/leaderboard-submission.md",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
{"entry": "harbor run", "setting": "-m/--model", "values": "the agent's model", "default": "",
"nullius": "fixed:the run's model - sent through the relay to the agent chosen"},
{"entry": "harbor run", "setting": "-k/--n-attempts", "values": "attempts per task", "default": "1",
"nullius": "fixed:the run's attempts (cli run --repeat k, the builder's Attempts per task) - each a trial of its own, the job reporting pass^k and pass@k; 1 by default, Harbor's"},
"nullius": "fixed:the run's attempts (cli run --attempts k, the builder's Attempts per task) - each a trial of its own, the job reporting pass^k and pass@k; 1 by default, Harbor's"},
{"entry": "harbor run", "setting": "--timeout-multiplier and the agent/verifier/setup/build multipliers", "values": "", "default": "1.0",
"nullius": "fixed:1.0 - each task's own timeouts as task.toml declares them (version 2)"},
{"entry": "task.toml", "setting": "agent.timeout_sec", "values": "600 to 12000", "default": "per task",
Expand Down
34 changes: 34 additions & 0 deletions app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl
self.workspace = rec.get("workspace", "") # the workspace its job was added to, the one a restart adds it back to
# the trials the log has said already, (job, trial), so a restart says each once; none in a record from before
self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()}
self._settled: set = set() # the kept tries whose trial's copy was settled (settle_retried)
if record:
# a run still going reads its log now, to go on from it; one that ended (most of them, and
# history only grows) when its page asks - start-up read every one, seconds and hundreds of MB
Expand Down Expand Up @@ -1597,9 +1598,42 @@ def _mirror(self, run: Optional[Callable] = None, timeout: float = 900.0) -> Non
rc, out = self._copy_back(run, timeout)
if rc != 0:
self.log(" the job did not sync back: " + out.strip()[-300:])
else:
settle_retried(self.jobs, self._settled)
self._trial_lines()


def settle_retried(jobs: str, seen: Optional[set] = None) -> None:
"""A standard run's retry moves a trial's folder into ``retried/<trial>.try-N``
on the host and writes the trial again (``run.run_again``). The copy here is
taken without deleting, so the trial's own folder still holds the earlier
try's files the retry did not write again: each one the same, by size and
time, as the kept try's copy goes - what the retry wrote, and anything
written here, stays. ``seen``: the kept tries settled already, each
copied whole by the time it is first seen (the try had ended)."""
for job in os.listdir(jobs) if os.path.isdir(jobs) else ():
kept = os.path.join(jobs, job, "retried")
if not os.path.isdir(kept):
continue
for k in os.listdir(kept):
trial, sep, n = k.rpartition(".try-")
src, dest = os.path.join(kept, k), os.path.join(jobs, job, trial)
if not sep or not n.isdigit() or not os.path.isdir(src) or not os.path.isdir(dest) or src in (seen or ()):
continue
if seen is not None:
seen.add(src)
for d, dirs, files in os.walk(src):
here = os.path.join(dest, os.path.relpath(d, src))
for f in files:
a, b = os.path.join(d, f), os.path.join(here, f)
try:
x, y = os.lstat(a), os.lstat(b)
except OSError:
continue
if (x.st_size, int(x.st_mtime)) == (y.st_size, int(y.st_mtime)) and not os.path.isdir(b):
os.unlink(b)


def _probe(run: Callable, argv: List[str], timeout: float) -> tuple:
"""hosts.test wants (rc, stdout, stderr); the execution runner gives (rc, output)."""
rc, out = run(argv, timeout)
Expand Down
9 changes: 5 additions & 4 deletions app/nullius/judge/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,19 +104,20 @@ def ask_states(states: dict, key: str, post: Optional[Callable] = None, calibrat


def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, post: Optional[Callable] = None,
calibration: Optional[dict] = None) -> Optional[dict]:
calibration: Optional[dict] = None, started: Optional[str] = None) -> Optional[dict]:
"""`ask_states`, then the merged record saved under the trial's key -
its answers over the version's earlier ones. Returns the record as
stored. The pre-flight reads the workspace's own calibration unless one
is given, so a judge that counts more tokens per character than the rule
of thumb is refused on its own evidence."""
of thumb is refused on its own evidence. `started`: the trial's own
start, which the record keeps (`store.save`)."""
if calibration is None:
calibration = store.calibration(ws_dir)
rec = ask_states(states, key, post=post, calibration=calibration)
if rec is not None:
# a call over some of the kinds keeps the earlier readings for the others, as the answers are kept
lane = getattr(key, "lane", "system-one") # the LLM judge's records apart from System One's
earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane) or {}
earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane, started=started) or {}
rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})}
store.save(ws_dir, job_id, trial_name, rec, lane=lane)
store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started)
return rec
35 changes: 25 additions & 10 deletions app/nullius/judge/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,30 @@ def _versions(version: Optional[str]) -> tuple:
return (version,) if version else (Q.VERSION, *Q.EARLIER)


def _pick(data: dict, version: Optional[str]) -> Optional[dict]:
def _of(rec: Optional[dict], started: Optional[str]) -> bool:
"""Whether a record judged the try that started at ``started``: a retry
runs an attempt again under its trial's name, and what was asked of the
try before is not this one's. A record from before tries were told
apart, or a check that names no start, takes it as it is."""
return not started or not rec or rec.get("trial_started_at") in (None, started)


def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> Optional[dict]:
for v in _versions(version):
rec = data.get(v)
if rec:
if rec and _of(rec, started):
return rec
return None


def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one") -> Optional[dict]:
def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one",
started: Optional[str] = None) -> Optional[dict]:
"""The record under `version`; with none named, the current version or
the newest earlier one that has a record."""
the newest earlier one that has a record. `started`: the trial's own
start, so a record of an earlier try under its name is none of its."""
try:
with open(_path(ws_dir, job_id, trial_name, lane), encoding="utf-8") as fh:
return _pick(json.load(fh) or {}, version)
return _pick(json.load(fh) or {}, version, started)
except (OSError, ValueError):
return None

Expand All @@ -59,7 +69,8 @@ def is_current(record: Optional[dict]) -> bool:
return bool(record) and record.get("questions") == Q.VERSION


def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one") -> None:
def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one",
started: Optional[str] = None) -> None:
"""The record under its question version; other versions kept. A
record's answers merge over the version's earlier answers, so judging
one question later does not drop the others."""
Expand All @@ -72,7 +83,9 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "s
data = {}
version = record.get("questions", Q.VERSION)
earlier = data.get(version) or {}
merged = dict(record)
if not _of(earlier, started):
earlier = {} # an earlier try's: none of its answers are this one's
merged = dict(record, **({"trial_started_at": started} if started else {}))
merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})}
data[version] = merged
tmp = path + ".tmp"
Expand All @@ -81,9 +94,11 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "s
os.replace(tmp, path)


def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one") -> dict:
def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one",
started: Optional[dict] = None) -> dict:
"""Every judged trial of a job: {trial_name: record}, each the current
version's record or the newest earlier one, as `load` picks it."""
version's record or the newest earlier one, as `load` picks it.
`started`: {trial_name: its start}, as `load` takes it."""
d = os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id))
out = {}
try:
Expand All @@ -95,7 +110,7 @@ def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str =
continue
try:
with open(os.path.join(d, n), encoding="utf-8") as fh:
rec = _pick(json.load(fh) or {}, version)
rec = _pick(json.load(fh) or {}, version, (started or {}).get(n[:-5]))
except (OSError, ValueError):
continue
if rec:
Expand Down
Loading
Loading