diff --git a/.claude/skills/import-agent/SKILL.md b/.claude/skills/import-agent/SKILL.md index f56f5d28..ae7dfc11 100644 --- a/.claude/skills/import-agent/SKILL.md +++ b/.claude/skills/import-agent/SKILL.md @@ -193,7 +193,7 @@ run only if your instructions already say yes. ## 4. Prove 1. `cli conform KEY --model --unsandboxed` (or `--backend - docker`): every kit it can take; `--port PORT` for one kit, `--repeat 3` + docker`): every kit it can take; `--port PORT` for one kit, `--attempts 3` to hold a flaky check by the majority. 2. **The relay saw its calls:** `cat jobs_standard/*__kit__*__*/*/agent/calls.jsonl | head -3`. Empty diff --git a/.claude/skills/run-analysis/scripts/runs.py b/.claude/skills/run-analysis/scripts/runs.py index 949e7f05..95e6d87f 100644 --- a/.claude/skills/run-analysis/scripts/runs.py +++ b/.claude/skills/run-analysis/scripts/runs.py @@ -108,6 +108,11 @@ def find_jobs(root, max_depth=6): if isinstance(data, dict) and "task_name" in data: jobs[os.path.dirname(cur)].append(cur) dirs[:] = [] # nothing of interest below a trial + continue + if "config.json" in files and "retried" in dirs: + # a Nullius standard job keeps an attempt's tries before its last in retried/.try-N: + # none is a trial of the job, nor a job of its own + dirs.remove("retried") return dict(jobs) diff --git a/AGENTS.md b/AGENTS.md index 3985b757..afee95de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ and its resources. The source layout stays under `app/`. `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), `standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`, - `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its + `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `retries` (what an attempt is tried again after, and how many times), `pace` (how soon the next unit starts after a rate limit), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its `payload` snapshots the running backend and effective catalogue for a remote host; conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: @@ -72,7 +72,7 @@ and its resources. The source layout stays under `app/`. (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), - `paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request, never by a run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine + `paths` (user config, data and cache locations), `prices` (the user's model price list: fetched on request and before a capped run; what a cost is estimated from), `migrate` (explicit copying of legacy checkout storage), `settings`, `hosts`, `constraints` (what Nullius may use on each machine: one table, the container and systemd scope probes, the warnings), `execution`, `progress` (what a run is doing, one line per stage), `prerequisites` (what a recipe needs of each machine it runs on, asked before Run and again as it starts), `onboarding` (the Getting started checklist: what this machine says is done, and where each step is done), `gitrefs` (a repository's branches and tags, by `git ls-remote`), `secrets` (a benchmark's licences, kept per benchmark outside the repository, and the Hugging Face token of Data providers), `sandbox` (Docker's state, and the Docker Hub login Data providers uses) and `redact` at the top. - `app/templates/`, `app/static/` — the interface, grouped by the same capability diff --git a/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json b/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json index def442a8..79ddef45 100644 --- a/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/humaneval/upstream-settings.json @@ -11,14 +11,14 @@ "setting": "--k", "values": "1,10,100", "default": "1,10,100", - "nullius": "fixed:pass@1 to pass@n with n the run's attempts (cli run --repeat n, the builder's Attempts per problem) - the job's result.json reports it, 1 - C(n-c, k)/C(n, k) as estimate_pass_at_k reads it; one attempt by default, pass@1" + "nullius": "fixed:pass@1 to pass@n with n the run's attempts (cli run --attempts n, the builder's Attempts per problem) - the job's result.json reports it, 1 - C(n-c, k)/C(n, k) as estimate_pass_at_k reads it; one attempt by default, pass@1" }, { "entry": "README", "setting": "num_samples_per_task", "values": "200", "default": "200", - "nullius": "fixed:the run's attempts (--repeat), 1 by default - 200 agent runs per problem are offered, never assumed" + "nullius": "fixed:the run's attempts (--attempts), 1 by default - 200 agent runs per problem are offered, never assumed" }, { "entry": "evaluate_functional_correctness", diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py index f54c8669..cdcd00c2 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py +++ b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py @@ -29,7 +29,7 @@ retries, its user simulator), its max steps and max errors the run's ``max_steps`` and ``max_errors`` options (``bounds``). It draws the trial's seed from the base seed as it does for every run: the unit's first attempt is - trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1 + trial 0 of a one-trial run, its attempt N (``--attempts``) trial N-1 at the seed run_tasks draws for that trial (``trial``), and each attempt is its ``run_single_task``: an ``Orchestrator`` built by its ``build_text_orchestrator`` at that seed, run by its diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json index 9747780f..6f27dc7d 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json @@ -115,7 +115,7 @@ "limits": { "turn_seconds": 7200, "turns": 100, - "attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it." + "attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --attempts 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it." }, "setups": { "original": { diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json index 9f4f9ec4..5aea2dde 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json @@ -46,7 +46,7 @@ "setting": "--num-trials", "values": "an integer from 1", "default": "1 (DEFAULT_NUM_TRIALS)", - "nullius": "fixed:1 trial per attempt - a run's --repeat K (the builder's Attempts per unit) runs each unit K times, attempt N being the upstream's trial N-1 at the seed run_tasks draws for it (limits.attempts); the job's result.json carries pass^k" + "nullius": "fixed:1 trial per attempt - a run's --attempts K (the builder's Attempts per unit) runs each unit K times, attempt N being the upstream's trial N-1 at the seed run_tasks draws for it (limits.attempts); the job's result.json carries pass^k" }, { "entry": "tau2 run (src/tau2/cli.py)", @@ -606,7 +606,7 @@ "setting": "trials (4 or more)", "values": "an integer", "default": "4", - "nullius": "fixed:1 a unit's attempt - --repeat 4 runs the leaderboard's four trials (limits.attempts)" + "nullius": "fixed:1 a unit's attempt - --attempts 4 runs the leaderboard's four trials (limits.attempts)" }, { "entry": "docs/leaderboard-submission.md", diff --git a/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json b/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json index b781ad53..ac5ba7de 100644 --- a/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json @@ -12,7 +12,7 @@ {"entry": "harbor run", "setting": "-m/--model", "values": "the agent's model", "default": "", "nullius": "fixed:the run's model - sent through the relay to the agent chosen"}, {"entry": "harbor run", "setting": "-k/--n-attempts", "values": "attempts per task", "default": "1", - "nullius": "fixed:the run's attempts (cli run --repeat k, the builder's Attempts per task) - each a trial of its own, the job reporting pass^k and pass@k; 1 by default, Harbor's"}, + "nullius": "fixed:the run's attempts (cli run --attempts k, the builder's Attempts per task) - each a trial of its own, the job reporting pass^k and pass@k; 1 by default, Harbor's"}, {"entry": "harbor run", "setting": "--timeout-multiplier and the agent/verifier/setup/build multipliers", "values": "", "default": "1.0", "nullius": "fixed:1.0 - each task's own timeouts as task.toml declares them (version 2)"}, {"entry": "task.toml", "setting": "agent.timeout_sec", "values": "600 to 12000", "default": "per task", diff --git a/app/nullius/execution.py b/app/nullius/execution.py index aa8991ab..bd022e74 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -447,6 +447,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl self.workspace = rec.get("workspace", "") # the workspace its job was added to, the one a restart adds it back to # the trials the log has said already, (job, trial), so a restart says each once; none in a record from before self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()} + self._settled: set = set() # the kept tries whose trial's copy was settled (settle_retried) if record: # a run still going reads its log now, to go on from it; one that ended (most of them, and # history only grows) when its page asks - start-up read every one, seconds and hundreds of MB @@ -1597,9 +1598,42 @@ def _mirror(self, run: Optional[Callable] = None, timeout: float = 900.0) -> Non rc, out = self._copy_back(run, timeout) if rc != 0: self.log(" the job did not sync back: " + out.strip()[-300:]) + else: + settle_retried(self.jobs, self._settled) self._trial_lines() +def settle_retried(jobs: str, seen: Optional[set] = None) -> None: + """A standard run's retry moves a trial's folder into ``retried/.try-N`` + on the host and writes the trial again (``run.run_again``). The copy here is + taken without deleting, so the trial's own folder still holds the earlier + try's files the retry did not write again: each one the same, by size and + time, as the kept try's copy goes - what the retry wrote, and anything + written here, stays. ``seen``: the kept tries settled already, each + copied whole by the time it is first seen (the try had ended).""" + for job in os.listdir(jobs) if os.path.isdir(jobs) else (): + kept = os.path.join(jobs, job, "retried") + if not os.path.isdir(kept): + continue + for k in os.listdir(kept): + trial, sep, n = k.rpartition(".try-") + src, dest = os.path.join(kept, k), os.path.join(jobs, job, trial) + if not sep or not n.isdigit() or not os.path.isdir(src) or not os.path.isdir(dest) or src in (seen or ()): + continue + if seen is not None: + seen.add(src) + for d, dirs, files in os.walk(src): + here = os.path.join(dest, os.path.relpath(d, src)) + for f in files: + a, b = os.path.join(d, f), os.path.join(here, f) + try: + x, y = os.lstat(a), os.lstat(b) + except OSError: + continue + if (x.st_size, int(x.st_mtime)) == (y.st_size, int(y.st_mtime)) and not os.path.isdir(b): + os.unlink(b) + + def _probe(run: Callable, argv: List[str], timeout: float) -> tuple: """hosts.test wants (rc, stdout, stderr); the execution runner gives (rc, output).""" rc, out = run(argv, timeout) diff --git a/app/nullius/judge/batch.py b/app/nullius/judge/batch.py index 9a7a55d4..4b207638 100644 --- a/app/nullius/judge/batch.py +++ b/app/nullius/judge/batch.py @@ -104,19 +104,20 @@ def ask_states(states: dict, key: str, post: Optional[Callable] = None, calibrat def ask_and_store(ws_dir: str, job_id: str, trial_name: str, states: dict, key, post: Optional[Callable] = None, - calibration: Optional[dict] = None) -> Optional[dict]: + calibration: Optional[dict] = None, started: Optional[str] = None) -> Optional[dict]: """`ask_states`, then the merged record saved under the trial's key - its answers over the version's earlier ones. Returns the record as stored. The pre-flight reads the workspace's own calibration unless one is given, so a judge that counts more tokens per character than the rule - of thumb is refused on its own evidence.""" + of thumb is refused on its own evidence. `started`: the trial's own + start, which the record keeps (`store.save`).""" if calibration is None: calibration = store.calibration(ws_dir) rec = ask_states(states, key, post=post, calibration=calibration) if rec is not None: # a call over some of the kinds keeps the earlier readings for the others, as the answers are kept lane = getattr(key, "lane", "system-one") # the LLM judge's records apart from System One's - earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane) or {} + earlier = store.load(ws_dir, job_id, trial_name, version=rec["questions"], lane=lane, started=started) or {} rec["hygiene"] = {**(earlier.get("hygiene") or {}), **rec.get("hygiene", {})} - store.save(ws_dir, job_id, trial_name, rec, lane=lane) + store.save(ws_dir, job_id, trial_name, rec, lane=lane, started=started) return rec diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index 30394c8f..93e012b5 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -36,20 +36,30 @@ def _versions(version: Optional[str]) -> tuple: return (version,) if version else (Q.VERSION, *Q.EARLIER) -def _pick(data: dict, version: Optional[str]) -> Optional[dict]: +def _of(rec: Optional[dict], started: Optional[str]) -> bool: + """Whether a record judged the try that started at ``started``: a retry + runs an attempt again under its trial's name, and what was asked of the + try before is not this one's. A record from before tries were told + apart, or a check that names no start, takes it as it is.""" + return not started or not rec or rec.get("trial_started_at") in (None, started) + + +def _pick(data: dict, version: Optional[str], started: Optional[str] = None) -> Optional[dict]: for v in _versions(version): rec = data.get(v) - if rec: + if rec and _of(rec, started): return rec return None -def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one") -> Optional[dict]: +def load(ws_dir: str, job_id: str, trial_name: str, version: Optional[str] = None, lane: str = "system-one", + started: Optional[str] = None) -> Optional[dict]: """The record under `version`; with none named, the current version or - the newest earlier one that has a record.""" + the newest earlier one that has a record. `started`: the trial's own + start, so a record of an earlier try under its name is none of its.""" try: with open(_path(ws_dir, job_id, trial_name, lane), encoding="utf-8") as fh: - return _pick(json.load(fh) or {}, version) + return _pick(json.load(fh) or {}, version, started) except (OSError, ValueError): return None @@ -59,7 +69,8 @@ def is_current(record: Optional[dict]) -> bool: return bool(record) and record.get("questions") == Q.VERSION -def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one") -> None: +def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "system-one", + started: Optional[str] = None) -> None: """The record under its question version; other versions kept. A record's answers merge over the version's earlier answers, so judging one question later does not drop the others.""" @@ -72,7 +83,9 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "s data = {} version = record.get("questions", Q.VERSION) earlier = data.get(version) or {} - merged = dict(record) + if not _of(earlier, started): + earlier = {} # an earlier try's: none of its answers are this one's + merged = dict(record, **({"trial_started_at": started} if started else {})) merged["answers"] = {**(earlier.get("answers") or {}), **(record.get("answers") or {})} data[version] = merged tmp = path + ".tmp" @@ -81,9 +94,11 @@ def save(ws_dir: str, job_id: str, trial_name: str, record: dict, lane: str = "s os.replace(tmp, path) -def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one") -> dict: +def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = "system-one", + started: Optional[dict] = None) -> dict: """Every judged trial of a job: {trial_name: record}, each the current - version's record or the newest earlier one, as `load` picks it.""" + version's record or the newest earlier one, as `load` picks it. + `started`: {trial_name: its start}, as `load` takes it.""" d = os.path.join(_root(ws_dir, lane), _SAFE.sub("_", job_id)) out = {} try: @@ -95,7 +110,7 @@ def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = continue try: with open(os.path.join(d, n), encoding="utf-8") as fh: - rec = _pick(json.load(fh) or {}, version) + rec = _pick(json.load(fh) or {}, version, (started or {}).get(n[:-5])) except (OSError, ValueError): continue if rec: diff --git a/app/nullius/prices.py b/app/nullius/prices.py index e5653a7e..19c8d957 100644 --- a/app/nullius/prices.py +++ b/app/nullius/prices.py @@ -5,7 +5,7 @@ configuration, ``prices.json`` (``NULLIUS_PRICES`` names another: the file a host is sent), in two parts: -``models`` a public list, fetched on request (``update``) and kept whole +``models`` a public list, fetched on request and before a capped run (``update``) and kept whole with where it came from and when: LiteLLM's ``model_prices_and_context_window.json``, keyed by the model names the providers themselves use @@ -17,7 +17,7 @@ the model's output limit in tokens. A model is priced by exactly its name or not at all: a name that only starts like another's is another model. -Nothing here runs during a run: a run reads the file, never the network. +A run reads the file; ``cli run`` fetches the list first when the run has a spending cap. nullius prices # where the list came from, how many models, the user's own rows nullius prices list MODEL ... # those models' rows diff --git a/app/nullius/progress.py b/app/nullius/progress.py index b6a0c08a..f56c4d10 100644 --- a/app/nullius/progress.py +++ b/app/nullius/progress.py @@ -224,6 +224,8 @@ def fold(events: Iterable[dict]) -> dict: - ``units``: ``n`` to run, the ``noun``, how many ended and how: passed, failed, scored, errors; ``running`` the ones started and not ended. A run of several parts counts every part's: ``n`` grows as each begins. + A retry's end (``unit retried``, then its ``unit done``) takes the place + of the try before it: ``n`` counts attempts, not tries. - ``current``: the stage running now, else None; ``ended``: ``done``, ``refused`` (with ``detail``) or None while the run goes on. """ @@ -251,6 +253,11 @@ def fold(events: Iterable[dict]) -> dict: outcome = ev.get("outcome") bucket = {"passed": "passed", "failed": "failed", "scored": "scored"}.get(outcome, "errors") units[bucket] += 1 + elif state == "retried" and units["done"] and units["errors"]: + # a retry ended: it takes the place of its try before, an error, in the count + units["done"] -= 1 + units["errors"] -= 1 + started_units -= 1 units["running"] = max(0, started_units - units["done"]) continue if st in ("done", "refused"): diff --git a/app/nullius/providers.py b/app/nullius/providers.py index 7521faa7..f2f80427 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -367,8 +367,12 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): def _get(url: str, headers: dict, opener=None, timeout: float = 12) -> bytes: """One GET, never another method: no redirect followed, so a key stays at its endpoint; at most 8 MiB read.""" req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": "nullius", **headers}) - with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=timeout) as response: - content = response.read(_MOST + 1) + try: + with (opener or urllib.request.build_opener(_NoRedirect()).open)(req, timeout=timeout) as response: + content = response.read(_MOST + 1) + except urllib.error.HTTPError as e: + e.close() # its reply read by no one: closed, not left to the collector + raise if len(content) > _MOST: raise ValueError("response too large") return content diff --git a/app/nullius/runs/attempts.py b/app/nullius/runs/attempts.py index 33dff5d3..5394f930 100644 --- a/app/nullius/runs/attempts.py +++ b/app/nullius/runs/attempts.py @@ -1,7 +1,7 @@ """A task's attempts in one job. An attempt is one run of a task, counted whatever its result: Harbor's -``n_attempts`` and the standard's ``--repeat`` say how many. A retry runs an +``n_attempts`` and the standard's ``--attempts`` say how many. A retry runs an attempt again after the infrastructure or the agent failed, never because the task was failed, and only its last try is kept as the attempt's trial: Harbor deletes the others, the standard moves them into the job's ``retried/``. So a diff --git a/app/nullius/runs/live.py b/app/nullius/runs/live.py index 0932ea38..a0500b7b 100644 --- a/app/nullius/runs/live.py +++ b/app/nullius/runs/live.py @@ -218,6 +218,7 @@ class TrialState: started_at: datetime | None = None attempt: int = 1 #: its attempt; above 1 when Harbor retried it under the same name retried: list = field(default_factory=list) #: the outcomes its earlier attempts ended with + cost_before: float | None = None #: what its earlier attempts cost, counted in the board's total finished_at: datetime | None = None #: (ts, {token field: value}) per report — only the keys that report #: carried — for the rate window; see Board.tokens_per_sec. @@ -531,7 +532,7 @@ def _totals(trials: list) -> dict: for f in TOKEN_FIELDS: vals = [getattr(t, f) for t in trials if getattr(t, f) is not None] out[f] = sum(vals) if vals else None - costs = [t.cost_usd for t in trials if t.cost_usd is not None] + costs = [c for t in trials for c in (t.cost_usd, t.cost_before) if c is not None] # an earlier attempt's paid for too out["cost_usd"] = sum(costs) if costs else None # absent, never zero return out @@ -607,6 +608,13 @@ def trial_of(e: Event) -> TrialState: t.retried.append(t.outcome) t.outcome, t.reward, t.score, t.parts, t.finished_at = None, None, None, None, None t.phases_done, t.durations, t.started_at = [], {}, e.ts + # its counters are the new attempt's own: what the one before spent is not reported by this one, + # but it was paid for, and the board's cost goes on counting it + if t.cost_usd is not None: + t.cost_before = (t.cost_before or 0.0) + t.cost_usd + for key in COUNTERS: + setattr(t, key, None) + t._series = [] if t.phase and t.phase not in t.phases_done and t.phase != d["phase"]: t.phases_done.append(t.phase) t.phase, t.since = d["phase"], e.ts # the last phase event wins diff --git a/app/nullius/runs/parse.py b/app/nullius/runs/parse.py index 4d6e7290..8d657c8c 100644 --- a/app/nullius/runs/parse.py +++ b/app/nullius/runs/parse.py @@ -30,7 +30,7 @@ # The shape this module produces. A workspace caches parsed Jobs keyed on it, # so bump it when a Trial or Job field is added, renamed or means something # new — otherwise a cache written by the old parse is read back as current. -SCHEMA = "15" +SCHEMA = "17" # What a rate-limit record in `agent/gateway.log` looks like. The agent's SDK # retries a 429 below the harness (an agent's `max_retries` reaching its @@ -53,6 +53,28 @@ GATEWAY_SCAN_BYTES = 8 * 1024 * 1024 +def _retried_cost(trial_dir: str, n: int) -> float | None: + """What an attempt's ``n`` tries before its last cost, summed (each one's + ``retried/.try-N``, as ``tries`` reads them); None when none + recorded a cost.""" + if not n: + return None + job_dir, folder = os.path.split(trial_dir) + costs = [c for k in range(1, min(n, 99) + 1) + for c in [_token_totals(verifier._read_json(os.path.join(job_dir, "retried", f"{folder}.try-{k}", "result.json")) or {}) + .get("cost_usd")] + if isinstance(c, (int, float)) and not isinstance(c, bool)] + return sum(costs) if costs else None + + +def _retries_of(res: dict) -> int: + """``standard.retries`` of a standard trial's result or job's config: the + retries the trial took, or each attempt could take. 0 otherwise.""" + std = res.get("standard") if isinstance(res.get("standard"), dict) else {} + n = std.get("retries") + return n if isinstance(n, int) and not isinstance(n, bool) and n > 0 else 0 + + def count_rate_limits(path: str) -> int | None: """Rate-limit records in a gateway log, or None when there is no log. @@ -148,6 +170,16 @@ class Trial: # the run survived, which no result.json field records (`count_rate_limits`). # None when the trial has no `agent/gateway.log`: not measured, not zero. n_rate_limited_retries: int | None = None + # How many times the runner retried this attempt after an infrastructure or + # agent failure before this, its last try (the standard's --retries: + # `standard.retries`). 0 when it did not, or does not say: Harbor retries + # under the trial's own name and keeps no count in it. + n_retries: int = 0 + # What the tries before this one cost, summed: kept in the job's + # `retried/` (`tries`), never a trial of their own, but paid for all the + # same, so the job's cost counts them (`Job.total_cost_usd`). None when + # none recorded a cost. + retried_cost_usd: float | None = None # A graded verdict (an F1, partial credit), beside the pass or fail the # reward says: the runner records it when its benchmark scores that way. # A benchmark that gives a score and no pass or fail leaves the reward @@ -275,6 +307,11 @@ class Job: # A Nullius standard job (config ``standard``): its trials are written as each unit ends, so the # live board links one as soon as it has - unlike a runner's, imported at its end. standard: bool = False + # The retries a Nullius standard job allowed each attempt (config + # ``standard.retries``, its --retries); each attempt says how many it took + # (Trial.n_retries). 0 for every other job: Harbor counts its retries for + # the whole job only (``n_retries``). + retries: int = 0 # The launch a Nullius standard job was one shard of (config # ``standard.launch``): the shards of one launch, run on several machines, # are grouped as one on the runs page (launches.group). "" for every other job. @@ -391,8 +428,9 @@ def total_cache_tokens(self) -> int: @property def total_cost_usd(self): - """None when no trial reported a cost — not every agent does.""" - costs = [t.cost_usd for t in self.trials if t.cost_usd is not None] + """The trials' costs and their earlier tries' (``Trial.retried_cost_usd``); + None when none reported a cost — not every agent does.""" + costs = [c for t in self.trials for c in (t.cost_usd, t.retried_cost_usd) if c is not None] return sum(costs) if costs else None @property @@ -677,6 +715,8 @@ def parse_trial(job_id: str, trial_dir: str) -> Trial | None: has_trajectory=os.path.isfile(traj_path), has_trace=os.path.isfile(trace_path), n_rate_limited_retries=count_rate_limits(os.path.join(trial_dir, "agent", "gateway.log")), + n_retries=_retries_of(res), + retried_cost_usd=_retried_cost(trial_dir, _retries_of(res)), env_type=((res.get("config") or {}).get("environment") or {}).get("type", ""), started_at=_iso(res.get("started_at")), finished_at=_iso(res.get("finished_at")), @@ -684,6 +724,24 @@ def parse_trial(job_id: str, trial_dir: str) -> Trial | None: ) +def tries(trial: Trial) -> list[tuple[int, Trial]]: + """``(number, trial)`` for each try of an attempt, Try 1 first and + ``trial``, its last and the one the job counts, last; [] when no try + before it was kept. The standard moves each into the job's ``retried/`` + as ``.try-N`` before it runs the attempt again; Harbor keeps none. + A try's ``trial_name`` is ``.try-N``: how a page addresses it.""" + if not trial.n_retries: + return [] + job_dir, folder = os.path.split(trial.dir) + out = [] + # a record's count is read, not trusted: a page addresses at most try 99 (two digits) + for n in range(1, min(trial.n_retries, 99) + 1): + t = parse_trial(trial.job_id, os.path.join(job_dir, "retried", f"{folder}.try-{n}")) + if t: + out.append((n, replace(t, trial_name=f"{trial.trial_name}.try-{n}"))) + return out + [(trial.n_retries + 1, trial)] if out else [] + + def harbor_command(cfg: dict, env_type: str = "") -> str: """Reconstruct the `harbor run` invocation a job's config.json records. @@ -932,6 +990,7 @@ def parse_job(job_dir: str, reuse=None, trials: bool = True) -> Job: job.merged_from = [s["job"] for s in merged if isinstance(s, dict) and isinstance(s.get("job"), str) and s["job"]] if isinstance(merged, list) else [] job.budget = _budget(res.get("budget")) job.standard = bool(std) + job.retries = _retries_of(cfg) # a runner's job was never a `harbor run`: there is no command to rebuild job.harbor_command = "" if job.runner else redact(harbor_command( cfg, next((t.env_type for t in job.trials if t.env_type), ""))) diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 951868ac..36b36fa8 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,7 +6,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION - PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] + PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--attempts K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] @@ -34,7 +34,7 @@ version of an agent or a capability (``new version``: the default's pins, with the version replaced, and nothing proven); ``fit`` says, for one benchmark, what can run it and why the rest cannot; ``conform`` runs the kits and writes what passed into the -definition - the port, and each skill of its kit shown or not (``--repeat K``: +definition - the port, and each skill of its kit shown or not (``--attempts K``: each check K times, held by a majority); ``run`` runs a benchmark into a job under ``--jobs`` (``jobs_standard/``), which the application opens like any other; ``fidelity`` drives an agent over ACP and through its own command, compares the requests its model was sent, and @@ -74,6 +74,7 @@ import sys import tempfile +from nullius.standard.retries import MAX_RETRIES, RETRIES from nullius import constraints as limits, paths, prices as price_list, progress, providers, secrets, settings # The legacy module command needs saved path overrides before install.HOME # and other runtime defaults are computed, just like the unified CLI does. @@ -488,6 +489,17 @@ def prices(given) -> dict: return out +def _retries(value: str) -> int: + """``--retries``: a whole number from 0 to ``MAX_RETRIES``.""" + try: + n = int(value) + except ValueError: + raise argparse.ArgumentTypeError(f"{value!r} is not a whole number") + if not 0 <= n <= MAX_RETRIES: + raise argparse.ArgumentTypeError(f"{n} is not between 0 and {MAX_RETRIES}") + return n + + def cmd_conform(a): o = _other(a.other) try: @@ -533,13 +545,23 @@ def cmd_run(a): if why: print("refused: constraints:", why) return 2 + # a cap counts each call at its model's price: the public list fetched first (Settings, Model prices); a machine + # that cannot reach it runs uncapped where a model has no price, and says so (run.Run) + unreachable = None + if a.max_usd is not None or a.max_usd_per_unit is not None: + try: + price_list.update() + except ValueError as e: + unreachable = str(e) held, spill = licences(b, from_run=a.from_run) try: job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit, concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, - licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, - prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, + licences=held, bench_options=options(a.bench_option), repeat=a.repeat, retries=a.retries, + remove_images=a.remove_images, + prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, prices_unreachable=unreachable, + constraints=given, timeouts=timeouts(a.timeout), max_turns=a.max_turns, max_calls=a.max_calls, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), @@ -688,7 +710,7 @@ def main(argv=None): elif name == "conform": p.add_argument("--port", type=definitions.names.port, action="append", help="only this port's kit (repeatable)") p.add_argument("--stamp", help="the time stamp the kit jobs are named with (default: now)") - p.add_argument("--repeat", type=int, default=1, metavar="K", + p.add_argument("--attempts", "--repeat", dest="repeat", type=int, default=1, metavar="K", help="run each check K times; a check passes when it passes in most of them (default 1)") else: p.add_argument("--dataset") @@ -699,8 +721,12 @@ def main(argv=None): p.add_argument("-j", "--concurrency", type=int, default=4) p.add_argument("--judge-model", help="a benchmark scored by a model: this judge instead of its official one") p.add_argument("--simulator-model", help="a benchmark whose user a model plays: this model instead of its official one") - p.add_argument("--repeat", type=int, default=1, metavar="K", - help="run each unit K times, each an attempt of its own (the upstream's --num-trials, pass^k); default 1") + p.add_argument("--attempts", "--repeat", dest="repeat", type=int, default=1, metavar="K", + help="run each unit K times, each an attempt counted whatever its result " + "(the upstream's --num-trials, pass^k); default 1") + p.add_argument("--retries", type=_retries, default=RETRIES, metavar="N", + help="run a failed attempt again, up to N times, after the other units: never after a timeout, " + f"a wrong key or model, spent credit or a cap; 0 to {MAX_RETRIES}, default {RETRIES}") p.add_argument("--remove-images", action="store_true", help="remove each image this run pulls or builds when its last unit ends (default: kept for the next run)") p.add_argument("--name") diff --git a/app/nullius/standard/conform.py b/app/nullius/standard/conform.py index ea378a28..8cfbe4e8 100644 --- a/app/nullius/standard/conform.py +++ b/app/nullius/standard/conform.py @@ -28,7 +28,7 @@ from typing import Callable, Dict, List, Optional from nullius.runs import parse -from nullius.standard import definitions, ports, run +from nullius.standard import definitions, ports, retries, run STAMP = "%Y-%m-%d__%H-%M-%S" @@ -125,7 +125,10 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] job_dir = run.run(kit, other, model=model, provider=provider, jobs_root=jobs_root, concurrency=2, name=job_name(stamp or time.strftime(STAMP), kit["key"], definitions.job_word(other)), say=lambda s: say(" " + s), env=env, proving=True, unsandboxed=unsandboxed, - backend=backend, options=options, repeat=repeat) + backend=backend, options=options, repeat=repeat, + # a unit the provider's rate limit refused runs once more, after its pause: a proof says + # what the agent does, never how busy the endpoint was + retries=1, retry_if=retries.rate_limited) o = outcome(kit, _runs(parse.parse_job(job_dir)), repeat) got[p] = {"passed": o["passed"], "ok": o["ok"], "on": time.strftime("%Y-%m-%d"), "model": model, "provider": provider, "version": str((other.get("upstream") or {}).get("version") or ""), diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index e2e75da2..04ca51a2 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -462,7 +462,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), - serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None) -> dict: + serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None, + retries: int = 0, retryable: bool = True) -> dict: """Write one unit's trial and return its result. ``serving``: the run's pin of serving providers by model; ``standard.served`` says who served ``served_calls`` (the unit's calls on every route of its relay: its @@ -472,7 +473,9 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began (its setup, relay and container from ``t0``); ``checking``: the check's - start and end (its own step in the container, then the driver's score).""" + start and end (its own step in the container, then the driver's score). + ``retryable``: False for an exception a further try meets again + whatever its name (``retries``), which its record then says.""" os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True) os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True) tot = _usage_total(calls) @@ -490,7 +493,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), **({"environment": environment} if environment else {}), **({"image": image} if image else {}), - **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {})}} + **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {}), + **({"retries": retries} if retries else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score @@ -509,7 +513,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st _write(os.path.join(trial_dir, "verifier", "score.json"), json.dumps(dict({k: v for k, v in score.items() if k != "parts"}, says=bench["score"]["says"]), indent=1)) if exception: - res["exception_info"] = {"exception_type": exception[0], "exception_message": exception[1][:2000]} + res["exception_info"] = {"exception_type": exception[0], "exception_message": exception[1][:2000], + **({} if retryable else {"retryable": False})} _write(os.path.join(trial_dir, "agent", "calls.jsonl"), "".join(json.dumps(c) + "\n" for c in calls)) _write(os.path.join(trial_dir, "agent", "gateway.log"), "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) @@ -527,7 +532,7 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str, provider: str, dataset: str, units: List[str], concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", boat_size: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, - bench_options: Optional[dict] = None, + retries: int = 0, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None): @@ -538,7 +543,10 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str several machines (``shard``), which ``merge`` reads back. ``units_file``: the file a run's units were listed in (``run.units_file``), its name, its sha256 and its units, said only for such a run. ``repeat``: how - many times each unit was run, said only when more than once. ``judge`` and + many times each unit was run, each an attempt, said only when more than + once; ``retries``: how many times an attempt could be tried again after a + failure a further try might mend (``retries.retryable``), always said. + ``judge`` and ``simulator``: the models that graded and that played the user, when the benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. @@ -559,7 +567,7 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), - **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), + **({"repeat": repeat} if repeat > 1 else {}), "retries": retries, **({"prices": prices} if prices else {}), **({"constraints": constraints} if constraints else {}), **({"timeouts": timeouts} if timeouts else {}), **({"max_turns": max_turns} if max_turns else {}), **({"max_calls": max_calls} if max_calls else {}), @@ -618,6 +626,11 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(v, 4) for k, v in got["at"].items()}} +def n_retries(results: List[dict]) -> int: + """The retries a job's attempts took, all together: Harbor's ``stats.n_retries``, from each trial's ``standard.retries``.""" + return sum(int((r.get("standard") or {}).get("retries") or 0) for r in results) + + def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None, served: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, @@ -630,6 +643,7 @@ def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dic with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, - "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, + "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0, + "n_retries": n_retries(results)}, **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {}), **({"served": who} if who else {})}, f, indent=1) diff --git a/app/nullius/standard/jsonl.py b/app/nullius/standard/jsonl.py index 98bf61fd..892d2a7d 100644 --- a/app/nullius/standard/jsonl.py +++ b/app/nullius/standard/jsonl.py @@ -24,6 +24,10 @@ class Failed(Exception): """The program answered with an error, or did not answer as it should.""" +class NoReply(Failed): + """The program did not answer in its time: a run tries no unit again for it (``retries``).""" + + class Proc: def __init__(self, cmd: List[str], cwd: str, env: dict, log: str, stopped=None): self.cmd, self.cwd, self.env, self.log = cmd, cwd, env, log @@ -60,7 +64,7 @@ def request(self, op: str, timeout: float = 600, **kw) -> dict: line = wait_reply(self.lines, timeout, self.stopped) except queue.Empty: self.close() - raise Failed(f"{op}: no reply in {int(timeout)} s") + raise NoReply(f"{op}: no reply in {int(timeout)} s") if line is None: raise Failed(f"{op}: the program ended without replying ({self._tail()})") try: diff --git a/app/nullius/standard/ledger.py b/app/nullius/standard/ledger.py index d6134017..140eb392 100644 --- a/app/nullius/standard/ledger.py +++ b/app/nullius/standard/ledger.py @@ -242,8 +242,22 @@ def release(self, booking: str) -> None: if doc["held"].pop(booking, None) is not None: self._write(doc) + def unit_left(self, unit: str) -> Optional[float]: + """What ``unit`` may still spend under its own cap, every try of it + counted (``forget`` drops the count once no try follows); None + without a cap per unit. Read without the lock.""" + try: + doc = self._read() + except (OSError, ValueError, KeyError, TypeError) as e: + raise Unavailable(f"the spending cap's ledger could not be read: {type(e).__name__}: {e}") + each = doc.get("unit_max_usd") + if each is None: + return None + u = (doc.get("units") or {}).get(unit) or {} + return 0.0 if u.get("closed") else each - (u.get("spent_usd") or 0.0) - (u.get("unsettled_usd") or 0.0) + def forget(self, unit: str) -> None: - """A unit that ended: its own count goes, so the file stays the size of the units in flight.""" + """A unit that ended, and no try of it follows: its own count goes, so the file stays the size of the units in flight.""" with self._locked() as doc: if doc["units"].pop(unit, None) is not None: self._write(doc) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 3bdc33b4..a2fd8cf7 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -42,7 +42,7 @@ ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")), ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")), ("backend", ("standard", "backend")), ("serving providers", ("standard", "served_by")), - ("number of shards", ("standard", "shard", "of"))) + ("number of shards", ("standard", "shard", "of")), ("retries per attempt", ("standard", "retries"))) class Refused(Exception): @@ -69,6 +69,12 @@ def _at(d, path): return d +def _same(cfg: dict, path) -> object: + """What a shard's config says at ``path`` (``SAME``); a shard written before ``retries`` was recorded allowed none.""" + got = _at(cfg, path) + return got or 0 if path == ("standard", "retries") else got + + def _listed(cfg: dict) -> Optional[tuple]: """The units a shard's list holds (``standard.units_file``), sorted; None for a shard of no list.""" units = _at(cfg, ("standard", "units_file", "units")) @@ -102,6 +108,15 @@ def _trials(job_dir: str) -> List[str]: if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json"))) +def _tries(job_dir: str, trial: str) -> List[str]: + """The tries of ``trial`` before its last, kept in the job's ``retried/`` (``run.run_again``): their folders' names.""" + kept = os.path.join(job_dir, "retried") + if not os.path.isdir(kept): + return [] + return sorted(n for n in os.listdir(kept) if n.startswith(trial + ".try-") and n[len(trial) + 5:].isdigit() + and os.path.isfile(os.path.join(kept, n, "result.json"))) + + def _link_tree(src: str, dest: str) -> None: """``src`` into ``dest``, each file hard-linked (copied across devices).""" for d, dirs, files in os.walk(src): @@ -147,9 +162,9 @@ def check(job_dirs: List[str], force: bool = False) -> dict: + " (--force does not merge them)") differ = [] for label, path in SAME: - seen = {json.dumps(_at(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content + seen = {json.dumps(_same(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content if len(seen) > 1: - differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards)) + differ.append(f"{label}: " + ", ".join(f"{s['name']} {_same(s['config'], path)!r}" for s in shards)) if differ and not force: raise Refused("the shards are not of one launch - " + "; ".join(differ) + " (--force merges them anyway)") # every trial by its unit: two of one unit are never merged, force or not @@ -243,7 +258,8 @@ def result_of(shards: list, results: List[dict]) -> dict: return {"started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None, "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, - "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, + "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0, + "n_retries": job.n_retries(results)}, "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), "scored": len(scores), "mean_score": job.mean_score(results)}, **({"budget": budget} if budget else {})} @@ -262,16 +278,18 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: results = [] for s in shards: for t in s["trials"]: - src, dest = os.path.join(s["dir"], t), os.path.join(out, t) - _link_tree(src, dest) - # the trial's record is its own copy, naming the merged job and the shard it came from - res = _read(os.path.join(src, "result.json")) - std = dict(res.get("standard") or {}, job=name, shard_job=s["name"]) - res["standard"] = std - os.unlink(os.path.join(dest, "result.json")) - with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f: - json.dump(res, f, indent=1) - results.append(res) + # the trial, and each of its tries before it a retry kept (``retried/``), as the pages read them + for src, dest in [(os.path.join(s["dir"], t), os.path.join(out, t))] + [ + (os.path.join(s["dir"], "retried", k), os.path.join(out, "retried", k)) for k in _tries(s["dir"], t)]: + _link_tree(src, dest) + # the record is its own copy, naming the merged job and the shard it came from + res = _read(os.path.join(src, "result.json")) + std = dict(res.get("standard") or {}, job=name, shard_job=s["name"]) + res["standard"] = std + os.unlink(os.path.join(dest, "result.json")) + with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f: + json.dump(res, f, indent=1) + results.append(_read(os.path.join(out, t, "result.json"))) summary = result_of(shards, results) planned = summary["n_total_trials"] with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f: diff --git a/app/nullius/standard/pace.py b/app/nullius/standard/pace.py index 715aedca..f48356fc 100644 --- a/app/nullius/standard/pace.py +++ b/app/nullius/standard/pace.py @@ -4,10 +4,12 @@ provider; a unit whose last model call was refused that way ends as ``RateLimitError``. Starting the next unit at once only sends it into the same limit, so after such an end no new unit starts for a while: a pause that -doubles while units keep ending rate-limited (``FIRST`` seconds, up to -``LONGEST``) and resets on a unit that ends any other way. Units already -running are left alone; the concurrency asked for is unchanged, only how soon -the next unit starts. +doubles while units keep ending rate-limited (``FIRST`` seconds, a provider's +limits being mostly per minute, up to ``LONGEST``) and resets on a unit that +ends any other way - or the pause the provider said (``Retry-After``) when it +is longer. Units already running are left alone; the concurrency asked for is +unchanged, only how soon the next unit starts. A retry of a rate-limited +attempt waits the same pause from that attempt's end (``run.run_again``). """ from __future__ import annotations @@ -15,7 +17,7 @@ import time from typing import Optional -FIRST = 5.0 +FIRST = 60.0 LONGEST = 300.0 RATE_LIMITED = "RateLimitError" @@ -29,16 +31,26 @@ def __init__(self, *stops: threading.Event, first: Optional[float] = None, longe self.until = 0.0 self._lock = threading.Lock() - def ended(self, exception_type: Optional[str]) -> Optional[float]: + def ended(self, exception_type: Optional[str], said: Optional[float] = None) -> Optional[float]: """A unit's end: the pause it starts, in seconds, when it ended - rate-limited; None otherwise (and the pause is reset).""" + rate-limited - ``said``, the provider's own, when that is longer; None + otherwise (and the pause is reset).""" with self._lock: if exception_type != RATE_LIMITED: self.pause = 0.0 return None self.pause = min(self.longest, self.pause * 2 if self.pause else self.first) - self.until = max(self.until, time.time() + self.pause) - return self.pause + wait = max(self.pause, said or 0.0) + self.until = max(self.until, time.time() + wait) + return wait + + def due(self, ended_at: float, said: Optional[float] = None) -> float: + """A rate-limited unit tried again: nothing starts before its pause, + counted from its end (``ended_at``), is over - the doubled pause, at + least ``FIRST``, or ``said`` when longer. The seconds left.""" + with self._lock: + self.until = max(self.until, ended_at + max(self.first, self.pause, said or 0.0)) + return max(0.0, self.until - time.time()) def wait(self) -> None: """Until the pause is over, or the run is stopped.""" diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318e..20682da6 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -49,7 +49,7 @@ from typing import Callable, Dict, List, Optional from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, run, shard as shards +from nullius.standard import definitions, install, jsonl, retries, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -216,7 +216,7 @@ def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jo job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images) + remove_images=remove_images, retries=1, retry_if=retries.rate_limited) # as conform's j = parse.parse_job(job_dir) n = len(j.trials) errored = sum(1 for t in j.trials if t.exception_type) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 678ab3f8..1ee2509c 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -44,6 +44,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay, run +from nullius.standard.retries import MAX_RETRIES CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -461,6 +462,11 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, repeat = 1 if lb and repeat < lb.get("repeat", 1): add("limits", f"Like the leaderboard runs each {(b.get('units') or {}).get('noun', 'unit')} at least {lb['repeat']} times.") + # each attempt run again up to N times when it ends in a failure a further try may mend (standard.retries) + retries = 0 if recipe.get("retries") in (None, "") else _int(recipe.get("retries"), 0, MAX_RETRIES) + if retries is None: + add("limits", f"Retries per attempt: a whole number from 0 to {MAX_RETRIES}.") + retries = 0 sim_model = str(recipe.get("simulator_model") or "").strip() judge_model = str(recipe.get("judge_model") or "").strip() like = {"paper": "Like the paper", "leaderboard": "Like the leaderboard"}.get(model_choice) @@ -645,7 +651,8 @@ def command_for(d: dict, p: dict, index: int) -> str: words = ["nullius", "standard", "run", definitions.ref_of(b), o["_ref"], *(["--model", m] if m else []), "--provider", provider, "--dataset", d["id"], *(["--limit", str(limit)] if isinstance(limit, int) else []), "-j", str(conc), *[w for k, v in options_for_pick(d, p).items() for w in ("--bench-option", f"{k}={v}")], - *(["--repeat", str(repeat)] if repeat > 1 else []), + *(["--attempts", str(repeat)] if repeat > 1 else []), + *(["--retries", str(retries)] if retries else []), *(["--max-usd", f"{share:.6f}".rstrip("0").rstrip(".")] if share and spends(p) else []), *(["--max-usd-per-unit", f"{per_unit:.6f}".rstrip("0").rstrip(".")] if per_unit and spends(p) else []), *[w for k, v in forced.items() for w in ("--timeout", f"{k}={v:g}")], @@ -750,6 +757,7 @@ def row(p: dict) -> list: + (" of each dataset" if many and isinstance(limit, int) else "") + (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "") + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "") + + (f" · up to {retries} {'retry' if retries == 1 else 'retries'} an attempt" if retries else "") + (" · a setup counts " + next(p["o"]["units"]["nouns"] for p in picks if p.get("setup")) if any(p.get("setup") for p in picks) and not all(p.get("setup") for p in picks) else "")], *([[STEP["limits"], "Spending cap", f"${ledger.usd(cap)}" diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 1923088c..82320991 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -102,6 +102,41 @@ def _said(status: int, body: bytes) -> str: return f"HTTP {status}: {m or ''}".strip()[:300] +def _code(body: bytes) -> Optional[str]: + """An error reply's own code (OpenAI's ``code`` or ``type``), None when it gives none.""" + try: + e = json.loads(body).get("error") + except Exception: + return None + if not isinstance(e, dict): + return None + code = e.get("code") if isinstance(e.get("code"), str) else e.get("type") + return code[:80] if isinstance(code, str) and code else None + + +def _retry_after(headers) -> Optional[float]: + """Seconds the provider says to wait (``Retry-After``, in seconds or as a date; ``retry-after-ms``), None when unsaid.""" + got = {n.lower(): v for n, v in headers or ()} + try: + if "retry-after-ms" in got: + return max(0.0, float(got["retry-after-ms"]) / 1000) + if "retry-after" in got: + v = got["retry-after"].strip() + try: + return max(0.0, float(v)) + except ValueError: + from email.utils import parsedate_to_datetime + return max(0.0, parsedate_to_datetime(v).timestamp() - time.time()) + except (TypeError, ValueError, OverflowError): + return None + return None + + +# a provider's words for a model it does not have: no second call finds it +_NO_MODEL = re.compile(r"\b(not a valid model|invalid model|unknown model|no such model)\b" + r"|\bmodels?\b[^.]{0,80}?\b(does not exist|not exist|not found)\b", re.I) + + def _usage(u: Optional[dict]) -> Optional[dict]: """Chat Completions', Responses' and Messages' usage, in one shape: ``input`` counts every prompt token, ``cached`` those of them a cache @@ -797,6 +832,11 @@ def record(body): rec["cost_usd"], rec["cost_from"] = cost, how else: rec["error"] = _said(status, body) + code, wait = _code(body), _retry_after(back) + if code: + rec["error_code"] = code + if wait is not None: + rec["retry_after"] = wait # the provider's word on when to call again (pace) if booking is not None: rec["booked_usd"] = booked try: @@ -991,10 +1031,13 @@ def failure(calls: List[dict]) -> Optional[tuple]: last = calls[-1] s, said = last.get("status"), last.get("error") or "" if s == 429: - return "RateLimitError", said + # OpenAI's spent credit comes as a 429 too: a usage limit, no pace mends it + return ("ApiUsageLimitError" if last.get("error_code") == "insufficient_quota" else "RateLimitError"), said + if s == 402 and not last.get("refused"): + return "ApiUsageLimitError", said # the provider's credit spent (the relay's own refusals say so above, or are its own) if s in (401, 403): return "AgentAuthenticationError", said - if s == 404 and "model" in said.lower(): + if (s == 404 and "model" in said.lower()) or (s == 400 and _NO_MODEL.search(said)): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said diff --git a/app/nullius/standard/retries.py b/app/nullius/standard/retries.py new file mode 100644 index 00000000..a3d09459 --- /dev/null +++ b/app/nullius/standard/retries.py @@ -0,0 +1,48 @@ +"""What an attempt is tried again after, and how many times (``cli run --retries``). + +An attempt is one run of a unit, counted whatever its result (``--attempts``). +A retry runs an attempt again after a failure a further try may mend: after +the run's other units, as many at once as the run's own, up to ``--retries`` times (0 by +default, ``MAX_RETRIES`` at most). Each try before is kept under the job's +``retried/``; only the last is the attempt's trial and counts. + +Which failures, as Harbor's retry decides it (its ``RetryConfig``): any +exception but those a second try meets again - a wrong key or model, a usage +limit, a time the agent or the benchmark ran out of - and here a missing +secret, a setup this machine cannot give (the trial's ``exception_info`` says +``retryable: false``), the spending caps and a stopped run. A time run out +while the provider refused or failed the calls is the provider's failure +(``relay.failure``), retried; a rate limit's retry waits the provider's +pause (``pace``). A task the agent failed carries no exception and is never +retried. An attempt that spent its own spending cap, all its tries together, +is not tried again. +""" +from __future__ import annotations + +from typing import Optional + +from nullius.standard import ledger, pace, relay + +RETRIES = 0 +MAX_RETRIES = 9 + +NOT_RETRIED = frozenset({ + # Harbor's RetryConfig.exclude_exceptions: a result, or a mistake a second try does not mend + "AgentTimeoutError", "VerifierTimeoutError", "RewardFileNotFoundError", "RewardFileEmptyError", + "VerifierOutputParseError", "ApiUsageLimitError", "AgentSafetyRefusalError", "AgentAuthenticationError", + "ModelNotFoundError", + # this runner's own: a secret no second try provides, the spending caps and the calls cap, a stopped run + "EnvironmentSecretMissingError", ledger.ERROR, ledger.UNIT_ERROR, relay.CALLS_ERROR, "CancelledError", +}) + + +def retryable(res: Optional[dict]) -> bool: + """Whether an attempt's result is a failure a retry may mend.""" + info = (res or {}).get("exception_info") or {} + et = info.get("exception_type") + return bool(et) and et not in NOT_RETRIED and info.get("retryable") is not False + + +def rate_limited(res: Optional[dict]) -> bool: + """Whether an attempt ended refused by the provider's rate limit: what a proof run tries again after.""" + return ((res or {}).get("exception_info") or {}).get("exception_type") == pace.RATE_LIMITED diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index a2978c1b..3d9f0f68 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -75,6 +75,7 @@ from nullius.redact import redact from nullius.harbor import feed as live_feed from nullius.standard import acp, boat, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards +from nullius.standard.retries import RETRIES, retryable # the totals a unit's end carries onto the live board, from its trial's agent_result FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd") @@ -160,7 +161,7 @@ def units_file(path: str) -> dict: counts = Counter(ids) twice = [u for u in counts if counts[u] > 1] if twice: - raise Refused(f"--units {path} lists {', '.join(twice)} more than once: list each unit once (--repeat K runs each K times)") + raise Refused(f"--units {path} lists {', '.join(twice)} more than once: list each unit once (--attempts K runs each K times)") return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} @@ -232,7 +233,7 @@ def option_env(values: dict) -> dict: # The endpoint refusing the run itself, not one unit: a key it rejects, a model it does not know. -REFUSALS = ("AgentAuthenticationError", "ModelNotFoundError") +REFUSALS = ("AgentAuthenticationError", "ModelNotFoundError", "ApiUsageLimitError") REFUSED_IN_A_ROW = 3 # a unit still running when the run is stopped: Harbor's name for a trial a stopped job cut short CANCELLED = "CancelledError" @@ -415,10 +416,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, proving: bool = False, unsandboxed: bool = False, judge_model: Optional[str] = None, backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, - launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None, + launch: Optional[str] = None, repeat: int = 1, retries: int = RETRIES, + retry_if: Optional[Callable[[dict], bool]] = None, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, - max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, + max_usd_per_unit: Optional[float] = None, prices_unreachable: Optional[str] = None, + units_file: Optional[dict] = None, constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): """``proving``: the run is a conformance kit's, the one run that may @@ -441,7 +444,13 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas units. ``repeat``: each unit run that many times (a conformance check run for confidence, ``conform``), each run a trial of its own tagged - ``u0042r2``; once by default, tagged ``u0042`` as always. + ``u0042r2``; once by default, tagged ``u0042`` as always. Each is an + attempt, counted whatever its result. + ``retries``: how many times an attempt that ended in a failure a + further try may mend (``retry_if``, ``retries.retryable`` unless + given: a proof run's is ``retries.rate_limited``) runs again, after + the others and as many at once as the concurrency; each try before + is kept under the job's ``retried/``. 0 by default: none. ``simulator_model``: a benchmark whose user a model plays (``simulator``): this model instead of its official one. ``prices``: this run's own prices, by model, over the user's list @@ -580,7 +589,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.embed_model = providers.served_as(needs["embeddings"][0], provider) if needs["embeddings"] else None self.rerank_model = providers.served_as(needs["reranker"][0], provider) if needs["reranker"] else None # What a call's cost is estimated from when the provider does not say it: the user's price list - # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price. + # (nullius.prices, fetched before a capped run by ``cli run``), this run's own prices over it. Read once, here. self.run_prices = dict(prices or {}) self.prices = {**price_list.table(), **self.run_prices} self.max_usd, self.unit_max_usd, self.ledger = None, None, None @@ -598,11 +607,18 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.max_usd = cap else: self.unit_max_usd = cap + self.cap_dropped = None # why the spending cap asked for is not applied if self.max_usd is not None or self.unit_max_usd is not None: unpriced = [m for m in self.models_called() if m not in self.prices] - if unpriced: + if unpriced and prices_unreachable: + # the price list could not be fetched before the run (``cli run``): no price, no cap, and the user told + self.cap_dropped = (f"no spending cap: the price list could not be fetched, and " + f"{', '.join(unpriced)} {'has' if len(unpriced) == 1 else 'have'} no price") + self.cap_why = prices_unreachable + self.max_usd = self.unit_max_usd = None + elif unpriced: raise Refused("a spending cap needs a price for every model the run calls, and there is none for " - + ", ".join(unpriced) + ": nullius prices update, nullius prices set MODEL INPUT OUTPUT, " + + ", ".join(unpriced) + ": nullius prices set MODEL INPUT OUTPUT, " "or --price MODEL=INPUT,OUTPUT") on_host = needs["host"] if self.upstream and backend == "container": @@ -638,10 +654,19 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) + self.provider_env = env # where the provider's model list is asked (``known_models``) self.key_header = "authorization" if keyless else key_header(provider, other, env) self.limit, self.concurrency = limit, limits.concurrency(max(1, int(concurrency)), self.constraints) self.units_file = units_file self.repeat = max(1, int(repeat or 1)) + self.retries = max(0, int(retries or 0)) + self.retry_if = retry_if or retryable + # an attempt a try may follow keeps what that try needs: its own image (removed when no try follows, + # ``release_image``), its unit's count of what it spent (``ledger.forget``), and a rate limit's end + # (when it ended, when the provider said to call again), which its retry's pause counts from + self._held_images: Dict[str, str] = {} + self._counted: Dict[str, str] = {} + self._waits: Dict[str, tuple] = {} # an own image this run pulled or built stays for the next run, unless the user asks it removed # when its last unit ends (never with a repeat): {image: how many units are using it now} self.keep_images = not remove_images or self.repeat > 1 @@ -749,7 +774,10 @@ def start(self) -> str: if (self.max_usd is not None or self.unit_max_usd is not None) and self.base: # one ledger for the job, in a folder of its own: all a unit's relay container is given of the job besides its record self.ledger = ledgers.Ledger.create(os.path.join(self.job_dir, ".work", "ledger", "ledger.json"), self.max_usd, self.unit_max_usd) + if self.cap_dropped: + self.say(f"warning: {self.cap_dropped} ({self.cap_why})") try: + self.known_models() self.ready() except Refused as e: # said in the job, which holds nothing else: the runs list reads why there (progress.refusal) @@ -806,7 +834,8 @@ def start(self) -> str: why = self.boat.starts(len(units) * self.repeat) if why: self.say(why) - p.emit("units", "start", n=len(units), of=len(everything), noun=self.bench["units"]["nouns"]) + p.emit("units", "start", n=len(units) * self.repeat, of=len(everything), noun=self.bench["units"]["nouns"], + **({"detail": self.cap_dropped} if self.cap_dropped else {})) job.write_config(self.job_dir, job=self.name, bench=self.bench, other=self.other, model=self.model, provider=self.provider, dataset=self.ds["id"], units=units, concurrency=self.concurrency, granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend, @@ -816,7 +845,7 @@ def start(self) -> str: # a shard of a list splits the listed units: they are its whole shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, - repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, + repeat=self.repeat, retries=self.retries, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, timeouts=self.timeouts, max_turns=self.forced_turns, max_calls=self.max_calls, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run @@ -836,7 +865,7 @@ def start(self) -> str: self.loopback = None self.pace = pacing.Pace(self._stopped, self._halt) self._running = 0 - again: Dict[str, dict] = {} # a rate-limited unit's second attempt, by its trial's name + again: Dict[str, dict] = {} # an attempt's last try, by its trial's name, when it was retried def work_unit(iur): self.pace.wait() # a unit ended rate-limited: the next waits its pause out @@ -872,7 +901,7 @@ def work_unit(iur): res = future.result() if res is not None: # None: a unit not begun once the endpoint refused the run results.append(res) - self.run_again(results, work, work_unit, again) + self.run_again(results, work, work_unit, again, pool) except BaseException: self._stopped.set() self._halt.set() @@ -882,6 +911,7 @@ def work_unit(iur): finally: # a Stop or Ctrl-C: the units not begun are not begun, those running end with their agents pool.shutdown(wait=True, cancel_futures=True) + self.let_go() begun = [not f.cancelled() and f.exception() is None and f.result() is not None for f in futures] results = [again.get(f.result()["trial_name"], f.result()) for f, ran in zip(futures, begun) if ran] for d in self.drivers: @@ -923,54 +953,140 @@ def work_unit(iur): if self.bench["score"]["kind"] == "pass": ok = sum(1 for w in rewards if w.get("reward") == 1.0) p.emit("done", n=len(results), passed=ok, seconds=round(time.time() - t0, 1)) - self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {ok} passed") + self.say(f"done: {self.counted(results)}, {ok} passed") else: scores = [w["score"] for w in rewards if "score" in w] mean = job.mean_score(results) p.emit("done", n=len(results), scored=len(scores), mean=round(mean, 3) if mean is not None else None, seconds=round(time.time() - t0, 1)) - self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {len(scores)} scored" + self.say(f"done: {self.counted(results)}, {len(scores)} scored" + (f", mean {mean:.3f}" if mean is not None else "")) return self.job_dir + def counted(self, results: List[dict]) -> str: + """What ended, as the run's last line says it: ``3 problems``, and + ``6 attempts`` beside them when each was attempted more than once.""" + n = len({(r.get("standard") or {}).get("unit") for r in results}) + words = self.bench["units"] + return f"{n} {words['noun'] if n == 1 else words['nouns']}" + (f", {len(results)} attempts" if self.repeat > 1 else "") + def paced(self, res: Optional[dict]) -> None: """A unit's end, told to the run's pace: one that ended rate-limited - holds the next unit back, said with how many units run meanwhile.""" + holds the next unit back - as long as the provider said, when it said + longer - said with how many units run meanwhile.""" if res is None: return - pause = self.pace.ended((res.get("exception_info") or {}).get("exception_type")) + _, ready = self._waits.get(res.get("trial_name"), (None, None)) + pause = self.pace.ended((res.get("exception_info") or {}).get("exception_type"), + said=ready - time.time() if ready else None) if pause is not None: - self.progress.emit("units", "progress", detail=f"rate limited: no unit starts for {pause:g} s; " + self.progress.emit("units", "progress", detail=f"rate limited: no unit starts for {pause:.3g} s; " f"{self._running} running of {self.concurrency} at once") - def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, dict]) -> None: - """Each unit that ended rate-limited, run once more after the others, - one at a time and paced as the rest: its first attempt moved into the - job's ``retried/`` (kept whole, its calls and its cost with it, where - no reader takes it for a trial of its own), its second written under - the trial's own name and counted (into ``done``, as each ends). Nothing - is run again once the run is stopped or halted; the spending cap goes - on counting every call.""" + def tag_of(self, i: int, r: int = 1) -> str: + """A unit's attempt as its relay's routes and the ledger name it: ``u0042``, ``u0042r2`` with attempts.""" + return f"u{i:04d}" + (f"r{r}" if self.repeat > 1 else "") + + def cap_spent(self, i: int, r: int = 1) -> bool: + """Whether an attempt spent its own spending cap, all its tries + together: no try follows then. A ledger that cannot be read refuses + every capped call, so none follows either.""" + if getattr(self, "ledger", None) is None or self.unit_max_usd is None: + return False + try: + left = self.ledger.unit_left(self.tag_of(i, r)) + except ledgers.Unavailable: + return True + return left is not None and left <= 0 + + def let_go(self, name: Optional[str] = None) -> None: + """What an attempt's last try held for a try that does not follow - + ``name``'s, or every attempt's: its image released, its count dropped + from the ledger.""" + for n in [name] if name is not None else list({**self._held_images, **self._counted}): + image = self._held_images.pop(n, None) + if image: + self.release_image(image) + tag = self._counted.pop(n, None) + if tag is not None and getattr(self, "ledger", None) is not None and self.unit_max_usd is not None: + try: + self.ledger.forget(tag) # the unit ended, no try of it to follow: its own count goes + except ledgers.Unavailable: + pass + + def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, dict], pool) -> None: + """Each attempt that ended in a failure a further try may mend + (``retry_if``: a rate limit, a provider or setup error, an agent + crash), run again after the others, on the run's own ``pool`` (as + many at once as its concurrency) and paced as the rest - a rate + limit's retry not before the provider's pause is over - up to + ``retries`` times. Each try before is moved into the job's + ``retried/`` as ``.try-N`` (kept whole, its calls and its cost + with it, where no reader takes it for a trial of its own); the last is + written under the trial's own name and counted (into ``done``, as each + ends), its ``result.json`` saying how many retries it took. Nothing is + run again once the run is stopped or halted, nor an attempt that spent + its own spending cap, all its tries together.""" by_name = {self.trial_name(*iur): iur for iur in work} - todo = [by_name[r["trial_name"]] for r in results - if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name] - if not todo: - return - self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time") - self.say(f"running again, one at a time: {len(todo)} rate-limited") - for iur in todo: + latest = {r["trial_name"]: r for r in results if r["trial_name"] in by_name} + for retry in range(1, self.retries + 1): if self._stopped.is_set() or self._halt.is_set(): - break - name = self.trial_name(*iur) - keep = os.path.join(self.job_dir, "retried", name) - os.makedirs(os.path.dirname(keep), exist_ok=True) - os.replace(os.path.join(self.job_dir, name), keep) - self.feed.emit("trial.queued", name, agent=self.other["key"], position=0) - res = work_unit(iur) - if res is not None: - done[name] = res - elif not os.path.exists(os.path.join(self.job_dir, name)): - os.replace(keep, os.path.join(self.job_dir, name)) # not begun after all: the first attempt stands + return + # each attempt whose last try ended with a try to follow (``unit``: what it held stays held) + todo = [name for name, res in latest.items() if name in self._counted and self.retry_if(res)] + if not todo: + return + self.progress.emit("units", "progress", detail=f"trying {len(todo)} again: retry {retry} of {self.retries}") + self.say(f"trying again: {len(todo)}, retry {retry} of {self.retries}") + tries = {pool.submit(self.try_again, name, by_name[name], retry, work_unit): name for name in todo} + pending = set(tries) + while pending: + # timed, as the first pass: a Stop's signal is run in this thread + ended, pending = wait(pending, timeout=0.5, return_when=FIRST_EXCEPTION) + for future in ended: + res = future.result() + if res is not None: + done[tries[future]] = latest[tries[future]] = res + + def try_again(self, name: str, iur: tuple, retry: int, work_unit) -> Optional[dict]: + """One attempt's next try (``run_again``): the try before moved into + ``retried/.try-``, the next run under the trial's name. + A try that does not end - not begun, or stopped under it - leaves the + try before where it was, the attempt as it stood; what the stopped + try had written goes.""" + here = os.path.join(self.job_dir, name) + keep = os.path.join(self.job_dir, "retried", f"{name}.try-{retry}") + ended, ready = self._waits.pop(name, (None, None)) + if ended is not None: # a rate limit's: not before the provider's pause, from its end + left = self.pace.due(ended, ready - ended if ready else None) + if left > 0: + self.progress.emit("units", "progress", detail=f"rate limited: trying again in {left:.0f} s") + self.pace.wait() # the attempt stands as it is while it waits + if self._stopped.is_set() or self._halt.is_set(): + return None + os.makedirs(os.path.dirname(keep), exist_ok=True) + os.replace(here, keep) + held = self._held_images.pop(name, None) # the try before's image, kept for this one + self.feed.emit("trial.queued", name, agent=self.other["key"], position=0) + res = None + try: + res = work_unit((*iur, retry)) + if res is not None and self._stopped.is_set() and (res.get("exception_info") or {}).get("exception_type") == CANCELLED: + res = None # stopped under it: the try before says more than the Stop + finally: + if held: + self.release_image(held) + if res is None: + if os.path.lexists(here): + gone = keep + ".stopped" + os.replace(here, gone) + shutil.rmtree(gone, ignore_errors=True) + os.replace(keep, here) + try: + os.rmdir(os.path.dirname(keep)) # retried/ holds no try now: it goes + except OSError: + pass + return res def ready(self) -> None: """What the run needs before its first unit, each a stage said as it @@ -1086,13 +1202,13 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non if self.feed is not None: self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout) - def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: + def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()): return None # the endpoint refused the run: nothing more is begun if getattr(self, "ledger", None) is not None and self.ledger.closed(): self._halt.set() # the spending cap refused a call: every later one is refused too return None - tag = f"u{i:04d}" + (f"r{r}" if self.repeat > 1 else "") + tag = self.tag_of(i, r) self.local.attempt = r trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) @@ -1111,33 +1227,35 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: setup: dict = {} transcript: List[dict] = [] rl, environment, box, changes = None, None, None, None + image, finished = None, False # its own image; whether it ended, or was broken off (a Stop) + retry_ok = True # whether its failure is one a further try may mend try: try: # a setup's unit needs none: the upstream's answerer reads its data itself setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.phase_limit("setup", DRIVER_SECONDS), **self.attempt_kw()) - except jsonl.Failed as e: - raise _Ended("EnvironmentSetupError", str(e)) + except jsonl.Failed as e: # a driver that did not answer in its time is not asked again + raise _Ended("EnvironmentSetupError", str(e), retryable=not isinstance(e, jsonl.NoReply)) environment, env_def = self.unit_environment(setup) why = self.not_run(environment, env_def) - if why: - raise _Ended(*why) + if why: # this machine cannot give it: no further try can + raise _Ended(*why, retryable=False) own = definitions.own_image(env_def) if "internet" in setup and (not isinstance(setup["internet"], bool) or not own): - raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image") + raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image", retryable=False) if self.other["_kind"] == "agent" and self.backend == "boat" and (mcp_machine.wanted(setup) or setup.get("mcp")): - raise _Ended("EnvironmentSetupError", "not run: " + boat.MCP_REFUSAL) + raise _Ended("EnvironmentSetupError", "not run: " + boat.MCP_REFUSAL, retryable=False) if self.other["_kind"] == "agent" and self.backend == "local" and mcp_machine.wanted(setup): try: # the benchmark's own servers: the agent starts them from the driver's environment setup = dict(setup, mcp=mcp_machine.entries(self.bench, setup, install.driver_dir(self.bench), self.machine_env(trial))) except mcp_machine.Unknown as e: - raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}") + raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}", retryable=False) if "env" in setup and not own: - raise _Ended("EnvironmentSetupError", "the unit's setup: env requires an own image") + raise _Ended("EnvironmentSetupError", "the unit's setup: env requires an own image", retryable=False) if setup.get("verify") is not None and not own: - raise _Ended("EnvironmentSetupError", "the unit's setup: verify runs in an own image, and this unit has none") + raise _Ended("EnvironmentSetupError", "the unit's setup: verify runs in an own image, and this unit has none", retryable=False) if setup.get("prepare") is not None and not own: - raise _Ended("EnvironmentSetupError", "the unit's setup: prepare runs in an own image, and this unit has none") + raise _Ended("EnvironmentSetupError", "the unit's setup: prepare runs in an own image, and this unit has none", retryable=False) try: rl = self.unit_relay(tag, trial, setup, env_def, own) except (container.Unavailable, OSError) as e: @@ -1182,12 +1300,15 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e)) + raise _Ended("VerifierError", str(e), retryable=not isinstance(e, jsonl.NoReply)) + finished = True except _Ended as e: exc = (e.kind, e.said) traj = traj or e.trajectory + retry_ok, finished = e.retryable, True except Exception as e: # anything else is this unit's error, never the whole job's exc = (type(e).__name__, f"{type(e).__name__}: {e}\n" + "".join(traceback.format_exc().splitlines(True)[-6:])) + finished = True finally: if checking[0] is not None and checking[1] is None: checking = (checking[0], time.time()) @@ -1195,7 +1316,9 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: # the home it wrote as root given back, then the container gone box.give_back([home, *getattr(rl, "unit_rw", [])]) box.stop() - self.release_image(box.image) + image = box.image + if not finished: # broken off: no try of it follows + self.release_image(image) if rl is not None: rl.stop() for ln in getattr(rl, "mcp_listeners", None) or []: @@ -1214,7 +1337,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it - score = None + score, retry_ok = None, True # a time run out under refused calls is the provider's: tried again cut = self.cap_reached(rl, tag) # at scoring too: its judge's call, or a refusal its agent made something else of if cut: exc, score = cut, None @@ -1238,7 +1361,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), - environment=environment or "", serving=getattr(self, "serving", None), + environment=environment or "", retries=retry, retryable=retry_ok, serving=getattr(self, "serving", None), # who served the unit: its agent's calls and its judge's and simulated user's routes served_calls=calls + ([c for side in ("judge", "simulator") for c in rl.calls(f"{tag}-{side}")] if rl is not None else []), @@ -1246,7 +1369,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" self.say(f" {unit}: {verdict}") - self.said_unit(unit, score, exc) + self.said_unit(unit, score, exc, retry) if self.feed is not None: self.feed.emit("trial.end", name, task=unit, agent=self.other["key"], **job.feed_outcome(self.bench["score"]["kind"], score, exc), @@ -1260,13 +1383,27 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: elif exc and exc[0] == ledgers.UNIT_ERROR: with self._said: self._over.append(self.attempt_name(unit, r)) # its own cap: the others go on - if getattr(self, "ledger", None) is not None and self.unit_max_usd is not None: - try: - self.ledger.forget(tag) # the unit ended: its own count goes from the ledger - except ledgers.Unavailable: - pass + if exc and exc[0] == pacing.RATE_LIMITED: # when it ended, and when the provider said to call again + side = [c for x in ("judge", "simulator") for c in rl.calls(f"{tag}-{x}")] if rl is not None else [] + self._waits[name] = (time.time(), self.provider_ready(calls + side)) + # a try may follow: its image and its count of what it spent stay for it (``let_go`` when none does) + self._held_images.pop(name, None) + self._counted[name] = tag + if image: + self._held_images[name] = image + if not (retry < self.retries and self.retry_if(res) and not self._stopped.is_set() + and not (getattr(self, "_halt", None) is not None and self._halt.is_set()) and not self.cap_spent(i, r)): + self.let_go(name) # its rate limit's end stays: the next unit's pause reads it (``paced``) return res + @staticmethod + def provider_ready(calls: List[dict]) -> Optional[float]: + """When the provider said its refused calls could come again + (``Retry-After``, ``relay``): the latest; None when it never said.""" + said = [c["t1"] + c["retry_after"] for c in calls + if c.get("status") == 429 and isinstance(c.get("retry_after"), (int, float)) and isinstance(c.get("t1"), (int, float))] + return max(said) if said else None + def cap_reached(self, rl, tag: str) -> Optional[tuple]: """A spending cap's refusal among a unit's calls - its agent's, its judge's or its simulated user's - as what the unit ends with: cut @@ -1320,12 +1457,17 @@ def note_refusal(self, exc: Optional[tuple]) -> None: else: self._refused.clear() - def said_unit(self, unit: str, score: Optional[dict], exc: Optional[tuple]) -> None: + def said_unit(self, unit: str, score: Optional[dict], exc: Optional[tuple], retry: int = 0) -> None: """A unit's end, as a progress line: the i-th to end of the run's n, - and how - passed or failed (a pass benchmark), scored, or the error.""" + and how - passed or failed (a pass benchmark), scored, or the error. + A retry's end takes the place of its try before, an error, which + leaves the count (``unit retried``): the run's n counts attempts.""" with self._said: - self._ended += 1 + if not retry: + self._ended += 1 i = self._ended + if retry: + self.progress.emit("unit", "retried", unit=unit) score = score or {} if exc: how = {"outcome": "error", "error": exc[0]} @@ -1342,7 +1484,7 @@ def unit_environment(self, setup: dict): try: return definitions.environment_of(self.bench, setup, self.ds) except ValueError as e: - raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}") + raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}", retryable=False) def not_run(self, name: Optional[str], env_def: Optional[dict]) -> Optional[tuple]: """Why a unit cannot be run in its environment here, as (kind, said), @@ -1440,6 +1582,24 @@ def benchmark_calls(self) -> List[dict]: calls += [json.loads(line) for line in f if line.strip()] return calls + def known_models(self) -> None: + """Refused before anything is installed when the provider lists its + models and one this run calls is not among them: every call to it + would be refused the same way. A list that cannot be read checks + nothing; the first refusals then stop the run (``note_refusal``). + A name's ``:suffix`` (OpenRouter's ``:nitro``, ``:floor``) routes the + model it follows.""" + if not self.base: + return + try: + listed = {m["id"] for m in providers.list_models(self.provider, getattr(self, "provider_env", os.environ))} + except ValueError: + return + unknown = [m for m in self.models_called() if m not in listed and m.rpartition(":")[0] not in listed] + if listed and unknown: + raise Refused(f"{providers.label(self.provider, getattr(self, 'provider_env', os.environ))} lists no model " + f"{', '.join(unknown)}: check the name") + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the @@ -1480,7 +1640,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] setup["mcp"], doors, listeners = mcp_machine.doors(self.bench, setup, install.driver_dir(self.bench), self.machine_env(trial), work, work) except mcp_machine.Unknown as e: - raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}") + raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}", retryable=False) from . import web_services try: doors.extend(web_services.doors(setup)) @@ -1622,10 +1782,11 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: try: reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=self.phase_limit("agent", DRIVER_SECONDS)) except jsonl.Failed as e: - raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e)) + raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e), retryable=not isinstance(e, jsonl.NoReply)) if reply.get("solve") is not None: if box is None: - raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none") + raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none", + retryable=False) if name: # the reference solution, run under its own limit self.phase(name, unit, "agent_run", self.step_limit("agent", reply["solve"])) done = self.in_box(box, reply["solve"], collect=False, what="the benchmark's oracle") @@ -1634,7 +1795,8 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: f.write(done["output"]) if done["exit"] != 0: raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: its solution " - + ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}")) + + ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}"), + retryable=not done["timed_out"]) return str(reply.get("answer") or "") # -- an own image: the unit's container ------------------------------------ @@ -1701,10 +1863,10 @@ def own_box(self, tag: str, setup: dict, rl, home: str): if setup.get("workdir") is not None and not (isinstance(setup["workdir"], str) and setup["workdir"].startswith("/")): bad.append("workdir: an absolute path inside the image") if bad: - raise _Ended("EnvironmentSetupError", "the unit's setup: " + "; ".join(bad)) + raise _Ended("EnvironmentSetupError", "the unit's setup: " + "; ".join(bad), retryable=False) why = container.unusable() if why: - raise _Ended("EnvironmentSetupError", f"not run: its own image needs docker, and {why}") + raise _Ended("EnvironmentSetupError", f"not run: its own image needs docker, and {why}", retryable=False) box = None try: image = self.own_image(spec) @@ -1724,7 +1886,7 @@ def own_box(self, tag: str, setup: dict, rl, home: str): for name in runtimes: why = box.runtime_problem(name) if why: - raise _Ended("EnvironmentSetupError", "not run: " + why) + raise _Ended("EnvironmentSetupError", "not run: " + why, retryable=False) return box except (container.Unavailable, ValueError, OSError) as e: if box is not None: @@ -1846,7 +2008,7 @@ def prepare(self, step: dict, box, trial: str) -> None: if done["exit"] != 0: raise _Ended("EnvironmentSetupError", "the benchmark's preparation " + ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}") - + ": " + done["output"].strip()[-300:]) + + ": " + done["output"].strip()[-300:], retryable=not done["timed_out"]) def verify(self, step: dict, box, trial: str) -> dict: """The benchmark's check, run in the unit's container once the agent @@ -1989,7 +2151,7 @@ def left() -> float: raise _Ended("AgentTimeoutError", f"the agent did not answer {e} in time", job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}")) except jsonl.Failed as e: - raise _Ended("VerifierError", "the benchmark's next turn: " + str(e)) + raise _Ended("VerifierError", "the benchmark's next turn: " + str(e), retryable=not isinstance(e, jsonl.NoReply)) finally: c.close() raw = "".join(json.dumps(e) + "\n" for e in events) @@ -2045,7 +2207,7 @@ def send(op, **kw): for q in plan.get("recall") or []] except jsonl.Failed as e: raise _Ended("NonZeroAgentExitCodeError", "the capability: " + str(e), - job.trajectory_from_exchanges(exchanges, [], f, self.model, f"{self.name}/{unit}")) + job.trajectory_from_exchanges(exchanges, [], f, self.model, f"{self.name}/{unit}"), retryable=not isinstance(e, jsonl.NoReply)) finally: proc.close() try: @@ -2053,7 +2215,7 @@ def send(op, **kw): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) except jsonl.Failed as e: raise _Ended("VerifierError", "the benchmark's reader: " + str(e), - job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}")) + job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}"), retryable=not isinstance(e, jsonl.NoReply)) transcript.append({"role": "capability", "recalled": recalled}) return out.get("answer"), job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}") @@ -2081,15 +2243,17 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): + (" - the original's models do not reason" if o["key"] == "model" else ""), job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) raise _Ended("NonZeroAgentExitCodeError", f"{o['label']}: " + str(e), - job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}"), retryable=not isinstance(e, jsonl.NoReply)) transcript.append({"role": "upstream", "setup": o["key"], "asked": self.asked, "sent": self.model}) return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") class _Ended(Exception): - def __init__(self, kind, said, trajectory=None): + """A unit's end in an error: ``kind``, Harbor's exception name; ``retryable`` False for one a further try + meets again whatever its name (a setup this machine cannot give, a time run out: ``retries``).""" + def __init__(self, kind, said, trajectory=None, retryable=True): super().__init__(said) - self.kind, self.said, self.trajectory = kind, said, trajectory + self.kind, self.said, self.trajectory, self.retryable = kind, said, trajectory, retryable def run(bench: dict, other: dict, **kw) -> str: diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 4248df2a..cb849255 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -189,8 +189,9 @@ keepEndpoint(r, raw); if ((Number.isInteger(raw.limit) && raw.limit >= 1) || raw.limit === '') r.limit = raw.limit; if ((Number.isInteger(raw.concurrency) && raw.concurrency >= 1) || raw.concurrency === '') r.concurrency = raw.concurrency; - // attempts per unit and the judge's and user simulator's models: a reopened recipe runs as it was saved + // attempts per unit, retries per attempt, and the judge's and user simulator's models: a reopened recipe runs as it was saved if (Number.isInteger(raw.repeat) && raw.repeat >= 1 && raw.repeat <= 32) r.repeat = raw.repeat; + if (Number.isInteger(raw.retries) && raw.retries >= 0 && raw.retries <= 9) r.retries = raw.retries; // the run's spending cap in USD, shared by its jobs (recipe.py): an amount above zero, or none if (typeof raw.max_usd === 'number' && Number.isFinite(raw.max_usd) && raw.max_usd > 0) r.max_usd = raw.max_usd; // and each unit's own: a unit that reaches it is cut off there, the run going on @@ -376,7 +377,7 @@ const r = {version: 1, harness: 'harbor', name: 'harbor-experiment', title: 'Harbor experiment', benchmark: '', source: '', dataset: '', path: '', generatorRoot: '', agents: [], harbor: {kind: 'release', version: '', extras: []}, customAgents: [], model_choice: 'any', provider: firstAvailable(providersOf({}), providerOf({}, schema.defaultModel || '')), model: '', userModel: '', evalModel: '', keyEnv: (schema.credentialVars && schema.credentialVars.key) || 'API_TOKEN', urlEnv: schema.credentialVars ? (schema.credentialVars.url || '') : 'API_URL', - tasks: '', concurrency: 1, retries: 3, + tasks: '', concurrency: 1, retries: 0, timeoutMultiplier: 1, agentMinutes: '', attempts: '', setupTimeoutMultiplier: '', verifierTimeoutMultiplier: '', buildTimeoutMultiplier: '', harborDev: '', include: '', output: 'jobs', where: 'local', adapterPaths: {}, params: defaults(), custom: []}; adoptModel(r, r.provider, schema.defaultModel || ''); @@ -513,15 +514,16 @@ if (callsModel && benchmarkCallsModel(r) && providerOf(r, r.model || r.userModel) !== 'openai' && !r.urlEnv) add(STEP.model, 'A non-OpenAI model needs an endpoint source variable.'); if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(r.name)) add(STEP.limits, 'Recipe name may contain letters, numbers, dots, underscores and hyphens.'); if (!r.output.trim()) add(STEP.limits, 'Enter the output directory.'); - for (const key of ['tasks','concurrency','retries']) { - const min = key === 'retries' ? 0 : 1; + for (const key of ['tasks','concurrency']) { if (key === 'tasks' && r.tasks === '') continue; // empty: every task, no limit written - if (!Number.isInteger(Number(r[key])) || Number(r[key]) < min) add(STEP.limits, key + ' must be an integer of at least ' + min + ', or empty for every task.'); + if (!Number.isInteger(Number(r[key])) || Number(r[key]) < 1) add(STEP.limits, key + ' must be an integer of at least 1, or empty for every task.'); } + // empty: none, as the standard's field reads it + if (r.retries !== '' && r.retries != null && (!Number.isInteger(Number(r.retries)) || Number(r.retries) < 0)) add(STEP.limits, 'Retries per attempt must be a whole number of at least 0, or empty.'); if (!Number.isFinite(Number(r.timeoutMultiplier)) || !(Number(r.timeoutMultiplier) > 0)) add(STEP.agent, 'Timeout multiplier must be positive.', 'Common'); // Common trial settings (Harbor's own per-trial flags): empty means the flag is not written. for (const [key, label] of COMMON) if (r[key] !== '' && (!Number.isFinite(Number(r[key])) || !(Number(r[key]) > 0))) add(STEP.agent, label + ' must be a positive number, or empty.', 'Common'); - if (r.attempts !== '' && (!Number.isInteger(Number(r.attempts)) || Number(r.attempts) < 1)) add(STEP.agent, 'Repeats per task must be a whole number of at least 1, or empty.', 'Common'); + if (r.attempts !== '' && (!Number.isInteger(Number(r.attempts)) || Number(r.attempts) < 1)) add(STEP.agent, 'Attempts per task must be a whole number of at least 1, or empty.', 'Common'); if (r.agentMinutes !== '' && r.agentMinutes != null) { if (!Number.isFinite(Number(r.agentMinutes)) || !(Number(r.agentMinutes) > 0)) add(STEP.limits, 'Max agent time per trial must be a positive number of minutes.'); else if (!declaredTimeouts(r)) add(STEP.limits, 'No run has recorded this benchmark\'s declared timeouts, so minutes cannot be turned into a multiplier here; set the multiplier directly.'); @@ -597,7 +599,7 @@ parts.push(...agentCustom(r, id).map(p=>'--ak '+q(p.key+'='+kwargValue(customValue(p))))); // The reference agent's tool timeout follows the shared MCP timeout, so both agents wait the same. if (id === 'tau3-llm-agent' && r.params.mcp_timeout_s !== '' && r.params.mcp_timeout_s != null) parts.push('--ak '+q('tool_timeout_sec='+Number(r.params.mcp_timeout_s))); - parts.push(...(r.tasks === '' ? [] : ['--n-tasks '+r.tasks]),'--n-concurrent '+r.concurrency,'--max-retries '+r.retries, + parts.push(...(r.tasks === '' ? [] : ['--n-tasks '+r.tasks]),'--n-concurrent '+r.concurrency,'--max-retries '+(r.retries === '' || r.retries == null ? 0 : Number(r.retries)), '--agent-timeout-multiplier '+r.timeoutMultiplier,'--job-name '+q(r.name+(r.agents.length>1?'-'+id:'')), '--jobs-dir '+(runner ? '"$NULLIUS_JOBS"' : path(r.output))); if (r.include) parts.push('--include-task-name '+q(r.include)); parts.push(...commonFlags(r)); diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index c6ca0d70..73dd6d8b 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -150,7 +150,7 @@ }; const commonSettings = rec => `

Common settings

Every trial, whichever agent runs it.

` + `
${timeLimits(rec)}
` - + `
${field('Repeats per task', 'attempts', rec.attempts, {type: 'number', min: 1, placeholder: '1', hint: 'Every task is run this many times, pass or fail (Harbor\'s --n-attempts). Retries after a failure are on Limits.'})}
` + + `
${field('Attempts per task', 'attempts', rec.attempts, {type: 'number', min: 1, placeholder: '1'})}
` ; // One tab per selected agent: the parameters this catalogue knows for it, // its extra --ak list, and what else it needs. @@ -685,7 +685,8 @@ return '

Scope

' + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '
' - + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + + '
' + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + + field('Retries per attempt', 'retries', rec.retries ?? '', {type: 'number', min: 0, max: 9, placeholder: '0'}) + '
' // a conversation's most agent turns (recipe.py: run --max-turns): only a benchmark that holds conversations has turns + ((b.needs || []).includes('agent.conversation/v1') ? field('Turns per conversation', 'turns', rec.turns ?? '', {type: 'number', min: 1, placeholder: String((b.timeouts || {}).turns || ''), @@ -834,7 +835,7 @@ html='

Scope & execution

'; const allTasks=(()=>{const ds=schema.datasets.find(d=>d.benchmark===rec.benchmark);return ds&&ds.tasks?'all '+ds.tasks:'all';})(); html+=`
${field('Number of tasks','tasks',rec.tasks,{type:'number',min:1,placeholder:allTasks,hint:'Empty: '+allTasks+'.'})}${field('Concurrent trials','concurrency',rec.concurrency,{type:'number',min:1})}
`; - html+=`
${field('Retries per trial','retries',rec.retries,{type:'number',min:0,hint:'Run a trial again after an error, up to this many times.'})}${field('Include task names · optional glob','include',rec.include,{placeholder:'django*'})}
` + html+=`
${field('Retries per attempt','retries',rec.retries,{type:'number',min:0,placeholder:'0'})}${field('Include task names · optional glob','include',rec.include,{placeholder:'django*'})}
` +harborExecutableFields(rec); html+=field('Output directory','output',rec.output,{}); } diff --git a/app/static/runs/board.css b/app/static/runs/board.css index 90d346a7..aa2bf810 100644 --- a/app/static/runs/board.css +++ b/app/static/runs/board.css @@ -274,7 +274,7 @@ .rp .tscroll { position: relative; overflow-x: auto; } /* auto layout: the other columns keep their widths as minimums and Task takes the rest; past that the table scrolls */ .rp table.tt { width: 100%; border-collapse: collapse; } -.rp .w-out { width: 206px; } .rp .w-cause { width: 168px; } .rp .w-flags { width: 132px; } .rp .w-chk { width: 64px; } .rp .w-ins { width: 78px; } .rp .w-tok { width: 80px; } +.rp .w-out { width: 206px; } .rp .w-cause { width: 168px; } .rp .w-flags { width: 132px; } .rp .w-chk { width: 64px; } .rp .w-ins { width: 78px; } .rp .w-tok { width: 80px; } .rp .w-rt { width: 64px; } .rp .w-tm { width: 68px; } .rp .w-env { width: 54px; } .rp .w-ag { width: 58px; } .rp .w-ver { width: 56px; } .rp .w-go { width: 36px; } .rp .tt th { font: 400 12px/1.3 var(--sans); color: var(--text-faint); text-align: left; padding: 0 10px 7px; white-space: nowrap; } .rp .tt thead tr.grp th { padding-top: 12px; padding-bottom: 0; } diff --git a/app/static/runs/live-board.css b/app/static/runs/live-board.css index 4ef11b10..b8639c34 100644 --- a/app/static/runs/live-board.css +++ b/app/static/runs/live-board.css @@ -172,7 +172,7 @@ } @media (prefers-reduced-motion: reduce) { .lb *, .view-tabs * { animation: none !important; transition: none !important; } } -/* a trial Harbor runs again under its own name: which attempt, and how the last one ended */ +/* an attempt retried under its own name (Harbor's max_retries, the standard's --retries): how often, and how the try before ended */ .lb-retry { display: block; color: var(--patch_failed_tests, var(--text-dim)); font-size: 11.5px; } /* an ended trial, kept below the running ones */ .lb-ended .lb-outcome { display: inline-flex; align-items: center; gap: 6px; font-size: 12.5px; color: var(--text-dim); } diff --git a/app/static/runs/live-board.js b/app/static/runs/live-board.js index 1db5bd18..359e2604 100644 --- a/app/static/runs/live-board.js +++ b/app/static/runs/live-board.js @@ -200,7 +200,7 @@ const k = PHASES.indexOf(t.phase), p = pressure(t), tok = tokensOf(t), line = t.activity || t.last_line; const dim = opts.filter && t.outcome !== opts.filter ? ' dim' : ''; return ` - ${esc(shortTask(t))}${agents.length > 1 ? `${esc(t.agent || '')}` : ''}${t.attempt > 1 ? `attempt ${t.attempt}, after ${esc((t.retried || []).slice(-1)[0] || 'an error').replace('_', ' ')}` : ''} + ${esc(shortTask(t))}${agents.length > 1 ? `${esc(t.agent || '')}` : ''}${t.attempt > 1 ? `retried ${t.attempt === 2 ? 'once' : (t.attempt - 1) + ' times'}, after ${esc((t.retried || []).slice(-1)[0] || 'an error').replace('_', ' ')}` : ''} ${PHASES.map((_, j) => ``).join('')}${esc(PHASE_LABEL[t.phase] || t.phase || '—')} ${clock(t.elapsed_sec)} of ${t.timeout_sec ? clock(t.timeout_sec) : '—'} ${has.steps ? `${t.step == null ? '—' : esc(t.step)}` : ''} diff --git a/app/static/style.css b/app/static/style.css index 83fa94b0..87529940 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1032,10 +1032,25 @@ tr.control-row td { color: var(--text-dim); } .attempt-tab:not(.all)::before { content: ""; width: 8px; height: 8px; border-radius: 2px; flex: none; } .attempt-tab.oc-unknown::before { background: var(--unknown); } .attempt-tab:hover { color: var(--text); border-color: var(--text-faint); } -.attempt-tab[aria-current="page"] { color: var(--text); background: var(--accent-soft); border-color: var(--accent); font-weight: 600; } +.attempt-tab[aria-current] { color: var(--text); background: var(--accent-soft); border-color: var(--accent); font-weight: 600; } .attempt-tab:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.attempt-retried { width: 12px; height: 12px; margin-left: -.15rem; flex: none; fill: none; stroke: currentColor; stroke-width: 1.6; stroke-linecap: round; stroke-linejoin: round; } +/* an attempt's tries, when its runner kept the ones before its last: tabs of + their own under the attempt's, underlined, the last the one that counts */ +.try-tabs { display: flex; flex-wrap: wrap; gap: 0 1.2rem; width: fit-content; max-width: 100%; margin-top: .6rem; padding-right: .4rem; border-bottom: 1px solid var(--border); } +.try-tab { + display: inline-flex; align-items: center; gap: .45rem; padding: .3rem .1rem .45rem; margin-bottom: -1px; + border-bottom: 2px solid transparent; color: var(--text-dim); font-size: .8rem; text-decoration: none; white-space: nowrap; +} +.try-tab::before { content: ""; width: 8px; height: 8px; border-radius: 2px; flex: none; } +.try-tab.oc-unknown::before { background: var(--unknown); } +.try-tab:hover { color: var(--text); } +.try-tab[aria-current="page"] { color: var(--text); border-bottom-color: var(--accent); font-weight: 600; } +.try-tab:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.try-dur { color: var(--text-faint); font-weight: 400; font-variant-numeric: tabular-nums; } .attempt-table .num { text-align: right; font-variant-numeric: tabular-nums; } .attempt-table tr[aria-current="true"] td { background: var(--accent-soft); } +.attempt-table tr.replaced td { opacity: .55; } /* outcome tag inside a table row */ .tag { diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 962e4998..97a571b7 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -365,13 +365,13 @@

Trials

- {% if show_cause %}{% endif %}{% if show_flags %}{% endif %}{% if show_ins %}{% endif %} + {% if job.retries %}{% endif %}{% if show_cause %}{% endif %}{% if show_flags %}{% endif %}{% if show_ins %}{% endif %} {% if vocab.gained %}{% endif %}{% if vocab.held %}{% endif %} - + {% if n_judge %}{% endif %} {% if vocab.gained or vocab.held %}{% endif %} @@ -381,6 +381,7 @@

Trials

+ {% if job.retries %}{% endif %} {% if show_cause %}{% endif %} {% if show_flags %}{% endif %} {% if show_ins %}{% endif %} @@ -405,6 +406,7 @@

Trials

+ {% if job.retries %}{% endif %} {% if show_cause %}{% endif %} {% if show_flags %}{% endif %} {% if show_ins %}{% endif %} @@ -437,6 +439,7 @@

Trials

{{ t.exception_type }}{{ t.exception_message }} {%- else %}{{ t.exception_type }}{% endif %}{% endif %} + {% if job.retries %}{% endif %} {% if show_cause %}', row)[0] for row in re.findall(r"(.*?)", html, re.S)[1:]] self.assertEqual(rewards, ["1", "0.67"]) + def test_a_retried_attempt_shows_each_try_it_kept(self): + run = written_job(self.root, "retried-run", [("0001__u0000r1", "t", 0, 1, None), ("0001__u0000r2", "t", 4, 1, None), + ("0001__u0000r3", "t", 2, 0, None)]) + # the standard's record of a retry: the run allowed an attempt one, and + # attempt 2 took it after the others, its first try (rate limited) kept + # under retried/ + job_dir = self.root / "retried-run" + cfg = json.loads((job_dir / "config.json").read_text()); cfg["standard"] = {"benchmark": "t", "retries": 1} + (job_dir / "config.json").write_text(json.dumps(cfg)) + path = job_dir / "0001__u0000r2" / "result.json" + res = json.loads(path.read_text()) + write_json(job_dir / "retried" / "0001__u0000r2.try-1" / "result.json", dict( + res, verifier_result=None, exception_info={"exception_type": "RateLimitError", "exception_message": "429"})) + res["standard"] = {"unit": "t", "retries": 1}; path.write_text(json.dumps(res)) + run = parse.parse_job(str(job_dir)) + self.assertEqual((run.retries, len(run.trials)), (1, 3)) # a kept try is no trial of the job + self.stack.enter_context(patch.object(workspace, "load_jobs", return_value=[run])) + + job = self.get("/job/retried-run") + self.assertIn('', job) + self.assertEqual(re.findall(r'', job), + ['0/1', '1/1', '0/1']) + + def try_tabs(html): + return [(oc, unescape(href), bool(cur), n) for oc, href, cur, n in re.findall( + r']*?href="([^"]*)"[^>]*?( aria-current="page")?>Try (\d)', html, re.S)] + + kept, first = "/trial/retried-run/0001__u0000r2", "/trial/retried-run/0001__u0000r2.try-1" + page = self.get(kept) + self.assertEqual(try_tabs(page), [("rate_limited", first, False, "1"), ("resolved", kept, True, "2")]) + self.assertIn('title="Attempt 2: Resolved on try 2" aria-current="page">Attempt 2Rate limited', page) + # the page is Try 1's: its attempt's tab, which leads to the attempt's last try, is where it is, not it + self.assertEqual([cur for _, cur in tabs(page)], [False, False, False, False]) + self.assertIn('title="Attempt 2: Resolved on try 2" aria-current="true">Attempt 2Retries', table) + rows = re.findall(r']*)?>\s*([^<]*).*?(.*?)', table, re.S) + self.assertEqual([(bool(rep), unescape(href), label, retries) for rep, href, label, retries in rows], [ + (False, "/trial/retried-run/0001__u0000r1", "Attempt 1", '0/1'), + (True, first, "Attempt 2 · Try 1", '0/1'), + (False, kept, "Attempt 2 · Try 2", '1/1'), + (False, "/trial/retried-run/0001__u0000r3", "Attempt 3", '0/1')]) + + def test_no_retries_column_without_the_runs_limit(self): + # a trial that says it was retried, in a run that allowed none: Harbor + # says no attempt's retries, so there is no count to put over a limit + run = written_job(self.root, "unlimited-run", [("u__a", "u", 0, 1, None), ("u__b", "u", 2, 0, None)]) + path = self.root / "unlimited-run" / "u__a" / "result.json" + res = json.loads(path.read_text()); res["standard"] = {"unit": "u", "retries": 1}; path.write_text(json.dumps(res)) + run = parse.parse_job(str(self.root / "unlimited-run")) + self.assertEqual(run.retries, 0) + self.stack.enter_context(patch.object(workspace, "load_jobs", return_value=[run])) + self.assertNotIn("Retries", self.get("/job/unlimited-run")) + self.assertNotIn("run-retries", self.get("/job/unlimited-run")) + self.assertNotIn("Retries", self.get("/trial/unlimited-run/u__a/attempts")) + def test_an_unknown_trial_has_no_attempts_page(self): self.get("/trial/repeat-run/nope/attempts", status=404) diff --git a/app/ui_tests/test_command_builder_state.py b/app/ui_tests/test_command_builder_state.py index da05ec95..3a85c5c2 100644 --- a/app/ui_tests/test_command_builder_state.py +++ b/app/ui_tests/test_command_builder_state.py @@ -125,6 +125,7 @@ def test_common_trial_flags_are_written_for_every_agent_and_only_when_set(self): self.assertEqual(out['e'], []) bad = self.runs("r.attempts=0;r.verifierTimeoutMultiplier='x'")['e'] self.assertEqual(sorted(g for g, _ in bad), ['Common', 'Common']) + self.assertIn('Attempts per task must be a whole number of at least 1, or empty.', [m for _, m in bad]) # the field's own name def test_an_agents_own_kwarg_reaches_that_agent_only(self): out = self.runs("r.custom=[{scope:'agent',key:'shared',type:'string',value:'s'},{scope:'agent',key:'mine',type:'number',value:'7',agent:'oracle'},{scope:'agent',key:'mine',type:'number',value:'8',agent:'custom-0'}]") @@ -133,6 +134,24 @@ def test_an_agents_own_kwarg_reaches_that_agent_only(self): self.assertIn("--ak 'mine=8'", out['runs']['run-custom-0']); self.assertNotIn("--ak 'mine=7'", out['runs']['run-custom-0']) self.assertIn("--ak 'mine=7'", out['runs']['run-oracle']); self.assertNotIn("--ak 'mine=8'", out['runs']['run-oracle']) + def test_a_harbor_recipe_retries_nothing_unless_asked(self): + for text in self.runs()['runs'].values(): + self.assertIn('--max-retries 0', text) + for text in self.runs("r.retries=2")['runs'].values(): + self.assertIn('--max-retries 2', text) + # an emptied field is none, as the standard's reads it: never a flag without its value + out = self.runs("r.retries=''") + self.assertEqual(out['e'], []) + for text in out['runs'].values(): + self.assertIn('--max-retries 0', text) + self.assertEqual([m for _, m in self.runs("r.retries=-1")['e']], ['Retries per attempt must be a whole number of at least 0, or empty.']) + + def test_a_standard_recipes_retries_survive_reopening(self): + out = self.js('const r=C.createStandard("gsm8k");r.retries=2;' + 'process.stdout.write(JSON.stringify([C.fromJSON(JSON.stringify(r)).retries,' + 'C.fromJSON(JSON.stringify(Object.assign(r,{retries:10}))).retries,"retries" in C.createStandard("gsm8k")]));') + self.assertEqual(out, [2, None, False]) + def test_the_common_fields_and_agent_kwargs_survive_a_saved_recipe(self): out = self.js('const r=C.create("harbor");r.attempts=2;r.verifierTimeoutMultiplier=1.5;r.custom=[{scope:"agent",key:"k",type:"string",value:"v",agent:"nop"}];' 'const b=C.fromJSON(JSON.stringify(r));process.stdout.write(JSON.stringify([b.attempts,b.verifierTimeoutMultiplier,b.setupTimeoutMultiplier,b.custom[0].agent]));') diff --git a/app/ui_tests/test_reconcile_routes.py b/app/ui_tests/test_reconcile_routes.py index d7991318..59be52b1 100644 --- a/app/ui_tests/test_reconcile_routes.py +++ b/app/ui_tests/test_reconcile_routes.py @@ -9,6 +9,8 @@ import tempfile import unittest from contextlib import ExitStack +from dataclasses import replace +from datetime import datetime, timezone from html import unescape from unittest.mock import patch @@ -62,6 +64,19 @@ def test_nothing_judged_states_so_and_the_job_page_links_only_once_judged(self): self.seed() self.assertIn('href="/job/r-agent/reconcile"', self.get("/job/r-agent")) + def test_a_record_of_an_earlier_try_is_not_its_retrys(self): + """A retry runs an attempt again under its trial's name: what was asked + of the try before is neither shown nor counted for it.""" + at = datetime(2026, 11, 17, 11, 0, tzinfo=timezone.utc) + job = replace(self.job, trials=[replace(t, started_at=at) for t in self.job.trials]) + self.stack.enter_context(patch.object(web.shared, "_jobs_for_request", return_value=[job, self.nop])) + store.save(self.ws.dir, "r-agent", "failed__x", record("environment_error", 0.9, 0.85), started="2026-11-17T10:00:00+00:00") + self.assertIn("Judged 0 · 5 not judged", text_of(self.get("/job/r-agent/reconcile"))) + self.assertNotIn("classifier vs judge", self.get("/job/r-agent")) + store.save(self.ws.dir, "r-agent", "failed__x", record("environment_error", 0.9, 0.85), started=at.isoformat()) + self.assertIn("Judged 1 · 4 not judged", text_of(self.get("/job/r-agent/reconcile"))) + self.assertIn('href="/job/r-agent/reconcile"', self.get("/job/r-agent")) + def test_disagreements_first_gated_and_suggested_apart_counts_stated_outcome_unchanged(self): self.seed() before = [(t.trial_name, t.outcome, t.reward) for t in self.job.trials] diff --git a/app/web/judge.py b/app/web/judge.py index b81488ce..33145f27 100644 --- a/app/web/judge.py +++ b/app/web/judge.py @@ -62,9 +62,20 @@ def ws_dir(): return ws.dir if ws else None +def started_of(trial) -> str | None: + """A trial's start, as its judgement keeps it: a retry runs an attempt + again under the same name, and an earlier try's record is not its own.""" + return trial.started_at.isoformat() if trial.started_at else None + + +def started_by_name(job) -> dict: + """{store key: start} for every trial of a job: what `store.for_job` checks its records against.""" + return {store._SAFE.sub("_", t.trial_name): started_of(t) for t in job.trials} + + def judgement_of(job, trial, lane: str = "system-one") -> dict | None: d = ws_dir() - return store.load(d, job.job_id, trial.trial_name, lane=lane) if d else None + return store.load(d, job.job_id, trial.trial_name, lane=lane, started=started_of(trial)) if d else None def panel(job, trial, lane: str = "system-one") -> dict: @@ -245,7 +256,8 @@ def judge_trial(job, trial, api_key: str, post=None) -> dict: traj = trajectory.load(trial.trajectory_path) if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) - return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post) + return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, _trial_ids_and_states(trial, traj), api_key, post=post, + started=started_of(trial)) def _guard(): @@ -285,7 +297,7 @@ def judge_job_route(job_id): abort(404) lane = request.form.get("lane") if request.form.get("lane") in client.LANES else "system-one" d = ws_dir() - judged = store.for_job(d, job_id, lane=lane) if d else {} + judged = store.for_job(d, job_id, lane=lane, started=started_by_name(job)) if d else {} todo = [t for t in job.trials if t.has_trajectory and store._SAFE.sub("_", t.trial_name) not in judged] _guard() k = key(lane) @@ -305,7 +317,7 @@ def job_summary(job) -> dict: the board's record -> judge lanes (`nullius.judge.lanes`), whose status for each trial is also in `by_trial`.""" d = ws_dir() - judged = store.for_job(d, job.job_id) if d else {} + judged = store.for_job(d, job.job_id, started=started_by_name(job)) if d else {} flag_ids = [q for q in Q.group_ids("trial") if Q.QUESTIONS[q].get("flag")] board = lanes.board(job, judged, key=lambda name: store._SAFE.sub("_", name)) by_trial = {} @@ -338,7 +350,7 @@ def job_summary(job) -> dict: def _llm_count(d, job) -> dict: """The LLM judge's own count for the job's button: it judges apart from System One, so a trial judged by one is still unjudged by the other.""" - judged = store.for_job(d, job.job_id, lane="llm") if d else {} + judged = store.for_job(d, job.job_id, lane="llm", started=started_by_name(job)) if d else {} return {"unjudged": len([t for t in job.trials if store._SAFE.sub("_", t.trial_name) not in judged]), "keyed": bool(key("llm"))} @@ -373,7 +385,7 @@ def side_of(job, trial) -> dict | None: current record: the expectation, the level it rounds to, the judge's confidence, the provenance. None until scored under the current vocabulary.""" d = ws_dir() - rec = store.load(d, job.job_id, trial.trial_name) if d else None + rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) if d else None if not rec or not store.is_current(rec): return None a = (rec.get("answers") or {}).get(SIDE_QUESTION) @@ -432,7 +444,7 @@ def judge_side(job, trial, api_key: str, post=None) -> dict | None: if not trial.has_trajectory: raise client.JudgeError(0, "this trial has no trajectory to judge") d = ws_dir() - rec = store.load(d, job.job_id, trial.trial_name) + rec = store.load(d, job.job_id, trial.trial_name, started=started_of(trial)) have = (rec.get("answers") or {}) if store.is_current(rec) else {} ids = [q for q in side_ids(trial) if q not in have] if not ids: @@ -440,7 +452,8 @@ def judge_side(job, trial, api_key: str, post=None) -> dict | None: traj = trajectory.load(trial.trajectory_path) if traj.error: raise client.JudgeError(0, "the trajectory could not be read: " + traj.error) - return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post) + return batch.ask_and_store(d, job.job_id, trial.trial_name, _states_for(trial, traj, ids), api_key, post=post, + started=started_of(trial)) def _sides_todo(view) -> list: @@ -525,7 +538,8 @@ def judge_control(side, jobs, api_key: str, post=None) -> dict | None: job, trial = _control_trial(side, jobs) if trial is None: raise client.JudgeError(0, f"{side.control} trial {side.trial_name} is not in this workspace") - return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post) + return batch.ask_and_store(ws_dir(), job.job_id, trial.trial_name, control_states(side, trial), api_key, post=post, + started=started_of(trial)) def triage_summary(report, jobs) -> dict: diff --git a/app/web/reconcile.py b/app/web/reconcile.py index 9da34259..4bb71e78 100644 --- a/app/web/reconcile.py +++ b/app/web/reconcile.py @@ -11,7 +11,7 @@ from nullius.runs import classify from .core import app -from . import shared +from . import judge, shared def ws_dir(): @@ -31,7 +31,7 @@ def threshold_requested() -> float: def view_of(job, threshold: float) -> dict: d = ws_dir() - records = store.for_job(d, job.job_id) if d else {} + records = store.for_job(d, job.job_id, started=judge.started_by_name(job)) if d else {} return reconcile.reconcile(job, records, threshold, key=lambda name: store._SAFE.sub("_", name)) diff --git a/app/web/settings.py b/app/web/settings.py index b5844aed..f7601ae6 100644 --- a/app/web/settings.py +++ b/app/web/settings.py @@ -352,7 +352,7 @@ def settings_sandbox_logout(): @app.route("/settings/prices/update", methods=["POST"]) def settings_prices_update(): """The public price list fetched and kept (``nullius.prices``): the one - fetch, asked for here and never by a run. Same guards as Save.""" + fetch, asked for here (a capped run's ``cli run`` asks too). Same guards as Save.""" _sandbox_guard("Prices are updated") try: prices.update() diff --git a/app/web/traces.py b/app/web/traces.py index ffffbaf4..6bda5a51 100644 --- a/app/web/traces.py +++ b/app/web/traces.py @@ -3,9 +3,10 @@ import json import os +import re import threading from collections import OrderedDict -from flask import abort, flash, jsonify, redirect, render_template, request, url_for +from flask import abort, flash, g, jsonify, redirect, render_template, request, url_for from markupsafe import Markup, escape from nullius.traces import behavior, charts, conversation, trace, trajectory from nullius.runs import artifacts, attempts as task_attempts, parse @@ -72,16 +73,47 @@ def _gold_localization(job, trial, traj): return None +# An attempt's try before its last, as a page addresses it (parse.tries) +_TRY_RE = re.compile(r"^(?P.+)\.try-(?P[1-9][0-9]?)$") + + def _find_trial(job, trial_name): return next((x for x in job.trials if x.trial_name == trial_name), None) -def _trial_evidence(job_id, trial_name): - """Resolve URL identifiers through parsed runs, never as filesystem paths.""" +def _tries(attempt): + """``parse.tries`` of an attempt, read once a request: a try's page finds + its try in the same list its tabs show.""" + seen = g.setdefault("tries", {}) + key = (attempt.job_id, attempt.trial_name) + if key not in seen: + seen[key] = parse.tries(attempt) + return seen[key] + + +def _find_try(job, name): + m = _TRY_RE.match(name) + kept = _find_trial(job, m["trial"]) if m else None + return next((t for _, t in _tries(kept) if t.trial_name == name), None) if kept else None + + +def _attempt_of(job, trial): + """The trial a try is one of: the attempt's last, the one the job counts. + A trial of the job is its own.""" + if any(t is trial for t in job.trials): + return trial + m = _TRY_RE.match(trial.trial_name) + return (_find_trial(job, m["trial"]) if m else None) or trial + + +def _trial_evidence(job_id, trial_name, tries=False): + """Resolve URL identifiers through parsed runs, never as filesystem paths. + ``tries``: an attempt's earlier try too, ``.try-N``: evidence to + read, never a trial to judge.""" job = shared._find_job(shared._jobs_for_request(), job_id) if job is None: abort(404) - trial = _find_trial(job, trial_name) + trial = _find_trial(job, trial_name) or (_find_try(job, trial_name) if tries else None) if trial is None: abort(404) return job, trial @@ -139,7 +171,7 @@ def _parsed_trace(path, filt): @app.route("/trial///artifact/") def artifact_page(job_id, trial_name, key): - job, trial = _trial_evidence(job_id, trial_name) + job, trial = _trial_evidence(job_id, trial_name, tries=True) try: record = artifacts.read(trial.dir, key, request.args.get("page", 1)) except (LookupError, OSError): @@ -152,7 +184,7 @@ def artifact_page(job_id, trial_name, key): @app.route("/trial///conversation") def conversation_data(job_id, trial_name): - _, t = _trial_evidence(job_id, trial_name) + _, t = _trial_evidence(job_id, trial_name, tries=True) traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None payload = conversation.page( traj, query=request.args.get("q", ""), kind=request.args.get("kind", "all"), @@ -167,7 +199,7 @@ def conversation_data(job_id, trial_name): @app.route("/trial///trace-span") def trace_span_data(job_id, trial_name): - _, t = _trial_evidence(job_id, trial_name) + _, t = _trial_evidence(job_id, trial_name, tries=True) tr, _ = _requested_trace(t) try: payload = conversation.span_detail( @@ -183,7 +215,8 @@ def trace_span_data(job_id, trial_name): @app.route("/trial//") def trial(job_id, trial_name): - j, t = _trial_evidence(job_id, trial_name) + j, t = _trial_evidence(job_id, trial_name, tries=True) + attempt = _attempt_of(j, t) # t itself, or the attempt's last try when t is an earlier one traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None ok = traj is not None and not traj.error bench_judge = bench_parts = None @@ -225,18 +258,23 @@ def trial(job_id, trial_name): "traces/trial.html", job=j, trial=t, traj=traj, loc=loc, bench_judge=bench_judge, bench_parts=bench_parts, trace=tr, charts=ch, viewing=viewing, chart_filter=cfilt, default_save_name=SAVED_TRACE_DEFAULT, - artifacts=records, attempts=task_attempts.of(j, t), - judgement=_judgement_panel(j, t), llm_judgement=_judgement_panel(j, t, "llm"), csrf=shared._csrf_token(), + artifacts=records, attempt=attempt, attempts=task_attempts.of(j, attempt), tries=_tries(attempt), + # an earlier try is evidence only: the job counts its last, and only that is judged + judgement=_judgement_panel(j, t) if attempt is t else None, + llm_judgement=_judgement_panel(j, t, "llm") if attempt is t else None, csrf=shared._csrf_token(), ) @app.route("/trial///attempts") def trial_attempts(job_id, trial_name): - """Every attempt at this trial's task, one line each.""" - j, t = _trial_evidence(job_id, trial_name) + """Every attempt at this trial's task, one line each, and a line for each + earlier try its runner kept.""" + j, t = _trial_evidence(job_id, trial_name, tries=True) + t = _attempt_of(j, t) # a task tried once has no tab that leads here; its one line still answers attempts = task_attempts.of(j, t) or task_attempts.Attempts((t,)) - return render_template("traces/attempts.html", job=j, trial=t, attempts=attempts) + tries = {a.trial_name: _tries(a) for a in attempts.trials if a.n_retries} + return render_template("traces/attempts.html", job=j, trial=t, attempts=attempts, tries=tries) def _judgement_panel(job, trial, lane="system-one"): @@ -254,7 +292,7 @@ def save_trace(job_id, trial_name): is not reproducible, so nothing here may overwrite one. """ shared._require_csrf() - _, t = _trial_evidence(job_id, trial_name) + _, t = _trial_evidence(job_id, trial_name, tries=True) # The zoom rides along only so the page comes back as it was; the file written # is the filtered tree whole — a zoom is a way of looking, not a change. diff --git a/docs/agents/add-manually.md b/docs/agents/add-manually.md index 67d3a126..93b54324 100644 --- a/docs/agents/add-manually.md +++ b/docs/agents/add-manually.md @@ -79,7 +79,7 @@ Keep the whole copied definition, `model.env` and `options` included. ```bash title="Run conformance · Container backend" nullius standard conform my-agent \ - --provider openai --model gpt-5-mini --backend container --repeat 3 + --provider openai --model gpt-5-mini --backend container --attempts 3 ``` 3. Which benchmarks it fits: @@ -99,7 +99,7 @@ contribute it to the bundled catalogue, copy its whole folder into and refresh `python3 app/tests/catalogue_docs.py --write`. !!! warning "Conformance makes paid model calls" - Each check is a trial; `--repeat 3` runs each three times. + Each check is a trial; `--attempts 3` runs each three times. ## Another agent diff --git a/docs/builder.md b/docs/builder.md index 7651d574..032d0ae5 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -27,7 +27,7 @@ Then [read its results](results.md). | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | -| Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | +| Limits | Task limit (empty is every task), a task-name glob, concurrency, attempts per unit, retries per attempt, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | | Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | diff --git a/docs/cli/conform.md b/docs/cli/conform.md index e3b1cced..07fa24db 100644 --- a/docs/cli/conform.md +++ b/docs/cli/conform.md @@ -4,7 +4,7 @@ Run each port's conformance kit; a port enters `offers` only through a pass, rec ```bash title="Conform" nullius standard conform [@version] --model M [--provider P] [--port PORT]... \ - [--unsandboxed | --backend container] [--option NAME=VALUE]... [--repeat K] [--jobs DIR] [--stamp STAMP] [--from-run] + [--unsandboxed | --backend container] [--option NAME=VALUE]... [--attempts K] [--jobs DIR] [--stamp STAMP] [--from-run] ``` | Flag | Does | @@ -14,13 +14,13 @@ nullius standard conform [@version] --model M [--provider P] [--port PORT | `--port PORT` | Only this port's kit (repeatable) | | `--unsandboxed`, `--backend` | As for a run: as you on this machine, or each check in a container | | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | -| `--repeat K` | Each check K times (`u0002r1` to `u0002rK`, one kit job); a majority passes it. Default 1 | +| `--attempts K` | Each check K times (`u0002r1` to `u0002rK`, one kit job); a majority passes it. Default 1 | | `--jobs`, `--stamp` | Where kit jobs go (default `jobs_standard/`), and their time stamp | | `--from-run` | Started by Nullius: `OPENAI_API_KEY` only, saved settings unread | | Recorded | Detail | | --- | --- | -| `proven` | Date, score, model, `kit_version`, skills shown, the kit job; with `--repeat`, `"repeat": K`, each count (`"3/3"`), each skill's `"passed"` | +| `proven` | Date, score, model, `kit_version`, skills shown, the kit job; with `--attempts`, `"repeat": K`, each count (`"3/3"`), each skill's `"passed"` | | Skills | A benchmark's `needs_skills` fits only an agent whose proof shows them | | Replaced | A pass on the current kit replaces an older proof; a fail takes the port back; a `fidelity` record stays | diff --git a/docs/cli/run.md b/docs/cli/run.md index 4471c961..6ce9f6be 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -8,7 +8,7 @@ nullius standard run [@version] [--model M] [--dataset D] [-- [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS]... [--max-turns N] [--max-calls N] \ [--constraint KEY=VALUE]... \ [--served-by [MODEL=]SLUG[,SLUG...]]... \ - [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] + [--attempts K] [--retries N] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` ```bash title="Examples" @@ -38,8 +38,9 @@ nullius standard run locomo model --model gpt-4o-mini | `--max-calls N` | The most model calls a unit's agent may make: [below](#a-cap-on-model-calls) | | `--constraint KEY=VALUE` | What the machine allows the run (repeatable): `cpus`, `memory_mb`, `processes`, `cpu_set`, `concurrency`, `gpu_ids` (indices, empty for none); [constraints](../settings/run-hosts.md#constraints). Containers take them as flags, the run's processes on the machine as a systemd scope. A recipe passes each machine's own | | `--served-by [MODEL=]SLUG[,SLUG...]` | On OpenRouter, the serving providers that may answer a model (repeatable): [below](#serving-providers) | -| `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | -| `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--repeat`); by default they stay for the next run | +| `--attempts K` | Each unit K times; `result.json` carries `pass_hat_k` | +| `--retries N` | Run a failed attempt again, up to N times: [below](#retries) | +| `--remove-images` | Remove each image the run pulls or builds when its last unit ends (never with `--attempts`); by default they stay for the next run | | `--judge-model` | Instead of the official judge | | `--simulator-model` | Instead of the official simulated user | | `--name` | The job's name | @@ -50,6 +51,7 @@ nullius standard run locomo model --model gpt-4o-mini | Rule | Detail | | --- | --- | | The pair must fit | Else refused, naming the missing port | +| The model | Refused when the provider's model list lacks it | | Keys | Environment, then user `settings.json`; the agent never gets one | | A data provider's token | `HF_TOKEN` from the environment, then Settings → Data providers, for a benchmark that declares Hugging Face; only its driver gets it. Not with `--from-run` | | A control | No `--model`, no `--unsandboxed`; only a judge is called | @@ -89,17 +91,31 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | Rule | Detail | | --- | --- | -| Prices first | Every model the run calls needs a price ([price list](../settings/model-prices.md) or `--price`), else the run is refused | +| Prices first | The [price list](../settings/model-prices.md) is fetched first; a model it lacks needs `--price`, else the run is refused. Offline and unpriced: no cap, and the run says so | | What is counted | Each reply's cost, and an estimate for each call in flight | | At the run's cap | The call that does not fit is refused by the relay (HTTP 402), and every call after it | | Its units | One cut off is the outcome Spending cap, outside the adjusted rate; those not begun are not run | -| At a unit's cap | Its own calls and its simulated user's count, its judge's do not. A unit that reaches its cap is an Agent error; the run goes on | +| At a unit's cap | Its own calls and its simulated user's count, over all its tries; its judge's do not. A unit that reaches its cap is an Agent error; the run goes on | | Not exact | A call that costs more than its estimate can take the spend past a cap by up to the difference | | Cached input | Estimated as if not cached, so a unit can be cut off short of its cap | | The result | `result.json` carries `budget`, and the job page shows it | | From the builder | *Limits*: one cap shared in equal parts by the run's jobs, and one per unit | | Shards | A shard's cap is its own: [Shard and merge](shard-and-merge.md) | +## Retries + +`--retries N` (at most 9) runs a failed attempt again after the other units; the last try counts. + +| Failure | Retried | +| --- | --- | +| Rate limit | After the [pause](../standard.md#a-run) | +| Provider, network or setup error; agent crash; setup timeout | Yes | +| Run or check out of time | Only if the provider was refusing the calls | +| Wrong key, spent credit | No; three in a row stop the run | +| A setup this machine lacks, a missing secret, a cap reached, Stop, failed task | No | + +`conform` and `prove` retry a rate limit once. + ## A list of units `--units tasks.txt` runs only the units the file lists. diff --git a/docs/field-guide.md b/docs/field-guide.md index 7da51fbb..29f6d7e8 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -141,7 +141,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `total_input_tokens` | Prompt tokens across the run. | derived | any | | `total_cache_tokens` | The part of those served from cache, as each trial's `result.json` records it. | derived | any | | `total_output_tokens` | Completion tokens. | derived | any | -| `total_cost_usd` | The trials' recorded costs, summed: an estimate, not a bill. Shows — when no trial recorded one. | derived | any | +| `total_cost_usd` | The trials' recorded costs and their retried tries', summed: an estimate, not a bill. Shows — when none recorded one. | derived | any | | `duration_min` | The whole run, start to finish. | derived | any | | `budget` | A standard run's spending cap, from `result.json`: `max_usd` and `spent_usd`, whether it `stopped` the run, the units `cut_off` and `not_run`, and with a cap per unit `unit_max_usd` and `over_unit`. Empty without a cap. | measured | any | | `agent_label` | The agent's name: the class of a `module:Class` path, or the trials' own name when the config gives none. | measured | any | @@ -149,6 +149,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `is_control` | An `oracle` or `nop` run: a check on the tasks, kept out of rankings and totals. Its row reads as a verdict, not a rate. A standard run is one only when its benchmark declares that control. An imported runner's arm without that declaration is not a control. On a benchmark that scores, a control's score of 1 reads as accepted and 0 as rejected. | measured | any | | `is_running` | The job records running or pending trials. Numbers are partial; a managed execution's terminal status takes precedence on the page. | measured | any | | `n_planned / n_retries` | Planned trials from the run record or task count, and retries recorded by the harness. Repeated attempts are separate trials, not retries. | measured | any | +| `retries` | Retries a standard run allows each attempt (`--retries`); 0 elsewhere. | measured | any | | `dataset` | The dataset folder, `name@version` for a registry fetch, or the task names' shared prefix. | measured | any | ## Trial metrics, one row per task attempt {#trials} @@ -171,6 +172,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | | `n_steps / n_tool_calls` | Trajectory steps and tool calls; empty placeholder calls are not counted. | measured | any | | `exception_type` | What ended the trial, if anything. It decides the infra / agent split. | measured | any | +| `n_retries` | Retries a standard run's attempt took. | measured | any | | `n_rate_limited_retries` | 429s counted in `agent/gateway.log`: on Harbor, the ones the agent's SDK retried and survived; on a standard run, every model call the relay saw refused with 429. No log reads *not recorded*, never 0. | inferred | any | ## Standard run records diff --git a/docs/standard.md b/docs/standard.md index 0e1ba2c4..3437a731 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -89,7 +89,7 @@ A kit names its skills in its `standard.json`; any other check is basic. A proof from before skills were recorded reads as every skill shown on the current kit. A benchmark that relies on a skill says `"needs_skills": {"agent.tools/v1": ["decoys"]}`, each one its port's kit names (`cli check` refuses another); an agent that has not shown it does not fit. -`cli conform --repeat K` (default 1) runs each check K times, each a trial (`u0002r1` … `u0002rK`); a check holds when a majority pass. The proof records `"repeat": 3`, `"checks": {"one-tool": "3/3", "arguments": "2/3"}` and each skill's `"passed": "1/3"`. +`cli conform --attempts K` (default 1) runs each check K times, each a trial (`u0002r1` … `u0002rK`); a check holds when a majority pass. The proof records `"repeat": 3`, `"checks": {"one-tool": "3/3", "arguments": "2/3"}` and each skill's `"passed": "1/3"`. ### The tools kit @@ -145,7 +145,7 @@ record); verifier/reward.txt and score.json | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass | -| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | +| Rate limits | After a `rate_limited` end no unit starts for 60 s, or the provider's `Retry-After`, doubling to 300 s while units keep ending so | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. The key goes as the agent's protocol says: `x-api-key` and `anthropic-version` for `messages`, else a bearer header. Provider `anthropic` serves `messages` only ([protocols](settings/model-providers.md#protocols)) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | | A cap on model calls (`--max-calls`) | Agents only. A unit's relay counts its agent route's answered calls and refuses the one past the cap: the agent's error, as a timeout is ([Run a benchmark](cli/run.md#a-cap-on-model-calls)) | @@ -607,7 +607,8 @@ already accepts arbitrary models needs only the `original` method. | `units` | A setup may count its own units; `units` is sent `setup` | | `answer` | Runs the upstream's answerer through the unit's relay; `score` is sent `setup` too. Local processes or the Original setup driver container | | Recorded | `standard.setup` in the config and each trial: its name, the model `asked` and `sent`, `comparable_with_paper` | -| Attempts | `cli run --repeat K` runs each unit K times; `result.json` carries `pass_hat_k` (C(c, k) / C(n, k) averaged over the units, an errored unit left out) | +| Attempts | `cli run --attempts K` runs each unit K times; `result.json` carries [`pass_hat_k`](field-guide.md#attempts) | +| Retries | `cli run --retries N` runs a failed attempt again: [which failures](cli/run.md#retries) | ### Every setting, inventoried
Judge{% if tq %}instructions scored {{ tq.judged }}/{{ tq.tasks }}{% endif %}TestsTask OutcomeRetriesCause{% if show_flags %} · flags{% endif %}FlagsInstruction{{ attempt_count(group, false) }} {{ instruction(t) }}{% if t.n_retries %}{{ t.n_retries }}/{{ job.retries }}{% else %}0/{{ job.retries }}{% endif %} {%- if t.outcome == 'resolved' %}not asked {%- elif jr and jr.cause %}{{ jr.cause }} diff --git a/app/templates/traces/_attempts.html b/app/templates/traces/_attempts.html index 857039b4..a35aaee6 100644 --- a/app/templates/traces/_attempts.html +++ b/app/templates/traces/_attempts.html @@ -1,19 +1,36 @@ -{# One tab per attempt at a task, and All attempts. `current` is the trial on - screen, or none on the All attempts view. A tab keeps the evidence tab the +{# One tab per attempt at a task, and All attempts. `current` is the attempt + on screen, or none on the All attempts view; `shown`, the try on screen when + it is one before the attempt's last: the attempt's tab is then where the + page is, not the page (aria-current="true"). A tab keeps the evidence tab the reader is on (conversation.js rewrites `tab` on every [data-keep-tab]). #} -{% macro attempt_tabs(job, attempts, current=none, workspace=none) -%} +{% macro attempt_tabs(job, attempts, current=none, workspace=none, shown=none) -%} {%- endmacro %} +{# An attempt's tries, when its runner kept the ones before its last + (parse.tries): one tab each, the last the one that counts. `current` is the + try on screen. #} +{% macro try_tabs(job, tries, current, workspace=none) -%} + +{%- endmacro %} + {# How a run did at one task over its attempts, as its adjusted rate reads them: passes over the attempts that ran, infra counted apart. `word` false leaves out "passed", for a row with no room. #} diff --git a/app/templates/traces/attempts.html b/app/templates/traces/attempts.html index 5560d1c2..2eff8602 100644 --- a/app/templates/traces/attempts.html +++ b/app/templates/traces/attempts.html @@ -17,23 +17,28 @@
- + {% if job.retries %}{% endif %} {% for a in attempts.trials %} - {% set used = (a.n_input_tokens or 0) + (a.n_output_tokens or 0) %} - - - - + {% set k = loop.index %} + {#- an attempt whose runner kept its earlier tries: a line each, the last the one that counts #} + {% for n, t in tries.get(a.trial_name) or [(none, a)] %} + {% set used = (t.n_input_tokens or 0) + (t.n_output_tokens or 0) %} + + + + {% if job.retries %}{% endif %} + - - + + {% endfor %} + {% endfor %}
AttemptOutcomeRewardTokensAttemptOutcomeRetriesRewardTokens AgentTotal
Attempt {{ loop.index }}{{ a.outcome_label }} - {%- if a.score is not none %} {{ '%.2f'|format(a.score) }}{% endif %} - {%- if a.exception_type %} {{ a.exception_type }}{% endif %}{{ '—' if a.reward is none else ('%g'|format(a.reward) if a.reward in (0, 1) else '%.2f'|format(a.reward)) }}Attempt {{ k }}{% if n %} · Try {{ n }}{% endif %}{{ t.outcome_label }} + {%- if t.score is not none %} {{ '%.2f'|format(t.score) }}{% endif %} + {%- if t.exception_type %} {{ t.exception_type }}{% endif %}{% if t.n_retries %}{{ t.n_retries }}/{{ job.retries }}{% else %}0/{{ job.retries }}{% endif %}{{ '—' if t.reward is none else ('%g'|format(t.reward) if t.reward in (0, 1) else '%.2f'|format(t.reward)) }} {{ used|tokens(job.kind) }}{{ a.dur_agent|secs }}{{ a.dur_total|secs }}{{ t.dur_agent|secs }}{{ t.dur_total|secs }}
diff --git a/app/templates/traces/trial.html b/app/templates/traces/trial.html index 22b0fa32..13e19f9e 100644 --- a/app/templates/traces/trial.html +++ b/app/templates/traces/trial.html @@ -23,7 +23,7 @@ {%- endmacro %} {% from 'traces/_judgement.html' import judgement_panel %} -{% from 'traces/_attempts.html' import attempt_tabs, attempt_count %} +{% from 'traces/_attempts.html' import attempt_tabs, attempt_count, try_tabs %} {% block body %} @@ -52,9 +52,12 @@

{{ trial.task_base }}

{% if attempts %}
{{ attempt_count(attempts) }} - {{ attempt_tabs(job, attempts, trial, request.args.get('workspace')) }} + {{ attempt_tabs(job, attempts, attempt, request.args.get('workspace'), trial) }}
{% endif %} +{% if tries %}{{ try_tabs(job, tries, trial, request.args.get('workspace')) }}{% endif %} +{# an earlier try of an attempt: evidence only, never judged #} +{% set replaced = trial.trial_name != attempt.trial_name %}
{# The check cards speak the dataset's language: labels come from its @@ -102,7 +105,7 @@

{{ trial.task_base }}

{% endif %}
@@ -298,6 +301,7 @@

Files the agent edited

No trajectory recorded for this trial.

{% endif %} +{% if not replaced %}
{% if job.runner and bench_parts %} {#- scored part by part by the benchmark's own script: the score is the mean of these, each as the script wrote it #} @@ -336,6 +340,7 @@

{{ job.runner }}'s judge gives the
{{ judgement_panel(llm_judgement, job, trial, csrf) }}
+{% endif %}
{# The task's instruction, scored on its own text: once per dataset and task, the same for every attempt, nothing of any run read. An unclear instruction caps every agent on this task - a finding about the diff --git a/app/tests/__init__.py b/app/tests/__init__.py index b6dcb03c..1af55bad 100644 --- a/app/tests/__init__.py +++ b/app/tests/__init__.py @@ -20,3 +20,34 @@ if str(APP) not in sys.path: sys.path.insert(0, str(APP)) + +# No test reaches past this machine. A run asks its provider for its model list +# before it starts, and a capped run fetches the price list: asked of a real +# address here, each fails as on a machine offline, and what it serves is +# skipped. A test of either serves its own on the loopback, or patches it. +import urllib.parse as _urls # noqa: E402 +import urllib.request as _request # noqa: E402 + +from nullius import providers as _providers # noqa: E402 + +_LOCAL = ("127.0.0.1", "localhost", "::1") +_get, _urlopen = _providers._get, _request.urlopen + + +def _local(url): + if (_urls.urlsplit(url).hostname or "") not in _LOCAL: + raise OSError(f"the test suite reaches no address past this machine: {url}") + + +def _local_get(url, headers, opener=None, timeout=12): + if opener is None: + _local(url) + return _get(url, headers, opener, timeout) + + +def _local_urlopen(url, *a, **kw): + _local(url.full_url if isinstance(url, _request.Request) else url) + return _urlopen(url, *a, **kw) + + +_providers._get, _request.urlopen = _local_get, _local_urlopen diff --git a/app/tests/field_contract.json b/app/tests/field_contract.json index 96a13252..36c6c419 100644 --- a/app/tests/field_contract.json +++ b/app/tests/field_contract.json @@ -319,6 +319,7 @@ "raw_solve_rate", "reasoning_effort", "reports_regressions", + "retries", "runner", "runner_commit", "runner_items", @@ -365,6 +366,7 @@ "n_output_tokens", "n_parts", "n_rate_limited_retries", + "n_retries", "outcome", "outcome_label", "p2p_fail", @@ -376,6 +378,7 @@ "patch_exists", "phases", "reasoning_effort", + "retried_cost_usd", "reward", "score", "source", diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index 3831a02a..3c102026 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -659,6 +659,37 @@ def test_a_run_that_failed_before_any_part_was_refused(self): self.assertIsNone(alive.refused("std-r")) # still going: nothing to say yet +class RetriedOnHost(unittest.TestCase): + """A host's retry moved a trial into retried/ and wrote it again; the copy + here, taken without deleting, keeps the earlier try's files under the trial.""" + + def test_the_earlier_trys_files_leave_the_trials_copy(self): + import tempfile + jobs = tempfile.mkdtemp(prefix="nullius-settle-") + self.addCleanup(shutil.rmtree, jobs, True) + trial, kept = Path(jobs, "job", "t__u0001"), Path(jobs, "job", "retried", "t__u0001.try-1") + for d in (trial / "agent", kept / "agent"): + d.mkdir(parents=True) + def put(path, text, at): + path.write_text(text) + os.utime(path, (at, at)) + put(kept / "agent" / "trace.json", "try 1's spans", 1_700_000_000) # the try, copied whole + put(trial / "agent" / "trace.json", "try 1's spans", 1_700_000_000) # its copy under the trial: stale + put(kept / "result.json", '{"try": 1}', 1_700_000_000) + put(trial / "result.json", '{"try": 2}', 1_700_000_900) # the retry's own, written again + put(trial / "agent" / "view.json", "a view saved here", 1_700_000_500) # never on the host + seen = set() + execution.settle_retried(jobs, seen) + self.assertFalse((trial / "agent" / "trace.json").exists()) + self.assertEqual((trial / "result.json").read_text(), '{"try": 2}') + self.assertTrue((trial / "agent" / "view.json").exists()) + self.assertEqual((kept / "agent" / "trace.json").read_text(), "try 1's spans") # the kept try itself is untouched + self.assertEqual(seen, {str(kept)}) + put(trial / "agent" / "trace.json", "try 1's spans", 1_700_000_000) + execution.settle_retried(jobs, seen) # settled once: its copy was whole then + self.assertTrue((trial / "agent" / "trace.json").exists()) + + class TasksOnHost(unittest.TestCase): def test_two_folders_of_one_name_never_share_a_copy_and_one_folder_keeps_its_own(self): a, b = execution._tasks_place("/data/projA/tasks", "r1"), execution._tasks_place("/data/projB/tasks", "r2") diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index fd39414e..8ca868ad 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -244,6 +244,23 @@ def test_v1_and_v2_coexist_and_the_read_prefers_the_current(self): self.assertEqual(store.for_job(d, "job")["t"]["questions"], "judgement.questions.v2") self.assertFalse(store.is_current(None)) + def test_a_record_is_the_try_it_judged_and_no_retry_under_its_name(self): + """A retry runs an attempt again under its trial's name: what was asked + of the try before is not shown, counted nor merged into the retry's.""" + d = str(self.tmp) + first, retry = "2026-11-17T10:00:00+00:00", "2026-11-17T11:00:00+00:00" + store.save(d, "job", "t", self.record(questions.VERSION, task_completed={"noul": 0.2}), started=first) + self.assertEqual(store.load(d, "job", "t", started=first)["trial_started_at"], first) + self.assertIsNone(store.load(d, "job", "t", started=retry)) + self.assertEqual(store.for_job(d, "job", started={"t": retry}), {}) + self.assertIsNotNone(store.load(d, "job", "t")) # a read that names no start takes it as it is + store.save(d, "job", "t", self.record(questions.VERSION, failure_cause={"choice": "x"}), started=retry) + got = store.load(d, "job", "t", started=retry) + self.assertEqual(sorted(got["answers"]), ["failure_cause"]) # none of the try before's answers + # a record from before tries were told apart is taken as it is + store.save(d, "job", "old", self.record(questions.VERSION, task_completed={"noul": 0.5})) + self.assertIsNotNone(store.load(d, "job", "old", started=retry)) + class Batch(unittest.TestCase): """The seam a surface asks through: questions grouped by state, one call diff --git a/app/tests/test_live.py b/app/tests/test_live.py index 0666408b..aa57ffdb 100644 --- a/app/tests/test_live.py +++ b/app/tests/test_live.py @@ -228,6 +228,24 @@ def at(sec): return (start + timedelta(seconds=sec)).isoformat().replace("+00:00 flipt = next(t for t in board_of(*lines).snapshot()["trials"] if t["trial"] == "flipt") self.assertEqual((flipt["outcome"], flipt["attempt"]), ("resolved", 2)) # the last attempt is its result + def test_a_retried_trial_reports_its_new_attempts_counters_alone(self): + # The standard's totals come on trial.end only: the attempt before reported 10k tokens and a cost, + # and its retry has reported nothing yet - so it shows none, and no rate is made of the two ends + start = datetime(2026, 10, 8, 14, 0, 0, tzinfo=timezone.utc) + def at(sec): return (start + timedelta(seconds=sec)).isoformat().replace("+00:00", "Z") + lines = [ev(1, "job.start", trial=None, ts=at(0), n_total=1, n_concurrent=1), + ev(2, "trial.phase", trial="x", ts=at(0), phase="agent_run"), + ev(3, "trial.end", trial="x", ts=at(10), outcome="agent_error", n_input_tokens=10000, n_output_tokens=500, cost_usd=0.2), + ev(4, "trial.phase", trial="x", ts=at(11), phase="agent_run")] + b = board_of(*lines, now=start + timedelta(seconds=20)) + x = b.trials[0] + self.assertEqual((x.attempt, x.n_input_tokens, x.n_output_tokens, x.cost_usd), (2, None, None, None)) + lines.append(ev(5, "trial.end", trial="x", ts=at(25), outcome="resolved", reward=1.0, n_input_tokens=30000, n_output_tokens=900)) + b = board_of(*lines, now=start + timedelta(seconds=30)) + self.assertEqual((b.trials[0].n_input_tokens, b.trials[0].cost_usd), (30000, None)) + self.assertEqual(b.totals()["cost_usd"], 0.2) # the attempt before was paid for: the board's cost counts it + self.assertIsNone(b.tokens_per_sec()) # one report of the attempt's own: no baseline, no rate + def test_eta_counts_down_while_a_trial_runs(self): # the same median, read later: a running trial's time so far comes off its share, so the ETA moves start = datetime(2026, 9, 12, 18, 0, 0, tzinfo=timezone.utc) diff --git a/app/tests/test_parse.py b/app/tests/test_parse.py index 4f237cbb..a08af26d 100644 --- a/app/tests/test_parse.py +++ b/app/tests/test_parse.py @@ -741,5 +741,26 @@ def test_no_trials_is_no_rate_either(self): self.assertIsNone(job.adjusted_solve_rate) +class KeptTries(unittest.TestCase): + def test_a_recorded_retry_count_is_read_not_trusted(self): + """``standard.retries`` says how many tries a page looks for: a record + that says a billion looks for 99 at most, never a billion.""" + with tempfile.TemporaryDirectory() as d: + os.makedirs(os.path.join(d, "job", "x")) + with open(os.path.join(d, "job", "x", "result.json"), "w") as f: + json.dump({"task_name": "b/t", "trial_name": "x", "standard": {"retries": 10 ** 9}}, f) + t = parse.parse_trial("job", os.path.join(d, "job", "x")) + real, looked = parse.parse_trial, [] + + def read(*a, **kw): + looked.append(a) + if len(looked) > 100: + raise AssertionError("looked past try 99") # fails at once where a billion would never end + return real(*a, **kw) + with mock.patch.object(parse, "parse_trial", side_effect=read): + self.assertEqual((t.n_retries, parse.tries(t)), (10 ** 9, [])) + self.assertEqual(len(looked), 99) + + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_progress.py b/app/tests/test_progress.py index dde576bb..90aafd02 100644 --- a/app/tests/test_progress.py +++ b/app/tests/test_progress.py @@ -115,6 +115,17 @@ def test_units_are_counted_as_they_end(self): u = got["units"] self.assertEqual((u["n"], u["noun"], u["done"], u["passed"], u["failed"], u["errors"], u["running"]), (10, "tasks", 3, 1, 1, 1, 0)) + def test_a_retry_takes_the_place_of_its_try_in_the_count(self): + """Two attempts, one rate-limited and run again: 2 of 2 at the end, no + error - the retry's end replaces its try's, never 3 of 2.""" + first = [ev("units", "start", n=2), ev("unit", "start", unit="a"), ev("unit", "start", unit="b"), + ev("unit", "done", outcome="passed"), ev("unit", "done", outcome="error", error="RateLimitError"), + ev("units", "progress", detail="trying 1 again, one at a time: retry 1 of 1"), ev("unit", "start", unit="b")] + u = progress.fold(first)["units"] + self.assertEqual((u["n"], u["done"], u["errors"], u["running"]), (2, 2, 1, 1)) # the retry running, the try's error still said + u = progress.fold(first + [ev("unit", "retried", unit="b"), ev("unit", "done", outcome="passed")])["units"] + self.assertEqual((u["n"], u["done"], u["passed"], u["errors"], u["running"]), (2, 2, 2, 0, 0)) + def test_a_run_of_several_parts_counts_every_parts_units_and_goes_on_after_one_ends(self): """nop then oracle, three units each: 6 of 6 at the end, never 6 of 3, and the first part's end is not the run's.""" diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index cb61ee16..ca360130 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -6,8 +6,9 @@ import shutil import tempfile import unittest +from unittest import mock -from nullius.standard import conform, definitions, install, match +from nullius.standard import conform, definitions, install, match, retries FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard") @@ -32,7 +33,10 @@ def check(self, kind, key, which): return got, definitions.get(kind, key, self.root) def test_a_kit_that_passes_is_recorded_with_its_job(self): - got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) + with mock.patch.object(conform.run, "run", wraps=conform.run.run) as ran: + got, d = self.check("capability", "fakemem", ["cap.memory/v1"]) + # a rate-limited unit runs once more, after its pause, and nothing else: a proof is never the endpoint's busy minute + self.assertEqual((ran.call_args.kwargs["retries"], ran.call_args.kwargs["retry_if"]), (1, retries.rate_limited)) self.assertTrue(got["cap.memory/v1"]["ok"]) rec = d["proven"]["cap.memory/v1"] self.assertEqual(rec["passed"], "2/2") diff --git a/app/tests/test_standard_ledger.py b/app/tests/test_standard_ledger.py index 9cdbfd75..90002105 100644 --- a/app/tests/test_standard_ledger.py +++ b/app/tests/test_standard_ledger.py @@ -67,6 +67,22 @@ def test_a_unit_that_reaches_its_own_cap_is_refused_from_then_on_and_the_run_goe self.assertEqual((doc["units"]["u1"]["spent_usd"], doc["units"]["u1"]["closed"], doc["of"].get(a)), (0.02, True, None)) self.assertAlmostEqual(led.state()["spent_usd"], 0.02) + def test_a_units_count_covers_every_try_of_it_until_none_follows(self): + """A retry books under its attempt's own unit: what the tries before it + spent stays counted, so the cap is the attempt's, all its tries + together - until no try follows and the count goes.""" + led = ledger.Ledger.create(self.path, None, 0.50) + self.assertEqual(led.unit_left("u1"), 0.50) + a, _ = led.reserve(0.10, "u1") + led.settle(a, 0.45) # try 1 spent 0.45, then crashed + self.assertAlmostEqual(led.unit_left("u1"), 0.05) # what its retry may spend + b, state = led.reserve(0.10, "u1") + self.assertEqual((b, state["by"]), (None, "unit")) # never a fresh 0.50 for the retry + self.assertEqual(led.unit_left("u1"), 0.0) # spent: no try follows + led.forget("u1") + self.assertEqual(led.unit_left("u1"), 0.50) + self.assertIsNone(ledger.Ledger.create(os.path.join(self.tmp, "run", "ledger.json"), 1.0).unit_left("u1")) # no cap per unit + def test_a_unit_that_has_spent_nothing_is_not_refused_on_what_other_units_replies_were(self): """Refused on that average, it would count against its agent with no call sent: 8 of 16 problems did, on a real run.""" led = ledger.Ledger.create(self.path, None, 0.00015) diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index a6758ccd..3ab1a31e 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -900,5 +900,38 @@ def test_no_reply_in_time_is_raised(self): q.close() +class Failure(unittest.TestCase): + """What a unit whose last call failed ends with: named so a retry is asked + only of what a second call may mend (``retries``).""" + + def test_a_spent_credit_and_an_unknown_model_are_named_as_no_retry_mends(self): + def named(status, said, **kw): + return relay.failure([{"status": status, "error": said, **kw}])[0] + self.assertEqual(named(429, "HTTP 429: Rate limit reached for requests"), "RateLimitError") + # OpenAI's spent credit is a 429 too; Gemini says "exceeded your current quota" for its per-minute limit + self.assertEqual(named(429, "HTTP 429: You exceeded your current quota", error_code="insufficient_quota"), "ApiUsageLimitError") + self.assertEqual(named(429, "HTTP 429: You exceeded your current quota, please check your plan"), "RateLimitError") + self.assertEqual(named(402, "HTTP 402: Insufficient credits"), "ApiUsageLimitError") + self.assertEqual(named(402, "HTTP 402: no price for the model", refused="price"), "ApiError") # the relay's own, as before + for said in ("HTTP 400: xyz is not a valid model ID", "HTTP 400: Invalid model: foo", "HTTP 400: Model Not Exist", + "HTTP 404: models/x is not found for API version v1beta"): + with self.subTest(said=said): + self.assertEqual(named(int(said[5:8]), said), "ModelNotFoundError") + for said in ("HTTP 400: This model's maximum context length is 8192 tokens", "HTTP 400: The model does not support tools"): + with self.subTest(said=said): + self.assertEqual(named(400, said), "ApiError") + + def test_the_providers_word_on_when_to_call_again_is_read(self): + self.assertEqual(relay._retry_after([("Retry-After", "20")]), 20.0) + self.assertEqual(relay._retry_after([("retry-after-ms", "1500")]), 1.5) + self.assertEqual(relay._retry_after([("Retry-After", "Wed, 21 Oct 2015 07:28:00 GMT")]), 0.0) # a date gone by + self.assertIsNone(relay._retry_after([("Retry-After", "soon")])) + self.assertIsNone(relay._retry_after([])) + self.assertEqual(relay._code(b'{"error": {"message": "x", "type": "insufficient_quota", "code": "insufficient_quota"}}'), + "insufficient_quota") + self.assertIsNone(relay._code(b'{"error": {"message": "x", "code": 429}}')) + self.assertIsNone(relay._code(b"not json")) + + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_standard_retries.py b/app/tests/test_standard_retries.py new file mode 100644 index 00000000..dda11ead --- /dev/null +++ b/app/tests/test_standard_retries.py @@ -0,0 +1,67 @@ +"""What an attempt is retried after: Harbor's retry rule, and this runner's own stops.""" + +import time +import unittest + +from nullius.standard import pace, retries + + +def ended(exception_type): + return {"exception_info": {"exception_type": exception_type}} if exception_type else {"verifier_result": {"rewards": {"reward": 0.0}}} + + +class Retryable(unittest.TestCase): + def test_infrastructure_and_agent_crashes_are_retried(self): + for et in ("RateLimitError", "ApiError", "NetworkConnectionError", "EnvironmentSetupError", "AgentSetupTimeoutError", + "NonZeroAgentExitCodeError", "OutputLengthExceededError", "VerifierError", "SomethingNewError"): + with self.subTest(et=et): + self.assertTrue(retries.retryable(ended(et))) + + def test_a_result_a_wrong_setting_a_cap_or_a_stop_is_not(self): + for et in ("AgentTimeoutError", "VerifierTimeoutError", "AgentSafetyRefusalError", "AgentAuthenticationError", + "ModelNotFoundError", "ApiUsageLimitError", "EnvironmentSecretMissingError", "SpendingCapError", + "UnitSpendingCapError", "UnitCallCapError", "CancelledError"): + with self.subTest(et=et): + self.assertFalse(retries.retryable(ended(et))) + + def test_a_failure_its_record_says_no_try_mends_is_not(self): + # a setup this machine cannot give, a driver that did not answer in its time: named as any setup or + # verifier error, told apart by the record + self.assertFalse(retries.retryable({"exception_info": {"exception_type": "EnvironmentSetupError", "retryable": False}})) + self.assertTrue(retries.retryable({"exception_info": {"exception_type": "EnvironmentSetupError"}})) + + def test_a_proof_run_tries_a_rate_limit_again_and_nothing_else(self): + self.assertTrue(retries.rate_limited(ended("RateLimitError"))) + for et in ("ApiError", "NonZeroAgentExitCodeError", "AgentSetupTimeoutError", None): + with self.subTest(et=et): + self.assertFalse(retries.rate_limited(ended(et))) + + def test_a_task_the_agent_failed_is_not(self): + self.assertFalse(retries.retryable(ended(None))) + self.assertFalse(retries.retryable(None)) + + def test_none_by_default(self): + self.assertEqual((retries.RETRIES, retries.MAX_RETRIES), (0, 9)) + + +class Pause(unittest.TestCase): + def test_a_rate_limit_holds_new_units_a_minute_or_as_long_as_the_provider_said(self): + self.assertEqual(pace.FIRST, 60.0) # a provider's limits are mostly per minute + p = pace.Pace() + t = time.time() + self.assertEqual(p.ended("RateLimitError"), 60.0) + self.assertEqual(p.ended("RateLimitError", said=500.0), 500.0) # longer than the doubled 120: the provider's + self.assertEqual(p.pause, 120.0) + self.assertGreaterEqual(p.until, t + 500) + self.assertIsNone(p.ended("ApiError")) + self.assertEqual(p.pause, 0.0) + + def test_a_retry_waits_its_pause_from_its_own_end(self): + now = time.time() + self.assertAlmostEqual(pace.Pace().due(now - 50), 10, delta=1) # a minute from its end, 50 s ago + self.assertEqual(pace.Pace().due(now - 120), 0.0) # over already: at once + self.assertAlmostEqual(pace.Pace().due(now - 50, said=200.0), 150, delta=1) # the provider said longer + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 34f4684b..bd8a3dd3 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import re import os import shutil import tempfile @@ -14,8 +15,9 @@ from datetime import datetime from unittest import mock +from nullius import progress from nullius.runs import classify, live, parse -from nullius.standard import acp, container, definitions, install, ledger, pace as pacing, relay, run +from nullius.standard import acp, container, definitions, install, jsonl, ledger, pace as pacing, relay, run ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") # A conversation whose user a model plays: it opens, speaks once more, then is done. @@ -101,6 +103,21 @@ class Echo(BaseHTTPRequestHandler): status = 200 refused = None # a model that gets the rate limit while every other is answered limited = 0 # how many calls, from now, get the rate limit before the model answers again + retry_after = None # what a refused call says of when to call again (Retry-After), when it says + listed = None # the models GET /models lists; None: no list, as a provider without one + + def do_GET(self): + if Echo.listed is None or not self.path.endswith("/models"): + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + return + out = json.dumps({"data": [{"id": m} for m in Echo.listed]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) def log_message(self, *a): pass @@ -112,6 +129,8 @@ def do_POST(self): if Echo.status != 200 or body.get("model") == Echo.refused or limited: out = json.dumps({"error": {"message": "Rate limit reached for requests"}}).encode() self.send_response(Echo.status if Echo.status != 200 else 429) + if Echo.retry_after is not None: + self.send_header("Retry-After", Echo.retry_after) else: said = body["messages"][-1]["content"] out = json.dumps({"choices": [{"message": {"role": "assistant", "content": "you said: " + said}}], @@ -138,7 +157,7 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self._home = install.HOME install.HOME = os.path.join(self.tmp, "home") - Echo.status, Echo.refused, Echo.limited = 200, None, 0 + Echo.status, Echo.refused, Echo.limited, Echo.retry_after, Echo.listed = 200, None, 0, None, None self.addCleanup(setattr, pacing, "FIRST", pacing.FIRST) pacing.FIRST = 0.5 # a rate-limited unit's pause, short enough for a test @@ -409,6 +428,13 @@ def test_a_repeated_runs_budget_names_each_attempt(self): self.assertEqual(result["budget"]["not_run"], [f"m{i} (attempt 1)" for i in range(4, 8)] + [f"m{i} (attempt 2)" for i in range(8)]) self.assertEqual((result["n_total_trials"], result["stats"]["n_cancelled_trials"]), (16, 12)) # as many as the trials not run + def test_the_last_line_counts_units_and_attempts_apart(self): + nouns = definitions.get("benchmark", "fakebench", ROOT)["units"]["nouns"] + _, _, _, said = self.calling(repeat=2) + self.assertTrue(any(s.startswith(f"done: 8 {nouns}, 16 attempts, ") for s in said), said) + _, _, _, said = self.calling() + self.assertTrue(any(s.startswith(f"done: 8 {nouns}, ") and "attempts" not in s for s in said), said) + def test_each_threads_driver_has_a_route_of_its_own_on_the_benchmarks_relay(self): """A cap's refusal of a driver's own call (an option's embeddings) is the unit's that driver served, never one beside it.""" r = run.Run.__new__(run.Run) @@ -511,7 +537,7 @@ def test_a_failed_unit_whose_last_call_got_a_429_is_rate_limited(self): concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True) (t,) = parse.parse_job(job_dir).trials self.assertEqual((t.exception_type, t.outcome, t.reward), ("RateLimitError", classify.RATE_LIMITED, None)) - self.assertEqual([c["status"] for c in self.relay_calls(job_dir)], [429, 429]) # run once more after a rate limit + self.assertEqual([c["status"] for c in self.relay_calls(job_dir)], [429]) # tried once: no retry by default def test_a_cap_refusal_in_a_passed_unit_is_still_the_caps(self): """The run's cap and the unit's own end a unit wherever their refusal falls, its last call included.""" @@ -600,36 +626,165 @@ def prompt(client, sid, text, timeout=600): with self.subTest(agent_timeout=agent_timeout), mock.patch.object(acp.Client, "prompt", prompt): got, _ = self.outcomes(self.go("fakebench", "fake", dataset="responding", name=f"rate-limit-{agent_timeout}")) - self.assertEqual(failed_prompts, ["model hi", "model hi"]) # run once more, after the others + self.assertEqual(failed_prompts, ["model hi"]) # tried once: no retry by default self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) self.assertEqual(got["good"].outcome, classify.RESOLVED) # made no call: its verdict stands - def test_a_rate_limited_unit_holds_the_next_back_and_is_run_again_once(self): + def test_a_rate_limited_unit_holds_the_next_back_and_is_retried(self): Echo.limited = 1 # the first call is refused, every later one answered b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, - concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding", retries=1) got, j = self.outcomes(job_dir) - self.assertEqual(len(j.trials), 3) # one trial per unit: the first attempt is not one of them - self.assertEqual(got["model"].outcome, classify.FAILED_VERIFICATION) # its second attempt counts: a verdict, no error - (name,) = os.listdir(os.path.join(job_dir, "retried")) + self.assertEqual(len(j.trials), 3) # one trial per unit: the try before its retry is not one of them + self.assertEqual(got["model"].outcome, classify.FAILED_VERIFICATION) # its retry counts: a verdict, no error + self.assertEqual(got["model"].n_retries, 1) + (kept,) = os.listdir(os.path.join(job_dir, "retried")) + name, _, number = kept.rpartition(".try-") self.assertTrue(name.startswith("model__")) - with open(os.path.join(job_dir, "retried", name, "result.json")) as f: - first = json.load(f) # the first attempt, kept whole beside the job's trials + self.assertEqual(number, "1") + with open(os.path.join(job_dir, "retried", kept, "result.json")) as f: + first = json.load(f) # the first try, kept whole beside the job's trials self.assertEqual(first["exception_info"]["exception_type"], "RateLimitError") with open(os.path.join(job_dir, name, "result.json")) as f: second = json.load(f) + self.assertEqual(second["standard"]["retries"], 1) ended, began = (datetime.fromisoformat(x.replace("Z", "+00:00")) for x in (first["finished_at"], second["started_at"])) self.assertGreaterEqual((began - ended).total_seconds(), 0.4) # the pause, before it starts again with open(os.path.join(job_dir, "progress.jsonl")) as f: said = [json.loads(line) for line in f] self.assertTrue(any("rate limited: no unit starts for 0.5 s" in (e.get("detail") or "") for e in said)) + # the retry's end took its try's place: three attempts, none an error, the run's n never grown by the retry + units = progress.fold(said)["units"] + self.assertEqual((units["n"], units["done"], units["errors"], units["running"]), (3, 3, 0, 0)) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["stats"]["n_retries"], 1) + self.assertEqual(j.n_retries, 1) + + def test_an_attempt_is_retried_up_to_its_retries_and_never_by_default(self): + Echo.status = 429 # every call refused: each try ends rate-limited + real_prompt = acp.Client.prompt + + def prompt(client, sid, text, timeout=600): + if text == "model hi": + tries.append(text) + return real_prompt(client, sid, text, timeout=timeout) + + for retries in (2, 0): + tries = [] + with self.subTest(retries=retries), mock.patch.object(acp.Client, "prompt", prompt): + job_dir = self.go("fakebench", "fake", dataset="responding", name=f"retries-{retries}", retries=retries) + got, _ = self.outcomes(job_dir) + self.assertEqual(len(tries), 1 + retries) + self.assertEqual((got["model"].outcome, got["model"].n_retries), (classify.RATE_LIMITED, retries)) + kept = sorted(os.listdir(os.path.join(job_dir, "retried"))) if retries else [] + self.assertEqual([k.rpartition(".try-")[2] for k in kept if k.startswith("model__")], [str(n) for n in range(1, retries + 1)]) + self.assertEqual(os.path.exists(os.path.join(job_dir, "retried")), retries > 0) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["retries"], retries) + # what each attempt's retries are counted over on the pages + self.assertEqual(parse.parse_job(job_dir).retries, retries) + # each try, as the pages number and address it: Try 1 first, the trial last + name = got["model"].trial_name + self.assertEqual([(n, t.trial_name, t.outcome, t.n_retries) for n, t in parse.tries(got["model"])], + [(n, name if n > retries else f"{name}.try-{n}", classify.RATE_LIMITED, n - 1) + for n in range(1, retries + 2)] if retries else []) + + def test_an_agent_crash_is_retried_and_a_timeout_is_not(self): + real_prompt = acp.Client.prompt + + def prompt(client, sid, text, timeout=600): + if text == "model hi": + tries.append(text) + if kind == "timeout": + raise acp.Timeout("no answer in time", client.events) + if len(tries) == 1: + raise acp.AgentFailed("the agent's process exited with 1") + return real_prompt(client, sid, text, timeout=timeout) + + for kind in ("crash", "timeout"): + tries = [] + with self.subTest(kind=kind), mock.patch.object(acp.Client, "prompt", prompt): + job_dir = self.go("fakebench", "fake", dataset="responding", name=f"retried-{kind}", retries=1) + got, _ = self.outcomes(job_dir) + if kind == "crash": + self.assertEqual(len(tries), 2) # the crash, then the retry that answered + self.assertEqual(got["model"].outcome, classify.FAILED_VERIFICATION) + (kept,) = os.listdir(os.path.join(job_dir, "retried")) + with open(os.path.join(job_dir, "retried", kept, "result.json")) as f: + self.assertEqual(json.load(f)["exception_info"]["exception_type"], "NonZeroAgentExitCodeError") + else: + self.assertEqual(len(tries), 1) # a timeout is the agent's result: never retried + self.assertEqual(got["model"].outcome, classify.AGENT_ERROR) + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + + def test_retries_run_as_many_at_once_as_the_run(self): + """Every unit crashes on its first try: their retries share the run's + concurrency (2 here), so two of them are running at once.""" + real_prompt = acp.Client.prompt + lock, seen = threading.Lock(), {} + + def prompt(client, sid, text, timeout=600): + with lock: + seen[text] = seen.get(text, 0) + 1 + first = seen[text] == 1 + if first: + raise acp.AgentFailed("the agent's process exited with 1") + time.sleep(0.6) # each retry takes long enough to meet another + return real_prompt(client, sid, text, timeout=timeout) + + with mock.patch.object(acp.Client, "prompt", prompt): + job_dir = self.go("fakebench", "fake", dataset="responding", name="retried-together", retries=1) + spans = [] + for name in os.listdir(job_dir): + path = os.path.join(job_dir, name, "result.json") + if os.path.isfile(path): + with open(path) as f: + res = json.load(f) + if (res.get("standard") or {}).get("retries"): + spans.append(tuple(datetime.fromisoformat(res[k].replace("Z", "+00:00")) for k in ("started_at", "finished_at"))) + self.assertGreaterEqual(len(spans), 2, spans) + self.assertTrue(any(a[0] < b[1] and b[0] < a[1] for i, a in enumerate(spans) for b in spans[i + 1:]), spans) + + def test_a_model_the_provider_does_not_list_is_refused_before_anything_starts(self): + """Every call to it would be refused the same way: no install, no unit, + the job saying why. A :suffix routes the model it follows; a provider + with no list is not checked.""" + Echo.listed = ["m-1"] + b, o = definitions.get("benchmark", "fakebench", ROOT), definitions.get("agent", "fake", ROOT) + kw = dict(provider="custom", jobs_root=self.tmp, concurrency=2, env=self.env, say=lambda s: None, unsandboxed=True) + with self.assertRaises(run.Refused) as e: + run.run(b, o, model="m-typo", name="unlisted", **kw) + self.assertIn("lists no model m-typo: check the name", str(e.exception)) + job_dir = os.path.join(self.tmp, "unlisted") + self.assertFalse([n for n in os.listdir(job_dir) if "__u" in n]) # no unit begun + with open(os.path.join(job_dir, "progress.jsonl")) as f: + self.assertTrue(any(json.loads(line).get("stage") == "refused" for line in f)) + self.assertTrue(os.path.isfile(os.path.join(run.run(b, o, model="m-1:nitro", name="routed", **kw), "result.json"))) + Echo.listed = None + self.assertTrue(os.path.isfile(os.path.join(run.run(b, o, model="m-typo", name="no-list", **kw), "result.json"))) + + def test_a_capped_run_whose_price_list_could_not_be_fetched_runs_uncapped_and_says_so(self): + said = [] + b, o = definitions.get("benchmark", "fakebench", ROOT), definitions.get("agent", "fake", ROOT) + kw = dict(model="m-1", provider="custom", jobs_root=self.tmp, concurrency=2, env=self.env, unsandboxed=True, max_usd=0.5) + job_dir = run.run(b, o, name="uncapped", say=said.append, prices_unreachable="could not fetch the price list: offline", **kw) + self.assertIn("warning: no spending cap: the price list could not be fetched, and m-1 has no price " + "(could not fetch the price list: offline)", said) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertNotIn("budget", json.load(f)) + with open(os.path.join(job_dir, "progress.jsonl")) as f: + units = [e for e in map(json.loads, f) if e.get("stage") == "units" and e.get("state") == "start"] + self.assertEqual(units[0]["detail"], "no spending cap: the price list could not be fetched, and m-1 has no price") + with self.assertRaises(run.Refused) as e: # the list reached, the model not in it + run.run(b, o, name="capped", say=lambda s: None, **kw) + self.assertIn("a spending cap needs a price for every model the run calls, and there is none for m-1", str(e.exception)) def test_a_stopped_run_runs_nothing_again(self): Echo.status = 429 b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, - concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding", retries=1) real = r.unit def unit(*a): @@ -642,6 +797,102 @@ def unit(*a): self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + with open(os.path.join(job_dir, "progress.jsonl")) as f: + self.assertFalse(any("again" in (json.loads(line).get("detail") or "") for line in f)) # no round said that never ran + + def test_a_stop_during_a_retry_leaves_the_try_before_as_the_attempt(self): + """Stop is a SystemExit in the CLI (cli._exit_on_term), and a retry runs + on the run's own thread, where it lands: whether the retry was waiting + or running, the attempt keeps the try it had, counted.""" + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + for where in ("waiting", "running"): + with self.subTest(where=where): + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding", retries=1, + name=f"stopped-{where}") + + def unit(i, u, rr=1, retry=0, real=r.unit, r=r, where=where): + if retry: + if where == "running": # broken off as it ran: a folder, no result + os.makedirs(os.path.join(r.job_dir, r.trial_name(i, u, rr), "workspace")) + raise SystemExit(143) + return real(i, u, rr, retry) + r.unit = unit + with self.assertRaises(SystemExit): + r.start() + got, j = self.outcomes(r.job_dir) + self.assertEqual(len(j.trials), 3) + self.assertEqual((got["model"].outcome, got["model"].n_retries), (classify.RATE_LIMITED, 0)) + self.assertFalse(os.path.exists(os.path.join(r.job_dir, "retried"))) # the try before is back; nothing beside it + with open(os.path.join(r.job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["stats"]["n_errored_trials"], 1) + + def test_an_attempt_that_spent_its_own_cap_is_not_tried_again(self): + """The cap per unit is the attempt's, all its tries together: one that + spent it is not run again, one that did not is.""" + real_prompt = acp.Client.prompt + + def prompt(client, sid, text, timeout=600): + out = real_prompt(client, sid, text, timeout=timeout) + if text == "model hi": + tries.append(text) + if len(tries) == 1: + raise acp.AgentFailed("the agent's process exited with 1") # after its one call, paid for + return out + + prices = {"m-1": {"input": 1.0, "output": 10.0, "max_output": 10}} # a call: 11 in, 3 out, 0.000041 USD + for cap, again in ((0.00004, False), (0.001, True)): + tries = [] + with self.subTest(cap=cap), mock.patch.object(acp.Client, "prompt", prompt): + job_dir = self.go("fakebench", "fake", dataset="responding", name=f"capped-{again}", retries=1, + prices=prices, max_usd_per_unit=cap) + self.assertEqual(len(tries), 2 if again else 1) + self.assertEqual(os.path.exists(os.path.join(job_dir, "retried")), again) + got, j = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.FAILED_VERIFICATION if again else classify.AGENT_ERROR) + # the run's cost: the counted try's and, retried, the try before it, both paid for + self.assertAlmostEqual(j.total_cost_usd, 0.000041 * (2 if again else 1)) + + def test_a_driver_that_did_not_answer_in_its_time_is_not_asked_again(self): + """A score with no reply in its time is the benchmark's: a retry would run + the agent again, paid for, to meet the same wait. One that broke is retried.""" + real = jsonl.Proc.request + + def request(proc, op, timeout=600, **kw): + if op == "score" and kw.get("unit") == "model" and not scored: + scored.append(op) + raise (jsonl.NoReply if kind == "no-reply" else jsonl.Failed)(f"score: {kind}") + return real(proc, op, timeout=timeout, **kw) + + for kind in ("no-reply", "broke"): + scored = [] + with self.subTest(kind=kind), mock.patch.object(jsonl.Proc, "request", request): + job_dir = self.go("fakebench", "fake", dataset="responding", name=f"driver-{kind}", retries=1) + retried = os.path.exists(os.path.join(job_dir, "retried")) + self.assertEqual(retried, kind == "broke") + if not retried: + got, _ = self.outcomes(job_dir) + with open(os.path.join(got["model"].dir, "result.json")) as f: + self.assertEqual(json.load(f)["exception_info"]["retryable"], False) + + def test_a_rate_limits_retry_waits_as_long_as_the_provider_said(self): + Echo.limited, Echo.retry_after = 1, "2" # the first call refused: call again in 2 s, not the 0.5 s pause + job_dir = self.go("fakebench", "fake", dataset="responding", name="said-wait", retries=1) + (kept,) = os.listdir(os.path.join(job_dir, "retried")) + with open(os.path.join(job_dir, "retried", kept, "result.json")) as f: + first = json.load(f) + with open(os.path.join(job_dir, kept.rpartition(".try-")[0], "result.json")) as f: + second = json.load(f) + ended, began = (datetime.fromisoformat(x.replace("Z", "+00:00")) for x in (first["finished_at"], second["started_at"])) + self.assertGreaterEqual((began - ended).total_seconds(), 1.5) + + def test_without_a_retry_the_providers_pause_still_holds_the_next_unit_back(self): + Echo.limited, Echo.retry_after = 1, "2" + job_dir = self.go("fakebench", "fake", dataset="responding", name="said-pause") # no retries: the default + with open(os.path.join(job_dir, "progress.jsonl")) as f: + pauses = [float(m[1]) for line in f if (m := re.search(r"no unit starts for ([0-9.]+) s", json.loads(line).get("detail") or ""))] + self.assertTrue(pauses and pauses[0] >= 1.5, pauses) # the provider's 2 s, not the 0.5 s pause def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") diff --git a/app/tests/test_standard_setups.py b/app/tests/test_standard_setups.py index a1f42aee..be62b957 100644 --- a/app/tests/test_standard_setups.py +++ b/app/tests/test_standard_setups.py @@ -342,7 +342,7 @@ def test_like_the_leaderboard_runs_its_user_simulator_options_and_attempts(self) model_choice="leaderboard") out = self.compiled(**tau) self.assertEqual(out["errors"], []) - for words in ("--model their-model", "--simulator-model gpt-5.2", "--bench-option user_reasoning_effort=low", "--repeat 4"): + for words in ("--model their-model", "--simulator-model gpt-5.2", "--bench-option user_reasoning_effort=low", "--attempts 4"): self.assertIn(words, out["script"]) self.assertNotIn("--judge-model", out["script"]) self.assertIn("--simulator-model openai/gpt-5.2", self.compiled(**{**tau, "provider": "openrouter"})["script"]) diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index c19da339..fdd3bdcc 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -119,6 +119,42 @@ def test_the_merged_job_reads_as_one_run_of_every_trial(self): self.assertTrue(os.path.samefile(os.path.join(src, "workspace", "solution.py"), os.path.join(got["out"], "t-1__u0001", "workspace", "solution.py"))) + def retried(self, dirs, retries=1): + """The launch allowed each attempt one retry, and the first shard's first attempt took it: its try + before kept under retried/, as a run leaves it. The attempt's name.""" + for d in dirs: + cfg = read(os.path.join(d, "config.json")) + cfg["standard"]["retries"] = retries + with open(os.path.join(d, "config.json"), "w") as f: + json.dump(cfg, f) + t = sorted(n for n in os.listdir(dirs[0]) if "__u" in n)[0] + res = read(os.path.join(dirs[0], t, "result.json")) + first = dict(res, exception_info={"exception_type": "RateLimitError", "exception_message": "HTTP 429"}) + os.makedirs(os.path.join(dirs[0], "retried", t + ".try-1")) + with open(os.path.join(dirs[0], "retried", t + ".try-1", "result.json"), "w") as f: + json.dump(first, f) + res["standard"]["retries"] = 1 + with open(os.path.join(dirs[0], t, "result.json"), "w") as f: + json.dump(res, f) + return t + + def test_the_merged_job_keeps_each_shards_earlier_tries(self): + dirs = self.shards() + t = self.retried(dirs) + got = merge.merge(dirs, os.path.join(self.tmp, "out", "run")) + self.assertEqual(read(os.path.join(got["out"], "retried", t + ".try-1", "result.json"))["standard"]["job"], "run") + j = parse.parse_job(got["out"]) + self.assertEqual((len(j.trials), j.retries, j.n_retries), (10, 1, 1)) # a kept try is no trial; the retry is counted + attempt = next(x for x in j.trials if x.trial_name == t) + self.assertEqual([(n, x.outcome) for n, x in parse.tries(attempt)], [(1, "rate_limited"), (2, attempt.outcome)]) + + def test_shards_that_allowed_different_retries_are_not_one_launch(self): + dirs = self.shards() + self.retried(dirs[:1], retries=2) + with self.assertRaises(merge.Refused) as e: + merge.merge(dirs, os.path.join(self.tmp, "never")) + self.assertIn("retries per attempt", str(e.exception)) + def test_the_merged_job_carries_the_shards_spending_caps_as_one(self): """Each shard ran under its share of the run's cap: the merged job says the whole cap, what all of them spent, and every unit a cap cut off or left out.""" diff --git a/app/tests/test_standard_skills.py b/app/tests/test_standard_skills.py index 6a072720..a89dba2f 100644 --- a/app/tests/test_standard_skills.py +++ b/app/tests/test_standard_skills.py @@ -149,7 +149,7 @@ def test_problems_checks_a_kits_basic_and_skills(self): class Repeat(unittest.TestCase): - """``conform --repeat``: each check a trial of its own per run, the record counting them.""" + """``conform --attempts``: each check a trial of its own per run, the record counting them.""" def setUp(self): self.tmp = tempfile.mkdtemp() self.root = os.path.join(self.tmp, "catalogue") diff --git a/app/tests/test_standard_ui.py b/app/tests/test_standard_ui.py index 4750b174..17bb32f7 100644 --- a/app/tests/test_standard_ui.py +++ b/app/tests/test_standard_ui.py @@ -5,6 +5,7 @@ from __future__ import annotations import os +import re import shutil import tempfile import unittest @@ -33,6 +34,16 @@ def a_recipe(**kw): class Backend(unittest.TestCase): + def test_attempts_and_retries_reach_the_command_only_when_set(self): + for kw, flags in (({}, []), ({"retries": 0}, []), ({"retries": 2, "repeat": 3}, ["--attempts 3", "--retries 2"])): + with self.subTest(kw=kw): + out = recipe.compile(a_recipe(unsandboxed=True, **kw), defs()) + self.assertEqual(out["errors"], []) + self.assertEqual(sorted(set(re.findall(r"--attempts \S+|--retries \S+", out["script"]))), flags) + self.assertEqual(any("up to 2 retries an attempt" in str(row) for row in out["review"]), bool(flags)) + bad = recipe.compile(a_recipe(unsandboxed=True, retries=10), defs())["errors"] + self.assertIn("Retries per attempt: a whole number from 0 to 9.", [e["message"] for e in bad]) + def test_a_container_needs_no_yes_and_says_so_in_the_command(self): out = recipe.compile(a_recipe(backend="container"), defs(), docker_unusable=lambda: None) self.assertEqual(out["errors"], []) diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index 6ed5e162..40223bbb 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -243,9 +243,9 @@ def test_a_run_without_a_list_is_the_run_it_was(self): self.assertEqual(self.trials(job), ["good__u0000", "model__u0003", "slow__u0002", "wrong__u0001"]) self.assertEqual(self.started(job), DATASET) cfg = read(os.path.join(job, "config.json")) - # the keys a control's run recorded before there were lists, and no other + # the keys a control's run recorded before there were lists, the retries every run now records, and no other self.assertEqual(sorted(cfg["standard"]), ["backend", "benchmark", "control", "kind", "needs", "permissions_granted", - "plugged", "provider", "unsandboxed"]) + "plugged", "provider", "retries", "unsandboxed"]) self.assertEqual(cfg["datasets"], [{"name": "fakebench/all", "n_tasks": 4}]) limited = self.go(limit=2, name="limited") self.assertEqual(self.trials(limited), ["good__u0000", "wrong__u0001"]) @@ -253,8 +253,8 @@ def test_a_run_without_a_list_is_the_run_it_was(self): class RateLimited(unittest.TestCase): - """A listed unit that ends rate-limited is run once more after the others, as any unit is: under its own - trial's name, its first attempt kept in the job's ``retried/``. A unit the list leaves out is never run.""" + """A listed unit that ends rate-limited is retried after the others, as any unit is: under its own + trial's name, its first try kept in the job's ``retried/``. A unit the list leaves out is never run.""" @classmethod def setUpClass(cls): @@ -288,10 +288,10 @@ def test_a_listed_unit_that_ends_rate_limited_is_run_again_under_its_own_name(se b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) job = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), concurrency=1, env=self.env, say=lambda s: None, - unsandboxed=True, dataset="responding", units_file=run.units_file(path)) + unsandboxed=True, dataset="responding", units_file=run.units_file(path), retries=1) self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000", "model__u0002"]) - self.assertEqual(os.listdir(os.path.join(job, "retried")), ["model__u0002"]) - first = read(os.path.join(job, "retried", "model__u0002", "result.json")) + self.assertEqual(os.listdir(os.path.join(job, "retried")), ["model__u0002.try-1"]) + first = read(os.path.join(job, "retried", "model__u0002.try-1", "result.json")) self.assertEqual(first["exception_info"]["exception_type"], "RateLimitError") self.assertFalse(read(os.path.join(job, "model__u0002", "result.json")).get("exception_info")) # its second counts self.assertEqual(read(os.path.join(job, "result.json"))["n_total_trials"], 2) @@ -329,6 +329,19 @@ def file(self, text): f.write(text) return path + def test_a_capped_run_fetches_the_price_list_first(self): + offline = "could not fetch the price list from the list's address: offline" + with mock.patch.object(cli.price_list, "update", side_effect=ValueError(offline)) as update, \ + mock.patch.object(cli.run, "run_and_report", return_value=("job", False)) as ran: + self.cli("--max-usd", "1") + update.assert_called_once_with() + self.assertEqual(ran.call_args.kwargs["prices_unreachable"], offline) # the run goes on uncapped, saying why + with mock.patch.object(cli.price_list, "update") as update, \ + mock.patch.object(cli.run, "run_and_report", return_value=("job", False)) as ran: + self.cli() + update.assert_not_called() # no cap: nothing to price + self.assertIsNone(ran.call_args.kwargs["prices_unreachable"]) + def test_the_command_runs_the_listed_units_with_and_without_from_run(self): path = self.file("# a pair\nslow\ngood\n") for extra in ([], ["--from-run"]): diff --git a/app/ui_tests/__init__.py b/app/ui_tests/__init__.py index 73a8e3db..1afdef12 100644 --- a/app/ui_tests/__init__.py +++ b/app/ui_tests/__init__.py @@ -1,5 +1,6 @@ """Flask integration checks, separate from the framework-free data suite.""" +import tests # noqa: F401 (no test reaches past this machine: tests/__init__.py) from nullius import execution, prerequisites # The suite runs where docker may not answer, and a run here asks it before it starts diff --git a/app/ui_tests/test_attempt_routes.py b/app/ui_tests/test_attempt_routes.py index f4ba89df..fbedcf4c 100644 --- a/app/ui_tests/test_attempt_routes.py +++ b/app/ui_tests/test_attempt_routes.py @@ -6,11 +6,14 @@ from contextlib import ExitStack from html import unescape from pathlib import Path +import json import re import tempfile import unittest from unittest.mock import patch +from werkzeug.exceptions import NotFound + from nullius.runs import parse from nullius import workspace from .test_routes import web, write_json @@ -175,6 +178,80 @@ def test_all_attempts_writes_a_reward_as_the_score_beside_it(self): rewards = [re.findall(r'

([^<]*)
Retries(.*?)