diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index 59e1ba5e..b4f9d233 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -229,6 +229,17 @@ def _agent(): return name, AGENTS["agents"][name] +def with_steps(argv, steps): + """The script's argv with the run's ``max_steps`` option as its --max_steps: the agent's own (its + invocation's, else its script's default) at ``default``.""" + if steps == "default": + return argv + if "--max_steps" in argv: + i = argv.index("--max_steps") + return argv[:i + 1] + [steps] + argv[i + 2:] + return argv + ["--max_steps", steps] + + def _fill(value, slots): for k, v in slots.items(): value = value.replace("{" + k + "}", v) @@ -269,6 +280,7 @@ def answer(unit, dir, setup, model, base_url, key, **_): del common[i:i + (1 if flag == "--headless" else 2)] argv = [sys.executable, os.path.join(UP, spec["script"])] + [_fill(a, slots) for a in common + spec["argv"] + spec.get("model_argv", {}).get(model, [])] + argv = with_steps(argv, os.environ.get("NULLIUS_OPTION_MAX_STEPS", "default")) extra = {k: _fill(v, slots) for k, v in spec["env"].items()} if spec.get("aws"): extra.update(AGENTS["aws"]) diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 8c01261e..53f55d38 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -176,6 +176,33 @@ ] } }, + "max_steps": { + "label": "Steps per task", + "summary": "The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it.", + "setups": [ + "original", + "model" + ], + "default": "default", + "pattern": "default|[1-9][0-9]{0,3}", + "limit": true, + "by": { + "agent": { + "m3": "100", + "prompt": "15", + "claude": "50", + "gpt54": "50", + "openai_cua": "50", + "qwen": "50", + "os_symphony": "50", + "evocua": "50", + "autoglm": "50", + "autoglm_v": "50", + "mano": "15", + "muse_spark": "100" + } + } + }, "vm_provider": { "label": "VM provider", "summary": "--provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own.", diff --git a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json index c3c02423..99e0a49c 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json @@ -60,7 +60,7 @@ "setting": "--max_steps", "values": "a number", "default": "100", - "nullius": "fixed:100 - its default and its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_m3.py", @@ -340,7 +340,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_autoglm.py", @@ -501,7 +501,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_autoglm_v.py", @@ -676,7 +676,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:15 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv.py", @@ -914,7 +914,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_claude.py", @@ -1131,7 +1131,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_evocua.py", @@ -1348,7 +1348,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_gpt54.py", @@ -1565,7 +1565,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:15 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_mano.py", @@ -1824,7 +1824,7 @@ "setting": "--max_steps", "values": "int", "default": "100", - "nullius": "fixed:100 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_muse_spark.py", @@ -2125,7 +2125,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_openaicua.py", @@ -2342,7 +2342,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_os_symphony.py", @@ -2748,7 +2748,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_qwen.py", diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py index a0a2606c..f54c8669 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py +++ b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py @@ -24,9 +24,10 @@ executes nothing itself: it passes each call to this driver turn - the conversation is the upstream's: its batch runner, ``runner.batch.run_tasks``, run on the unit's one task in a thread of - this driver, with a ``TextRunConfig`` at the upstream's defaults (max - steps, max errors, no timeout, one trial, its seed 300, three - retries, its user simulator). It draws the trial's seed from the + this driver, with a ``TextRunConfig`` at the upstream's defaults (no + timeout, one trial, its seed 300, three + retries, its user simulator), its max steps and max errors the run's + ``max_steps`` and ``max_errors`` options (``bounds``). It draws the trial's seed from the base seed as it does for every run: the unit's first attempt is trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1 at the seed run_tasks draws for that trial (``trial``), and each @@ -406,10 +407,18 @@ def retrieval(): return os.environ.get("NULLIUS_OPTION_RETRIEVAL") or None +def bounds(): + """The conversation's bounds the run set, the upstream's --max-steps and --max-errors: its ``max_steps`` + and ``max_errors`` options; the upstream's defaults (200, 10) otherwise.""" + return {k: int(os.environ[v]) for k, v in (("max_steps", "NULLIUS_OPTION_MAX_STEPS"), ("max_errors", "NULLIUS_OPTION_MAX_ERRORS")) + if os.environ.get(v)} + + def run_config(**kw): - """The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one.""" + """The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one, + and the conversation's bounds (bounds).""" return TextRunConfig(domain=domain(), task_split_name=split(), - **({"retrieval_config": retrieval()} if retrieval() else {}), **kw) + **({"retrieval_config": retrieval()} if retrieval() else {}), **bounds(), **kw) def user_config(simulator): diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json index d8d7e614..9747780f 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json @@ -106,14 +106,14 @@ }, "score": { "kind": "pass", - "says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200, max errors 10, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)", + "says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200 and max errors 10 unless the run sets lower or other ones, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)", "judge": { "model": "gpt-4.1-2025-04-14", "says": "the upstream's NL-assertion evaluator (evaluator_nl_assertions.py, its prompt, its call and its own model name), reached through the judge's route, at temperature 0, asked only for a task whose reward_basis names NL_ASSERTION and that has assertions" } }, "limits": { - "turn_seconds": 600, + "turn_seconds": 7200, "turns": 100, "attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it." }, @@ -311,6 +311,28 @@ "default": "The upstream's own arguments: temperature 0, no effort sent." } }, + "max_steps": { + "label": "Steps per conversation", + "summary": "The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met.", + "setups": [ + "original", + "yours" + ], + "default": "200", + "pattern": "[1-9][0-9]?|1[0-9]{2}|200", + "limit": true + }, + "max_errors": { + "label": "Tool errors per conversation", + "summary": "The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's.", + "setups": [ + "original", + "yours" + ], + "default": "10", + "pattern": "[1-9][0-9]{0,3}", + "limit": true + }, "upstream_agent": { "summary": "Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools.", "setups": [ diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json index 821c3c4f..9f4f9ec4 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json @@ -158,21 +158,21 @@ "setting": "--max-steps", "values": "an integer", "default": "200 (DEFAULT_MAX_STEPS)", - "nullius": "fixed:200 - the upstream's default, the leaderboard's; limits.turns (100) lets its limit be the one met" + "nullius": "option:max_steps" }, { "entry": "tau2 run (src/tau2/cli.py)", "setting": "--max-errors", "values": "an integer", "default": "10 (DEFAULT_MAX_ERRORS)", - "nullius": "fixed:10 - the upstream's default" + "nullius": "option:max_errors" }, { "entry": "tau2 run (src/tau2/cli.py)", "setting": "--timeout", "values": "seconds", "default": "none", - "nullius": "fixed:none - the upstream's default; a run's turn limit (600 s an agent turn, 7200 s an original conversation) ends a unit that hangs, as the unit's error" + "nullius": "fixed:none - the upstream's default; a run's agent limit (7200 s a conversation, as an original one; run --timeout agent=SECONDS forces another) ends a unit that hangs, as the unit's error" }, { "entry": "tau2 run (src/tau2/cli.py)", diff --git a/app/nullius/runs/classify.py b/app/nullius/runs/classify.py index ec8b7acf..8346df4f 100644 --- a/app/nullius/runs/classify.py +++ b/app/nullius/runs/classify.py @@ -85,6 +85,7 @@ _AGENT_EXCEPTIONS = { "agenttimeouterror", # the agent spent its budget — that is a result "unitspendingcaperror", # the unit spent its own spending cap (the standard's relay): the same, in money + "unitcallcaperror", # the unit's agent made the model calls the run allowed it: the same, in steps "agentsafetyrefusalerror", # the agent (or its provider) chose to refuse "contextwindowexceedederror", # the agent filled its own context "contextlengthexceedederror", # same, from Harbor's LLM layer diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 9a803b58..951868ac 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,7 +7,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -446,17 +446,22 @@ def cmd_new(a): return 0 -def options(pairs) -> dict: +def options(pairs, flag: str = "--option", form: str = "NAME=VALUE") -> dict: """``--option name=value``, repeatable, as {name: value}.""" out = {} for p in pairs or []: name, eq, value = p.partition("=") if not eq or not name: - sys.exit(f"--option {p!r}: NAME=VALUE") + sys.exit(f"{flag} {p!r}: {form}") out[name.strip()] = value.strip() return out +def timeouts(pairs) -> dict: + """``--timeout PHASE=SECONDS``, repeatable, as {phase: seconds}; ``run`` checks the phases and the values.""" + return options(pairs, "--timeout", "PHASE=SECONDS") + + def constraint_values(pairs) -> dict: """``--constraint KEY=VALUE``, repeatable: what the machine allows the run (``nullius.constraints``).""" out = {} @@ -535,6 +540,7 @@ def cmd_run(a): boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, + timeouts=timeouts(a.timeout), max_turns=a.max_turns, max_calls=a.max_calls, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) @@ -714,6 +720,14 @@ def main(argv=None): "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on") + p.add_argument("--timeout", action="append", metavar="PHASE=SECONDS", + help="a unit's time limit for one phase, forced over the task's, the benchmark's and the default " + "(repeatable): setup, image, agent (its whole work on the unit), verify, score") + p.add_argument("--max-turns", type=int, metavar="N", + help="the most agent turns in a conversation, over the benchmark's limits.turns") + p.add_argument("--max-calls", type=int, metavar="N", + help="the most model calls a unit's agent may make, the whole unit through: past it the relay refuses " + "the next call and the unit is the agent's error, as a timeout is") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bacddab5..ea7f6d13 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + max_calls: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -752,6 +753,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--prices", json.dumps(prices, separators=(",", ":"))] if ledger: a += ["--ledger", ledger] + if max_calls: + a += ["--max-calls", str(max_calls)] for m in side_models: a += ["--side-model", m] # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's @@ -1022,8 +1025,9 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, - serving: Optional[Dict[str, List[str]]] = None): + serving: Optional[Dict[str, List[str]]] = None, max_calls: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app + self.max_calls = max_calls # the run's cap on its agent's model calls, when it has one self.ledger = ledger # the file of the run's spending cap, when it has one self.serving = dict(serving or {}) # by model, the serving providers that may answer it self.servers = list(servers) # licence servers, host:port each: a forwarder apiece @@ -1051,7 +1055,7 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger, serving=self.serving) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, max_calls=self.max_calls) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) @@ -1433,16 +1437,17 @@ def own_tag(bench: dict, rel: str, path: str) -> str: return f"nullius/own-{bench['key']}-{word}:{h.hexdigest()[:12]}" -def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) -> str: +def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None, timeout: float = 3600) -> str: """A unit's own image: ``pull``ed by its digest, or built from the folder ``build`` names in the benchmark's upstream (``root``) - the task's own Dockerfile, its own ``FROM``, its folder the whole build context - the first time a unit asks for it, and kept by its tag after. Two units - asking at once wait for one build. No secret is given to a build.""" + asking at once wait for one build. No secret is given to a build. + ``timeout``: the most seconds the pull or the build may take.""" if spec.get("pull"): ref = spec["pull"] if not present(ref): - r = _docker(["pull", ref], timeout=3600, check=False) + r = _docker(["pull", ref], timeout=timeout, check=False) if r.returncode != 0: raise Unavailable(f"pulling {ref}: {(r.stderr or r.stdout).strip()[-300:]}") _built[ref] = True @@ -1458,7 +1463,7 @@ def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) if present(tag): return tag # --force-rm: a build that fails (a task's Dockerfile fetching what is gone) leaves no container behind - r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=3600, check=False) + r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=timeout, check=False) if log_dir: os.makedirs(log_dir, exist_ok=True) with open(os.path.join(log_dir, "own-" + tag.split(":")[0].split("/")[-1] + ".log"), "a") as f: diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index bf5bcf86..8bf1516f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -468,7 +468,9 @@ def _bench_options(d: dict) -> List[str]: ``summary``, its ``values``, a ``default`` a run gets when it chooses none, and optionally the ``datasets`` it applies to, a line ``about`` each value, what a value ``needs``: an ``embeddings`` model called through the run's relay, ``host`` requirements or ``files`` by name - and - ``when`` it applies: another option's values (a retriever's, only with a retrieval).""" + ``when`` it applies: another option's values (a retriever's, only with a retrieval). ``limit``: + a bound the upstream has of its own (its steps, its errors), offered with the run's limits; ``by``: + its default under each value of another option (an agent's own steps), for a default of ``default``.""" opts = d.get("options") if opts is None: return [] @@ -486,6 +488,13 @@ def _bench_options(d: dict) -> List[str]: on = o.get("setups") if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours",) for x in on)): bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)") + if "limit" in o and (not isinstance(o["limit"], bool) or "pattern" not in o): + bad.append(f"{w}.limit: true for a bound the upstream has of its own, a number its pattern allows") + by = o.get("by") + if by is not None and not (isinstance(by, dict) and len(by) == 1 and all( + k != name and k in opts and isinstance(m, dict) and set(m) == set(opts[k].get("values") or [None]) + and all(option_problem(o, v) is None for v in m.values()) for k, m in by.items())): + bad.append(f"{w}.by: another option -> its every value -> this one's default under it") when = o.get("when") or {} if not _when_valid(name, when, opts): bad.append(f"{w}.when: another option -> the values of it this one applies under") diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 51a74390..c682a739 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -529,8 +529,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None): + constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, + max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -543,6 +543,10 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any. + ``timeouts``, ``max_turns`` and ``max_calls``: the time limits by phase, + in seconds, the most turns of a conversation and the most model calls of + a unit's agent the run forced (``run --timeout``, ``--max-turns``, + ``--max-calls``), when it forced any. ``served_by``: by model, the serving providers the run let answer it (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: @@ -557,6 +561,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"units_file": units_file} if units_file else {}), **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), **({"constraints": constraints} if constraints else {}), + **({"timeouts": timeouts} if timeouts else {}), **({"max_turns": max_turns} if max_turns else {}), + **({"max_calls": max_calls} if max_calls else {}), **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 36aaf7d7..678ab3f8 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -43,7 +43,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay +from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay, run CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -129,7 +129,7 @@ def _bench_page(b: dict) -> dict: "controls": controls, # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], - "reference_agent": x.get("reference_agent", ""), + "reference_agent": x.get("reference_agent", ""), "turn_seconds": x.get("turn_seconds"), "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} @@ -140,9 +140,15 @@ def _bench_page(b: dict) -> dict: # what the upstream's leaderboard fixes besides the ranked model (Like the leaderboard) "leaderboard": b.get("leaderboard") or None, "attempts": (b.get("limits") or {}).get("attempts", ""), + # what a unit's time limits and a conversation's length are when the recipe forces none + "timeouts": {"setup": run.DRIVER_SECONDS, "image": run.IMAGE_SECONDS, "score": run.DRIVER_SECONDS, + "agent": (b.get("limits") or {}).get("turn_seconds") or run.TURN_SECONDS, + "turns": (b.get("limits") or {}).get("turns") or run.MAX_TURNS, + "catalogue": [k for k, f in (("agent", "turn_seconds"), ("turns", "turns")) if (b.get("limits") or {}).get(f)]}, "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [], "setups": o.get("setups") or ["yours"], - **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}} + **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, + "limit": bool(o.get("limit")), "by": o.get("by") or {}} for n, o in (b.get("options") or {}).items()}, "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]} for n, r in hostreq.all_of(b).items()}, @@ -288,6 +294,10 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, the whole run: its jobs that call a model - each dataset with each participant, on each machine - take equal shares (``--max-usd``), so none spends another's and the amount typed is the most the run spends. + ``timeouts``: {phase: minutes}, each phase's time limit forced on every + job (``--timeout``), empty for the default; ``turns``: the most agent + turns in a conversation (``--max-turns``); ``max_calls``: the most model + calls a unit's agent may make (``--max-calls``). ``max_usd_per_unit``: each unit's own cap (``--max-usd-per-unit``), given whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it @@ -493,6 +503,29 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, elif per_unit < 1e-6: # what the command line can say: a millionth of a dollar add("limits", f"The spending cap per {noun} is too small.") per_unit = None + # the time limits the recipe forces, in minutes by phase (run --timeout, in seconds), and a conversation's length + forced: Dict[str, float] = {} + given_t = recipe.get("timeouts") if isinstance(recipe.get("timeouts"), dict) else {} + for k, v in given_t.items(): + if v in ("", None): + continue + if k not in run.TIMEOUT_PHASES or isinstance(v, bool) or not isinstance(v, (int, float)) \ + or not 0 < round(v * 60, 1) <= run.MAX_TIMEOUT_SECONDS: + add("limits", f"A time limit is a number of minutes above zero, at most {run.MAX_TIMEOUT_SECONDS // 60}, or empty for the default.") + continue + forced[k] = round(v * 60, 1) + calls = recipe.get("max_calls") # the most model calls a unit's agent may make (run --max-calls) + if calls in ("", None): + calls = None + elif _int(calls, 1, 1000000) is None: + add("limits", "Model calls per unit: a whole number of at least 1, or empty for no cap.") + calls = None + turns = recipe.get("turns") + if turns in ("", None) or "agent.conversation/v1" not in (b.get("needs") or []): # only a conversation has turns + turns = None + elif _int(turns, 1, 100000) is None: + add("limits", "Turns per conversation: a whole number of at least 1, or empty for the default.") + turns = None limit = recipe.get("limit") if limit not in ("", None) and _int(limit, 1) is None: add("limits", "The number of units is a whole number of at least 1, or empty for all of them.") @@ -615,6 +648,9 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--repeat", str(repeat)] if repeat > 1 else []), *(["--max-usd", f"{share:.6f}".rstrip("0").rstrip(".")] if share and spends(p) else []), *(["--max-usd-per-unit", f"{per_unit:.6f}".rstrip("0").rstrip(".")] if per_unit and spends(p) else []), + *[w for k, v in forced.items() for w in ("--timeout", f"{k}={v:g}")], + *(["--max-turns", str(turns)] if turns else []), + *(["--max-calls", str(calls)] if calls and spends(p) and o.get("_kind") != "capability" else []), *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], @@ -719,6 +755,9 @@ def row(p: dict) -> list: *([[STEP["limits"], "Spending cap", f"${ledger.usd(cap)}" + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []), *([[STEP["limits"], f"Spending cap per {noun}", f"${ledger.usd(per_unit)}"]] if per_unit else []), + *([[STEP["limits"], "Time limits", " · ".join(f"{k} {v / 60:g} min" for k, v in forced.items())]] if forced else []), + *([[STEP["limits"], "Turns per conversation", str(turns)]] if turns else []), + *([[STEP["limits"], f"Model calls per {noun}", str(calls)]] if calls else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" + (f" · {boat_size} VMs" if backend == "boat" else "")]] diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 13d28ba0..1923088c 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -28,6 +28,14 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +Another is a run's cap on a unit's model calls (``max_calls``): the agent's +own route (the unit's tag alone, never its judge's or its simulated user's) +counts each call the provider answered, and once the count reaches the cap +the next call is refused with a 402 the record says ``refused: calls``. A +call the provider refused (a 429, an error) is not counted, so a rate limit +never reads as the agent's steps. Calls in flight together may pass it by +the few that were already sent. + A run may pin, for a model, the serving providers that may answer it (``serving``, OpenRouter's routing): the relay then puts ``provider: {order, allow_fallbacks: false}`` on every request it sends as @@ -562,7 +570,7 @@ class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, - facts: Optional[bool] = None): + facts: Optional[bool] = None, max_calls: Optional[int] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -585,8 +593,12 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s put on every such request in place of the caller's own. ``facts``: a pinned Chat Completions or Responses request asks OpenRouter for its routing facts, where its reply names who served it; by default when - ``base`` is OpenRouter's.""" + ``base`` is OpenRouter's. + ``max_calls``: the most model calls the agent's own route of a unit + may make (:func:`agent_route`), each answered call counted.""" self.fields = fields + self.max_calls = max_calls + self.answered: Dict[str, int] = {} # the calls each agent route had answered, for max_calls self.serving = {m: list(s) for m, s in (serving or {}).items()} host = (urllib.parse.urlsplit(base).hostname or "").lower() self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) @@ -707,6 +719,13 @@ def _forward(self, h: BaseHTTPRequestHandler): routing_asked = req.get("provider") req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if self.max_calls and agent_route(tag) and h.command == "POST" and isinstance(req, dict) and req.get("model") and not _free(rest): + with self.lock: + made = self.answered.get(tag, 0) + if made >= self.max_calls: + self._refuse(h, tag, rest, req, asked, f"the unit's cap of {self.max_calls} model calls was reached: " + "no further call of its agent is sent", "calls") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -763,6 +782,9 @@ def record(body): rec["fields"] = request_fields(req) cost = None if status == 200: + if self.max_calls and agent_route(tag) and isinstance(req, dict) and req.get("model") and not _free(rest): + with self.lock: + self.answered[tag] = self.answered.get(tag, 0) + 1 rec["reply"] = read_reply(rest, body, streamed) # who served it, how it ended, an error under the 200: beside what was read, never in its place try: @@ -874,7 +896,9 @@ def _refuse(self, h: BaseHTTPRequestHandler, tag: str, rest: str, req, asked, wh with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") - body = json.dumps({"type": "error", "error": {"type": "spending_cap", "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable"}.get(by, "spending_cap"), + body = json.dumps({"type": "error", "error": {"type": "call_cap" if by == "calls" else "spending_cap", + "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable", + "calls": "unit_call_cap"}.get(by, "spending_cap"), "message": "nullius relay: " + why}}).encode() h.send_response(402) for n, v in (("Content-Type", "application/json"), ("X-Nullius-Refused", by), ("Content-Length", str(len(body)))): @@ -920,6 +944,12 @@ def stop(self): self.pool.close() +def agent_route(tag: str) -> bool: + """Whether a route is a unit's agent's own (``u0042``, ``u0042r2``): its + judge's and its simulated user's carry a suffix (``-judge``, ``-simulator``).""" + return "-" not in tag + + def _free(rest: str) -> bool: """A route that names a model and is not billed: a token count.""" return rest.split("?")[0].rstrip("/").endswith("/count_tokens") @@ -935,6 +965,11 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True +# what a unit whose agent reached the run's cap on model calls carries (``max_calls``): an agent error, counted +# against the agent, as a timeout is (runs.classify) +CALLS_ERROR = "UnitCallCapError" + + def failure(calls: List[dict]) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the @@ -948,6 +983,9 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" + many = next((c for c in calls if c.get("refused") == "calls"), None) + if many: # its agent made the calls it was allowed: the agent's, as its time running out is + return CALLS_ERROR, many.get("error") or "" if not calls or calls[-1].get("status") == 200: return None last = calls[-1] @@ -989,6 +1027,7 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--max-calls", type=int, help="the most model calls a unit's agent route may make") ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", help="the serving providers that may answer a request sent as MODEL, in order, and no other; " "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " @@ -1019,7 +1058,7 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, max_calls=a.max_calls) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 0b52ef2f..a2978c1b 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -106,6 +106,10 @@ def driver_path() -> str: IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) MAX_AGENT_SECONDS = 6 * 3600 # the longest a unit's own agent limit (setup's agent_seconds) may ask MAX_TURNS = 20 +# The phases of a unit whose time limit a run may set (``Run(timeouts=)``, ``cli run --timeout PHASE=SECONDS``): +# its environment's setup, its own image's pull or build, the agent's whole work on it, the check, the score +TIMEOUT_PHASES = ("setup", "image", "agent", "verify", "score") +MAX_TIMEOUT_SECONDS = 24 * 3600 # the longest any one phase may be given BACKENDS = definitions.names.BACKENDS _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) @@ -160,6 +164,24 @@ def units_file(path: str) -> dict: return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} +def forced_timeouts(given: Optional[dict]) -> Dict[str, float]: + """A run's own time limits (``--timeout PHASE=SECONDS``): {phase: seconds}, + each phase one of ``TIMEOUT_PHASES``, each above zero and at most + ``MAX_TIMEOUT_SECONDS``. Refused otherwise; none given is {}.""" + out: Dict[str, float] = {} + for k, v in (given or {}).items(): + if k not in TIMEOUT_PHASES: + raise Refused(f"--timeout {k}: a phase is one of {', '.join(TIMEOUT_PHASES)}") + try: + sec = float(v) if not isinstance(v, bool) else 0.0 + except (TypeError, ValueError): + sec = 0.0 + if not 0 < sec <= MAX_TIMEOUT_SECONDS: + raise Refused(f"--timeout {k}={v}: seconds above zero, at most {MAX_TIMEOUT_SECONDS}") + out[k] = sec + return out + + def key_header(provider: str, other: dict, env=os.environ) -> str: """How the agent's relay sends the key: by the protocol the agent (or capability) speaks on this provider. Refused, as ``recipe.compile`` refuses it, on a provider that serves none of them.""" @@ -385,6 +407,9 @@ def get(self, name: str) -> Optional[dict]: class Run: + timeouts: Dict[str, float] = {} # the run's own time limits by phase (__init__), none by default + forced_turns: Optional[int] = None + max_calls: Optional[int] = None # the run's cap on a unit's agent's model calls (__init__), none by default def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -394,8 +419,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): + constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, + max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -444,6 +469,15 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas constraints are checked on the engine (where those containers run), and the scope's on systemd: on the local backend a cap no scope holds is refused, on the container backend its drivers then run without one. + ``timeouts``: the run's own time limit for each phase of a unit, by + name (``TIMEOUT_PHASES``), in seconds, forced over whatever the task, + the setup, the benchmark or this module would give it. ``agent`` is + the agent's whole work on a unit, every turn of a conversation + included. ``max_turns``: the most agent turns in a conversation, over + the benchmark's ``limits.turns``. ``max_calls``: the most model calls + a unit's agent may make, the whole unit through (``relay.max_calls``): + a unit that reaches it ends as ``relay.CALLS_ERROR``, the agent's, + as a timeout is, and the run goes on. The job's config records all three. ``served_by``: by model, the serving providers that may answer it (:func:`served_by`), put on every request sent as that model by the run's relays (``relay.routing``) and recorded in the job's config. @@ -614,9 +648,19 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self._image_users: Dict[str, int] = {} self.turn_seconds = (turn_seconds or (other.get("turn_seconds") if self.upstream else None) or (bench.get("limits") or {}).get("turn_seconds") or TURN_SECONDS) + self.timeouts = forced_timeouts(timeouts) # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met - self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + if max_turns is not None and (isinstance(max_turns, bool) or not isinstance(max_turns, int) or not 1 <= max_turns <= 100000): + raise Refused(f"--max-turns {max_turns!r}: a whole number of turns, at least 1") + self.forced_turns = max_turns + if max_calls is not None and (isinstance(max_calls, bool) or not isinstance(max_calls, int) or not 1 <= max_calls <= 1000000): + raise Refused(f"--max-calls {max_calls!r}: a whole number of model calls, at least 1") + if max_calls is not None and other.get("_kind") == "capability": + # a capability's route carries the benchmark's reader too (produce): the cap would charge its calls to the capability + raise Refused("--max-calls caps an agent's model calls: a capability's unit route carries the benchmark's reader too") + self.max_calls = max_calls + self.max_turns = max_turns or (bench.get("limits") or {}).get("turns") or MAX_TURNS self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -773,6 +817,7 @@ def start(self) -> str: shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, + timeouts=self.timeouts, max_turns=self.forced_turns, max_calls=self.max_calls, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -1060,7 +1105,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: os.makedirs(home, exist_ok=True) t0 = time.time() self.progress.emit("unit", "start", unit=unit) - self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS) + self.phase(name, unit, "env_setup", None if self.upstream else self.phase_limit("setup", DRIVER_SECONDS)) produce, score, exc, traj, logs = (None, None), None, None, None, {} p0, checking = None, (None, None) # when the agent began; the check's start and end setup: dict = {} @@ -1069,7 +1114,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: try: try: # a setup's unit needs none: the upstream's answerer reads its data itself - setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=DRIVER_SECONDS, **self.attempt_kw()) + setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.phase_limit("setup", DRIVER_SECONDS), **self.attempt_kw()) except jsonl.Failed as e: raise _Ended("EnvironmentSetupError", str(e)) environment, env_def = self.unit_environment(setup) @@ -1100,14 +1145,14 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if rl is not None: rl.unit_secrets = self.agent_licences(env_def, rl) if own: - self.phase(name, unit, "env_setup", IMAGE_SECONDS) # its image, pulled or built when not here yet + self.phase(name, unit, "env_setup", self.phase_limit("image", IMAGE_SECONDS)) # its image, pulled or built when not here yet box = self.own_box(tag, setup, rl, home) if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts p0 = time.time() - self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" - else DRIVER_SECONDS if self.other["key"] == "oracle" else None) + self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control" + else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None) try: with self.agent_internet(setup, box, rl, trial): if self.other["_kind"] == "control": @@ -1127,13 +1172,13 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.phase_limit("score", DRIVER_SECONDS) + (self.step_limit("verify", step) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.phase_limit("score", DRIVER_SECONDS), **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: @@ -1157,7 +1202,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: ln.stop() calls = rl.calls(tag) if rl is not None else [] failed = relay.failure(calls) - if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR, relay.CALLS_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is @@ -1225,12 +1270,12 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: def cap_reached(self, rl, tag: str) -> Optional[tuple]: """A spending cap's refusal among a unit's calls - its agent's, its judge's or its simulated user's - as what the unit ends with: cut - off by the run's own limit, never a miss, before its own. None - without one.""" - if rl is None or getattr(self, "ledger", None) is None: + off by the run's own limit, never a miss, before its own; then the + cap on its agent's model calls (``max_calls``). None without one.""" + if rl is None or (getattr(self, "ledger", None) is None and not self.max_calls): return None cuts = [f for f in (relay.failure(rl.calls(t)) for t in (tag, f"{tag}-judge", f"{tag}-simulator")) if f] - return next((f for f in cuts if f[0] == ledgers.ERROR), None) or next((f for f in cuts if f[0] == ledgers.UNIT_ERROR), None) + return next((f for k in (ledgers.ERROR, ledgers.UNIT_ERROR, relay.CALLS_ERROR) for f in cuts if f[0] == k), None) def attempt_name(self, unit: str, r: int = 1) -> str: """A unit as the budget lists it: by its name, and its attempt when the run repeats its units.""" @@ -1419,7 +1464,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = self.key_header if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + max_calls=self.max_calls).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1448,7 +1494,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), serving=getattr(self, "serving", None), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), max_calls=self.max_calls, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1574,14 +1620,14 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: if self.other["key"] != "oracle": return "" try: - reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=DRIVER_SECONDS) + reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=self.phase_limit("agent", DRIVER_SECONDS)) except jsonl.Failed as e: raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e)) if reply.get("solve") is not None: if box is None: raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none") if name: # the reference solution, run under its own limit - self.phase(name, unit, "agent_run", float(reply["solve"].get("seconds") or self.turn_seconds)) + self.phase(name, unit, "agent_run", self.step_limit("agent", reply["solve"])) done = self.in_box(box, reply["solve"], collect=False, what="the benchmark's oracle") os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True) with open(os.path.join(os.path.dirname(ws), "agent", "solve.txt"), "w") as f: @@ -1595,14 +1641,15 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: @contextmanager def unit_vm(self, ws: str, home: str, trial: str, setup: dict): """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``), - living as long as its every turn may take.""" + living as long as its work may take: an agent's limit is its whole unit's, a capability's each add's and recall's.""" if getattr(self, "boat", None) is None: yield return - turns = self.max_turns if "agent.conversation/v1" in self.bench["needs"] else 1 + plan = setup.get("memory") or {} + requests = 1 if self.other["_kind"] == "agent" else max(1, len(plan.get("add") or []) + len(plan.get("recall") or [])) try: with self.boat.unit(trial, ws, home, stopped=self._stopped, inst=self.inst, here=self.other.get("_here"), - seconds=self.turn_limit(setup) * turns): + seconds=self.agent_limit(setup) * requests): yield except boat.Unavailable as e: raise _Ended("EnvironmentSetupError", str(e)) @@ -1721,11 +1768,12 @@ def own_image(self, spec: dict) -> str: first = tag not in self._envs self._envs.add(tag) self._image_users[tag] = self._image_users.get(tag, 0) + 1 + seconds = self.phase_limit("image", IMAGE_SECONDS) if not first: - return container.own_image(self.bench, spec, root, work) + return container.own_image(self.bench, spec, root, work, timeout=seconds) with self.progress.stage("environment", cached=container.present(tag), start=("pulling " if spec.get("pull") else "building ") + tag, name=tag): - return container.own_image(self.bench, spec, root, work) + return container.own_image(self.bench, spec, root, work, timeout=seconds) def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str] = None) -> dict: """A step (``verify``, ``solve``) run in the unit's container. For the @@ -1754,7 +1802,7 @@ def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str env = box.online() if step.get("network") else (getattr(box, "agent_proxy", None) if not collect else None) if step.get("collect"): box.run(["mkdir", "-p", step["collect"]], 60, user="0") - return box.run(step["run"], float(step.get("seconds") or self.turn_seconds), env=env) + return box.run(step["run"], self.step_limit("verify" if collect else "agent", step), env=env) except (container.Unavailable, ValueError) as e: raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: {e}") @@ -1781,7 +1829,7 @@ def prepare(self, step: dict, box, trial: str) -> None: box.write_in(dest, text) env = box.online() if step.get("network") else None try: - done = box.run(step["run"], float(step.get("seconds") or self.turn_seconds), user="0", env=env) + done = box.run(step["run"], self.step_limit("setup", step), user="0", env=env) finally: said = box.egress_said() if said: @@ -1840,10 +1888,22 @@ def judge(self, rl, tag: str) -> dict: return {"judge": {"model": self.judge_model, "base_url": rl.url(tag + "-judge", model=self.judge_model), "key": relay.PLACEHOLDER_KEY}} - def turn_limit(self, setup: dict) -> float: - """How long one turn of the agent may take: the unit's own limit when its ``setup`` names one + def phase_limit(self, phase: str, default: float) -> float: + """A phase's time limit: the run's own (``timeouts``) when it sets one, else ``default``.""" + return self.timeouts.get(phase) or default + + def step_limit(self, phase: str, step: dict) -> float: + """A step's time limit in the unit's container (``prepare``, ``solve``, ``verify``): the run's own + for its phase, else the step's ``seconds``, else the run's ``turn_seconds``.""" + return float(self.timeouts.get(phase) or step.get("seconds") or self.turn_seconds) + + def agent_limit(self, setup: dict) -> float: + """How long the agent's whole work on a unit may take, every turn of a conversation included: + the run's own (``timeouts["agent"]``), else the unit's when its ``setup`` names one (``agent_seconds``: a task that declares its agent timeout, as Terminal-Bench's task.toml does), else the run's ``turn_seconds``.""" + if self.timeouts.get("agent"): + return self.timeouts["agent"] own = setup.get("agent_seconds") if isinstance(setup, dict) else None if isinstance(own, (int, float)) and not isinstance(own, bool) and 0 < own <= MAX_AGENT_SECONDS: return float(own) @@ -1878,6 +1938,15 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})") prompt = setup["prompt"] answer = "" + # one limit for the agent's whole work on the unit, every turn and the simulated user's between them + limit = self.agent_limit(setup) + deadline = time.monotonic() + limit + + def left() -> float: + rest = deadline - time.monotonic() + if rest <= 0: + raise acp.Timeout(f"the agent did not end its work in {int(limit)} s", []) + return rest # a conversation's turns carry its simulated user's route and its judge's (each when it has one) sim = dict(self.simulator(rl, tag), **self.judge(rl, tag)) if "agent.conversation/v1" in self.bench["needs"] else {} opened = True @@ -1888,11 +1957,12 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name opened = not first.get("done") if opened: prompt = prompt + "\n\n" + first["message"] - for n in range(self.max_turns if opened else 0): + for _ in range(self.max_turns if opened else 0): transcript.append({"role": "user", "text": prompt}) - if n: # the limit is each turn's: the board's clock starts again with it - self.phase(name, unit, "agent_run", self.turn_limit(setup)) - turn = c.prompt(sid, prompt, timeout=self.turn_limit(setup)) + try: + turn = c.prompt(sid, prompt, timeout=left()) + except acp.Timeout as e: # said as the whole work's limit, not the seconds this turn had left + raise acp.Timeout(f"the agent did not end its work in {int(limit)} s", e.events) from e answer = turn.text transcript.append({"role": "agent", "text": turn.text, "stop": turn.stop}) if "agent.conversation/v1" not in self.bench["needs"]: @@ -1900,6 +1970,7 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name nxt = self.driver().request("turn", unit=unit, dir=ws, transcript=transcript, timeout=600, **sim, **self.attempt_kw()) if nxt.get("done"): break + left() # the simulated user's time is the conversation's too prompt = nxt["message"] if nxt.get("restart"): # the upstream started its conversation again (its own retry of an attempt that @@ -1931,10 +2002,10 @@ def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl t = time.time() try: with open(log, "a") as err: - r = run_group(cmd, self.turn_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + r = run_group(cmd, self.agent_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=err, text=True) except subprocess.TimeoutExpired: - raise _Ended("AgentTimeoutError", f"the agent did not end in {self.turn_limit(setup)} s") + raise _Ended("AgentTimeoutError", f"the agent did not end in {int(self.agent_limit(setup))} s") if r.returncode != 0: raise _Ended("NonZeroAgentExitCodeError", f"exit {r.returncode}") transcript.append({"role": "agent", "text": r.stdout}) @@ -1954,10 +2025,13 @@ def drive_feature(self, unit, ws, home, tag, setup, transcript, rl): proc = jsonl.Proc(cmd, cwd, env, os.path.join(os.path.dirname(ws), "agent", "capability.log"), stopped=self._stopped) os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True) exchanges: List[dict] = [] + # each add and each recall its own limit: a memory adds a unit's sessions one by one, each its own model calls, + # so a limit on the whole would grow with the haystack (LongMemEval's ~50 sessions a question) + limit = self.agent_limit(setup) def send(op, **kw): t0 = time.time() - reply = proc.request(op, timeout=self.turn_seconds, **kw) + reply = proc.request(op, timeout=limit, **kw) exchanges.append({"op": op, "request": kw, "reply": reply, "t0": t0, "t1": time.time()}) return reply @@ -1994,7 +2068,7 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes out = self.driver().request("answer", unit=unit, dir=ws, setup=o["key"], model=self.asked, **self.attempt_kw(), **self.simulator(rl, tag), **self.judge(rl, tag), - base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) + base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.agent_limit({})) except jsonl.Failed as e: calls = rl.calls(tag) silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index a58e76e4..4248df2a 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -80,6 +80,8 @@ } return r; } + // the phases of a unit whose time limit a standard recipe may force (run.py TIMEOUT_PHASES) + const STANDARD_PHASES = [['setup', 'Environment setup'], ['image', 'Image pull or build'], ['agent', 'Agent'], ['verify', 'Verifier'], ['score', 'Score']]; // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. const SERVED_ROLES = ['model', 'simulator', 'judge']; @@ -193,6 +195,14 @@ if (typeof raw.max_usd === 'number' && Number.isFinite(raw.max_usd) && raw.max_usd > 0) r.max_usd = raw.max_usd; // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; + // the time limits it forces, in minutes by phase (recipe.py: run --timeout), and a conversation's most turns + delete r.timeouts; + if (raw.timeouts && typeof raw.timeouts === 'object' && !Array.isArray(raw.timeouts)) { + const t = Object.fromEntries(Object.entries(raw.timeouts).filter(([k, v]) => STANDARD_PHASES.some(([p]) => p === k) && typeof v === 'number' && Number.isFinite(v) && v > 0)); + if (Object.keys(t).length) r.timeouts = t; + } + if (Number.isInteger(raw.turns) && raw.turns >= 1) r.turns = raw.turns; + if (Number.isInteger(raw.max_calls) && raw.max_calls >= 1) r.max_calls = raw.max_calls; // the most model calls of a unit's agent for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) @@ -833,5 +843,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index f3e37326..0e86bdab 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -99,6 +99,7 @@ #nullius-builder .nb-customagent input { font: 12.5px/1.4 var(--mono); padding: 5px 9px; border: 1px solid var(--nb-line); border-radius: 7px; background: var(--nb-surface); color: var(--nb-text); width: 26ch; } #nullius-builder .nb-agentline { display: flex; justify-content: space-between; gap: 12px; font-size: 12px; color: var(--nb-muted); margin-top: 4px; } #nullius-builder .nb-timelimits input { width: 10ch; color: var(--nb-text); border: 1px solid var(--nb-line); border-radius: 6px; background: var(--nb-surface); padding: 5px 8px; font-size: 13px; } +#nullius-builder .nb-sectionnext { margin-top: 36px; } #nullius-builder .nb-timelimits td { vertical-align: middle; } #nullius-builder .nb-timelimits .nb-mono { font: 12px var(--mono); color: var(--nb-muted); } #nullius-builder .nb-note code, #nullius-builder .nb-categoryhead code { font: 12px var(--mono); background: var(--nb-soft); border: 1px solid var(--nb-line); border-radius: 5px; padding: 2px 6px; color: var(--nb-text); white-space: nowrap; } @@ -261,3 +262,5 @@ #nullius-builder .std-served-slug { min-width: 0; overflow-wrap: anywhere; } #nullius-builder .std-served-pick .nb-remove { flex: none; padding: 0 6px; font-size: 14px; line-height: 1; } @media (max-width: 650px) { #nullius-builder .std-served-list { grid-column: 1; } #nullius-builder label.std-served:has(.std-served-list:not(:empty)) > .ui-field-copy { grid-row: auto; margin-top: 0; } } +/* a limit the benchmark or the agent declares: its placeholder reads as a value; Nullius's default stays grey */ +#nullius-builder .nb-field input.nb-declared::placeholder { color: var(--nb-text); opacity: 1; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 1574913e..c6ca0d70 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -511,7 +511,8 @@ function benchOptions(rec, b, scope = 'yours') { const local = ((schema.standardHere || {}).host || {})[b.key] || {}; const chosen = C.chosenOf(rec); - return Object.entries(b.options || {}).filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) + return Object.entries(b.options || {}).filter(([, o]) => !o.limit) // a bound of the upstream's own is with the limits + .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) // one that applies only under another's values (a top-k, only with a retrieval) is left out otherwise .filter(([, o]) => Object.entries(o.when || {}).every(([k, vs]) => vs.includes((rec.bench_options || {})[k] ?? (b.options[k] || {}).default))) .filter(([, o]) => { @@ -685,12 +686,63 @@ + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '' + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + // a conversation's most agent turns (recipe.py: run --max-turns): only a benchmark that holds conversations has turns + + ((b.needs || []).includes('agent.conversation/v1') + ? field('Turns per conversation', 'turns', rec.turns ?? '', {type: 'number', min: 1, placeholder: String((b.timeouts || {}).turns || ''), + declared: ((b.timeouts || {}).catalogue || []).includes('turns')}) : '') + // the most model calls a unit's agent may make, the whole unit through (recipe.py: run --max-calls); none calls no model + // an agent's alone: a capability's route carries the benchmark's reader too (run.py refuses it) + + (C.standardKeyless(rec) || (b.needs || []).some(n => n.startsWith('cap.')) ? '' : field('Model calls per ' + u.noun, 'max_calls', rec.max_calls ?? '', {type: 'number', min: 1, placeholder: 'none'})) // the whole run's, shared by its jobs (recipe.py); a run that calls no model spends nothing: no cap to offer + (C.standardKeyless(rec) ? '' : field('Spending cap (USD)', 'max_usd', rec.max_usd ?? '', {type: 'number', min: 0, placeholder: 'none'}) - + field('Spending cap per ' + u.noun + ' (USD)', 'max_usd_per_unit', rec.max_usd_per_unit ?? '', {type: 'number', min: 0, placeholder: 'none'})); + + field('Spending cap per ' + u.noun + ' (USD)', 'max_usd_per_unit', rec.max_usd_per_unit ?? '', {type: 'number', min: 0, placeholder: 'none'})) + + standardTimeLimits(rec, b, u) + benchLimits(rec, b); } return ''; } + // The bounds the benchmark's upstream has of its own (its steps, its errors: options with ``limit``), in a section + // of the benchmark's name after the run's scope and its time limits: those the chosen datasets and participants + // take, each empty at its default, shown greyed - an agent's own under ``by`` (OSWorld's agents' steps). + function benchLimits(rec, b) { + const chosen = C.chosenOf(rec), yours = chosen.some(k => !['original', 'model'].includes(k) && !(b.controls || []).some(x => x.key === k)); + const given = rec.bench_options || {}, of = k => given[k] ?? (b.options[k] || {}).default; + const rows = Object.entries(b.options || {}).filter(([, o]) => o.limit).sort(([, x], [, y]) => x.label.localeCompare(y.label)) + .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) + .filter(([, o]) => (o.setups || ['yours']).some(s => s === 'yours' ? yours : chosen.includes(s))) + .map(([n, o]) => { + const [k, by] = Object.entries(o.by || {})[0] || []; + const def = (k && by[of(k)]) || o.default; + return ``; + }); + return rows.length ? `

${esc(b.label)}

` + rows.join('') : ''; + } + // a time limit's row, laid out as every other field of the step: its name and its unit + const LIMIT_TIPS = {agent: 'A task that declares its own limit keeps it unless one is typed here.', verify: 'Each task\'s own check limit unless one is typed here.'}; + const minutesRow = (k, name, value, [placeholder, declared]) => ``; + // The agent's limit while its field is empty, as [minutes, declared]: by priority, the benchmark's (a setup's own, + // as its Original declares it, else its limits.turn_seconds), then the agent's, then Nullius's default. Participants + // that would get different ones read "per setup"; a check (a control) has its own. + function agentDefault(rec, b) { + const d = b.timeouts || {}, m = s => String(+(s / 60).toFixed(2)), benchwide = (d.catalogue || []).includes('agent'); + const ofOne = k => { const s = (b.setups || []).find(x => x.key === k); return s && s.turn_seconds ? [s.turn_seconds, true] : [d.agent, benchwide]; }; + const each = C.chosenOf(rec).filter(k => !(b.controls || []).some(x => x.key === k)).map(ofOne); + const all = each.length ? each : [[d.agent, benchwide]]; + if (all.some(x => x[0] !== all[0][0])) return ['per setup', true]; + return [m(all[0][0]), all[0][1]]; + } + // A unit's time limit for each phase, in minutes, forced on every participant (recipe.py: run --timeout); + // empty keeps the default: the task's own when it declares one, else the benchmark's, else Nullius's. + function standardTimeLimits(rec, b, u) { + const d = b.timeouts || {}, t = rec.timeouts || {}; + // each field shows, greyed, the value used while it is empty; a task that declares its own keeps it + const m = s => String(+(s / 60).toFixed(2)); + // [placeholder, declared]: the benchmark's or the agent's own reads as a value, Nullius's default stays grey + const def = {setup: [m(d.setup), false], image: [m(d.image), false], agent: agentDefault(rec, b), verify: ['per task', true], score: [m(d.score), false]}; + const label = (k, l) => k === 'agent' ? 'Agent, whole ' + u.noun : l; + return '

Time limits

' + + C.STANDARD_PHASES.map(([k, l]) => minutesRow(k, label(k, l), t[k], def[k])).join(''); + } function compileRunner() { const rec = r(), key = JSON.stringify(rec); if (key === runnerKey) return; @@ -711,7 +763,7 @@ refreshPlan(); }, 120); } - const field = (label,key,value,{type='text',hint='',list='',min='',max='',placeholder=''}={}) => ``; + const field = (label,key,value,{type='text',hint='',list='',min='',max='',placeholder='',declared=false}={}) => ``; const select = (label,key,value,options) => ``; const fixed = (label,value,whose='official') => ``; const note = (title,text) => `
${title}${text}
`; @@ -1034,6 +1086,7 @@ else if(['model','simulator_model','judge_model'].includes(d.field)){C.setModelOf(rec,d.field,value);syncServed();} // the field's pin goes with its model; its Served by follows, in place else rec[d.field]=value; if(el.list?.id==='nb-paths')completeDirectory(el.value); + }else if(d.stdMinutes){const t=Object.assign({},rec.timeouts);if(el.value===''||!(Number(el.value)>0))delete t[d.stdMinutes];else t[d.stdMinutes]=Number(el.value);if(Object.keys(t).length)rec.timeouts=t;else delete rec.timeouts;refreshPlan();return; }else if(d.minutes){C.setPhaseMinutes(rec,d.minutes,el.value===''?'':Number(el.value));const row=el.closest('tr');if(row){const m=row.querySelector('.nb-mono');const t=C.effectiveTimeouts(rec);const p=t&&t.phases.find(x=>x.key===d.minutes);if(m&&p)m.textContent='×'+p.multiplier;}refreshPlan();return;} else if(d.customAgent!==undefined){rec.customAgents[Number(d.customAgent)]=el.value;refreshPlan();return;} else if(d.harborRepo!==undefined){const h=rec.harbor;h.repo=el.value;h.commit='';h.revision='';applyScan();} @@ -1047,7 +1100,7 @@ // no cap, unsaid. It is said, and the field emptied, so that it shows what the recipe holds root.addEventListener('focusout',event=>{ const el=event.target; - if(el.type==='number'&&el.dataset.field&&el.validity&&el.validity.badInput){el.value='';window.nulliusNotify?.(`${el.getAttribute('aria-label')}: enter a number.`,{error:true,key:'bad-number'});} + if(el.type==='number'&&(el.dataset.field||el.dataset.stdMinutes||el.dataset.benchOptSelect)&&el.validity&&el.validity.badInput){el.value='';window.nulliusNotify?.(`${el.getAttribute('aria-label')}: enter a number.`,{error:true,key:'bad-number'});} }); // a provider picked in a Served by, or slugs typed there and Enter: they join its list, after the ones there root.addEventListener('nb-pick',event=>{const role=event.target.dataset.servedBy;if(!role)return;C.addServed(r(),role,event.detail);syncServed();refreshPlan();}); diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index ab016691..e0ce5639 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -47,7 +47,7 @@ def test_a_value_may_only_need_a_declared_requirement_and_a_known_dataset(self): class Scope(unittest.TestCase): def test_an_option_applies_only_to_its_datasets(self): self.assertEqual(definitions.bench_options_of(TAU, "airline", {"retrieval": "alltools"}), - ({"task_split": "base", "user_reasoning_effort": "default"}, [])) + ({"task_split": "base", "user_reasoning_effort": "default", "max_steps": "200", "max_errors": "10"}, [])) vals, bad = definitions.bench_options_of(TAU, "banking_knowledge", {"retrieval": "alltools"}) self.assertEqual((vals["retrieval"], "task_split" in vals, bad), ("alltools", False, [])) self.assertEqual(definitions.bench_options_of(TAU, "banking_knowledge")[0]["retrieval"], "bm25") @@ -296,14 +296,72 @@ def test_the_null_control_is_refused_as_it_starts_on_a_machine_without_docker(se self.assertIn("vm_provider=docker needs on this machine docker (docker not on PATH)", str(e.exception)) +class UpstreamBounds(unittest.TestCase): + """The bounds a benchmark's upstream has of its own - τ³-bench's steps and tool errors per conversation, + OSWorld's agents' steps - are options with ``limit``: a run sets them as any option, the builder shows + them with the limits, and an OSWorld agent's own steps are its default (``by``).""" + + def osworld_driver(self): + import importlib.util + b = definitions.get("benchmark", "osworld1") + with patch.dict(os.environ, {"NULLIUS_UPSTREAM": "/nonexistent"}), patch("sys.argv", ["driver.py"]): + spec = importlib.util.spec_from_file_location("osworld1_driver", os.path.join(b["_here"], "driver.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return b, mod + + def test_an_option_bound_is_a_number_and_its_default_by_another_option_names_each_value(self): + for key in ("tau3-bench", "osworld1"): + self.assertEqual(definitions.problems(definitions.get("benchmark", key), "benchmark"), [], key) + b = copy.deepcopy(definitions.get("benchmark", "osworld1")) + b["options"]["max_steps"]["by"]["agent"].pop("m3") + self.assertTrue(any("max_steps.by" in p for p in definitions.problems(b, "benchmark"))) + b = copy.deepcopy(definitions.get("benchmark", "osworld1")) + del b["options"]["max_steps"]["pattern"] + b["options"]["max_steps"]["values"] = ["default"] + self.assertTrue(any("max_steps.limit" in p for p in definitions.problems(b, "benchmark"))) + + def test_each_osworld_agents_own_steps_are_its_default_and_a_run_forces_another(self): + b, mod = self.osworld_driver() + own = b["options"]["max_steps"]["by"]["agent"] + for name, spec in mod.AGENTS["agents"].items(): + if "--max_steps" in spec["argv"]: + self.assertEqual(spec["argv"][spec["argv"].index("--max_steps") + 1], own[name], name) + argv = ["x.py", "--max_steps", "100", "--model", "m"] + self.assertEqual(mod.with_steps(argv, "default"), argv) + self.assertEqual(mod.with_steps(argv, "30"), ["x.py", "--max_steps", "30", "--model", "m"]) + self.assertEqual(mod.with_steps(["x.py"], "30"), ["x.py", "--max_steps", "30"]) + + def test_a_recipe_sets_a_bound_on_the_participants_it_applies_to(self): + r = {"version": 1, "harness": "standard", "name": "t", "title": "t", "benchmark": "tau3-bench", "datasets": ["airline"], + "others": ["original"], "model_choice": "any", "provider": "openrouter", "model": "vendor/m", "limit": 1, + "concurrency": 1, "where": "local", "bench_options": {"max_steps": "60", "max_errors": "4"}} + out = recipe.compile(r) + self.assertEqual(out["errors"], []) + self.assertIn("--bench-option max_steps=60", out["script"]) + self.assertIn("--bench-option max_errors=4", out["script"]) + for over in ("0", "201"): # above the upstream's 200, limits.turns would end it first + bad = recipe.compile(dict(r, bench_options={"max_steps": over}))["errors"] + self.assertTrue(bad, over) + + class TurnLimit(unittest.TestCase): def test_a_units_own_agent_limit_wins_within_bounds(self): r = run.Run.__new__(run.Run) r.turn_seconds = 900 - self.assertEqual(r.turn_limit({"agent_seconds": 3600}), 3600) - self.assertEqual(r.turn_limit({}), 900) + self.assertEqual(r.agent_limit({"agent_seconds": 3600}), 3600) + self.assertEqual(r.agent_limit({}), 900) for bad in (0, -5, True, "3600", run.MAX_AGENT_SECONDS + 1): - self.assertEqual(r.turn_limit({"agent_seconds": bad}), 900, bad) + self.assertEqual(r.agent_limit({"agent_seconds": bad}), 900, bad) + + def test_a_runs_own_limits_are_forced_over_the_task_and_the_step(self): + r = run.Run.__new__(run.Run) + r.turn_seconds = 900 + self.assertEqual((r.phase_limit("setup", 600), r.step_limit("verify", {"seconds": 120}), r.step_limit("verify", {})), (600, 120, 900)) + r.timeouts = {"agent": 60.0, "setup": 30.0, "verify": 45.0} + self.assertEqual(r.agent_limit({"agent_seconds": 3600}), 60) # the run's, over the task's own + self.assertEqual((r.phase_limit("setup", 600), r.phase_limit("score", 600)), (30, 600)) + self.assertEqual((r.step_limit("verify", {"seconds": 120}), r.step_limit("setup", {"seconds": 900})), (45, 30)) GPU = {"summary": "A CUDA GPU with enough memory.", "check": ["nvidia-smi"], "gpu_memory_mb": 12000} diff --git a/app/tests/test_standard_recipe.py b/app/tests/test_standard_recipe.py index 4d086c3a..9f6ea98a 100644 --- a/app/tests/test_standard_recipe.py +++ b/app/tests/test_standard_recipe.py @@ -35,6 +35,33 @@ def a_recipe(**kw): class Compile(unittest.TestCase): + def test_time_limits_are_forced_in_seconds_on_every_job_and_turns_only_on_a_conversation(self): + out = recipe.compile(a_recipe(timeouts={"agent": 30, "verify": 1.5, "setup": ""}, turns=40), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--timeout agent=1800 --timeout verify=90 ", out["script"]) + self.assertNotIn("--timeout setup", out["script"]) + self.assertNotIn("--max-turns", out["script"]) # fakebench holds no conversation + self.assertIn(["Time limits", "agent 30 min · verify 1.5 min"], [row[1:] for row in out["review"]]) + for bad in ({"agent": 0}, {"agent": -1}, {"lunch": 5}, {"agent": True}, {"agent": 24 * 60 + 1}, {"agent": 0.0001}): # the last rounds to no time + errors = recipe.compile(a_recipe(timeouts=bad), defs())["errors"] + self.assertEqual([e["step"] for e in errors], [recipe.STEP["limits"]], bad) + talk = dict(next(x for x in defs()["benchmark"] if x["key"] == "fakebench"), needs=["agent.task/v1", "agent.conversation/v1"]) + real = recipe.definitions.find + with patch.object(recipe.definitions, "find", lambda items, key: talk if key == "fakebench" else real(items, key)), \ + patch.object(recipe.match, "fit", lambda b, o: None): + out = recipe.compile(a_recipe(turns=40), defs()) + self.assertIn("--max-turns 40 ", out["script"]) + self.assertEqual(recipe.compile(a_recipe(turns=0), defs())["errors"][0]["step"], recipe.STEP["limits"]) + + def test_a_cap_on_model_calls_reaches_every_job_that_calls_a_model(self): + out = recipe.compile(a_recipe(max_calls=60), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--max-calls 60 ", out["script"]) + self.assertIn(["Model calls per task", "60"], [row[1:] for row in out["review"]]) + for bad in (0, -3, 1.5, True, "x"): + errors = recipe.compile(a_recipe(max_calls=bad), defs())["errors"] + self.assertEqual([e["step"] for e in errors], [recipe.STEP["limits"]], bad) + def test_a_fitting_pair_is_one_command_and_one_job(self): out = recipe.compile(a_recipe(), defs()) self.assertEqual(out["errors"], []) diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 1bd8ad9a..a6758ccd 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -92,6 +92,37 @@ def post(self, tag, path, body): except urllib.error.HTTPError as e: return e.code, e.read() + def test_a_units_agent_is_refused_past_its_cap_on_model_calls(self): + """``max_calls`` counts the answered calls of a unit's agent route alone: a refused + call (a 429) is not one, its judge's and simulated user's never are, and past the cap + the next call is the relay's own 402, which the unit ends with as the agent's.""" + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + r = relay.Relay(f"http://127.0.0.1:{self.up.server_address[1]}/v1", "sk-real", tmp, max_calls=2).start() + self.addCleanup(r.stop) + + def post(tag, body): + req = urllib.request.Request(r.url(tag) + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as got: + return got.status, got.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + ask = {"model": "m", "messages": [{"role": "user", "content": "hi"}]} + self.assertEqual(post("u0007", dict(ask, mode="429"))[0], 429) # refused upstream: not counted + self.assertEqual([post("u0007", ask)[0] for _ in range(2)], [200, 200]) + for side in ("u0007-judge", "u0007-simulator"): # never the agent's + self.assertEqual(post(side, ask)[0], 200) + status, body = post("u0007", ask) + self.assertEqual(status, 402) + self.assertEqual(json.loads(body)["error"]["code"], "unit_call_cap") + self.assertEqual(post("u0008", ask)[0], 200) # another unit counts its own + kind, said = relay.failure(r.calls("u0007")) + self.assertEqual(kind, relay.CALLS_ERROR) + self.assertIn("cap of 2 model calls", said) + self.assertEqual(classify.from_verdict(kind, None), classify.AGENT_ERROR) + def test_plain_reply_passes_through_and_the_real_key_goes_out(self): status, body = self.post("u1", "/chat/completions", {"model": "m", "messages": [{"role": "user", "content": "hi"}]}) self.assertEqual((status, json.loads(body)), (200, CHAT)) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec268..34f4684b 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -8,6 +8,7 @@ import shutil import tempfile import threading +import time import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from datetime import datetime @@ -37,6 +38,24 @@ def turn(unit, dir, transcript, simulator=None): "detail": {"said": [t["text"] for t in transcript if t["role"] == "agent"], "routes": ROUTES}}}) ''' +SLOW_CHAT_DRIVER = '''import time +from nullius.standard.jsonl import serve +def turn(unit, dir, transcript): + time.sleep(0.6) # each reply quick, the user slow: the conversation never ends by itself + return {"message": "more"} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "intro"}, "turn": turn, + "score": lambda unit, dir, transcript, answer: {"reward": 1.0, "passed": True, "detail": {}}}) +''' + +CALLING_CHAT_DRIVER = '''from nullius.standard.jsonl import serve +def turn(unit, dir, transcript): + if sum(1 for t in transcript if t["role"] == "user") >= 4: + return {"done": True} + return {"message": "swallow more"} # each turn, one model call of the agent's +serve({"units": lambda dataset: {"units": ["a", "b"]}, "setup": lambda unit, dir: {"prompt": "swallow hi"}, "turn": turn, + "score": lambda unit, dir, transcript, answer: {"reward": 1.0, "passed": True, "detail": {}}}) +''' + JUDGED_DRIVER = '''import json, urllib.request from nullius.standard.jsonl import serve def score(unit, dir, transcript, answer, judge=None): @@ -636,6 +655,28 @@ def test_a_feature_run_drives_the_feature_through_its_port(self): self.assertEqual(names, ["fakemem.add"] * 3 + ["fakemem.recall"]) self.assertIn("green", t["steps"][-1]["message"]) + def test_a_capabilitys_agent_limit_bounds_each_add_and_recall_not_the_whole(self): + """A memory adds a unit's sessions one by one: three adds of 0.6 s each pass an agent limit of + 1 s, which each fits and their sum does not. A cap on model calls is an agent's alone: a + capability's route carries the benchmark's reader too, so a capability run refuses it.""" + here = os.path.join(self.tmp, "slowmem") + os.makedirs(here) + src = definitions.get("capability", "fakemem", ROOT) + with open(os.path.join(src["_here"], "capability.py")) as f: + code = f.read() + with open(os.path.join(here, "capability.py"), "w") as f: + f.write("import time\n" + code.replace(' if req["op"] == "add":\n', ' if req["op"] == "add":\n time.sleep(0.6)\n')) + slow = dict(src, _here=here) + b = definitions.get("benchmark", "fakememo", ROOT) + jobs = os.path.join(self.tmp, "jobs") + job_dir = run.run(b, slow, model="m-1", provider="custom", jobs_root=jobs, concurrency=2, env=self.env, + say=lambda s: None, unsandboxed=True, timeouts={"agent": 1}) + got, _ = self.outcomes(job_dir) + self.assertTrue(all(t.outcome == classify.SCORED for t in got.values()), {u: t.outcome for u, t in got.items()}) + with self.assertRaisesRegex(run.Refused, "max-calls"): + run.run(b, src, model="m-1", provider="custom", jobs_root=jobs, env=self.env, say=lambda s: None, + unsandboxed=True, max_calls=5) + def test_a_pair_that_does_not_fit_is_refused_before_anything_starts(self): with self.assertRaisesRegex(run.Refused, "takes a capability"): self.go("fakememo", "fake") @@ -750,6 +791,53 @@ def test_a_driver_writes_beside_the_job_never_into_its_catalogue_folder(self): self.assertEqual(os.listdir(here), ["driver.py"]) self.assertTrue(os.path.isfile(os.path.join(job_dir, ".work", "driver", "data", ".cache"))) + def test_the_agents_limit_is_its_whole_conversation_and_a_run_may_force_it(self): + """``--timeout agent`` bounds the agent's whole work on a unit: every + turn of a conversation and the user's between them, though no one + reply comes near it. The config records what the run forced.""" + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_CHAT_DRIVER) + b = dict(definitions.get("benchmark", "fakebench", ROOT), _here=here, + needs=["agent.task/v1", "agent.conversation/v1"], driver={"command": ["python3", "{here}/driver.py"]}) + t0 = time.time() + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, proving=True, limit=1, + timeouts={"agent": 2, "score": 30}, max_turns=50) + self.assertLess(time.time() - t0, 30) # 50 turns of 0.6 s would be 30 s + by_unit, _ = self.outcomes(job_dir) + self.assertEqual(by_unit["a"].exception_type, "AgentTimeoutError") + self.assertIn("did not end its work in 2 s", by_unit["a"].exception_message) + with open(os.path.join(job_dir, "config.json")) as f: + std = json.load(f)["standard"] + self.assertEqual((std["timeouts"], std["max_turns"]), ({"agent": 2.0, "score": 30.0}, 50)) + for bad in ({"agent": 0}, {"agent": run.MAX_TIMEOUT_SECONDS + 1}, {"lunch": 5}, {"agent": True}, {"agent": "x"}): + with self.assertRaises(run.Refused, msg=bad): + run.forced_timeouts(bad) + + def test_a_units_agent_past_its_cap_on_model_calls_is_the_agents_and_is_not_checked(self): + """``max_calls`` counts the whole unit: four turns of one call each pass a cap of + three in the fourth, though no turn alone comes near it. The unit is the agent's + error, never scored; the next unit has calls of its own.""" + here = os.path.join(self.tmp, "calling") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(CALLING_CHAT_DRIVER) + b = dict(definitions.get("benchmark", "fakebench", ROOT), _here=here, + needs=["agent.task/v1", "agent.conversation/v1"], driver={"command": ["python3", "{here}/driver.py"]}) + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, proving=True, max_calls=3) + by_unit, _ = self.outcomes(job_dir) + for u in ("a", "b"): + self.assertEqual((by_unit[u].exception_type, by_unit[u].outcome), (relay.CALLS_ERROR, classify.AGENT_ERROR), u) + self.assertIsNone(by_unit[u].reward, u) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["max_calls"], 3) + with self.assertRaises(run.Refused): + run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + env=self.env, say=lambda s: None, unsandboxed=True, proving=True, max_calls=0) + def test_a_simulated_user_opens_the_conversation_and_speaks_through_its_route(self): """A benchmark whose user a model plays (``simulator``): its first turn, asked with nothing said, follows setup's prompt; every turn is @@ -773,7 +861,8 @@ def test_a_simulated_user_opens_the_conversation_and_speaks_through_its_route(se self.assertEqual(detail["routes"], ["another-user"] * 2) with open(os.path.join(job_dir, "events.jsonl")) as f: clocks = [e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "agent_run"] - self.assertEqual(len(clocks), len(detail["said"])) # the limit is each turn's: its clock starts with it + self.assertEqual(len(detail["said"]), 2) + self.assertEqual(len(clocks), 1) # one limit for the whole conversation: one clock, never restarted self.assertTrue(all(e["timeout_sec"] for e in clocks)) with open(os.path.join(trial, ".work", "relay", "u0000-simulator.jsonl")) as f: self.assertEqual([json.loads(l)["model"] for l in f], ["another-user"] * 2) diff --git a/app/tests/test_tau3_setups.py b/app/tests/test_tau3_setups.py index 5a940a14..60faf7c8 100644 --- a/app/tests/test_tau3_setups.py +++ b/app/tests/test_tau3_setups.py @@ -47,10 +47,10 @@ def test_the_original_takes_any_model_and_sends_it_as_given(self): def test_each_option_applies_where_tau2_run_applies_it_at_what_earlier_runs_did(self): opts = lambda ds, setup: definitions.bench_options_of(BENCH, ds, None, setup)[0] - self.assertEqual(opts("airline", "yours"), {"task_split": "base", "user_reasoning_effort": "default"}) - self.assertEqual(opts("airline", "original"), {"task_split": "base", "user_reasoning_effort": "default", + self.assertEqual(opts("airline", "yours"), {"task_split": "base", "user_reasoning_effort": "default", "max_steps": "200", "max_errors": "10"}) + self.assertEqual(opts("airline", "original"), {"task_split": "base", "user_reasoning_effort": "default", "max_steps": "200", "max_errors": "10", "agent_reasoning_effort": "default", "upstream_agent": "llm_agent"}) - self.assertEqual(opts("banking_knowledge", "yours"), {"retrieval": "bm25", "user_reasoning_effort": "default"}) + self.assertEqual(opts("banking_knowledge", "yours"), {"retrieval": "bm25", "user_reasoning_effort": "default", "max_steps": "200", "max_errors": "10"}) self.assertEqual(opts("banking_knowledge", "original")["retrieval"], "bm25") def test_attempts_are_told_apart(self): @@ -185,6 +185,16 @@ def test_a_reasoning_effort_replaces_the_default_arguments_as_the_cli_does(self) self.assertEqual((by["agent"].get("reasoning_effort"), "temperature" in by["agent"]), ("high", False)) self.assertEqual((by["user"].get("reasoning_effort"), "temperature" in by["user"]), ("low", False)) + def test_the_runs_bounds_are_the_conversations_as_max_steps_and_max_errors_set_them(self): + for options, want in (({}, (200, 10)), ({"max_steps": "7", "max_errors": "3"}, (7, 3))): + d = self.driver(**options) + ws = self.ws("b" + "".join(options.values())) + Fake.lines = ["Hello.", "Bye. ###STOP###"] + d.request("answer", unit="0", dir=ws, setup="original", simulator=self.user, + **{k: self.agent[k] for k in ("model", "base_url", "key")}) + dt = d.request("score", unit="0", dir=ws, transcript=[], answer="", setup="original")["detail"] + self.assertEqual((dt["max_steps"], dt["max_errors"]), want, options) + def test_attempt_n_is_the_upstreams_trial_n_minus_1_at_its_seed(self): d = self.driver() ws = self.ws("a2") diff --git a/app/ui_tests/test_command_builder_state.py b/app/ui_tests/test_command_builder_state.py index 46321a4f..da05ec95 100644 --- a/app/ui_tests/test_command_builder_state.py +++ b/app/ui_tests/test_command_builder_state.py @@ -16,6 +16,17 @@ def test_original_is_default_for_every_benchmark_with_a_reference(self): for key, others in out: self.assertEqual(others, ["original"], key) + def test_no_time_limit_is_forced_until_the_user_types_one(self): + """The benchmark's and the agent's own limits apply by themselves: choosing a setup or a + benchmark forces none on the recipe, and a saved recipe keeps exactly what it was saved with.""" + out = self.js('const r=C.createStandard("tau3-bench");const osw=C.createStandard("osworld1");' + 'const saved=JSON.parse(JSON.stringify(r));saved.timeouts={agent:45};' + 'process.stdout.write(JSON.stringify([r.others,"timeouts" in r,"timeouts" in osw,C.fromJSON(JSON.stringify(saved)).timeouts]));') + self.assertEqual(out[0], ["original"]) + self.assertFalse(out[1]) + self.assertFalse(out[2]) + self.assertEqual(out[3], {"agent": 45}) + def test_explicit_agent_is_preserved(self): out = self.js('const r=C.createStandard("gsm8k");' 'const agent=C.standardOthers(r).find(o=>!o.setup&&!o.control&&!o.why.length);' diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 66904def..66c51bf8 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -31,13 +31,14 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Version | Setups as published, whose model | Options | |---|---|---| -| `osworld1` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `thinking`, `vm_provider` | +| `osworld1` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `max_steps`, `thinking`, `vm_provider` | ## Options | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| | `agent` | **`m3`**, `prompt`, `claude`, `gpt54`, `openai_cua`, `qwen`, `os_symphony`, `evocua`, `autoglm`, `autoglm_v`, `mano`, `muse_spark` | Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit. | `original`, `model` | +| `max_steps` | pattern `default\|[1-9][0-9]{0,3}`, default `default` | The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it. | `original`, `model` | | `thinking` | **`adaptive`**, `off` | M3_THINKING_MODE: adaptive, as run_multienv_m3.py's own documented invocation exports it (the model decides whether to think; temperature 1, no top_p), or off, the agent's code default when the variable is unset. | `original`, `model` | | `vm_provider` | **`docker`** | --provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own. | `original`, `model`, `yours` | diff --git a/docs/benchmarks/supported/tau3-bench.md b/docs/benchmarks/supported/tau3-bench.md index f5928ba4..712b2b9c 100644 --- a/docs/benchmarks/supported/tau3-bench.md +++ b/docs/benchmarks/supported/tau3-bench.md @@ -32,13 +32,15 @@ Customer service with domain tools and a simulated user: airline, retail, teleco | Version | Setups as published, whose model | Options | |---|---|---| -| `tau3-bench` | Original · τ³-bench LLM agent · Like the paper, Like the leaderboard or Any of your choice | `agent_reasoning_effort`, `retrieval`, `task_split`, `upstream_agent`, `user_reasoning_effort` | +| `tau3-bench` | Original · τ³-bench LLM agent · Like the paper, Like the leaderboard or Any of your choice | `agent_reasoning_effort`, `max_errors`, `max_steps`, `retrieval`, `task_split`, `upstream_agent`, `user_reasoning_effort` | ## Options | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| | `agent_reasoning_effort` | **`default`**, `none`, `minimal`, `low`, `medium`, `high`, `xhigh` | How much reasoning the benchmark's own agent model uses, with `default` keeping the upstream settings. | `original` | +| `max_errors` | pattern `[1-9][0-9]{0,3}`, default `10` | The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's. | `original`, `yours` | +| `max_steps` | pattern `[1-9][0-9]?\|1[0-9]{2}\|200`, default `200` | The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met. | `original`, `yours` | | `retrieval` | **`bm25`**, `bm25_grep`, `grep_only`, `no_knowledge`, `full_kb`, `golden_retrieval`, `openai_embeddings`, `openai_embeddings_grep`, `alltools`, `terminal_use`, `terminal_use_write`, `bm25_reranker`, `bm25_reranker_grep`, `openai_embeddings_reranker`, `openai_embeddings_reranker_grep`, `qwen_embeddings`, `qwen_embeddings_grep`, `qwen_embeddings_reranker`, `qwen_embeddings_reranker_grep`, `alltools-qwen` | How the agent searches the bank's documents in banking_knowledge tasks. | `original`, `yours` | | `task_split` | **`base`**, `train`, `test`, `small`, `full` | Which group of tasks to run within the selected domain. | `original`, `yours` | | `upstream_agent` | **`llm_agent`**, `llm_agent_gt`, `llm_agent_solo` | Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools. | `original` | diff --git a/docs/builder.md b/docs/builder.md index 6d4ea85a..7651d574 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -27,7 +27,7 @@ Then [read its results](results.md). | Dataset | One or more; each dataset with each agent is a job of its own. Each shows its proofs. A missing or expired credential is named once, linked to **Settings → Data providers** (Hugging Face) or **Custom keys** | | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | -| Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit; a [list of units](cli/run.md#a-list-of-units) is set only on the command line | +| Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | | Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | diff --git a/docs/cli/run.md b/docs/cli/run.md index faa16b7a..4471c961 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -5,7 +5,8 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--units FILE] [--limit N] [-j N] \ [--unsandboxed | --backend container|boat] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ - [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE]... \ + [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS]... [--max-turns N] [--max-calls N] \ + [--constraint KEY=VALUE]... \ [--served-by [MODEL=]SLUG[,SLUG...]]... \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] ``` @@ -33,6 +34,8 @@ nullius standard run locomo model --model gpt-4o-mini | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | | `--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | | `--max-usd USD`, `--max-usd-per-unit USD` | A spending cap for the run, for each unit: [below](#a-spending-cap) | +| `--timeout PHASE=SECONDS`, `--max-turns N` | A unit's time limit for a phase, a conversation's most turns, forced: [below](#time-limits) | +| `--max-calls N` | The most model calls a unit's agent may make: [below](#a-cap-on-model-calls) | | `--constraint KEY=VALUE` | What the machine allows the run (repeatable): `cpus`, `memory_mb`, `processes`, `cpu_set`, `concurrency`, `gpu_ids` (indices, empty for none); [constraints](../settings/run-hosts.md#constraints). Containers take them as flags, the run's processes on the machine as a systemd scope. A recipe passes each machine's own | | `--served-by [MODEL=]SLUG[,SLUG...]` | On OpenRouter, the serving providers that may answer a model (repeatable): [below](#serving-providers) | | `--repeat K` | Each unit K times; `result.json` carries `pass_hat_k` | @@ -54,6 +57,32 @@ nullius standard run locomo model --model gpt-4o-mini | Progress | stderr, one line per stage (`[nullius] stage=install ...`, `stage=units n=10`, `stage=done`, `stage=refused`), also `progress.jsonl` | | The job | Opens in the app; `events.jsonl` feeds the live board | +## Time limits + +Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) forces one, the same for every participant, over the task's, the setup's, the benchmark's and the default. + +| Phase | Default | A unit past it | +| --- | --- | --- | +| `setup` | 600 s for the driver's `setup`; an own image's `prepare` step its `seconds` | Environment setup error | +| `image` | 3600 s: an own image's pull or build | Environment setup error | +| `agent` | The task's `agent_seconds`, else the Original setup's `turn_seconds`, else `limits.turn_seconds`, else 900 s: the agent's whole work on the unit, every turn of a conversation and the simulated user's between them; each add and each recall of a capability (its reader has `limits.turn_seconds`); an Original setup's answerer; the oracle | Agent timeout | +| `verify` | The verify step's `seconds`, else the agent's default | Verifier error | +| `score` | 600 s: the driver's `score` | Verifier error | + +`--max-turns N` caps a conversation's agent turns over `limits.turns` (default 20); the conversation is scored as it stands. `config.json` records what was forced under `standard.timeouts` and `standard.max_turns`. From the builder, *Limits* takes them in minutes. + +## A cap on model calls + +`--max-calls 60` lets a unit's agent make at most 60 model calls, the whole unit through, every turn of a conversation. Agents only: a capability's route carries the benchmark's reader too, so a capability run refuses it. Most agents make one call per step, so it bounds a looping agent where nothing else counts steps, and where a model with no price leaves no spending cap. + +| Rule | Detail | +| --- | --- | +| What is counted | Each call the provider answered on the unit's agent route. A refused call (a 429, an error) is not one; the judge's and the simulated user's never are | +| Past it | The relay answers the next call itself (HTTP 402, `unit_call_cap`); the unit ends as `UnitCallCapError`, an Agent error, as a timeout is, and is not checked. The run goes on | +| Not exact | Calls in flight together may pass it by the few already sent | +| The upstream's own | A step limit of the benchmark's (τ³-bench's `--max-steps`, an OSWorld agent's `--max_steps`) still holds; this cap is on top | +| The result | `config.json` records it under `standard.max_calls`; from the builder, *Limits* → *Model calls per unit* | + ## A spending cap `--max-usd 5` stops a run at 5 USD and `--max-usd-per-unit 0.50` one unit at 0.50. diff --git a/docs/field-guide.md b/docs/field-guide.md index 68d54817..cfe9e7d1 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -243,7 +243,7 @@ Thirteen classes. The five marked **yes** under Infrastructure leave the adjuste | `failed_verification` | Failed verification | no | The verifier said no, on a benchmark with no patch detail: Terminal-Bench, or one read only by its reward. | | `patch_not_applied` | Patch did not apply | no | Malformed context lines, or edits against the wrong revision. | | `no_patch` | No patch produced | no | Finished without changing anything. | -| `agent_error` | Agent error | no | Timed out, filled its context, truncated its output, refused, exited non-zero, or reached its own spending cap. A result, counted against the agent. | +| `agent_error` | Agent error | no | Timed out, filled its context, truncated its output, refused, exited non-zero, reached its own spending cap or its cap on model calls. A result, counted against the agent. | | `rate_limited` | Rate limited | yes | An HTTP 429 ended the trial. A standard unit that passed keeps its pass. | | `api_error` | API error | yes | Any other provider failure: overloaded, unreachable, out of quota, key rejected. A standard unit that passed keeps its pass. | | `setup_error` | Setup error | yes | The environment or image never came up, or the unit was not run for want of a secret or docker it needs, so the agent never ran. | diff --git a/docs/standard.md b/docs/standard.md index fbbfae5b..0e1ba2c4 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -121,8 +121,8 @@ benchmark[@version] × agent|capability × model × dataset × its options × li │ backend, or --backend container (see Safety); an option that needs the │ machine's programs is refused without them, one that embeds gets │ the run's own relay, started before the driver - ▼ for each unit, N at a time (its agent turn: the unit's agent_seconds, - │ else limits.turn_seconds) + ▼ for each unit, N at a time (the agent's whole work on it: --timeout agent, + │ else the unit's agent_seconds, else limits.turn_seconds) setup driver fills the unit's workspace, returns the prompt (+ MCP servers), or for a capability, what to add and what to recall produce agent: an ACP session in the workspace, model via the relay @@ -144,10 +144,11 @@ record); verifier/reward.txt and score.json | A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | +| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. The key goes as the agent's protocol says: `x-api-key` and `anthropic-version` for `messages`, else a bearer header. Provider `anthropic` serves `messages` only ([protocols](settings/model-providers.md#protocols)) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | +| A cap on model calls (`--max-calls`) | Agents only. A unit's relay counts its agent route's answered calls and refuses the one past the cap: the agent's error, as a timeout is ([Run a benchmark](cli/run.md#a-cap-on-model-calls)) | | Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | | Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | @@ -435,7 +436,8 @@ use the source folders and repository checks. | `score.judge` | `{model, says}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another | | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | -| `limits.turns` | At most this many agent turns in a conversation (default 20) | +| `limits.turns` | At most this many agent turns in a conversation (default 20; `cli run --max-turns` forces another) | +| `limits.turn_seconds` | The agent's whole work on a unit, every turn of a conversation included (default 900; a task's `agent_seconds` or `cli run --timeout agent=` before it) | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -472,6 +474,8 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | `about` | A line about each value | | `needs` | What a value needs: an `embeddings` model, a `reranker` model, `host` requirements, or named `files` | | `when` | Another option's values that make it apply | +| `limit` | `true`: a bound of the upstream's own (steps, errors). The builder shows it in Limits, under the benchmark's name | +| `by` | Its default under each value of another option, e.g. each OSWorld agent's own steps | | `setups` | `["original"]`, `["original", "model"]`: a setting of the paper's pipeline. Without it the option is the port's (`["yours"]`) | - **Temperature** in GSM8K and HumanEval's Original setups accepts a non-negative decimal, such as `0.5`. **Provider default** omits the parameter; the provider/model decides. Some models restrict overrides. In the builder, leave the input empty for provider default; the CLI accepts `--bench-option temperature=0.5` or `temperature=default`. This does not override an agent's own sampling settings.