diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index 59e1ba5e..b4f9d233 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -229,6 +229,17 @@ def _agent(): return name, AGENTS["agents"][name] +def with_steps(argv, steps): + """The script's argv with the run's ``max_steps`` option as its --max_steps: the agent's own (its + invocation's, else its script's default) at ``default``.""" + if steps == "default": + return argv + if "--max_steps" in argv: + i = argv.index("--max_steps") + return argv[:i + 1] + [steps] + argv[i + 2:] + return argv + ["--max_steps", steps] + + def _fill(value, slots): for k, v in slots.items(): value = value.replace("{" + k + "}", v) @@ -269,6 +280,7 @@ def answer(unit, dir, setup, model, base_url, key, **_): del common[i:i + (1 if flag == "--headless" else 2)] argv = [sys.executable, os.path.join(UP, spec["script"])] + [_fill(a, slots) for a in common + spec["argv"] + spec.get("model_argv", {}).get(model, [])] + argv = with_steps(argv, os.environ.get("NULLIUS_OPTION_MAX_STEPS", "default")) extra = {k: _fill(v, slots) for k, v in spec["env"].items()} if spec.get("aws"): extra.update(AGENTS["aws"]) diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 8c01261e..53f55d38 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -176,6 +176,33 @@ ] } }, + "max_steps": { + "label": "Steps per task", + "summary": "The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it.", + "setups": [ + "original", + "model" + ], + "default": "default", + "pattern": "default|[1-9][0-9]{0,3}", + "limit": true, + "by": { + "agent": { + "m3": "100", + "prompt": "15", + "claude": "50", + "gpt54": "50", + "openai_cua": "50", + "qwen": "50", + "os_symphony": "50", + "evocua": "50", + "autoglm": "50", + "autoglm_v": "50", + "mano": "15", + "muse_spark": "100" + } + } + }, "vm_provider": { "label": "VM provider", "summary": "--provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own.", diff --git a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json index c3c02423..99e0a49c 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json @@ -60,7 +60,7 @@ "setting": "--max_steps", "values": "a number", "default": "100", - "nullius": "fixed:100 - its default and its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_m3.py", @@ -340,7 +340,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_autoglm.py", @@ -501,7 +501,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_autoglm_v.py", @@ -676,7 +676,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:15 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv.py", @@ -914,7 +914,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_claude.py", @@ -1131,7 +1131,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_evocua.py", @@ -1348,7 +1348,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_gpt54.py", @@ -1565,7 +1565,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:15 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_mano.py", @@ -1824,7 +1824,7 @@ "setting": "--max_steps", "values": "int", "default": "100", - "nullius": "fixed:100 - its documented invocation, and its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_muse_spark.py", @@ -2125,7 +2125,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_openaicua.py", @@ -2342,7 +2342,7 @@ "setting": "--max_steps", "values": "int", "default": "15", - "nullius": "fixed:50 - its documented invocation" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_os_symphony.py", @@ -2748,7 +2748,7 @@ "setting": "--max_steps", "values": "int", "default": "50", - "nullius": "fixed:50 - its default" + "nullius": "option:max_steps" }, { "entry": "scripts/python/run_multienv_qwen.py", diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py index a0a2606c..f54c8669 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/driver.py +++ b/app/nullius/catalogue/benchmarks/tau3-bench/driver.py @@ -24,9 +24,10 @@ executes nothing itself: it passes each call to this driver turn - the conversation is the upstream's: its batch runner, ``runner.batch.run_tasks``, run on the unit's one task in a thread of - this driver, with a ``TextRunConfig`` at the upstream's defaults (max - steps, max errors, no timeout, one trial, its seed 300, three - retries, its user simulator). It draws the trial's seed from the + this driver, with a ``TextRunConfig`` at the upstream's defaults (no + timeout, one trial, its seed 300, three + retries, its user simulator), its max steps and max errors the run's + ``max_steps`` and ``max_errors`` options (``bounds``). It draws the trial's seed from the base seed as it does for every run: the unit's first attempt is trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1 at the seed run_tasks draws for that trial (``trial``), and each @@ -406,10 +407,18 @@ def retrieval(): return os.environ.get("NULLIUS_OPTION_RETRIEVAL") or None +def bounds(): + """The conversation's bounds the run set, the upstream's --max-steps and --max-errors: its ``max_steps`` + and ``max_errors`` options; the upstream's defaults (200, 10) otherwise.""" + return {k: int(os.environ[v]) for k, v in (("max_steps", "NULLIUS_OPTION_MAX_STEPS"), ("max_errors", "NULLIUS_OPTION_MAX_ERRORS")) + if os.environ.get(v)} + + def run_config(**kw): - """The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one.""" + """The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one, + and the conversation's bounds (bounds).""" return TextRunConfig(domain=domain(), task_split_name=split(), - **({"retrieval_config": retrieval()} if retrieval() else {}), **kw) + **({"retrieval_config": retrieval()} if retrieval() else {}), **bounds(), **kw) def user_config(simulator): diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json index d8d7e614..9747780f 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/standard.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/standard.json @@ -106,14 +106,14 @@ }, "score": { "kind": "pass", - "says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200, max errors 10, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)", + "says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200 and max errors 10 unless the run sets lower or other ones, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)", "judge": { "model": "gpt-4.1-2025-04-14", "says": "the upstream's NL-assertion evaluator (evaluator_nl_assertions.py, its prompt, its call and its own model name), reached through the judge's route, at temperature 0, asked only for a task whose reward_basis names NL_ASSERTION and that has assertions" } }, "limits": { - "turn_seconds": 600, + "turn_seconds": 7200, "turns": 100, "attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it." }, @@ -311,6 +311,28 @@ "default": "The upstream's own arguments: temperature 0, no effort sent." } }, + "max_steps": { + "label": "Steps per conversation", + "summary": "The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met.", + "setups": [ + "original", + "yours" + ], + "default": "200", + "pattern": "[1-9][0-9]?|1[0-9]{2}|200", + "limit": true + }, + "max_errors": { + "label": "Tool errors per conversation", + "summary": "The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's.", + "setups": [ + "original", + "yours" + ], + "default": "10", + "pattern": "[1-9][0-9]{0,3}", + "limit": true + }, "upstream_agent": { "summary": "Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools.", "setups": [ diff --git a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json index 821c3c4f..9f4f9ec4 100644 --- a/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json @@ -158,21 +158,21 @@ "setting": "--max-steps", "values": "an integer", "default": "200 (DEFAULT_MAX_STEPS)", - "nullius": "fixed:200 - the upstream's default, the leaderboard's; limits.turns (100) lets its limit be the one met" + "nullius": "option:max_steps" }, { "entry": "tau2 run (src/tau2/cli.py)", "setting": "--max-errors", "values": "an integer", "default": "10 (DEFAULT_MAX_ERRORS)", - "nullius": "fixed:10 - the upstream's default" + "nullius": "option:max_errors" }, { "entry": "tau2 run (src/tau2/cli.py)", "setting": "--timeout", "values": "seconds", "default": "none", - "nullius": "fixed:none - the upstream's default; a run's turn limit (600 s an agent turn, 7200 s an original conversation) ends a unit that hangs, as the unit's error" + "nullius": "fixed:none - the upstream's default; a run's agent limit (7200 s a conversation, as an original one; run --timeout agent=SECONDS forces another) ends a unit that hangs, as the unit's error" }, { "entry": "tau2 run (src/tau2/cli.py)", diff --git a/app/nullius/runs/classify.py b/app/nullius/runs/classify.py index ec8b7acf..8346df4f 100644 --- a/app/nullius/runs/classify.py +++ b/app/nullius/runs/classify.py @@ -85,6 +85,7 @@ _AGENT_EXCEPTIONS = { "agenttimeouterror", # the agent spent its budget — that is a result "unitspendingcaperror", # the unit spent its own spending cap (the standard's relay): the same, in money + "unitcallcaperror", # the unit's agent made the model calls the run allowed it: the same, in steps "agentsafetyrefusalerror", # the agent (or its provider) chose to refuse "contextwindowexceedederror", # the agent filled its own context "contextlengthexceedederror", # same, from Harbor's LLM layer diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 9a803b58..951868ac 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,7 +7,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -446,17 +446,22 @@ def cmd_new(a): return 0 -def options(pairs) -> dict: +def options(pairs, flag: str = "--option", form: str = "NAME=VALUE") -> dict: """``--option name=value``, repeatable, as {name: value}.""" out = {} for p in pairs or []: name, eq, value = p.partition("=") if not eq or not name: - sys.exit(f"--option {p!r}: NAME=VALUE") + sys.exit(f"{flag} {p!r}: {form}") out[name.strip()] = value.strip() return out +def timeouts(pairs) -> dict: + """``--timeout PHASE=SECONDS``, repeatable, as {phase: seconds}; ``run`` checks the phases and the values.""" + return options(pairs, "--timeout", "PHASE=SECONDS") + + def constraint_values(pairs) -> dict: """``--constraint KEY=VALUE``, repeatable: what the machine allows the run (``nullius.constraints``).""" out = {} @@ -535,6 +540,7 @@ def cmd_run(a): boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, + timeouts=timeouts(a.timeout), max_turns=a.max_turns, max_calls=a.max_calls, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) @@ -714,6 +720,14 @@ def main(argv=None): "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on") + p.add_argument("--timeout", action="append", metavar="PHASE=SECONDS", + help="a unit's time limit for one phase, forced over the task's, the benchmark's and the default " + "(repeatable): setup, image, agent (its whole work on the unit), verify, score") + p.add_argument("--max-turns", type=int, metavar="N", + help="the most agent turns in a conversation, over the benchmark's limits.turns") + p.add_argument("--max-calls", type=int, metavar="N", + help="the most model calls a unit's agent may make, the whole unit through: past it the relay refuses " + "the next call and the unit is the agent's error, as a timeout is") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") v.add_argument("benchmark") v.add_argument("--dataset", help="one dataset (default: each)") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bacddab5..ea7f6d13 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + max_calls: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -752,6 +753,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--prices", json.dumps(prices, separators=(",", ":"))] if ledger: a += ["--ledger", ledger] + if max_calls: + a += ["--max-calls", str(max_calls)] for m in side_models: a += ["--side-model", m] # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's @@ -1022,8 +1025,9 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, - serving: Optional[Dict[str, List[str]]] = None): + serving: Optional[Dict[str, List[str]]] = None, max_calls: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app + self.max_calls = max_calls # the run's cap on its agent's model calls, when it has one self.ledger = ledger # the file of the run's spending cap, when it has one self.serving = dict(serving or {}) # by model, the serving providers that may answer it self.servers = list(servers) # licence servers, host:port each: a forwarder apiece @@ -1051,7 +1055,7 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger, serving=self.serving) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, max_calls=self.max_calls) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) @@ -1433,16 +1437,17 @@ def own_tag(bench: dict, rel: str, path: str) -> str: return f"nullius/own-{bench['key']}-{word}:{h.hexdigest()[:12]}" -def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) -> str: +def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None, timeout: float = 3600) -> str: """A unit's own image: ``pull``ed by its digest, or built from the folder ``build`` names in the benchmark's upstream (``root``) - the task's own Dockerfile, its own ``FROM``, its folder the whole build context - the first time a unit asks for it, and kept by its tag after. Two units - asking at once wait for one build. No secret is given to a build.""" + asking at once wait for one build. No secret is given to a build. + ``timeout``: the most seconds the pull or the build may take.""" if spec.get("pull"): ref = spec["pull"] if not present(ref): - r = _docker(["pull", ref], timeout=3600, check=False) + r = _docker(["pull", ref], timeout=timeout, check=False) if r.returncode != 0: raise Unavailable(f"pulling {ref}: {(r.stderr or r.stdout).strip()[-300:]}") _built[ref] = True @@ -1458,7 +1463,7 @@ def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) if present(tag): return tag # --force-rm: a build that fails (a task's Dockerfile fetching what is gone) leaves no container behind - r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=3600, check=False) + r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=timeout, check=False) if log_dir: os.makedirs(log_dir, exist_ok=True) with open(os.path.join(log_dir, "own-" + tag.split(":")[0].split("/")[-1] + ".log"), "a") as f: diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index bf5bcf86..8bf1516f 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -468,7 +468,9 @@ def _bench_options(d: dict) -> List[str]: ``summary``, its ``values``, a ``default`` a run gets when it chooses none, and optionally the ``datasets`` it applies to, a line ``about`` each value, what a value ``needs``: an ``embeddings`` model called through the run's relay, ``host`` requirements or ``files`` by name - and - ``when`` it applies: another option's values (a retriever's, only with a retrieval).""" + ``when`` it applies: another option's values (a retriever's, only with a retrieval). ``limit``: + a bound the upstream has of its own (its steps, its errors), offered with the run's limits; ``by``: + its default under each value of another option (an agent's own steps), for a default of ``default``.""" opts = d.get("options") if opts is None: return [] @@ -486,6 +488,13 @@ def _bench_options(d: dict) -> List[str]: on = o.get("setups") if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours",) for x in on)): bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)") + if "limit" in o and (not isinstance(o["limit"], bool) or "pattern" not in o): + bad.append(f"{w}.limit: true for a bound the upstream has of its own, a number its pattern allows") + by = o.get("by") + if by is not None and not (isinstance(by, dict) and len(by) == 1 and all( + k != name and k in opts and isinstance(m, dict) and set(m) == set(opts[k].get("values") or [None]) + and all(option_problem(o, v) is None for v in m.values()) for k, m in by.items())): + bad.append(f"{w}.by: another option -> its every value -> this one's default under it") when = o.get("when") or {} if not _when_valid(name, when, opts): bad.append(f"{w}.when: another option -> the values of it this one applies under") diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 51a74390..c682a739 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -529,8 +529,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None): + constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, + max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -543,6 +543,10 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any. + ``timeouts``, ``max_turns`` and ``max_calls``: the time limits by phase, + in seconds, the most turns of a conversation and the most model calls of + a unit's agent the run forced (``run --timeout``, ``--max-turns``, + ``--max-calls``), when it forced any. ``served_by``: by model, the serving providers the run let answer it (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: @@ -557,6 +561,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"units_file": units_file} if units_file else {}), **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), **({"constraints": constraints} if constraints else {}), + **({"timeouts": timeouts} if timeouts else {}), **({"max_turns": max_turns} if max_turns else {}), + **({"max_calls": max_calls} if max_calls else {}), **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 36aaf7d7..678ab3f8 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -43,7 +43,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay +from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay, run CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -129,7 +129,7 @@ def _bench_page(b: dict) -> dict: "controls": controls, # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], - "reference_agent": x.get("reference_agent", ""), + "reference_agent": x.get("reference_agent", ""), "turn_seconds": x.get("turn_seconds"), "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} @@ -140,9 +140,15 @@ def _bench_page(b: dict) -> dict: # what the upstream's leaderboard fixes besides the ranked model (Like the leaderboard) "leaderboard": b.get("leaderboard") or None, "attempts": (b.get("limits") or {}).get("attempts", ""), + # what a unit's time limits and a conversation's length are when the recipe forces none + "timeouts": {"setup": run.DRIVER_SECONDS, "image": run.IMAGE_SECONDS, "score": run.DRIVER_SECONDS, + "agent": (b.get("limits") or {}).get("turn_seconds") or run.TURN_SECONDS, + "turns": (b.get("limits") or {}).get("turns") or run.MAX_TURNS, + "catalogue": [k for k, f in (("agent", "turn_seconds"), ("turns", "turns")) if (b.get("limits") or {}).get(f)]}, "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [], "setups": o.get("setups") or ["yours"], - **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}} + **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}, + "limit": bool(o.get("limit")), "by": o.get("by") or {}} for n, o in (b.get("options") or {}).items()}, "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]} for n, r in hostreq.all_of(b).items()}, @@ -288,6 +294,10 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, the whole run: its jobs that call a model - each dataset with each participant, on each machine - take equal shares (``--max-usd``), so none spends another's and the amount typed is the most the run spends. + ``timeouts``: {phase: minutes}, each phase's time limit forced on every + job (``--timeout``), empty for the default; ``turns``: the most agent + turns in a conversation (``--max-turns``); ``max_calls``: the most model + calls a unit's agent may make (``--max-calls``). ``max_usd_per_unit``: each unit's own cap (``--max-usd-per-unit``), given whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it @@ -493,6 +503,29 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, elif per_unit < 1e-6: # what the command line can say: a millionth of a dollar add("limits", f"The spending cap per {noun} is too small.") per_unit = None + # the time limits the recipe forces, in minutes by phase (run --timeout, in seconds), and a conversation's length + forced: Dict[str, float] = {} + given_t = recipe.get("timeouts") if isinstance(recipe.get("timeouts"), dict) else {} + for k, v in given_t.items(): + if v in ("", None): + continue + if k not in run.TIMEOUT_PHASES or isinstance(v, bool) or not isinstance(v, (int, float)) \ + or not 0 < round(v * 60, 1) <= run.MAX_TIMEOUT_SECONDS: + add("limits", f"A time limit is a number of minutes above zero, at most {run.MAX_TIMEOUT_SECONDS // 60}, or empty for the default.") + continue + forced[k] = round(v * 60, 1) + calls = recipe.get("max_calls") # the most model calls a unit's agent may make (run --max-calls) + if calls in ("", None): + calls = None + elif _int(calls, 1, 1000000) is None: + add("limits", "Model calls per unit: a whole number of at least 1, or empty for no cap.") + calls = None + turns = recipe.get("turns") + if turns in ("", None) or "agent.conversation/v1" not in (b.get("needs") or []): # only a conversation has turns + turns = None + elif _int(turns, 1, 100000) is None: + add("limits", "Turns per conversation: a whole number of at least 1, or empty for the default.") + turns = None limit = recipe.get("limit") if limit not in ("", None) and _int(limit, 1) is None: add("limits", "The number of units is a whole number of at least 1, or empty for all of them.") @@ -615,6 +648,9 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--repeat", str(repeat)] if repeat > 1 else []), *(["--max-usd", f"{share:.6f}".rstrip("0").rstrip(".")] if share and spends(p) else []), *(["--max-usd-per-unit", f"{per_unit:.6f}".rstrip("0").rstrip(".")] if per_unit and spends(p) else []), + *[w for k, v in forced.items() for w in ("--timeout", f"{k}={v:g}")], + *(["--max-turns", str(turns)] if turns else []), + *(["--max-calls", str(calls)] if calls and spends(p) and o.get("_kind") != "capability" else []), *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], @@ -719,6 +755,9 @@ def row(p: dict) -> list: *([[STEP["limits"], "Spending cap", f"${ledger.usd(cap)}" + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []), *([[STEP["limits"], f"Spending cap per {noun}", f"${ledger.usd(per_unit)}"]] if per_unit else []), + *([[STEP["limits"], "Time limits", " · ".join(f"{k} {v / 60:g} min" for k, v in forced.items())]] if forced else []), + *([[STEP["limits"], "Turns per conversation", str(turns)]] if turns else []), + *([[STEP["limits"], f"Model calls per {noun}", str(calls)]] if calls else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" + (f" · {boat_size} VMs" if backend == "boat" else "")]] diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 13d28ba0..1923088c 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -28,6 +28,14 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +Another is a run's cap on a unit's model calls (``max_calls``): the agent's +own route (the unit's tag alone, never its judge's or its simulated user's) +counts each call the provider answered, and once the count reaches the cap +the next call is refused with a 402 the record says ``refused: calls``. A +call the provider refused (a 429, an error) is not counted, so a rate limit +never reads as the agent's steps. Calls in flight together may pass it by +the few that were already sent. + A run may pin, for a model, the serving providers that may answer it (``serving``, OpenRouter's routing): the relay then puts ``provider: {order, allow_fallbacks: false}`` on every request it sends as @@ -562,7 +570,7 @@ class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, - facts: Optional[bool] = None): + facts: Optional[bool] = None, max_calls: Optional[int] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -585,8 +593,12 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s put on every such request in place of the caller's own. ``facts``: a pinned Chat Completions or Responses request asks OpenRouter for its routing facts, where its reply names who served it; by default when - ``base`` is OpenRouter's.""" + ``base`` is OpenRouter's. + ``max_calls``: the most model calls the agent's own route of a unit + may make (:func:`agent_route`), each answered call counted.""" self.fields = fields + self.max_calls = max_calls + self.answered: Dict[str, int] = {} # the calls each agent route had answered, for max_calls self.serving = {m: list(s) for m, s in (serving or {}).items()} host = (urllib.parse.urlsplit(base).hostname or "").lower() self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) @@ -707,6 +719,13 @@ def _forward(self, h: BaseHTTPRequestHandler): routing_asked = req.get("provider") req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if self.max_calls and agent_route(tag) and h.command == "POST" and isinstance(req, dict) and req.get("model") and not _free(rest): + with self.lock: + made = self.answered.get(tag, 0) + if made >= self.max_calls: + self._refuse(h, tag, rest, req, asked, f"the unit's cap of {self.max_calls} model calls was reached: " + "no further call of its agent is sent", "calls") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -763,6 +782,9 @@ def record(body): rec["fields"] = request_fields(req) cost = None if status == 200: + if self.max_calls and agent_route(tag) and isinstance(req, dict) and req.get("model") and not _free(rest): + with self.lock: + self.answered[tag] = self.answered.get(tag, 0) + 1 rec["reply"] = read_reply(rest, body, streamed) # who served it, how it ended, an error under the 200: beside what was read, never in its place try: @@ -874,7 +896,9 @@ def _refuse(self, h: BaseHTTPRequestHandler, tag: str, rest: str, req, asked, wh with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") - body = json.dumps({"type": "error", "error": {"type": "spending_cap", "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable"}.get(by, "spending_cap"), + body = json.dumps({"type": "error", "error": {"type": "call_cap" if by == "calls" else "spending_cap", + "code": {"unit": "unit_spending_cap", "price": "unpriced_model", "ledger": "ledger_unavailable", + "calls": "unit_call_cap"}.get(by, "spending_cap"), "message": "nullius relay: " + why}}).encode() h.send_response(402) for n, v in (("Content-Type", "application/json"), ("X-Nullius-Refused", by), ("Content-Length", str(len(body)))): @@ -920,6 +944,12 @@ def stop(self): self.pool.close() +def agent_route(tag: str) -> bool: + """Whether a route is a unit's agent's own (``u0042``, ``u0042r2``): its + judge's and its simulated user's carry a suffix (``-judge``, ``-simulator``).""" + return "-" not in tag + + def _free(rest: str) -> bool: """A route that names a model and is not billed: a token count.""" return rest.split("?")[0].rstrip("/").endswith("/count_tokens") @@ -935,6 +965,11 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True +# what a unit whose agent reached the run's cap on model calls carries (``max_calls``): an agent error, counted +# against the agent, as a timeout is (runs.classify) +CALLS_ERROR = "UnitCallCapError" + + def failure(calls: List[dict]) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the @@ -948,6 +983,9 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" + many = next((c for c in calls if c.get("refused") == "calls"), None) + if many: # its agent made the calls it was allowed: the agent's, as its time running out is + return CALLS_ERROR, many.get("error") or "" if not calls or calls[-1].get("status") == 200: return None last = calls[-1] @@ -989,6 +1027,7 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--max-calls", type=int, help="the most model calls a unit's agent route may make") ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", help="the serving providers that may answer a request sent as MODEL, in order, and no other; " "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " @@ -1019,7 +1058,7 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, max_calls=a.max_calls) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 0b52ef2f..a2978c1b 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -106,6 +106,10 @@ def driver_path() -> str: IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) MAX_AGENT_SECONDS = 6 * 3600 # the longest a unit's own agent limit (setup's agent_seconds) may ask MAX_TURNS = 20 +# The phases of a unit whose time limit a run may set (``Run(timeouts=)``, ``cli run --timeout PHASE=SECONDS``): +# its environment's setup, its own image's pull or build, the agent's whole work on it, the check, the score +TIMEOUT_PHASES = ("setup", "image", "agent", "verify", "score") +MAX_TIMEOUT_SECONDS = 24 * 3600 # the longest any one phase may be given BACKENDS = definitions.names.BACKENDS _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) @@ -160,6 +164,24 @@ def units_file(path: str) -> dict: return {"name": os.path.basename(path), "sha256": hashlib.sha256(data).hexdigest(), "units": ids} +def forced_timeouts(given: Optional[dict]) -> Dict[str, float]: + """A run's own time limits (``--timeout PHASE=SECONDS``): {phase: seconds}, + each phase one of ``TIMEOUT_PHASES``, each above zero and at most + ``MAX_TIMEOUT_SECONDS``. Refused otherwise; none given is {}.""" + out: Dict[str, float] = {} + for k, v in (given or {}).items(): + if k not in TIMEOUT_PHASES: + raise Refused(f"--timeout {k}: a phase is one of {', '.join(TIMEOUT_PHASES)}") + try: + sec = float(v) if not isinstance(v, bool) else 0.0 + except (TypeError, ValueError): + sec = 0.0 + if not 0 < sec <= MAX_TIMEOUT_SECONDS: + raise Refused(f"--timeout {k}={v}: seconds above zero, at most {MAX_TIMEOUT_SECONDS}") + out[k] = sec + return out + + def key_header(provider: str, other: dict, env=os.environ) -> str: """How the agent's relay sends the key: by the protocol the agent (or capability) speaks on this provider. Refused, as ``recipe.compile`` refuses it, on a provider that serves none of them.""" @@ -385,6 +407,9 @@ def get(self, name: str) -> Optional[dict]: class Run: + timeouts: Dict[str, float] = {} # the run's own time limits by phase (__init__), none by default + forced_turns: Optional[int] = None + max_calls: Optional[int] = None # the run's cap on a unit's agent's model calls (__init__), none by default def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -394,8 +419,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): + constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, + max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -444,6 +469,15 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas constraints are checked on the engine (where those containers run), and the scope's on systemd: on the local backend a cap no scope holds is refused, on the container backend its drivers then run without one. + ``timeouts``: the run's own time limit for each phase of a unit, by + name (``TIMEOUT_PHASES``), in seconds, forced over whatever the task, + the setup, the benchmark or this module would give it. ``agent`` is + the agent's whole work on a unit, every turn of a conversation + included. ``max_turns``: the most agent turns in a conversation, over + the benchmark's ``limits.turns``. ``max_calls``: the most model calls + a unit's agent may make, the whole unit through (``relay.max_calls``): + a unit that reaches it ends as ``relay.CALLS_ERROR``, the agent's, + as a timeout is, and the run goes on. The job's config records all three. ``served_by``: by model, the serving providers that may answer it (:func:`served_by`), put on every request sent as that model by the run's relays (``relay.routing``) and recorded in the job's config. @@ -614,9 +648,19 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self._image_users: Dict[str, int] = {} self.turn_seconds = (turn_seconds or (other.get("turn_seconds") if self.upstream else None) or (bench.get("limits") or {}).get("turn_seconds") or TURN_SECONDS) + self.timeouts = forced_timeouts(timeouts) # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met - self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + if max_turns is not None and (isinstance(max_turns, bool) or not isinstance(max_turns, int) or not 1 <= max_turns <= 100000): + raise Refused(f"--max-turns {max_turns!r}: a whole number of turns, at least 1") + self.forced_turns = max_turns + if max_calls is not None and (isinstance(max_calls, bool) or not isinstance(max_calls, int) or not 1 <= max_calls <= 1000000): + raise Refused(f"--max-calls {max_calls!r}: a whole number of model calls, at least 1") + if max_calls is not None and other.get("_kind") == "capability": + # a capability's route carries the benchmark's reader too (produce): the cap would charge its calls to the capability + raise Refused("--max-calls caps an agent's model calls: a capability's unit route carries the benchmark's reader too") + self.max_calls = max_calls + self.max_turns = max_turns or (bench.get("limits") or {}).get("turns") or MAX_TURNS self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -773,6 +817,7 @@ def start(self) -> str: shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, + timeouts=self.timeouts, max_turns=self.forced_turns, max_calls=self.max_calls, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -1060,7 +1105,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: os.makedirs(home, exist_ok=True) t0 = time.time() self.progress.emit("unit", "start", unit=unit) - self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS) + self.phase(name, unit, "env_setup", None if self.upstream else self.phase_limit("setup", DRIVER_SECONDS)) produce, score, exc, traj, logs = (None, None), None, None, None, {} p0, checking = None, (None, None) # when the agent began; the check's start and end setup: dict = {} @@ -1069,7 +1114,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: try: try: # a setup's unit needs none: the upstream's answerer reads its data itself - setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=DRIVER_SECONDS, **self.attempt_kw()) + setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.phase_limit("setup", DRIVER_SECONDS), **self.attempt_kw()) except jsonl.Failed as e: raise _Ended("EnvironmentSetupError", str(e)) environment, env_def = self.unit_environment(setup) @@ -1100,14 +1145,14 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if rl is not None: rl.unit_secrets = self.agent_licences(env_def, rl) if own: - self.phase(name, unit, "env_setup", IMAGE_SECONDS) # its image, pulled or built when not here yet + self.phase(name, unit, "env_setup", self.phase_limit("image", IMAGE_SECONDS)) # its image, pulled or built when not here yet box = self.own_box(tag, setup, rl, home) if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts p0 = time.time() - self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" - else DRIVER_SECONDS if self.other["key"] == "oracle" else None) + self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control" + else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None) try: with self.agent_internet(setup, box, rl, trial): if self.other["_kind"] == "control": @@ -1127,13 +1172,13 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.phase_limit("score", DRIVER_SECONDS) + (self.step_limit("verify", step) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.phase_limit("score", DRIVER_SECONDS), **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: @@ -1157,7 +1202,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: ln.stop() calls = rl.calls(tag) if rl is not None else [] failed = relay.failure(calls) - if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR, relay.CALLS_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is @@ -1225,12 +1270,12 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: def cap_reached(self, rl, tag: str) -> Optional[tuple]: """A spending cap's refusal among a unit's calls - its agent's, its judge's or its simulated user's - as what the unit ends with: cut - off by the run's own limit, never a miss, before its own. None - without one.""" - if rl is None or getattr(self, "ledger", None) is None: + off by the run's own limit, never a miss, before its own; then the + cap on its agent's model calls (``max_calls``). None without one.""" + if rl is None or (getattr(self, "ledger", None) is None and not self.max_calls): return None cuts = [f for f in (relay.failure(rl.calls(t)) for t in (tag, f"{tag}-judge", f"{tag}-simulator")) if f] - return next((f for f in cuts if f[0] == ledgers.ERROR), None) or next((f for f in cuts if f[0] == ledgers.UNIT_ERROR), None) + return next((f for k in (ledgers.ERROR, ledgers.UNIT_ERROR, relay.CALLS_ERROR) for f in cuts if f[0] == k), None) def attempt_name(self, unit: str, r: int = 1) -> str: """A unit as the budget lists it: by its name, and its attempt when the run repeats its units.""" @@ -1419,7 +1464,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = self.key_header if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + max_calls=self.max_calls).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1448,7 +1494,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), serving=getattr(self, "serving", None), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), max_calls=self.max_calls, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1574,14 +1620,14 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: if self.other["key"] != "oracle": return "" try: - reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=DRIVER_SECONDS) + reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=self.phase_limit("agent", DRIVER_SECONDS)) except jsonl.Failed as e: raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e)) if reply.get("solve") is not None: if box is None: raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none") if name: # the reference solution, run under its own limit - self.phase(name, unit, "agent_run", float(reply["solve"].get("seconds") or self.turn_seconds)) + self.phase(name, unit, "agent_run", self.step_limit("agent", reply["solve"])) done = self.in_box(box, reply["solve"], collect=False, what="the benchmark's oracle") os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True) with open(os.path.join(os.path.dirname(ws), "agent", "solve.txt"), "w") as f: @@ -1595,14 +1641,15 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: @contextmanager def unit_vm(self, ws: str, home: str, trial: str, setup: dict): """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``), - living as long as its every turn may take.""" + living as long as its work may take: an agent's limit is its whole unit's, a capability's each add's and recall's.""" if getattr(self, "boat", None) is None: yield return - turns = self.max_turns if "agent.conversation/v1" in self.bench["needs"] else 1 + plan = setup.get("memory") or {} + requests = 1 if self.other["_kind"] == "agent" else max(1, len(plan.get("add") or []) + len(plan.get("recall") or [])) try: with self.boat.unit(trial, ws, home, stopped=self._stopped, inst=self.inst, here=self.other.get("_here"), - seconds=self.turn_limit(setup) * turns): + seconds=self.agent_limit(setup) * requests): yield except boat.Unavailable as e: raise _Ended("EnvironmentSetupError", str(e)) @@ -1721,11 +1768,12 @@ def own_image(self, spec: dict) -> str: first = tag not in self._envs self._envs.add(tag) self._image_users[tag] = self._image_users.get(tag, 0) + 1 + seconds = self.phase_limit("image", IMAGE_SECONDS) if not first: - return container.own_image(self.bench, spec, root, work) + return container.own_image(self.bench, spec, root, work, timeout=seconds) with self.progress.stage("environment", cached=container.present(tag), start=("pulling " if spec.get("pull") else "building ") + tag, name=tag): - return container.own_image(self.bench, spec, root, work) + return container.own_image(self.bench, spec, root, work, timeout=seconds) def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str] = None) -> dict: """A step (``verify``, ``solve``) run in the unit's container. For the @@ -1754,7 +1802,7 @@ def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str env = box.online() if step.get("network") else (getattr(box, "agent_proxy", None) if not collect else None) if step.get("collect"): box.run(["mkdir", "-p", step["collect"]], 60, user="0") - return box.run(step["run"], float(step.get("seconds") or self.turn_seconds), env=env) + return box.run(step["run"], self.step_limit("verify" if collect else "agent", step), env=env) except (container.Unavailable, ValueError) as e: raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: {e}") @@ -1781,7 +1829,7 @@ def prepare(self, step: dict, box, trial: str) -> None: box.write_in(dest, text) env = box.online() if step.get("network") else None try: - done = box.run(step["run"], float(step.get("seconds") or self.turn_seconds), user="0", env=env) + done = box.run(step["run"], self.step_limit("setup", step), user="0", env=env) finally: said = box.egress_said() if said: @@ -1840,10 +1888,22 @@ def judge(self, rl, tag: str) -> dict: return {"judge": {"model": self.judge_model, "base_url": rl.url(tag + "-judge", model=self.judge_model), "key": relay.PLACEHOLDER_KEY}} - def turn_limit(self, setup: dict) -> float: - """How long one turn of the agent may take: the unit's own limit when its ``setup`` names one + def phase_limit(self, phase: str, default: float) -> float: + """A phase's time limit: the run's own (``timeouts``) when it sets one, else ``default``.""" + return self.timeouts.get(phase) or default + + def step_limit(self, phase: str, step: dict) -> float: + """A step's time limit in the unit's container (``prepare``, ``solve``, ``verify``): the run's own + for its phase, else the step's ``seconds``, else the run's ``turn_seconds``.""" + return float(self.timeouts.get(phase) or step.get("seconds") or self.turn_seconds) + + def agent_limit(self, setup: dict) -> float: + """How long the agent's whole work on a unit may take, every turn of a conversation included: + the run's own (``timeouts["agent"]``), else the unit's when its ``setup`` names one (``agent_seconds``: a task that declares its agent timeout, as Terminal-Bench's task.toml does), else the run's ``turn_seconds``.""" + if self.timeouts.get("agent"): + return self.timeouts["agent"] own = setup.get("agent_seconds") if isinstance(setup, dict) else None if isinstance(own, (int, float)) and not isinstance(own, bool) and 0 < own <= MAX_AGENT_SECONDS: return float(own) @@ -1878,6 +1938,15 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})") prompt = setup["prompt"] answer = "" + # one limit for the agent's whole work on the unit, every turn and the simulated user's between them + limit = self.agent_limit(setup) + deadline = time.monotonic() + limit + + def left() -> float: + rest = deadline - time.monotonic() + if rest <= 0: + raise acp.Timeout(f"the agent did not end its work in {int(limit)} s", []) + return rest # a conversation's turns carry its simulated user's route and its judge's (each when it has one) sim = dict(self.simulator(rl, tag), **self.judge(rl, tag)) if "agent.conversation/v1" in self.bench["needs"] else {} opened = True @@ -1888,11 +1957,12 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name opened = not first.get("done") if opened: prompt = prompt + "\n\n" + first["message"] - for n in range(self.max_turns if opened else 0): + for _ in range(self.max_turns if opened else 0): transcript.append({"role": "user", "text": prompt}) - if n: # the limit is each turn's: the board's clock starts again with it - self.phase(name, unit, "agent_run", self.turn_limit(setup)) - turn = c.prompt(sid, prompt, timeout=self.turn_limit(setup)) + try: + turn = c.prompt(sid, prompt, timeout=left()) + except acp.Timeout as e: # said as the whole work's limit, not the seconds this turn had left + raise acp.Timeout(f"the agent did not end its work in {int(limit)} s", e.events) from e answer = turn.text transcript.append({"role": "agent", "text": turn.text, "stop": turn.stop}) if "agent.conversation/v1" not in self.bench["needs"]: @@ -1900,6 +1970,7 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name nxt = self.driver().request("turn", unit=unit, dir=ws, transcript=transcript, timeout=600, **sim, **self.attempt_kw()) if nxt.get("done"): break + left() # the simulated user's time is the conversation's too prompt = nxt["message"] if nxt.get("restart"): # the upstream started its conversation again (its own retry of an attempt that @@ -1931,10 +2002,10 @@ def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl t = time.time() try: with open(log, "a") as err: - r = run_group(cmd, self.turn_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + r = run_group(cmd, self.agent_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=err, text=True) except subprocess.TimeoutExpired: - raise _Ended("AgentTimeoutError", f"the agent did not end in {self.turn_limit(setup)} s") + raise _Ended("AgentTimeoutError", f"the agent did not end in {int(self.agent_limit(setup))} s") if r.returncode != 0: raise _Ended("NonZeroAgentExitCodeError", f"exit {r.returncode}") transcript.append({"role": "agent", "text": r.stdout}) @@ -1954,10 +2025,13 @@ def drive_feature(self, unit, ws, home, tag, setup, transcript, rl): proc = jsonl.Proc(cmd, cwd, env, os.path.join(os.path.dirname(ws), "agent", "capability.log"), stopped=self._stopped) os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True) exchanges: List[dict] = [] + # each add and each recall its own limit: a memory adds a unit's sessions one by one, each its own model calls, + # so a limit on the whole would grow with the haystack (LongMemEval's ~50 sessions a question) + limit = self.agent_limit(setup) def send(op, **kw): t0 = time.time() - reply = proc.request(op, timeout=self.turn_seconds, **kw) + reply = proc.request(op, timeout=limit, **kw) exchanges.append({"op": op, "request": kw, "reply": reply, "t0": t0, "t1": time.time()}) return reply @@ -1994,7 +2068,7 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes out = self.driver().request("answer", unit=unit, dir=ws, setup=o["key"], model=self.asked, **self.attempt_kw(), **self.simulator(rl, tag), **self.judge(rl, tag), - base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) + base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.agent_limit({})) except jsonl.Failed as e: calls = rl.calls(tag) silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index a58e76e4..4248df2a 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -80,6 +80,8 @@ } return r; } + // the phases of a unit whose time limit a standard recipe may force (run.py TIMEOUT_PHASES) + const STANDARD_PHASES = [['setup', 'Environment setup'], ['image', 'Image pull or build'], ['agent', 'Agent'], ['verify', 'Verifier'], ['score', 'Score']]; // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. const SERVED_ROLES = ['model', 'simulator', 'judge']; @@ -193,6 +195,14 @@ if (typeof raw.max_usd === 'number' && Number.isFinite(raw.max_usd) && raw.max_usd > 0) r.max_usd = raw.max_usd; // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; + // the time limits it forces, in minutes by phase (recipe.py: run --timeout), and a conversation's most turns + delete r.timeouts; + if (raw.timeouts && typeof raw.timeouts === 'object' && !Array.isArray(raw.timeouts)) { + const t = Object.fromEntries(Object.entries(raw.timeouts).filter(([k, v]) => STANDARD_PHASES.some(([p]) => p === k) && typeof v === 'number' && Number.isFinite(v) && v > 0)); + if (Object.keys(t).length) r.timeouts = t; + } + if (Number.isInteger(raw.turns) && raw.turns >= 1) r.turns = raw.turns; + if (Number.isInteger(raw.max_calls) && raw.max_calls >= 1) r.max_calls = raw.max_calls; // the most model calls of a unit's agent for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) @@ -833,5 +843,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STANDARD_PHASES,STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index f3e37326..0e86bdab 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -99,6 +99,7 @@ #nullius-builder .nb-customagent input { font: 12.5px/1.4 var(--mono); padding: 5px 9px; border: 1px solid var(--nb-line); border-radius: 7px; background: var(--nb-surface); color: var(--nb-text); width: 26ch; } #nullius-builder .nb-agentline { display: flex; justify-content: space-between; gap: 12px; font-size: 12px; color: var(--nb-muted); margin-top: 4px; } #nullius-builder .nb-timelimits input { width: 10ch; color: var(--nb-text); border: 1px solid var(--nb-line); border-radius: 6px; background: var(--nb-surface); padding: 5px 8px; font-size: 13px; } +#nullius-builder .nb-sectionnext { margin-top: 36px; } #nullius-builder .nb-timelimits td { vertical-align: middle; } #nullius-builder .nb-timelimits .nb-mono { font: 12px var(--mono); color: var(--nb-muted); } #nullius-builder .nb-note code, #nullius-builder .nb-categoryhead code { font: 12px var(--mono); background: var(--nb-soft); border: 1px solid var(--nb-line); border-radius: 5px; padding: 2px 6px; color: var(--nb-text); white-space: nowrap; } @@ -261,3 +262,5 @@ #nullius-builder .std-served-slug { min-width: 0; overflow-wrap: anywhere; } #nullius-builder .std-served-pick .nb-remove { flex: none; padding: 0 6px; font-size: 14px; line-height: 1; } @media (max-width: 650px) { #nullius-builder .std-served-list { grid-column: 1; } #nullius-builder label.std-served:has(.std-served-list:not(:empty)) > .ui-field-copy { grid-row: auto; margin-top: 0; } } +/* a limit the benchmark or the agent declares: its placeholder reads as a value; Nullius's default stays grey */ +#nullius-builder .nb-field input.nb-declared::placeholder { color: var(--nb-text); opacity: 1; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 1574913e..c6ca0d70 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -511,7 +511,8 @@ function benchOptions(rec, b, scope = 'yours') { const local = ((schema.standardHere || {}).host || {})[b.key] || {}; const chosen = C.chosenOf(rec); - return Object.entries(b.options || {}).filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) + return Object.entries(b.options || {}).filter(([, o]) => !o.limit) // a bound of the upstream's own is with the limits + .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) // one that applies only under another's values (a top-k, only with a retrieval) is left out otherwise .filter(([, o]) => Object.entries(o.when || {}).every(([k, vs]) => vs.includes((rec.bench_options || {})[k] ?? (b.options[k] || {}).default))) .filter(([, o]) => { @@ -685,12 +686,63 @@ + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '' + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + // a conversation's most agent turns (recipe.py: run --max-turns): only a benchmark that holds conversations has turns + + ((b.needs || []).includes('agent.conversation/v1') + ? field('Turns per conversation', 'turns', rec.turns ?? '', {type: 'number', min: 1, placeholder: String((b.timeouts || {}).turns || ''), + declared: ((b.timeouts || {}).catalogue || []).includes('turns')}) : '') + // the most model calls a unit's agent may make, the whole unit through (recipe.py: run --max-calls); none calls no model + // an agent's alone: a capability's route carries the benchmark's reader too (run.py refuses it) + + (C.standardKeyless(rec) || (b.needs || []).some(n => n.startsWith('cap.')) ? '' : field('Model calls per ' + u.noun, 'max_calls', rec.max_calls ?? '', {type: 'number', min: 1, placeholder: 'none'})) // the whole run's, shared by its jobs (recipe.py); a run that calls no model spends nothing: no cap to offer + (C.standardKeyless(rec) ? '' : field('Spending cap (USD)', 'max_usd', rec.max_usd ?? '', {type: 'number', min: 0, placeholder: 'none'}) - + field('Spending cap per ' + u.noun + ' (USD)', 'max_usd_per_unit', rec.max_usd_per_unit ?? '', {type: 'number', min: 0, placeholder: 'none'})); + + field('Spending cap per ' + u.noun + ' (USD)', 'max_usd_per_unit', rec.max_usd_per_unit ?? '', {type: 'number', min: 0, placeholder: 'none'})) + + standardTimeLimits(rec, b, u) + benchLimits(rec, b); } return ''; } + // The bounds the benchmark's upstream has of its own (its steps, its errors: options with ``limit``), in a section + // of the benchmark's name after the run's scope and its time limits: those the chosen datasets and participants + // take, each empty at its default, shown greyed - an agent's own under ``by`` (OSWorld's agents' steps). + function benchLimits(rec, b) { + const chosen = C.chosenOf(rec), yours = chosen.some(k => !['original', 'model'].includes(k) && !(b.controls || []).some(x => x.key === k)); + const given = rec.bench_options || {}, of = k => given[k] ?? (b.options[k] || {}).default; + const rows = Object.entries(b.options || {}).filter(([, o]) => o.limit).sort(([, x], [, y]) => x.label.localeCompare(y.label)) + .filter(([, o]) => !o.datasets.length || C.datasetsOf(rec).some(id => o.datasets.includes(id))) + .filter(([, o]) => (o.setups || ['yours']).some(s => s === 'yours' ? yours : chosen.includes(s))) + .map(([n, o]) => { + const [k, by] = Object.entries(o.by || {})[0] || []; + const def = (k && by[of(k)]) || o.default; + return ``; + }); + return rows.length ? `