Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions app/nullius/catalogue/benchmarks/osworld1/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,17 @@ def _agent():
return name, AGENTS["agents"][name]


def with_steps(argv, steps):
"""The script's argv with the run's ``max_steps`` option as its --max_steps: the agent's own (its
invocation's, else its script's default) at ``default``."""
if steps == "default":
return argv
if "--max_steps" in argv:
i = argv.index("--max_steps")
return argv[:i + 1] + [steps] + argv[i + 2:]
return argv + ["--max_steps", steps]


def _fill(value, slots):
for k, v in slots.items():
value = value.replace("{" + k + "}", v)
Expand Down Expand Up @@ -269,6 +280,7 @@ def answer(unit, dir, setup, model, base_url, key, **_):
del common[i:i + (1 if flag == "--headless" else 2)]
argv = [sys.executable, os.path.join(UP, spec["script"])] + [_fill(a, slots) for a in
common + spec["argv"] + spec.get("model_argv", {}).get(model, [])]
argv = with_steps(argv, os.environ.get("NULLIUS_OPTION_MAX_STEPS", "default"))
extra = {k: _fill(v, slots) for k, v in spec["env"].items()}
if spec.get("aws"):
extra.update(AGENTS["aws"])
Expand Down
27 changes: 27 additions & 0 deletions app/nullius/catalogue/benchmarks/osworld1/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,33 @@
]
}
},
"max_steps": {
"label": "Steps per task",
"summary": "The agent script's --max_steps: the most actions it takes on a task. default: each agent's own, as its documented invocation runs it.",
"setups": [
"original",
"model"
],
"default": "default",
"pattern": "default|[1-9][0-9]{0,3}",
"limit": true,
"by": {
"agent": {
"m3": "100",
"prompt": "15",
"claude": "50",
"gpt54": "50",
"openai_cua": "50",
"qwen": "50",
"os_symphony": "50",
"evocua": "50",
"autoglm": "50",
"autoglm_v": "50",
"mano": "15",
"muse_spark": "100"
}
}
},
"vm_provider": {
"label": "VM provider",
"summary": "--provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own.",
Expand Down
24 changes: 12 additions & 12 deletions app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
"setting": "--max_steps",
"values": "a number",
"default": "100",
"nullius": "fixed:100 - its default and its documented invocation"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_m3.py",
Expand Down Expand Up @@ -340,7 +340,7 @@
"setting": "--max_steps",
"values": "int",
"default": "50",
"nullius": "fixed:50 - its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_autoglm.py",
Expand Down Expand Up @@ -501,7 +501,7 @@
"setting": "--max_steps",
"values": "int",
"default": "50",
"nullius": "fixed:50 - its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_autoglm_v.py",
Expand Down Expand Up @@ -676,7 +676,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:15 - its documented invocation, and its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv.py",
Expand Down Expand Up @@ -914,7 +914,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:50 - its documented invocation"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_claude.py",
Expand Down Expand Up @@ -1131,7 +1131,7 @@
"setting": "--max_steps",
"values": "int",
"default": "50",
"nullius": "fixed:50 - its documented invocation, and its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_evocua.py",
Expand Down Expand Up @@ -1348,7 +1348,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:50 - its documented invocation"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_gpt54.py",
Expand Down Expand Up @@ -1565,7 +1565,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:15 - its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_mano.py",
Expand Down Expand Up @@ -1824,7 +1824,7 @@
"setting": "--max_steps",
"values": "int",
"default": "100",
"nullius": "fixed:100 - its documented invocation, and its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_muse_spark.py",
Expand Down Expand Up @@ -2125,7 +2125,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:50 - its documented invocation"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_openaicua.py",
Expand Down Expand Up @@ -2342,7 +2342,7 @@
"setting": "--max_steps",
"values": "int",
"default": "15",
"nullius": "fixed:50 - its documented invocation"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_os_symphony.py",
Expand Down Expand Up @@ -2748,7 +2748,7 @@
"setting": "--max_steps",
"values": "int",
"default": "50",
"nullius": "fixed:50 - its default"
"nullius": "option:max_steps"
},
{
"entry": "scripts/python/run_multienv_qwen.py",
Expand Down
19 changes: 14 additions & 5 deletions app/nullius/catalogue/benchmarks/tau3-bench/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@
executes nothing itself: it passes each call to this driver
turn - the conversation is the upstream's: its batch runner,
``runner.batch.run_tasks``, run on the unit's one task in a thread of
this driver, with a ``TextRunConfig`` at the upstream's defaults (max
steps, max errors, no timeout, one trial, its seed 300, three
retries, its user simulator). It draws the trial's seed from the
this driver, with a ``TextRunConfig`` at the upstream's defaults (no
timeout, one trial, its seed 300, three
retries, its user simulator), its max steps and max errors the run's
``max_steps`` and ``max_errors`` options (``bounds``). It draws the trial's seed from the
base seed as it does for every run: the unit's first attempt is
trial 0 of a one-trial run, its attempt N (``--repeat``) trial N-1
at the seed run_tasks draws for that trial (``trial``), and each
Expand Down Expand Up @@ -406,10 +407,18 @@ def retrieval():
return os.environ.get("NULLIUS_OPTION_RETRIEVAL") or None


def bounds():
"""The conversation's bounds the run set, the upstream's --max-steps and --max-errors: its ``max_steps``
and ``max_errors`` options; the upstream's defaults (200, 10) otherwise."""
return {k: int(os.environ[v]) for k, v in (("max_steps", "NULLIUS_OPTION_MAX_STEPS"), ("max_errors", "NULLIUS_OPTION_MAX_ERRORS"))
if os.environ.get(v)}


def run_config(**kw):
"""The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one."""
"""The upstream's TextRunConfig for this domain, its retrieval config and task split set when the run chose one,
and the conversation's bounds (bounds)."""
return TextRunConfig(domain=domain(), task_split_name=split(),
**({"retrieval_config": retrieval()} if retrieval() else {}), **kw)
**({"retrieval_config": retrieval()} if retrieval() else {}), **bounds(), **kw)


def user_config(simulator):
Expand Down
26 changes: 24 additions & 2 deletions app/nullius/catalogue/benchmarks/tau3-bench/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -106,14 +106,14 @@
},
"score": {
"kind": "pass",
"says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200, max errors 10, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)",
"says": "the upstream's own batch runner, run_tasks, on the task: its Orchestrator holds the conversation by its rules (max steps 200 and max errors 10 unless the run sets lower or other ones, the user's stop) at the trial's seed run_tasks draws, an attempt that raises is made again (3 retries) and the last recorded INFRASTRUCTURE_ERROR, which is the unit's error, never a score; its evaluator (evaluate_simulation, ALL) scores the SimulationRun the Orchestrator returns: 0 unless the user ended it (the upstream's agent never does); else the product of the task's reward_basis - the database after the agent's calls, replayed by the upstream's environment, hashes as after the gold actions (DB), each required fact is in one of the agent's messages (COMMUNICATE), the judge finds each NL assertion met (NL_ASSERTION)",
"judge": {
"model": "gpt-4.1-2025-04-14",
"says": "the upstream's NL-assertion evaluator (evaluator_nl_assertions.py, its prompt, its call and its own model name), reached through the judge's route, at temperature 0, asked only for a task whose reward_basis names NL_ASSERTION and that has assertions"
}
},
"limits": {
"turn_seconds": 600,
"turn_seconds": 7200,
"turns": 100,
"attempts": "The upstream's batch runner draws one seed per trial from its base seed 300 (run_tasks: random.seed(300), then randint(0, 1000000) per trial, in order) and gives it to the agent and the user simulator: attempt N of a unit is its trial N-1 at that seed, as --num-trials runs it, so --repeat 4 is the leaderboard's four trials and pass^k is computed as the upstream's metrics compute it."
},
Expand Down Expand Up @@ -311,6 +311,28 @@
"default": "The upstream's own arguments: temperature 0, no effort sent."
}
},
"max_steps": {
"label": "Steps per conversation",
"summary": "The upstream's --max-steps: the most messages and tool calls of a conversation, the user's and the agent's together, at most 200: its default, the leaderboard's, which limits.turns leaves the one met.",
"setups": [
"original",
"yours"
],
"default": "200",
"pattern": "[1-9][0-9]?|1[0-9]{2}|200",
"limit": true
},
"max_errors": {
"label": "Tool errors per conversation",
"summary": "The upstream's --max-errors: the most tool calls that fail before a conversation ends. 10, its default, is the leaderboard's.",
"setups": [
"original",
"yours"
],
"default": "10",
"pattern": "[1-9][0-9]{0,3}",
"limit": true
},
"upstream_agent": {
"summary": "Whether the benchmark's own agent runs normally, receives the reference actions, or works alone with the user's tools.",
"setups": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,21 +158,21 @@
"setting": "--max-steps",
"values": "an integer",
"default": "200 (DEFAULT_MAX_STEPS)",
"nullius": "fixed:200 - the upstream's default, the leaderboard's; limits.turns (100) lets its limit be the one met"
"nullius": "option:max_steps"
},
{
"entry": "tau2 run (src/tau2/cli.py)",
"setting": "--max-errors",
"values": "an integer",
"default": "10 (DEFAULT_MAX_ERRORS)",
"nullius": "fixed:10 - the upstream's default"
"nullius": "option:max_errors"
},
{
"entry": "tau2 run (src/tau2/cli.py)",
"setting": "--timeout",
"values": "seconds",
"default": "none",
"nullius": "fixed:none - the upstream's default; a run's turn limit (600 s an agent turn, 7200 s an original conversation) ends a unit that hangs, as the unit's error"
"nullius": "fixed:none - the upstream's default; a run's agent limit (7200 s a conversation, as an original one; run --timeout agent=SECONDS forces another) ends a unit that hangs, as the unit's error"
},
{
"entry": "tau2 run (src/tau2/cli.py)",
Expand Down
1 change: 1 addition & 0 deletions app/nullius/runs/classify.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
_AGENT_EXCEPTIONS = {
"agenttimeouterror", # the agent spent its budget — that is a result
"unitspendingcaperror", # the unit spent its own spending cap (the standard's relay): the same, in money
"unitcallcaperror", # the unit's agent made the model calls the run allowed it: the same, in steps
"agentsafetyrefusalerror", # the agent (or its provider) chose to refuse
"contextwindowexceedederror", # the agent filled its own context
"contextlengthexceedederror", # same, from Harbor's LLM layer
Expand Down
20 changes: 17 additions & 3 deletions app/nullius/standard/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P]
PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION
PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--timeout PHASE=SECONDS ...] [--max-turns N] [--max-calls N] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID]
Expand Down Expand Up @@ -446,17 +446,22 @@ def cmd_new(a):
return 0


def options(pairs) -> dict:
def options(pairs, flag: str = "--option", form: str = "NAME=VALUE") -> dict:
"""``--option name=value``, repeatable, as {name: value}."""
out = {}
for p in pairs or []:
name, eq, value = p.partition("=")
if not eq or not name:
sys.exit(f"--option {p!r}: NAME=VALUE")
sys.exit(f"{flag} {p!r}: {form}")
out[name.strip()] = value.strip()
return out


def timeouts(pairs) -> dict:
"""``--timeout PHASE=SECONDS``, repeatable, as {phase: seconds}; ``run`` checks the phases and the values."""
return options(pairs, "--timeout", "PHASE=SECONDS")


def constraint_values(pairs) -> dict:
"""``--constraint KEY=VALUE``, repeatable: what the machine allows the run (``nullius.constraints``)."""
out = {}
Expand Down Expand Up @@ -535,6 +540,7 @@ def cmd_run(a):
boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch,
licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images,
prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given,
timeouts=timeouts(a.timeout), max_turns=a.max_turns, max_calls=a.max_calls,
**({"units_file": run.units_file(a.units)} if a.units is not None else {}),
**({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}),
**({"env": run_env(a.provider)} if a.from_run else {}))
Expand Down Expand Up @@ -714,6 +720,14 @@ def main(argv=None):
"other (repeatable; without MODEL=, the run's model); each call records which one did")
p.add_argument("--max-usd-per-unit", type=float, metavar="USD",
help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on")
p.add_argument("--timeout", action="append", metavar="PHASE=SECONDS",
help="a unit's time limit for one phase, forced over the task's, the benchmark's and the default "
"(repeatable): setup, image, agent (its whole work on the unit), verify, score")
p.add_argument("--max-turns", type=int, metavar="N",
help="the most agent turns in a conversation, over the benchmark's limits.turns")
p.add_argument("--max-calls", type=int, metavar="N",
help="the most model calls a unit's agent may make, the whole unit through: past it the relay refuses "
"the next call and the unit is the agent's error, as a timeout is")
v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed")
v.add_argument("benchmark")
v.add_argument("--dataset", help="one dataset (default: each)")
Expand Down
Loading
Loading