diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 81f8544d..f6793253 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -9,7 +9,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] - PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] + PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force] PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed @@ -603,11 +603,16 @@ def cmd_fidelity(a): def cmd_prove(a): b = _bench(a.benchmark) + jobs = options(a.job) + for c in jobs: + if c not in definitions.CONTROLS: + sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}") held, spill = licences(b, from_run=a.from_run) try: got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model, stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch, - env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images) + env=run_env(a.provider) if a.from_run else None, licences=held, + bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images) except run.Refused as e: print("refused:", e) return 2 @@ -717,6 +722,10 @@ def main(argv=None): v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)") v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only") v.add_argument("--launch", help="the launch id the control jobs of every shard record") + v.add_argument("--bench-option", action="append", metavar="NAME=VALUE", + help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs") + v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR", + help="hold this job, a control run already over the whole dataset (a merged launch), to its bound instead of running it") v.add_argument("--remove-images", action="store_true", help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)") g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bacddab5..9153d219 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None, + timeout: Optional[int] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -761,6 +762,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] + if timeout: + a += ["--timeout", str(timeout)] return a @@ -1022,10 +1025,11 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, - serving: Optional[Dict[str, List[str]]] = None): + serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one self.serving = dict(serving or {}) # by model, the serving providers that may answer it + self.timeout = timeout self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1051,7 +1055,8 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger, serving=self.serving) + hosts=self.hosts, ledger=self.ledger, serving=self.serving, + timeout=self.timeout) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index bf5bcf86..a67b0d5a 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -81,7 +81,7 @@ import re from typing import Callable, Dict, List, Optional, Tuple -from nullius.standard import ports +from nullius.standard import ports, relay from nullius import paths from nullius.catalogue import storage from . import names @@ -1035,6 +1035,25 @@ def control(bench: dict, which: str) -> Optional[dict]: "_kind": "control", "_ref": which, "_here": bench.get("_here", "")} +def comparable(orig: dict, model: str) -> bool: + """Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do.""" + c = orig.get("comparable", True) + return model in c if isinstance(c, list) else bool(c) + + +def upstream_name(orig: dict, model: str, provider: str) -> str: + """``model`` by the name the original's upstream knows, when given as ``provider`` lists it.""" + pre = (orig.get("providers") or {}).get(provider, "") + names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []} + return model if model in (orig.get("models") or []) else names.get(model, model) + + +def paper_models(orig: dict) -> List[str]: + """The original's models whose runs compare with the paper's.""" + c = orig.get("comparable", True) + return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c] + + def _leaderboard(d: dict) -> List[str]: """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the upstream's leaderboard fixes besides the ranked model, from its own submission @@ -1066,12 +1085,13 @@ def _leaderboard(d: dict) -> List[str]: def _setups(s) -> List[str]: """``setups``: {original: {summary, models, default, served_as?, providers, - side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s + side_models?, pin?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s ``models`` are the paper's, by the names the upstream's code knows them; ``side_models`` the ones its code asks besides the answerer (an extraction, a summary), sent as asked through served_as and the provider's prefix; ``providers`` the ones that serve them, each with the prefix its names - carry there; ``model`` needs ``original`` (its pipeline) and asks as one + carry there; ``pin`` {openrouter: {model: [provider slug]}}, the only + providers of each; ``model`` needs ``original`` (its pipeline) and asks as one of its models (``runs_as``).""" if s is None: return [] @@ -1090,7 +1110,7 @@ def _setups(s) -> List[str]: bad.append(f"{w}.summary: what runs, in a sentence") allowed = {"summary"} if k == "original": - allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"} + allowed |= {"models", "default", "served_as", "providers", "side_models", "pin", "units", "turn_seconds", "paper", "comparable", "reference_agent"} if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()): bad.append(f"{w}.reference_agent: the upstream agent's display name") models = x.get("models") @@ -1114,6 +1134,8 @@ def _setups(s) -> List[str]: pv = x.get("providers") if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()): bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}") + if "pin" in x: + bad += _pin(x["pin"], f"{w}.pin", pv if isinstance(pv, dict) else {}, set(models + side)) u = x.get("units") if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))): bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is") @@ -1122,8 +1144,11 @@ def _setups(s) -> List[str]: bad.append(f"{w}.turn_seconds: a whole number of seconds") if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()): bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase") - if "comparable" in x and not isinstance(x["comparable"], bool): - bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only") + c = x.get("comparable", True) + if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c) + or x.get("default") not in c): + bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; " + "or those of its models whose runs compare with the paper's, its default among them") else: allowed |= {"runs_as"} orig = s.get("original") if isinstance(s.get("original"), dict) else None @@ -1139,6 +1164,25 @@ def _setups(s) -> List[str]: return bad +def _pin(pin, w: str, providers: dict, models: Optional[set] = None) -> List[str]: + """``pin``: {provider: {model: [slug]}} for a setup (``models`` given), {provider: [slug]} for a judge.""" + if not isinstance(pin, dict) or not pin: + return [f"{w}: {{openrouter: ...}}, the providers that alone serve it"] + bad = [] + for p, v in pin.items(): + if p != "openrouter" or (providers and p not in providers): + bad.append(f"{w}: a pin is OpenRouter's provider routing, on a provider the benchmark runs on ({p!r})") + continue + pins = v if models is not None else {"_": v} + if not isinstance(pins, dict) or not pins: + bad.append(f"{w}.{p}: {{model: [provider slug, ...]}}") + continue + if models is not None and not all(m in models for m in pins): + bad.append(f"{w}.{p}: pins only its models and side_models") + bad += [f"{w}.{p}: " + b.removeprefix("_: ") for b in relay.serving_problems(pins)] + return bad + + # Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder). SETTINGS_FILE = "upstream-settings.json" _DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$") @@ -1204,7 +1248,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple return None, f"{bench['label']} has no {which} setup" orig, x = s["original"], s[which] model = (model or "").strip() - side = [] + side, pins = [], {} if which == "original" and "models" not in orig: # any model, sent as the provider lists it if not model: return None, f"the original {bench['label']} needs a model: its leaderboard ranks models" @@ -1215,7 +1259,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple model = model or orig["default"] if model not in orig["models"]: return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): " - f"{model} is not one - choose the model setup for another") + f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else "")) if provider and provider not in orig["providers"]: return None, (f"the paper's models are not served by {provider}: the original runs on " + " or ".join(orig["providers"])) @@ -1225,13 +1269,15 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple sent = served(model) # the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent] + pins = {served(m): list(only) for m, only in ((orig.get("pin") or {}).get(provider) or {}).items() + if m == model or m in (orig.get("side_models") or [])} else: if not model: return None, f"the model setup needs a model: the one to put in {orig['default']}'s place" asked, sent = x["runs_as"], model units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"} - return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", - "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True), + return {"model": {"side_models": side}, **({"pins": pins} if pins else {}), "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}", + "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model), "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which, "_here": bench.get("_here", "")}, "" @@ -1313,14 +1359,18 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") + if lim.get("run_again") is not None and not isinstance(lim["run_again"], bool): + bad.append("limits.run_again: false when a rate-limited unit is not run once more") a = lim.get("attempts") if a is not None and (not isinstance(a, str) or not a.strip()): bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), " "in a sentence - the driver is then sent attempt with setup, turn and answer") + if "gated" in d and not isinstance(d["gated"], bool): + bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept") s = d.get("score") or {} if s.get("kind") not in SCORES or not s.get("says"): bad.append(f"score: {{kind: one of {SCORES}, says}}") @@ -1332,6 +1382,8 @@ def problems(d: dict, kind: str) -> List[str]: j = s.get("judge") if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")): bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}") + elif j is not None and "pin" in j: + bad += _pin(j["pin"], "score.judge.pin", {}) sim = d.get("simulator") if sim is not None: if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"): diff --git a/app/nullius/standard/fidelity.py b/app/nullius/standard/fidelity.py index ca3a4ab7..35e133a3 100644 --- a/app/nullius/standard/fidelity.py +++ b/app/nullius/standard/fidelity.py @@ -174,7 +174,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa # records side by side, with the fields fidelity reads. r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model, side_models=(agent.get("model") or {}).get("side_models") or (), - key_header=r.key_header, fields=True).start() + key_header=r.key_header, fields=True, timeout=r.call_seconds).start() order = [(s, n) for n in range(1, passes + 1) for s in SIDES] got: Dict[str, List[dict]] = {s: [] for s in SIDES} solved: Dict[str, List[str]] = {} @@ -230,7 +230,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k except (run._Ended, jsonl.Failed) as e: r.say(f" {tag} {unit}: {e}") try: - score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600) + score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds) except jsonl.Failed: score = {} if os.path.isdir(os.path.join(base, "agent")): diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 6234e4e3..a4cd73c0 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -57,9 +57,9 @@ def _text(content) -> str: def _usage_total(calls: List[dict]) -> dict: - ok = [c for c in calls if c.get("status") == 200] + ok = [c for c in calls if c.get("status") == 200 or c.get("reply")] u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok) - costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))] + costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float)) and c.get("cost_from") != "failed"] return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"), "cost": sum(costs) if costs else None, "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None), @@ -169,6 +169,9 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili reply = c.get("reply") or {} st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"), "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"} + if c.get("status") == 200 and reply.get("tool_calls"): + st["tool_calls"] = [{"tool_call_id": f"call-{len(steps) + 1}-{k + 1}", "function_name": t.get("name") or "", + "arguments": _arguments(t.get("arguments"))} for k, t in enumerate(reply["tool_calls"])] if reply.get("usage"): st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"], "cached_tokens": reply["usage"]["cached"]} @@ -183,6 +186,17 @@ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capabili "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}} +def _arguments(said) -> dict: + """A tool call's arguments as a dict; text that is no JSON object goes under ``raw``.""" + try: + got = json.loads(said) if isinstance(said, str) else said + except ValueError: + got = None + if said is None: + return {} + return got if isinstance(got, dict) else {"raw": said if isinstance(said, str) else json.dumps(said)} + + def _control(other: dict) -> dict: """A control's run says so, and who wrote the control: the runs pages read a standard job named ``nop`` or ``oracle`` as a control only by this.""" @@ -317,7 +331,7 @@ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tu if at is not None: turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True, content={"input": st.get("message")})) - elif agent or st.get("tool_calls"): + elif agent or (st.get("tool_calls") and not st.get("model_name")): spans.extend(_step_spans(i + 1, st, source)) if not spans and not turns: return None @@ -497,6 +511,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st # benchmark a score and no pass or fail - the trial reads as Scored rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value} res["verifier_result"] = {"rewards": rewards} + if score.get("stopped"): + res["verifier_result"]["stopped"] = str(score["stopped"]) parts = score.get("parts") if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0: # a score over several parts (a conversation's questions): the job's mean weighs it by them @@ -515,9 +531,15 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls)) if trajectory is not None: _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1)) - timeline = trace_from_run(trajectory, calls, other, produce) - if timeline is not None: - _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + try: + timeline = trace_from_run(trajectory, calls, other, produce) + if timeline is not None: + _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1)) + except Exception as e: + try: + _write(os.path.join(trial_dir, "agent", "trace.error.txt"), type(e).__name__ + "\n") + except OSError: + pass for name, text in (extra_logs or {}).items(): _write(os.path.join(trial_dir, "agent", name), text) _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1)) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 3bdc33b4..bc12b9c5 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -19,7 +19,11 @@ comments, line endings and sha256 may differ from shard to shard, and the merged job records the first shard's list. Units no shard ran are not a refusal - a shard may be missing, or stopped early - and are said: -the shards absent, and the units a shard planned and has no trial for. +the shards absent, and the units a shard planned and has no trial for. A +trial whose record names another job, benchmark, version, control or agent +than its shard, or a job itself merged despite differences, is refused +unless ``force``. Shards run with different benchmark options keep them each +in ``merged_from``. Files are hard-linked into the merged job where the file system allows (a copy where it does not), so a merge of a thousand trials costs no second @@ -45,6 +49,11 @@ ("number of shards", ("standard", "shard", "of"))) +LISTED = 1000 +OWN = (("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), + ("control", ("standard", "control", "name")), ("agent or capability", ("standard", "plugged"))) + + class Refused(Exception): """The jobs cannot be merged as asked; the message says why.""" @@ -155,6 +164,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: # every trial by its unit: two of one unit are never merged, force or not by_unit: Dict[str, List[tuple]] = {} names: Dict[str, List[str]] = {} + foreign = [] for s in shards: s["trials"] = _trials(s["dir"]) s["results"] = [_read(os.path.join(s["dir"], t, "result.json")) for t in s["trials"]] @@ -162,6 +172,22 @@ def check(job_dirs: List[str], force: bool = False) -> dict: unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) + wrong = [label for label, path in OWN if _at(res, path) != _at(s["config"], path)] + if _at(res, ("standard", "job")) != (s["config"].get("job_name") or s["name"]): + wrong.append("job") + if wrong: + foreign.append(f"{s['name']}/{t} names another {', '.join(wrong)} than its job") + inherited = [f"{s['name']}: {w}" for s in shards for w in _at(s["config"], ("standard", "merged_despite")) or []] + if inherited: + if not force: + raise Refused("a job given was itself merged despite differences - " + "; ".join(inherited[:5]) + + " (--force merges it anyway, and the merge says so)") + differ += inherited + if foreign: + shown = f"{len(foreign)} trial(s) not of the job they are in: " + "; ".join(foreign[:5]) + (" ..." if len(foreign) > 5 else "") + if not force: + raise Refused(shown + " (--force merges them anyway, and the merge says so)") + differ.append(shown) twice = {u: w for u, w in by_unit.items() if len(w) > 1} if twice: shown = "; ".join(f"{u} in {', '.join(f'{j}/{t}' for j, t in w)}" for u, w in sorted(twice.items())[:10]) @@ -176,9 +202,10 @@ def check(job_dirs: List[str], force: bool = False) -> dict: if len(of) == 1: n = of.pop() have = sorted(s["shard"].get("index") for s in shards if s["shard"]) - absent = [i for i in range(1, (n or 0) + 1) if i not in have] - if absent: - notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") + if isinstance(n, int) and n <= LISTED: + absent = [i for i in range(1, n + 1) if i not in have] + if absent: + notes.append(f"shard{'s' if len(absent) > 1 else ''} {', '.join(f'{i}/{n}' for i in absent)} not in the merge") dup = sorted({i for i in have if have.count(i) > 1}) if dup: notes.append(f"shard{'s' if len(dup) > 1 else ''} {', '.join(f'{i}/{n}' for i in dup)} given more than once (their units differ)") @@ -280,8 +307,15 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: cfg["job_name"] = name std = cfg.setdefault("standard", {}) std.pop("shard", None) + own = len({json.dumps(_at(s["config"], ("standard", "benchmark_options")), sort_keys=True) for s in shards}) > 1 + if own: + std.pop("benchmark_options", None) std["merged_from"] = [{"job": s["name"], "trials": len(s["trials"]), - **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {})} for s in shards] + **({"shard": f"{s['shard'].get('index')}/{s['shard'].get('of')}"} if s["shard"] else {}), + **({"benchmark_options": _at(s["config"], ("standard", "benchmark_options")) or {}} if own else {}), + **({"merged_from": _at(s["config"], ("standard", "merged_from"))} + if _at(s["config"], ("standard", "merged_from")) else {})} + for s in shards] if got["units_total"] is not None: std["units_total"] = got["units_total"] if got["differ"]: diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318e..8b391429 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -30,6 +30,8 @@ pages pair them as a launch's controls); the units and leak checks talk to the driver directly and leave nothing behind. A control calls no model, so it needs no key - except a judged benchmark's, whose judge still grades. +``--bench-option`` reaches the checks' driver and each control's run; +``--job nop=DIR`` holds a merged launch of a control to its bound. A check that passes is recorded with its date, its counts and its job; one that fails is taken back, as ``conform`` takes back a port - what it reached kept @@ -41,6 +43,8 @@ """ from __future__ import annotations +import json +import math import os import re import shutil @@ -48,8 +52,7 @@ import time from typing import Callable, Dict, List, Optional -from nullius.runs import parse -from nullius.standard import definitions, install, jsonl, run, shard as shards +from nullius.standard import definitions, install, job, jsonl, run, shard as shards STAMP = "%Y-%m-%d__%H-%M-%S" SHORT = 12 # characters: a reference shorter than this is not looked for @@ -66,7 +69,8 @@ class _Driver: """The benchmark's driver, started as a run starts it, for the checks that talk to it directly (units, leak).""" - def __init__(self, bench: dict, ds: dict, work: str, licences=None): + def __init__(self, bench: dict, ds: dict, work: str, licences=None, options: Optional[dict] = None): + self.options = options if options is not None else options_for(bench, ds) self.up = install.upstream(bench) self.data = install.dataset(bench, ds, self.up) install.driver_packages(bench) # its driver.install, once: start() then finds it @@ -81,10 +85,15 @@ def start(self, n: int) -> jsonl.Proc: env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_VERSION": str(self.bench.get("_version") or ""), - **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults + **run.option_env(self.options)}) return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start() +def options_for(bench: dict, ds: dict, given: Optional[dict] = None, which=shutil.which) -> dict: + """The benchmark's options on ``ds`` for a control's run, at ``given`` or their defaults.""" + return run.resolve_bench_options(bench, ds["id"], given, "yours", which)[0] + + def units_check(driver: _Driver, ds: dict) -> dict: """Two drivers, one ``units`` call each: the same ids, unique, plain.""" got = [] @@ -209,44 +218,164 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str, provider: str, model: str, judge_model: Optional[str], env, concurrency: int, say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None, - licences=None, remove_images: bool = False) -> dict: + licences=None, options: Optional[dict] = None, job_dir: Optional[str] = None, + ids: Optional[List[str]] = None, remove_images: bool = False) -> dict: """Run one control over the dataset (or its first ``limit`` units, of - ``shard`` when given) and hold it to its bound: the pass rate, or the mean score.""" + ``shard`` when given) and hold it to its bound: the pass rate, or the mean score. + ``job_dir``: a control run already, held to the bound instead; it must cover ``ids``.""" other = definitions.control(bench, which) - job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, - jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s), - env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences, - remove_images=remove_images) - j = parse.parse_job(job_dir) - n = len(j.trials) - errored = sum(1 for t in j.trials if t.exception_type) + problems = [] + if job_dir: + job_dir = os.path.abspath(job_dir.rstrip("/")) + problems += job_problems(bench, ds, which, job_dir, ids or []) + cfg = _config(job_dir) + else: + job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency, + jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), + say=lambda s: say(" " + s), env=env, judge_model=judge_model, shard=shard, launch=launch, + licences=licences, bench_options=options, + remove_images=remove_images) + cfg = {} + field = FIELD[bench["score"]["kind"]] + results = [res for _, res in trials(job_dir)] + n = len(results) + errored = sum(1 for res in results if res.get("exception_info")) + values = [verdict(bench, res) for res in results if not res.get("exception_info")] + unscored = sum(1 for v in values if v is None) + values = [v for v in values if v is not None] + top = sum(1 for v in values if v >= 1.0) if bench["score"]["kind"] == "pass": - top = sum(1 for t in j.trials if t.reward == 1.0) value = top / n if n else 0.0 else: - scores = [t.score for t in j.trials if t.score is not None] - top = sum(1 for s in scores if s >= 1.0) - value = sum(scores) / len(scores) if scores else 0.0 + value = sum(values) / len(values) if values else 0.0 (bound, expected), = other["expect"].items() within = value <= expected if bound == "at_most" else value >= expected - problems = [] if not n: problems.append("no unit ran") if errored: problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials") + if unscored: + problems.append(f"{unscored} unit(s) ended with no {field} from 0 to 1 and no error: read the job's trials") if n and not within: problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}") rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected}, "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems} - if judge_model or (bench["score"].get("judge") or {}).get("model"): - rec["judge"] = judge_model or bench["score"]["judge"]["model"] + ran_with = _options_of(cfg) if cfg else (options or {}) + if ran_with: + rec["options"] = ran_with + judge = (cfg.get("standard") or {}).get("judge") if cfg else judge_model or (bench["score"].get("judge") or {}).get("model") + if judge: + rec["judge"] = judge return rec +FIELD = {"pass": "reward", "score": "score"} +UNITS_FAILED = "the units check failed" + + +def trials(job_dir: str) -> List[tuple]: + """(trial, its result.json) for each trial folder of a job; an unreadable record is {}.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if name.startswith(".") or not os.path.isfile(path): + continue + try: + with open(path, encoding="utf-8") as f: + res = json.load(f) + except (OSError, ValueError): + res = {} + out.append((name, res if isinstance(res, dict) else {})) + return out + + +def verdict(bench: dict, res: dict) -> Optional[float]: + """The trial's value in the field its benchmark's score kind names, when it is a finite number from 0 to 1.""" + v = (((res.get("verifier_result") or {}).get("rewards")) or {}).get(FIELD[bench["score"]["kind"]]) + if isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) or not 0.0 <= v <= 1.0: + return None + return float(v) + + +def _options_of(cfg: dict): + """The benchmark options a job ran with: one set, or each distinct set of its merged shards.""" + std = cfg.get("standard") or {} + if std.get("benchmark_options"): + return std["benchmark_options"] + sets, todo = [], list(std.get("merged_from") or []) + while todo: + m = todo.pop() + if m.get("benchmark_options") or not m.get("merged_from"): + sets.append(m.get("benchmark_options") or {}) + else: + todo += m.get("merged_from") or [] + distinct = {json.dumps(x, sort_keys=True): x for x in sets} + return [distinct[k] for k in sorted(distinct)] if any(sets) else [] + + +def _config(job_dir: str) -> dict: + try: + with open(os.path.join(job_dir, "config.json"), encoding="utf-8") as f: + got = json.load(f) + return got if isinstance(got, dict) else {} + except (OSError, ValueError): + return {} + + +def job_problems(bench: dict, ds: dict, which: str, job_dir: str, ids: List[str]) -> List[str]: + """Why ``job_dir`` is not one run of ``which`` over each unit of ``ids`` once.""" + cfg = _config(job_dir) + std = cfg.get("standard") or {} + if cfg.get("runner") != "standard": + return [f"{job_dir}: not a job of the Nullius standard"] + want = {"benchmark": bench["key"], "benchmark version": job._version(bench).get("benchmark_version"), + "control": which, "dataset": f"{bench['key']}/{ds['id']}"} + have = {"benchmark": std.get("benchmark"), "benchmark version": std.get("benchmark_version"), + "control": (std.get("control") or {}).get("name"), + "dataset": ((cfg.get("datasets") or [{}])[0] or {}).get("name")} + bad = [f"its {k} is {have[k]!r}, not {want[k]!r}" for k in want if have[k] != want[k]] + if std.get("merged_despite"): + bad.append("merged despite differences: " + "; ".join(std["merged_despite"])) + if (std.get("repeat") or 1) > 1: + bad.append(f"each unit ran {std['repeat']} times") + ran: Dict[str, int] = {} + shard_jobs = {m.get("job") for m in std.get("merged_from") or []} + alien: Dict[str, List[str]] = {} + for trial, res in trials(job_dir): + tstd = res.get("standard") if isinstance(res.get("standard"), dict) else {} + own = {"benchmark": tstd.get("benchmark"), "benchmark version": tstd.get("benchmark_version"), + "control": (tstd.get("control") or {}).get("name"), + "who ran it": (tstd.get("plugged"), (res.get("agent_info") or {}).get("name")), + "job": tstd.get("job")} + expect = {"benchmark": want["benchmark"], "benchmark version": want["benchmark version"], "control": which, + "who ran it": (which, which), "job": cfg.get("job_name")} + wrong = [k for k in expect if own[k] != expect[k]] + if shard_jobs and tstd.get("shard_job") not in shard_jobs: + wrong.append("shard") + for k in wrong: + alien.setdefault(k, []).append(trial) + u = str(tstd.get("unit") or trial) + ran[u] = ran.get(u, 0) + 1 + for k, names in alien.items(): + bad.append(f"{len(names)} trial(s) whose own record names another {k} than the job: {', '.join(names[:3])}") + known = set(ids) + missing = [u for u in ids if u not in ran] + extra = sorted(u for u in ran if u not in known) + twice = sorted(u for u, k in ran.items() if k > 1) + if missing: + bad.append(f"{len(missing)} of the dataset's {len(ids)} units have no trial: {', '.join(missing[:5])}" + (" ..." if len(missing) > 5 else "")) + if extra: + bad.append(f"{len(extra)} trial(s) of units the dataset does not have: {', '.join(extra[:5])}") + if twice: + bad.append(f"{len(twice)} unit(s) with more than one trial: {', '.join(twice[:5])}") + return [f"{os.path.basename(job_dir)}: {b}" for b in bad] + + def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs", provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None, env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print, shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None, + bench_options: Optional[dict] = None, jobs: Optional[Dict[str, str]] = None, remove_images: bool = False) -> Dict[str, dict]: """Every check on every dataset (or ``dataset``); with ``write``, what passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the @@ -254,13 +383,26 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = the units check all of them, and each record says which shard. ``remove_images``: each unit's own image is removed after it, so the oracle pulls it again (by default they stay). Returns - {dataset: {check: record}}, each record with ``ok`` and its ``problems``.""" + {dataset: {check: record}}, each record with ``ok`` and its ``problems``. + ``jobs``: {control: job directory}, held to its bound instead of run.""" env = env if env is not None else os.environ stamp = stamp or time.strftime(STAMP) + jobs = dict(jobs or {}) sets = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not sets: raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})] + if jobs: + if dataset is None and len(sets) > 1: + raise run.Refused("a control's job is a run on one dataset: name it (--dataset)") + if limit or (shard and shard[1] > 1): + raise run.Refused("a control's job is held over the whole dataset: no limit, no shard") + for c, d in jobs.items(): + if c not in declared: + raise run.Refused(f"{bench['label']} declares no {c} control") + if not os.path.isdir(d): + raise run.Refused(f"{d}: no such job") + opts = {ds["id"]: options_for(bench, ds, bench_options) for ds in sets} if not declared: say(f"{bench['label']} declares no controls: only its units and nothing else can be checked") got: Dict[str, dict] = {} @@ -270,11 +412,11 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = checks: Dict[str, dict] = {} try: try: - driver = _Driver(bench, ds, work, licences=licences) + driver = _Driver(bench, ds, work, licences=licences, options=opts[ds["id"]]) except install.InstallFailed as e: raise run.Refused(f"could not install what the proof needs: {e}") checks["units"] = units_check(driver, ds) - ids = checks["units"].pop("ids") + every = ids = checks["units"].pop("ids") if shard and shard[1] > 1: ids = [u for _, u in shards.select(ids, *shard)] ids = ids[:limit] if limit else ids @@ -285,10 +427,18 @@ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = finally: shutil.rmtree(work, ignore_errors=True) for c in declared: - checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, - model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, - shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, - remove_images=remove_images) + if not checks["units"]["ok"] and not jobs.get(c): + checks[c] = {"ok": False, "units": 0, "source": definitions.control(bench, c)["source"], + "problems": [UNITS_FAILED + ": the control was not run"]} + else: + checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider, + model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say, + shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences, + options=opts[ds["id"]], job_dir=jobs.get(c), ids=every, + remove_images=remove_images) + if not checks["units"]["ok"]: + checks[c]["problems"].append(UNITS_FAILED + ": a job's cover is held to the driver's ids, which did not pass") + checks[c]["ok"] = False _say(say, c, checks[c]) total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"] for name, r in checks.items(): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 5e5d759c..ee3c3edb 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -127,9 +127,10 @@ def _bench_page(b: dict) -> dict: # One Original procedure, with separate reference-participant and paper-model metadata. "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"], "reference_agent": x.get("reference_agent", ""), - "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}), - **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]), - "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}} + "comparable": k == "original" and b["setups"]["original"].get("comparable", True) is not False, "units": (b["setups"]["original"].get("units") or b.get("units") or {}), + **({"models": definitions.paper_models(x), "default": x["default"], "providers": list(x["providers"]), + "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in definitions.paper_models(x)} + for p, pre in x["providers"].items()}} if "models" in x else {"any": True, "providers": list(x["providers"])})} for k, x in (b.get("setups") or {}).items() if k == "original"], # its own options (a search, a mode): each value with its line and what it needs; and the host requirements @@ -340,6 +341,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("model", "Choose a model.") continue method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original" + if method == "original": + picked = definitions.upstream_name(orig, picked, provider_) o, why_not = definitions.setup(b, method, picked, provider_) if o is None: add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".") @@ -416,8 +419,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()): add("model", "No paper models are listed for this benchmark; choose Any of your choice.") elif model_choice == "paper" and needs_model and orig.get("models"): - if recipe.get("model") not in orig["models"]: - add("model", "Choose one of the paper's listed models.") + if recipe.get("model") not in definitions.paper_models(orig): + add("model", "Choose one of the paper's listed models, or Any of your choice for another of the original's.") elif provider_ not in orig.get("providers", {}): add("model", "The paper's models are not listed for this provider.") provider = recipe.get("provider") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index 13d28ba0..51f9673f 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -13,10 +13,12 @@ apart from the rest (:func:`reply_facts`), the reply also says, when it does, the serving provider that answered (``served_by``), how it ended (``end``) and an error it holds though its status was a success (``error``); reading -them never changes what the caller receives, nor the rest of the record. The +them never changes what the caller receives, nor the rest of the record but a failed Responses object's cost. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the -user's price list, ``nullius.prices``), marked as an estimate. +user's price list, ``nullius.prices``), marked as an estimate. A Responses object answered 200 that failed (its ``end``) with no usage +costs 0, marked as failed: the provider keeps no generation for it, and a +key's usage compared with the relay's sum would show one billed after all. What the relay changes on the way is kept small, so a call through it costs what the same call made directly would: the connection to the provider is @@ -41,6 +43,8 @@ """ from __future__ import annotations +import base64 +import binascii import hashlib import http.client import json @@ -77,6 +81,16 @@ # its next use, before any of the reply is read: worth one new connection. _STALE = (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError, ConnectionAbortedError) ANTHROPIC_VERSION = "2023-06-01" +ROUTE_REFUSED = (404, 503) +ROUTE_ERROR = "ApiProviderRouteError" +CONTEXT_ERROR = "ContextWindowExceededError" +STOP = "stop" +OVERFLOW = "context window" +COMPLETION = re.compile(r"^/(?:chat/completions|completions|messages|responses)$") +_WHOLE = ((b"\x89PNG\r\n\x1a\n", b"IEND\xaeB`\x82", "image/png"), (b"\xff\xd8\xff", b"\xff\xd9", "image/jpeg"), + (b"GIF87a", b";", "image/gif"), (b"GIF89a", b";", "image/gif")) +_PROVIDER = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._()/-]{0,79}") +_CONTEXT = re.compile(r"context[ _-]?(length|window|limit)|prompt is too long|token count .{0,40}exceeds", re.I) def _cut(text, n=4000): @@ -114,7 +128,7 @@ def _usage(u: Optional[dict]) -> Optional[dict]: inp += cached + written else: written = (det or {}).get("cache_write_tokens") or 0 - got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or 0} + got = {"input": inp, "output": out, "cached": cached, "reasoning": (rdet or {}).get("reasoning_tokens") or (rdet or {}).get("thinking_tokens") or 0} if written: got["cache_write"] = written hour = (u.get("cache_creation") or {}).get("ephemeral_1h_input_tokens") or 0 @@ -148,6 +162,25 @@ def _text_of_messages(msg: dict) -> (str, List[dict]): return "".join(text), calls +def _hollow(path: str, body: bytes) -> Optional[tuple]: + """A 200 to a model call that holds no reply: the status it stands for, 429 or 402 when an error it held says so, else + 502, and its kind alone, since its words can quote the request.""" + if not COMPLETION.search(path.split("?")[0]): + return None + try: + got = json.loads(body) + except ValueError: + return 502, "no reply" + if not isinstance(got, dict): + return 502, "no reply" + if read_reply(path, body, False): + return None + code = got["error"].get("code") if isinstance(got.get("error"), dict) else None + code = code if isinstance(code, int) and not isinstance(code, bool) and 400 <= code < 600 else None + return ((code if code in (402, 429) else 502), "an error" + (f" {code}" if code else "")) if got.get("error") \ + else (502, "no reply") + + def read_reply(path: str, body: bytes, streamed: bool) -> dict: """What a reply said: its text, its tool calls, its usage.""" if not streamed: @@ -202,7 +235,7 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: elif d.get("type") == "input_json_delta": args[i] = args.get(i, "") + (d.get("partial_json") or "") continue - if kind in ("response.completed", "response.done", "response.incomplete"): + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): t, c = _text_of_responses(ev.get("response") or {}) return {"text": t, "tool_calls": c, "usage": _usage((ev.get("response") or {}).get("usage"))} for ch in ev.get("choices") or []: @@ -417,6 +450,23 @@ def reply_facts(path: str, body: bytes, streamed: bool) -> dict: return {} +def _plain(name) -> str: + """A provider's name or slug, compared: lower case, letters and digits.""" + return re.sub(r"[^a-z0-9]", "", str(name).split("/")[0].lower()) + + +def held(provider: str, only: List[str]) -> Optional[bool]: + """True when the provider that answered is one of ``only``, False when it shares nothing with them, None when + it cannot be told: a name is not always its slug.""" + name = _plain(provider) + slugs = {_plain(p) for p in only} + if not name or name in slugs: + return True if name else None + if any(s.startswith(name) or name.startswith(s) for s in slugs if s): + return None + return False + + def _request_tail(req: dict) -> dict: """The part of a request a reader needs: the model, how many messages, the last two, abridged, and the reasoning effort it asked for (chat's @@ -442,6 +492,37 @@ def _request_tail(req: dict) -> dict: "reasoning_effort": effort if isinstance(effort, str) else None} +def _image(data: str) -> Optional[str]: + """An inline image's type when its bytes show a whole one: its head, and its end after it.""" + try: + raw = base64.b64decode(data, validate=True) + except (binascii.Error, ValueError): + return None + if raw[:4] == b"RIFF" and raw[8:12] == b"WEBP": + return "image/webp" if int.from_bytes(raw[4:8], "little") + 8 <= len(raw) else None + if raw[:2] == b"BM": + return "image/bmp" if int.from_bytes(raw[2:6], "little") <= len(raw) else None + return next((t for head, tail, t in _WHOLE if raw.startswith(head) and tail in raw[len(head):]), None) + + +def _images(x) -> List[dict]: + """Each inline image a request carries, as Messages or chat sends one: its declared type, the type its bytes + show whole (None when they show none) and its size, never a byte of it.""" + if isinstance(x, list): + return [i for v in x for i in _images(v)] + if not isinstance(x, dict): + return [] + src, url = x.get("source"), x.get("image_url") + url = url.get("url") if isinstance(url, dict) else url + if x.get("type") == "image" and isinstance(src, dict) and src.get("type") == "base64" and isinstance(src.get("data"), str): + said, data = src.get("media_type"), src["data"] + elif isinstance(url, str) and url.startswith("data:") and ";base64," in url: + said, data = url[5:].split(";", 1)[0], url.split(";base64,", 1)[1] + else: + return _images(list(x.values())) + return [{"type": str(said)[:40] if said else None, "is": _image(data), "bytes": len(data)}] + + def _digest(obj) -> str: text = obj if isinstance(obj, str) else json.dumps(obj, sort_keys=True) return hashlib.sha256(text.encode("utf-8", "replace")).hexdigest()[:16] @@ -498,8 +579,9 @@ class _Pool: KEEP = 16 # idle connections kept per provider; more are closed - def __init__(self, base: str): + def __init__(self, base: str, timeout: float = TIMEOUT): u = urllib.parse.urlsplit(base) + self.timeout = timeout self.https = u.scheme == "https" self.host, self.port = u.hostname, u.port or (443 if self.https else 80) self.prefix = u.path.rstrip("/") @@ -515,7 +597,7 @@ def take(self, fresh: bool = False): kind = http.client.HTTPSConnection if self.https else http.client.HTTPConnection with self.lock: self.opened += 1 - return kind(self.host, self.port, timeout=TIMEOUT), False + return kind(self.host, self.port, timeout=self.timeout), False def give(self, conn, r): """Back to the pool when the reply was read whole and the provider @@ -562,7 +644,7 @@ class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, - facts: Optional[bool] = None): + facts: Optional[bool] = None, timeout: float = TIMEOUT): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -585,7 +667,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s put on every such request in place of the caller's own. ``facts``: a pinned Chat Completions or Responses request asks OpenRouter for its routing facts, where its reply names who served it; by default when - ``base`` is OpenRouter's.""" + ``base`` is OpenRouter's. ``timeout``: the seconds one call may wait.""" self.fields = fields self.serving = {m: list(s) for m, s in (serving or {}).items()} host = (urllib.parse.urlsplit(base).hostname or "").lower() @@ -601,7 +683,7 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s self.models: Dict[str, str] = {} self.embedding_models: Dict[str, str] = {} self.lock = threading.Lock() - self.pool = _Pool(self.base) + self.pool = _Pool(self.base, timeout) self.server: Optional[ThreadingHTTPServer] = None os.makedirs(record_dir, exist_ok=True) @@ -674,8 +756,13 @@ def _headers(self, caller, tag: str = "", facts: bool = False) -> Dict[str, str] out["Authorization"] = "Bearer " + self.key return out + def _asked(self) -> bool: + return asked(self.dir) is not None + def _forward(self, h: BaseHTTPRequestHandler): - """One call of a caller, ``h``: sent on, passed back, recorded.""" + """One call of a caller, ``h``: sent on, passed back, recorded. A caller's own call + refused for its context window also leaves STOP beside the records; one made once + another asked by STOP is refused unsent and unrecorded.""" bits = h.path.split("/", 2) if len(bits) < 3 or not _TAG.match(bits[1]): h.send_error(404, "no route: //v1/...") @@ -707,6 +794,9 @@ def _forward(self, h: BaseHTTPRequestHandler): routing_asked = req.get("provider") req["provider"] = routing(pinned) raw = json.dumps(req).encode() + if not side and not self.models.get(tag) and self._asked(): + h.send_error(400, "the unit was asked to stop") + return booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and # reports no usage; one that names none (a cancellation, a body that is no request) has no price to book @@ -727,6 +817,7 @@ def _forward(self, h: BaseHTTPRequestHandler): t0 = time.time() conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed + cut_off = None try: # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) @@ -747,10 +838,17 @@ def _forward(self, h: BaseHTTPRequestHandler): except Exception as e: # cut off mid-reply status, kind, back, cut = 502, "application/json", [], True body = json.dumps({"error": {"message": "the endpoint's reply was cut off: " + self._said(e)}}).encode() + empty = _hollow(rest, body) if status == 200 and not streamed and h.command == "POST" \ + and not (isinstance(req, dict) and req.get("stream")) else None + if empty: + (status, inside), kind = empty, "application/json" + back = [(n, v) for n, v in back if n.lower() in ("retry-after", "retry-after-ms")] + body = json.dumps({"error": {"message": f"the endpoint answered 200 with {inside}"}}).encode() def record(body): rec = {"t0": t0, "t1": time.time(), "path": rest, "model": req.get("model") if isinstance(req, dict) else None, - "model_asked": asked, "status": status, "request": _request_tail(req) if isinstance(req, dict) else {}, + "model_asked": asked, "status": status, + "request": dict(_request_tail(req), sha256=_digest(req)) if isinstance(req, dict) else {}, "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True @@ -762,7 +860,7 @@ def record(body): if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None - if status == 200: + if status == 200 or cut_off: rec["reply"] = read_reply(rest, body, streamed) # who served it, how it ended, an error under the 200: beside what was read, never in its place try: @@ -770,16 +868,25 @@ def record(body): if k not in rec["reply"]}) except Exception: pass - cost, how = self._cost(rec["model"], rec["reply"].get("usage")) + cost, how = (0.0, "failed") if rec["reply"].get("end") == "failed" and not rec["reply"].get("usage") \ + else self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how - else: - rec["error"] = _said(status, body) + if status != 200: + rec["error"] = cut_off or _said(status, body) + sent = _images(req) if status in (400, 413, 422) else [] + if sent: + rec["images"] = sent + if empty: + rec["answered"] = 200 if booking is not None: rec["booked_usd"] = booked try: - if status == 200 or cut: # what it cost; unknown (no token count, cut off), the booking is kept whole - self.ledger.settle(booking, cost, rec["model"], ((rec.get("reply") or {}).get("usage") or {}).get("output")) + if status == 200 or cut or empty: # what it cost; unknown (no token count, cut off), the booking is kept whole + whole = not (cut_off or rec.get("cost_from") == "failed") + known = cost if whole or (cost is not None and cost > booked) else None + self.ledger.settle(booking, known, rec["model"], + ((rec.get("reply") or {}).get("usage") or {}).get("output") if whole else None) else: # an error of the provider's, or never reached: not billed self.ledger.release(booking) except ledgers.Unavailable: @@ -787,6 +894,9 @@ def record(body): with self.lock: with open(os.path.join(self.dir, tag + ".jsonl"), "a") as f: f.write(json.dumps(rec) + "\n") + if not side and not self.models.get(tag) and overflowed(rec): + with open(os.path.join(self.dir, STOP), "a") as f: + f.write(OVERFLOW + "\n") h.send_response(status) if not any(n.lower() == "content-type" for n, _ in back): @@ -802,6 +912,8 @@ def record(body): while True: chunk = r.read1(65536) if not chunk: + if r.length: + raise http.client.IncompleteRead(b"", r.length) whole = True break parts.append(chunk) @@ -810,8 +922,8 @@ def record(body): h.wfile.flush() except OSError: pass # the caller went away; the record still completes - except Exception: - pass + except Exception as e: + status, cut, cut_off = 502, True, "the endpoint's reply was cut off: " + self._said(e) record(b"".join(parts)) # before the connection closes: the caller's end of stream else: record(body) # before the caller has the reply, so a reader after it finds the call @@ -935,11 +1047,26 @@ def _gone(h: Optional[BaseHTTPRequestHandler]) -> bool: return True -def failure(calls: List[dict]) -> Optional[tuple]: +def asked(record_dir: str) -> Optional[str]: + """What a guard's STOP in ``record_dir`` says, or None: none there, or only the context window's.""" + try: + with open(os.path.join(record_dir, STOP)) as f: + said = next((ln.strip() for ln in f.read().splitlines() or [""] if ln.strip() != OVERFLOW), None) + except OSError: + return None + return None if said is None else said or STOP + + +def failure(calls: List[dict], words: bool = True, agent: bool = False, serving: Optional[Dict[str, List[str]]] = None) -> Optional[tuple]: """When a unit's model calls ended failing, the exception a trial carries for it - named so ``nullius.runs.classify`` sorts it as the infrastructure it was - and the endpoint's words. None when the last call - succeeded (a failure the agent recovered from is not the unit's). A call + succeeded (a failure the agent recovered from is not the unit's). + ``words`` False: the status alone, never the provider's text, which can quote + a gated task. ``agent``: the agent's last call over the model's context + window is ``ContextWindowExceededError``, its own, whatever side call followed, and a side call's 200 does not + recover the agent's own. A pinned call (``serving``) no endpoint took is + ``ApiProviderRouteError``. Calls are taken in the order they began. A call the run's spending cap refused is the unit's wherever it falls: once one is refused, every later one is.""" cut = next((c for c in calls if c.get("refused") == "cap"), None) @@ -948,21 +1075,76 @@ def failure(calls: List[dict]) -> Optional[tuple]: over = next((c for c in calls if c.get("refused") == "unit"), None) if over: # its own cap: the unit's doing, and the run goes on return ledgers.UNIT_ERROR, over.get("error") or "" - if not calls or calls[-1].get("status") == 200: + calls = _begun(calls) + ok = [i for i, c in enumerate(calls) if c.get("status") == 200] + after, side = max((i for i in ok if not (agent and calls[i].get("side"))), default=-1), max(ok, default=-1) + calls = [c for i, c in enumerate(calls) if i > after and c.get("status") != 200 and not (c.get("side") and i < side)] + if not calls: return None + own = next((c for c in reversed(calls) if not c.get("side")), None) if agent else None + if own is not None and overflowed(own): + said = (own.get("error") or "") if words else ("the endpoint answered 400 to the agent's call, longer than the model's " + "context window; its words are not kept, the benchmark's tasks being gated") + return CONTEXT_ERROR, said last = calls[-1] - s, said = last.get("status"), last.get("error") or "" + s = last.get("status") + said = (last.get("error") or "") if words else f"the endpoint answered {s}; its words are not kept, the benchmark's tasks being gated" + if (serving or {}).get(last.get("model")) and s in ROUTE_REFUSED: + return ROUTE_ERROR, _refusal(last, said, serving) if s == 429: return "RateLimitError", said if s in (401, 403): return "AgentAuthenticationError", said - if s == 404 and "model" in said.lower(): + if s == 404 and "model" in (last.get("error") or "").lower(): return "ModelNotFoundError", said if s == 502: return "NetworkConnectionError", said return "ApiError", said +def _begun(calls: List[dict]) -> List[dict]: + timed = all(isinstance(c.get("t0"), (int, float)) for c in calls) + return sorted(calls, key=lambda c: c["t0"]) if timed else list(calls) + + +def overflowed(call: dict) -> bool: + """A 400 for a request longer than the model's context window, by the provider's words.""" + return call.get("status") == 400 and bool(_CONTEXT.search(call.get("error") or "")) + + +def _refusal(call: dict, said: str, serving: Dict[str, List[str]]) -> str: + """A pinned route's refusal: its status first (route_dead reads it), the model, its pin, then ``said``.""" + text = f"{call.get('status')}: no endpoint of {call.get('model')} pinned to {', '.join(serving[call.get('model')])} took the call" + return f"{text}: {said}" if said else text + + +def routes_failure(routes: List[List[dict]], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """:func:`route_failure` of each route's calls on its own, the first found.""" + return next(filter(None, (route_failure(calls, serving, words) for calls in routes)), None) + + +def route_dead(exc: Optional[tuple]) -> bool: + """A pinned route with no endpoint at all (404): every later unit would fail the same way.""" + return bool(exc) and exc[0] == ROUTE_ERROR and str(exc[1]).startswith("404: ") + + +def route_failure(calls: List[dict], serving: Dict[str, List[str]], words: bool = True) -> Optional[tuple]: + """A pinned route (``serving``) that failed a unit, among one route's ``calls``: a model whose last call was + refused, or a call its reply says another provider served, by a name that reads as one.""" + last = {} + for c in _begun(calls): + last[c.get("model")] = c + by = (c.get("reply") or {}).get("served_by") + if serving.get(c.get("model")) and isinstance(by, str) and _PROVIDER.fullmatch(by) and held(by, serving[c.get("model")]) is False: + return ROUTE_ERROR, f"{c.get('status')}: {c.get('model')} pinned to {', '.join(serving[c.get('model')])} was served by {by}" + for c in last.values(): + if serving.get(c.get("model")) and c.get("status") in ROUTE_REFUSED: + said = (c.get("error") or "") if words else \ + f"the endpoint answered {c.get('status')}; its words are not kept, the benchmark's tasks being gated" + return ROUTE_ERROR, _refusal(c, said, serving) + return None + + # -- the relay as a process of its own: a container backend's sidecar ---------- def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL @@ -995,6 +1177,7 @@ def serve(argv=None) -> int: "or --routing given") ap.add_argument("--routing", action="store_true", help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") + ap.add_argument("--timeout", type=float, default=TIMEOUT, help="seconds one call may wait for the provider's reply") a = ap.parse_args(argv) serving, bad = parse_served_by(a.served_by, "") if serving and not bad: @@ -1019,7 +1202,8 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving, + timeout=a.timeout) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 20fbcc56..f996d668 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -51,6 +51,7 @@ """ from __future__ import annotations +import fcntl import hashlib import json import os @@ -66,7 +67,7 @@ from collections import Counter from concurrent.futures import FIRST_EXCEPTION, CancelledError, ThreadPoolExecutor, wait from contextlib import contextmanager -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, Tuple from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact @@ -107,6 +108,10 @@ def driver_path() -> str: _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe) +HELD: dict = {} +LIVE: dict = {} + + class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" @@ -381,7 +386,37 @@ def get(self, name: str) -> Optional[dict]: return {"kind": kind, "value": text} if text else None +def resolve_bench_options(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + which=shutil.which) -> Tuple[dict, dict]: + """The benchmark's options for a run on ``dataset`` and what their values need; refused before anything starts.""" + vals, bad = definitions.bench_options_of(bench, dataset, given, setup) + if bad: + raise Refused("; ".join(bad)) + needs = definitions.option_needs(bench, vals) + lacking = hostreq.problems(bench, needs["host"], which) + if lacking: + raise Refused(f"{', '.join(f'{k}={v}' for k, v in vals.items())} needs on this machine " + + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) + else ". Run it on a machine that has it")) + return vals, needs + + +def pins_of(bench: dict, other: dict, provider: str, judge_model: Optional[str]) -> Dict[str, List[str]]: + """{model as sent: [provider slug]}: the original setup's pins and the official judge's, on a provider that pins.""" + if provider != "openrouter": + return {} + pins = {m: list(o) for m, o in (other.get("pins") or {}).items()} if other.get("_kind") == "upstream" else {} + j = (bench.get("score") or {}).get("judge") or {} + only = (j.get("pin") or {}).get(provider) + if only and judge_model and judge_model == providers.served_as(j["model"], provider): + pins[judge_model] = list(only) + return pins + + class Run: + call_seconds: float = relay.TIMEOUT + def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None, limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, @@ -583,6 +618,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas if stray: raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + (", ".join(called) or "no model")) + self.route_pins = pins_of(bench, other, provider, self.judge_model) + self.serving.update(self.route_pins) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -599,6 +636,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS + self.score_seconds = (bench.get("limits") or {}).get("score_seconds") or DRIVER_SECONDS + self.call_seconds = (bench.get("limits") or {}).get("call_seconds") or relay.TIMEOUT self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None @@ -666,8 +705,25 @@ def driver(self) -> jsonl.Proc: return d def start(self) -> str: + lck, mine = os.path.join(self.job_dir, ".work", "job.lck"), object() + try: + return self._start(lck, mine) + finally: + if LIVE.get(lck) is mine: + del LIVE[lck] + + def _start(self, lck, mine) -> str: t0 = time.time() os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True) + held = HELD.get(lck) or open(lck, "a") + try: + if LIVE.setdefault(lck, mine) is not mine: + raise BlockingIOError + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused(f"another run under the job name {self.name} is running: " + "start this one once it ends, or under another name") from None + HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) @@ -870,11 +926,14 @@ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, job's ``retried/`` (kept whole, its calls and its cost with it, where no reader takes it for a trial of its own), its second written under the trial's own name and counted (into ``done``, as each ends). Nothing - is run again once the run is stopped or halted; the spending cap goes - on counting every call.""" + is run again once the run is stopped or halted, nor a unit a guard's + stop asked to stop; the spending cap goes on counting every call.""" + if (self.bench.get("limits") or {}).get("run_again") is False: + return by_name = {self.trial_name(*iur): iur for iur in work} todo = [by_name[r["trial_name"]] for r in results - if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name] + if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name + and relay.asked(os.path.join(self.job_dir, r["trial_name"], ".work", "relay")) is None] if not todo: return self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time") @@ -980,7 +1039,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), - unit_caps=False, serving=self.serving).start() # embeddings and rerank: a bearer key + unit_caps=False, serving=self.serving, timeout=self.call_seconds).start() # embeddings and rerank: a bearer key def runtimes_needed(self) -> List[str]: """What the agent needs in an own image that the image may not have: @@ -1018,6 +1077,11 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: trial = os.path.join(self.job_dir, self.trial_name(i, unit, r)) name = os.path.basename(trial) ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home") + earlier = os.path.join(trial, ".work", f"earlier.{time.time_ns()}") + for part in (os.path.join(".work", "relay"), "result.json", "verifier", "agent"): + if os.path.lexists(os.path.join(trial, part)): + os.makedirs(earlier, exist_ok=True) + os.replace(os.path.join(trial, part), os.path.join(earlier, os.path.basename(part))) os.makedirs(ws, exist_ok=True) # The workspace is a repository of its own: agents look up to the git root # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept @@ -1090,17 +1154,17 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: raise _Ended(cut[0], cut[1], traj) step = setup.get("verify") if box is not None else None checking = (time.time(), None) - self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0)) + self.phase(name, unit, "verify", self.score_seconds + (float(step.get("seconds") or self.turn_seconds) if step else 0)) verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read container.plain_only(os.path.join(trial, "verifier")) try: judge = self.judge(rl, tag) - score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS, + score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=self.score_seconds, **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {})) checking = (checking[0], time.time()) except jsonl.Failed as e: - raise _Ended("VerifierError", str(e)) + raise _Ended(_kind(e, "VerifierError"), str(e)) except _Ended as e: exc = (e.kind, e.said) traj = traj or e.trajectory @@ -1119,17 +1183,27 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: for ln in getattr(rl, "mcp_listeners", None) or []: ln.stop() calls = rl.calls(tag) if rl is not None else [] - failed = relay.failure(calls) + words = not self.bench.get("gated") + failed = relay.failure(calls, words, agent=True, serving=getattr(self, "route_pins", None)) if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0: # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is # the endpoint's, and a cap's refusal is the cap's failed = None - if not failed and rl is not None and exc is not None and exc[0] == "VerifierError": + if failed and failed[0] == relay.CONTEXT_ERROR and ((score or {}).get("stopped") + or self.upstream and exc and ": VerifierError: " in exc[1]): + failed = None + if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and (score or {}).get("stopped") not in (None, "", relay.OVERFLOW) \ + and rl is not None and relay.asked(rl.dir) is not None \ + and [c.get("status") for c in relay._begun(calls) if not c.get("side")][-1:] == [429]: + failed = None + if not failed and rl is not None and getattr(self, "route_pins", None): + failed = relay.routes_failure([calls] + [rl.calls(f"{tag}-{side}") for side in ("judge", "simulator")], self.route_pins, words) + if not failed and rl is not None and exc is not None and (exc[0] == "VerifierError" or not words): # the judge's or the simulated user's model failing (a 429, a refused request) is the # unit's infrastructure, as the agent's own would be - not a miss counted against it - failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None) + failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}"), words) for side in ("judge", "simulator"))), None) if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)): exc = failed # what the endpoint said outranks how the agent took it score = None @@ -1230,7 +1304,7 @@ def note_refusal(self, exc: Optional[tuple]) -> None: if getattr(self, "_halt", None) is None: return with self._said: - if exc and exc[0] in REFUSALS: + if exc and (exc[0] in REFUSALS or relay.route_dead(exc)): self._refused.append(exc) if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None: self.halted = exc @@ -1378,7 +1452,8 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = self.key_header if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1408,6 +1483,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, ledger=self.ledger_file(), serving=getattr(self, "serving", None), + timeout=self.call_seconds if self.call_seconds != relay.TIMEOUT else None, network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: @@ -1926,8 +2002,8 @@ def send(op, **kw): def drive_upstream(self, unit, ws, tag, transcript, rl): """The setup's unit: the driver runs the upstream's own answerer on it (``answer``), asked as ``asked`` through the unit's relay, which sends - the run's model in its place. A reply with no text - a reasoning model - that spent the upstream's token budget thinking - is said as such.""" + the run's model in its place. A last reply with no text or tool call - a reasoning model + that spent the upstream's token budget thinking - is said as such; a unit a guard stopped is cancelled.""" o = self.other try: # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes @@ -1936,7 +2012,18 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds) except jsonl.Failed as e: calls = rl.calls(tag) - silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()] + stop = relay.asked(rl.dir) + if stop and not _kind(e, None): + raise _Ended(CANCELLED, f"a guard asked the unit to stop ({stop[:200]}); {o['label']}: {e}", + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + own = [c for c in relay._begun(calls) if not c.get("side")][-1:] + failed = [((c["reply"].get("error") or {}).get("code") or "no code") for c in own if (c.get("reply") or {}).get("end") == "failed"] + kind = _kind(e, "ApiError" if failed else None) + if kind: + raise _Ended(kind, f"{o['label']}: {e}" + "".join(f"; {self.model}'s last reply failed ({c})" for c in failed), + job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}")) + silent = [c for c in own if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip() + and not (c.get("reply") or {}).get("tool_calls")] if silent: u = (silent[-1].get("reply") or {}).get("usage") or {} thought = u.get("reasoning") or 0 @@ -1951,6 +2038,12 @@ def drive_upstream(self, unit, ws, tag, transcript, rl): return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}") +def _kind(e, otherwise): + """A driver's own EnvironmentSetupError, VerifierError, CancelledError or CauseUnknownError keeps its kind.""" + m = re.match(r"\w+: (EnvironmentSetupError|VerifierError|CancelledError|CauseUnknownError): ", str(e)) + return m.group(1) if m else otherwise + + class _Ended(Exception): def __init__(self, kind, said, trajectory=None): super().__init__(said) diff --git a/app/tests/test_dependency_surface.py b/app/tests/test_dependency_surface.py index 12b6b2ee..d6cbd843 100644 --- a/app/tests/test_dependency_surface.py +++ b/app/tests/test_dependency_surface.py @@ -9,6 +9,7 @@ viewer independent of the runtime. """ +import fnmatch import importlib.util import json import os @@ -80,6 +81,26 @@ def test_every_backend_package_is_shipped_once(self): self.assertEqual({p for p in packages if p == "nullius" or p.startswith("nullius.")}, source) self.assertEqual({p.split(".")[0] for p in packages}, {"nullius", "_nullius_app"}) + def test_every_file_a_bundled_definition_runs_is_shipped(self): + """Every file a host's snapshot takes of a bundled entry is package data too.""" + from nullius.standard import payload + shipped = tomllib.loads((REPO / "pyproject.toml").read_text(encoding="utf-8"))["tool"]["setuptools"]["package-data"] + + def covered(package, relative): + parts = relative.split("/") + return any(len(g.split("/")) == len(parts) and all(map(fnmatch.fnmatchcase, parts, g.split("/"))) + for g in shipped[package]) + + base = APP / "nullius" + entries = [("nullius.catalogue", f"{kind}/{e.name}", payload._entry(e)) + for kind in ("agents", "benchmarks", "capabilities") + for e in sorted((base / "catalogue" / kind).iterdir()) if e.is_dir() and e.name != "__pycache__"] + entries += [("nullius.standard", f"kits/{e.name}", payload._entry(e, kit=True)) + for e in sorted((base / "standard" / "kits").iterdir()) if (e / "standard.json").is_file()] + missing = [f"{where}/{name}" for package, where, files in entries for name in files + if not covered(package, f"{where}/{name}")] + self.assertEqual(missing, []) + def test_every_requirement_is_on_the_allowlist(self): for req in requirements(): with self.subTest(req): diff --git a/app/tests/test_standard_context_window.py b/app/tests/test_standard_context_window.py new file mode 100644 index 00000000..49fcc160 --- /dev/null +++ b/app/tests/test_standard_context_window.py @@ -0,0 +1,318 @@ +"""An agent's last call over the model's window ends the unit as ContextWindowExceededError, whatever followed.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from nullius.runs import classify, parse +from nullius.standard import definitions, install, job, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +WORDS = "the gated task's own sentence" +OPENROUTER = f"HTTP 400: This endpoint's maximum context length is 1000 tokens. However, you requested 1200: {WORDS}" +ANTHROPIC = f"HTTP 400: prompt is too long: 1200 tokens > 1000 maximum: {WORDS}" +EMPTY = "HTTP 400: messages: at least one message is required" +AGENT_OK = {"status": 200} +OVERFLOW = {"status": 400, "error": OPENROUTER} +SIDE_EMPTY = {"status": 400, "error": EMPTY, "side": True} +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's runner ended on its simulated user's error") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' +STOPPED = ANSWERING.replace('raise RuntimeError("the upstream\'s runner ended on its simulated user\'s error")', + 'return {"answer": "DONE"}').replace('{"reward": 0.0, "detail": {}}', + '{"reward": 0.5, "detail": {}, "stopped": "context window"}') +UNREAD = '''from nullius.standard.jsonl import serve +class VerifierError(RuntimeError): + pass +def score(unit, dir, **_): + raise VerifierError("the evaluation did not read the VM") +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": lambda unit, dir, setup, model, base_url, key, **_: {"answer": "DONE"}, "score": score}) +''' +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Failure(unittest.TestCase): + def test_the_agents_overflow_outranks_the_side_call_after_it(self): + calls = [AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, agent=True), (relay.CONTEXT_ERROR, OPENROUTER)) + exc, said = relay.failure(calls, words=False, agent=True) + self.assertEqual(exc, "ContextWindowExceededError") + self.assertIn("answered 400", said) + self.assertIn("context window", said) + self.assertNotIn(WORDS, said) + self.assertNotIn("maximum context length", said) + + def test_anthropics_words_and_a_later_rate_limit(self): + calls = [AGENT_OK, {"status": 400, "error": ANTHROPIC}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_the_agents_overflow_as_the_last_call(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW], words=False, agent=True)[0], relay.CONTEXT_ERROR) + + def test_an_overflow_the_agent_recovered_from_is_not_the_units(self): + self.assertIsNone(relay.failure([OVERFLOW, AGENT_OK], agent=True)) + + def test_a_side_call_that_went_through_after_it_does_not_recover_the_agent(self): + self.assertEqual(relay.failure([AGENT_OK, OVERFLOW, {"status": 200, "side": True}], agent=True)[0], relay.CONTEXT_ERROR) + limited = [AGENT_OK, {"status": 429, "error": "slow down"}, {"status": 502, "error": "x"}, + {"status": 429, "error": "slow down"}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(limited, words=False, agent=True)[0], "RateLimitError") + self.assertIsNone(relay.failure(limited[1:3] + [AGENT_OK, {"status": 200, "side": True}], agent=True)) + self.assertIsNone(relay.failure(limited, words=False)) + calls = [AGENT_OK, OVERFLOW, {"status": 200, "side": True}, {"status": 429, "error": "slow down", "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], relay.CONTEXT_ERROR) + self.assertEqual(relay.failure(calls, words=False)[0], "RateLimitError") + calls = [{"status": 429, "error": "x"}, {"status": 502, "error": "x", "side": True}, {"status": 200, "side": True}] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "RateLimitError") + + def test_the_calls_are_read_in_the_order_they_began(self): + abandoned = {"status": 502, "error": "x", "t0": 1} + self.assertIsNone(relay.failure([dict(AGENT_OK, t0=2), abandoned], agent=True)) + self.assertEqual(relay.failure([{"status": 429, "error": "x", "t0": 2}, dict(AGENT_OK, t0=1)])[0], "RateLimitError") + self.assertEqual(relay.failure([dict(AGENT_OK, t0=1), {"status": 429, "error": "x"}], agent=True)[0], "RateLimitError") + + def test_another_400_of_the_agent_stays_the_infrastructures(self): + calls = [AGENT_OK, {"status": 400, "error": "HTTP 400: invalid image"}, SIDE_EMPTY] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_side_calls_overflow_is_not_the_agents(self): + calls = [AGENT_OK, dict(OVERFLOW, side=True)] + self.assertEqual(relay.failure(calls, words=False, agent=True)[0], "ApiError") + + def test_a_judges_or_simulators_route_is_never_the_agents_context(self): + calls = [AGENT_OK, OVERFLOW] + self.assertEqual(relay.failure(calls, words=False)[0], "ApiError") + self.assertEqual(relay.failure(calls)[0], "ApiError") + + def test_only_a_400_says_so(self): + self.assertTrue(relay.overflowed(OVERFLOW)) + self.assertTrue(relay.overflowed({"status": 400, "error": ANTHROPIC})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: input length and `max_tokens` exceed context " + "limit: 196000 + 8192 > 200000"})) + self.assertTrue(relay.overflowed({"status": 400, "error": "HTTP 400: The input token count (1200) exceeds the " + "maximum number of tokens allowed (1000)."})) + self.assertFalse(relay.overflowed(dict(OVERFLOW, status=500))) + self.assertFalse(relay.overflowed({"status": 400, "error": EMPTY})) + self.assertFalse(relay.overflowed({"status": 400})) + + def test_it_is_the_agents_result_not_the_infrastructure(self): + self.assertEqual(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.AGENT_ERROR) + self.assertNotIn(classify.from_verdict(relay.CONTEXT_ERROR, None), classify.INFRA_OUTCOMES) + fed = job.feed_outcome("score", None, (relay.CONTEXT_ERROR, "said")) + self.assertEqual((fed["outcome"], fed["exc"]), ("agent_error", relay.CONTEXT_ERROR)) + + +class Calls: + """The agent's calls ending on an overflow, then a refused side call.""" + def __init__(self, own, dir=os.devnull): + self.own, self.dir = own, dir + + def calls(self, tag): + return [] if tag.endswith(("-judge", "-simulator")) else list(self.own) + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Unit(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "paper") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, setups=SETUPS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def ran(self, own, gated=True, record=os.devnull): + b = dict(self.b, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"jobs-{len(os.listdir(self.tmp))}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(own, record)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f).get("exception_info"), parse.parse_job(job_dir) + + def test_the_unit_is_the_agents_context_said_by_status_alone(self): + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn("answered 400", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + (trial,) = ran.trials + self.assertEqual(trial.outcome, classify.AGENT_ERROR) + self.assertFalse(trial.is_infra) + self.assertEqual(ran.n_infra, 0) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY], gated=False) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + self.assertIn(WORDS, said["exception_message"]) + + def test_an_episode_the_driver_stopped_and_scored_keeps_its_score(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + said, ran = self.ran([AGENT_OK, OVERFLOW, OVERFLOW, OVERFLOW]) + self.assertIsNone(said) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + said, _ = self.ran([AGENT_OK, OVERFLOW, SIDE_EMPTY, {"status": 429, "error": "slow down"}]) + self.assertEqual(said["exception_type"], "RateLimitError") + + def test_an_episode_a_guard_stopped_and_the_driver_scored_keeps_its_score_after_a_429(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace('"stopped": "context window"', '"stopped": "batch cap"')) + record = os.path.join(self.tmp, "record") + os.makedirs(record) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("batch cap\n") + limited = {"status": 429, "error": "slow down"} + for calls in ([AGENT_OK, limited], [AGENT_OK, limited, {"status": 502, "error": "down", "side": True}]): + said, ran = self.ran(calls, record=record) + self.assertIsNone(said, calls) + self.assertEqual((ran.trials[0].outcome, ran.trials[0].reward), (classify.FAILED_VERIFICATION, 0.5)) + with open(os.path.join(ran.trials[0].dir, "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + said, _ = self.ran([AGENT_OK, limited]) + self.assertEqual(said["exception_type"], "RateLimitError") + for own in (AGENT_OK, {"status": 401, "error": "no"}, {"status": 502, "error": "down"}): + said, _ = self.ran([own, dict(limited, side=True)], record=record) + self.assertEqual(said["exception_type"], "RateLimitError", own) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("context window\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED) + with open(os.path.join(record, relay.STOP), "a") as f: + f.write("batch cap\n") + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(STOPPED.replace(', "stopped": "context window"', "")) + self.assertEqual(self.ran([AGENT_OK, limited], record=record)[0]["exception_type"], "RateLimitError") + + def test_an_episode_the_overflow_ended_whose_evaluation_failed_is_the_checks(self): + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD) + said, ran = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "VerifierError") + self.assertIn("the evaluation did not read the VM", said["exception_message"]) + with open(os.path.join(self.b["_here"], "driver.py"), "w") as f: + f.write(UNREAD.replace("raise VerifierError", "raise RuntimeError")) + said, _ = self.ran([AGENT_OK, OVERFLOW]) + self.assertEqual(said["exception_type"], "ContextWindowExceededError") + + def test_a_unit_run_again_under_its_jobs_name_starts_with_no_earlier_record_and_keeps_a_new_stop(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + record = os.path.join(jobs, "again", "a__u0000", ".work", "relay") + os.makedirs(record) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "verifier")) + os.makedirs(os.path.join(jobs, "again", "a__u0000", "agent")) + for name, text in (("u0000.jsonl", json.dumps({"status": 429, "error": "slow down"}) + "\n"), (relay.STOP, "batch cap\n"), + ("../../verifier/score.json", "{}"), ("../../result.json", "{}"), ("../../agent/old.txt", "")): + with open(os.path.join(record, name), "w") as f: + f.write(text) + seen = [] + + def phase(r, name, unit, phase, timeout=None): + if phase == "env_setup": + os.makedirs(record, exist_ok=True) + with open(os.path.join(record, relay.STOP), "w") as f: + f.write("key cap\n") + + def unit_relay(r, tag, trial, *a, **k): + with open(os.path.join(trial, ".work", "relay", relay.STOP)) as f: + seen.append((sorted(os.listdir(os.path.join(trial, ".work", "relay"))), f.read())) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "phase", phase), patch.object(run.Run, "unit_relay", unit_relay): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + self.assertEqual(seen, [([relay.STOP], "key cap\n")]) + (earlier,) = [d for d in os.listdir(os.path.dirname(record)) if d.startswith("earlier.")] + earlier = os.path.join(os.path.dirname(record), earlier) + with open(os.path.join(earlier, "relay", relay.STOP)) as f: + self.assertEqual(f.read(), "batch cap\n") + for part in (("verifier", "score.json"), ("result.json",), ("agent", "old.txt")): + self.assertTrue(os.path.isfile(os.path.join(earlier, *part))) + self.assertFalse(os.path.exists(os.path.join(jobs, "again", "a__u0000", "verifier", "score.json"))) + + def test_a_run_under_a_job_name_another_run_holds_is_refused_before_it_writes(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + jobs = os.path.join(self.tmp, "jobs") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + job_dir = os.path.join(jobs, "again") + lck = os.path.join(job_dir, ".work", "job.lck") + tried = subprocess.run([sys.executable, "-c", "import fcntl, sys\nfcntl.flock(open(sys.argv[1], 'a'), fcntl.LOCK_EX | fcntl.LOCK_NB)", lck], + capture_output=True) + self.assertNotEqual(tried.returncode, 0) + run.HELD.pop(lck).close() + before = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + holder = subprocess.Popen([sys.executable, "-c", "import fcntl, sys, time\nf = open(sys.argv[1], 'a')\n" + "fcntl.flock(f, fcntl.LOCK_EX)\nprint(flush=True)\ntime.sleep(30)", + lck], stdout=subprocess.PIPE) + self.addCleanup(holder.stdout.close) + self.addCleanup(holder.kill) + holder.stdout.readline() + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls([AGENT_OK])), \ + self.assertRaisesRegex(run.Refused, "another run under the job name again is running"): + run.run(self.b, o, model="paper-1", provider="custom", jobs_root=jobs, name="again", say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + after = {os.path.join(d, f): open(os.path.join(d, f), "rb").read() for d, _, fs in os.walk(job_dir) for f in fs} | {d: None for d, _, _ in os.walk(job_dir)} + self.assertEqual(after, before) + + def test_a_second_run_in_the_same_process_under_a_live_job_name_is_refused(self): + o, _ = definitions.setup(self.b, "original", "paper-1", "custom") + go = lambda: run.run(self.b, o, model="paper-1", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + name="live", say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + tried = [] + + def relay(r, *a, **k): + if not tried: + tried.append(None) + for again in (go, r.start): + try: + tried.append(again()) + except run.Refused as e: + tried.append(e) + return Calls([AGENT_OK]) + + with patch.object(run.Run, "unit_relay", relay): + go() + self.assertEqual(len(tried), 3) + for refused in tried[1:]: + self.assertIsInstance(refused, run.Refused) + self.assertIn("another run under the job name live is running", str(refused)) + go() + + def test_a_side_call_failing_after_the_agents_success_stays_the_infrastructures(self): + said, ran = self.ran([AGENT_OK, SIDE_EMPTY]) + self.assertEqual(said["exception_type"], "ApiError") + self.assertEqual(ran.trials[0].outcome, classify.API_ERROR) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_controls.py b/app/tests/test_standard_controls.py index e1c6b84e..7dd4f93c 100644 --- a/app/tests/test_standard_controls.py +++ b/app/tests/test_standard_controls.py @@ -166,6 +166,107 @@ def test_a_standard_job_named_oracle_without_the_record_is_no_control(self): self.assertFalse(parse.parse_job(j.dir).is_control) # a runner's arm called oracle stays an arm +SLOW_SCORE = '''import os, time +from nullius.standard.jsonl import serve +CHECKED = os.path.exists(os.path.join(os.path.dirname(os.path.abspath(__file__)), "checked")) +def setup(unit, dir): + return dict({"prompt": "hi"}, **({"verify": {"run": "true"}} if CHECKED else {})) +def score(unit, dir, transcript, answer, verified=None): + time.sleep(2) + return {"reward": 0.0, "passed": False, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "score": score}) +''' + + +class Box: + image = "" + + def give_back(self, paths): + pass + + def stop(self): + pass + + +class ScoreLimit(unittest.TestCase): + """A null control whose driver does its work in ``score``, waited for ``limits.score_seconds``.""" + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "slow") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(SLOW_SCORE) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def problems(self, limits): + return definitions.problems(dict(self.b, limits=limits), "benchmark") + + def trial(self, limits): + b = dict(self.b, limits=limits) + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", + jobs_root=os.path.join(self.tmp, "jobs"), env={}, say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f) + + def verify_phase(self): + jobs = os.path.join(self.tmp, "jobs") + with open(os.path.join(jobs, os.listdir(jobs)[0], "events.jsonl")) as f: + return next(e for e in map(json.loads, f) if e["event"] == "trial.phase" and e["phase"] == "verify") + + def test_the_limit_is_a_whole_number_of_seconds_from_1(self): + self.assertEqual(self.problems({"score_seconds": 1800}), []) + for v in (0, -1, 1.5, "600", True, 100001): + self.assertIn("limits.score_seconds: a whole number from 1", self.problems({"score_seconds": v}), v) + + def test_run_again_is_true_or_false(self): + for v in (False, True, None): + self.assertEqual(self.problems({"run_again": v}), [], v) + for v in (0, "no"): + self.assertIn("limits.run_again: false when a rate-limited unit is not run once more", self.problems({"run_again": v}), v) + + def test_score_is_waited_for_600_seconds_or_the_benchmarks_own_limit(self): + r = run.Run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", env={}) + self.assertEqual(r.score_seconds, 600) + r = run.Run(dict(self.b, limits={"score_seconds": 1800}), definitions.control(self.b, "nop"), model="", + provider="openai", env={}) + self.assertEqual(r.score_seconds, 1800) + + def test_a_score_past_the_limit_is_the_verifiers_error_and_one_within_it_scores(self): + late = self.trial({"score_seconds": 1}) + self.assertEqual(late["exception_info"]["exception_type"], "VerifierError") + self.assertIn("score: no reply in 1 s", late["exception_info"]["exception_message"]) + self.assertIsNone(late.get("verifier_result")) + on_time = self.trial({"score_seconds": 10}) + self.assertIsNone(on_time.get("exception_info")) + self.assertEqual(on_time["verifier_result"]["rewards"]["reward"], 0.0) + + def test_the_verify_phase_carries_the_score_limit_and_an_own_images_check(self): + self.trial({"score_seconds": 10}) + self.assertEqual(self.verify_phase()["timeout_sec"], 10) + shutil.rmtree(os.path.join(self.tmp, "jobs")) + open(os.path.join(self.b["_here"], "checked"), "w").close() + checked = [] + with patch.object(run.Run, "unit_environment", lambda r, setup: ("task", {"image": "own"})), \ + patch.object(run.Run, "not_run", lambda r, e, d: None), \ + patch.object(run.Run, "unit_relay", lambda r, *a, **k: None), \ + patch.object(run.Run, "own_box", lambda r, *a: Box()), \ + patch.object(run.Run, "box_changes", lambda r, box: ""), \ + patch.object(run.Run, "release_image", lambda r, image: None), \ + patch.object(run.Run, "verify", lambda r, step, box, trial: checked.append(step) or {"exit": 0}): + got = self.trial({"score_seconds": 10}) + self.assertIsNone(got.get("exception_info")) + self.assertEqual(checked, [{"run": "true"}]) + self.assertEqual(self.verify_phase()["timeout_sec"], 10 + run.TURN_SECONDS) + + class Prove(unittest.TestCase): def setUp(self): self.tmp = tempfile.mkdtemp() @@ -281,6 +382,254 @@ def test_units_that_may_not_be_redistributed_are_counted_never_named(self): self.assertIn("held-unit-2", b["proven"]["leaky"]["failed"]["leak"]["problems"][0]) +class ProveOptions(unittest.TestCase): + """The benchmark's options reach the checks' driver and each control's run; a merged control job is held as one.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "kits", "agent-task-v1") + shutil.copytree(os.path.join(definitions.KITS, "agent-task-v1"), here) + with open(os.path.join(here, "driver.py")) as f: + src = f.read() + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(src.replace('return {"units": list(WORDS)}', + 'import os\n return {"units": list(WORDS) if os.environ.get("NULLIUS_OPTION_SITES") == "on" else []}')) + with open(os.path.join(here, "standard.json")) as f: + d = json.load(f) + d["options"] = {"sites": {"summary": "The sites.", "values": ["off", "on"], "default": "off"}, + "lane": {"summary": "A lane.", "values": ["a", "b"], "default": "a"}, + "tool": {"summary": "A tool.", "values": ["none", "x"], "default": "none", + "needs": {"x": {"host": ["tool"]}}}} + d["host"] = {"tool": {"summary": "A program this machine lacks.", "check": ["nullius-no-such-program-k5"]}} + with open(os.path.join(here, "standard.json"), "w") as f: + json.dump(d, f) + self.b, bad = definitions._read(os.path.join(here, "standard.json"), "benchmark", "agent-task-v1") + assert not bad, bad + self.jobs = os.path.join(self.tmp, "jobs") + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def prove(self, **kw): + kw.setdefault("stamp", "2026-01-02__03-04-05") + return prove.prove(self.b, jobs_root=self.jobs, env={}, say=lambda s: None, **kw) + + def config(self, job_dir): + with open(os.path.join(job_dir, "config.json")) as f: + return json.load(f)["standard"] + + def nop_shard(self, i, n, lane): + return run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=self.jobs, env={}, + say=lambda s: None, shard=(i, n), launch="L", bench_options={"sites": "on", "lane": lane}) + + def test_the_options_reach_the_units_driver_and_each_controls_run(self): + got = self.prove(bench_options={"sites": "on"}, write=False)["kit"] + self.assertTrue(all(r["ok"] for r in got.values()), got) + self.assertEqual((got["units"]["units"], got["nop"]["passed"], got["oracle"]["passed"]), (3, "0/3", "3/3")) + self.assertEqual(self.config(os.path.join(self.jobs, got["nop"]["job"]))["benchmark_options"], + {"sites": "on", "lane": "a", "tool": "none"}) + self.assertEqual(got["nop"]["options"], {"sites": "on", "lane": "a", "tool": "none"}) + dflt = self.prove(write=False, stamp="2026-01-02__03-04-06")["kit"] + self.assertEqual(dflt["units"]["problems"], ["units: no ids"]) + self.assertEqual(dflt["nop"]["problems"], ["the units check failed: the control was not run"]) + + def test_a_name_or_a_value_the_benchmark_does_not_allow_is_refused_before_anything_runs(self): + was = json.dumps(self.b.get("proven")) + for given, why in (({"sites": "maybe"}, "'maybe' is not one of off, on"), ({"nope": "x"}, "has no option 'nope'")): + with self.assertRaises(run.Refused) as e: + self.prove(bench_options=given) + self.assertIn(why, str(e.exception)) + self.assertFalse(os.path.exists(self.jobs)) + self.assertEqual(json.dumps(self.b.get("proven")), was) + + def test_an_option_whose_value_needs_a_program_this_machine_lacks_is_refused_before_any_install(self): + with patch.object(install, "upstream", side_effect=AssertionError("installed")) as up: + with self.assertRaises(run.Refused) as e: + self.prove(bench_options={"sites": "on", "tool": "x"}) + self.assertIn("nullius-no-such-program-k5 not on PATH", str(e.exception)) + up.assert_not_called() + self.assertFalse(os.path.exists(self.jobs)) + + def test_a_job_is_not_recorded_when_the_units_check_fails(self): + from nullius.standard import merge + whole = merge.merge([self.nop_shard(i, 4, "a") for i in (1, 3)], os.path.join(self.jobs, "nop-merged"))["out"] + self.b["datasets"][0]["units"] = 4 + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertFalse(got["units"]["ok"]) + self.assertFalse(got["nop"]["ok"]) + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", got["nop"]["problems"]) + kept = self.b["proven"]["kit"] + self.assertNotIn("nop", kept) + self.assertNotIn("units", kept) + self.assertEqual(kept["failed"]["nop"]["job"], "nop-merged") + self.assertIn("the units check failed: a job's cover is held to the driver's ids, which did not pass", + kept["failed"]["nop"]["problems"]) + + def test_a_control_is_not_run_nor_kept_when_the_units_check_fails(self): + self.assertIn("nop", self.b["proven"]["kit"]) + self.b["datasets"][0]["units"] = 4 + got = self.prove(bench_options={"sites": "on"})["kit"] + self.assertFalse(got["units"]["ok"]) + for c in ("nop", "oracle"): + self.assertEqual(got[c]["problems"], ["the units check failed: the control was not run"]) + self.assertNotIn(c, self.b["proven"]["kit"]) + self.assertIn(c, self.b["proven"]["kit"]["failed"]) + self.assertFalse(os.path.exists(self.jobs) and any("prove-" in x for x in os.listdir(self.jobs))) + + def test_a_trial_of_another_job_cannot_be_merged_into_a_proof(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, "a") for i in (1, 3)] + trial = next(t for t in sorted(os.listdir(shards[0])) if os.path.isfile(os.path.join(shards[0], t, "result.json"))) + path = os.path.join(shards[0], trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + with self.assertRaises(merge.Refused) as e: + merge.merge(shards, os.path.join(self.jobs, "never")) + self.assertIn("names another job than its job", str(e.exception)) + forced = merge.merge(shards, os.path.join(self.jobs, "forced"), force=True)["out"] + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": forced}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_a_merge_of_merges_keeps_what_any_of_them_was_merged_despite(self): + from nullius.standard import merge + good_shard, bad_shard = (self.nop_shard(i, 4, "a") for i in (1, 3)) + trial = next(t for t in sorted(os.listdir(bad_shard)) if os.path.isfile(os.path.join(bad_shard, t, "result.json"))) + path = os.path.join(bad_shard, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["job"] = "old-unrelated-job" + with open(path, "w") as f: + json.dump(res, f) + good = merge.merge([good_shard], os.path.join(self.jobs, "good"))["out"] + bad = merge.merge([bad_shard], os.path.join(self.jobs, "bad"), force=True)["out"] + with self.assertRaises(merge.Refused): + merge.merge([good, bad], os.path.join(self.jobs, "never")) + both = merge.merge([good, bad], os.path.join(self.jobs, "both"), force=True)["out"] + std = self.config(both) + self.assertTrue(any(w.startswith("bad: ") and "not of the job they are in" in w for w in std["merged_despite"])) + self.assertEqual([m.get("merged_from", [{}])[0].get("job") for m in std["merged_from"]], [os.path.basename(good_shard), os.path.basename(bad_shard)]) + r = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": both}, write=False)["kit"]["nop"] + self.assertFalse(r["ok"]) + self.assertTrue(any("merged despite differences" in p and "not of the job they are in" in p for p in r["problems"]), + r["problems"]) + + def test_the_cli_hands_its_options_and_jobs_to_the_proof(self): + from nullius import progress + from nullius.standard import cli + seen = {} + with patch.object(cli, "_bench", return_value=self.b), patch.object(cli, "licences", return_value=(None, None)), \ + patch.object(cli, "_exit_on_term"), patch.object(progress, "ECHO", None), \ + patch.object(prove, "prove", side_effect=lambda b, **kw: seen.update(kw) or {}): + self.assertEqual(cli.main(["prove", "agent-task-v1", "--from-run", "--dataset", "kit", "--bench-option", "sites=on", + "--bench-option", "lane=b", "--job", "nop=/x/merged"]), 0) + self.assertEqual((seen["bench_options"], seen["jobs"]), ({"sites": "on", "lane": "b"}, {"nop": "/x/merged"})) + with self.assertRaises(SystemExit): + cli.main(["prove", "agent-task-v1", "--from-run", "--job", "reference=/x"]) + + def test_a_merged_launch_is_held_to_the_bound_as_one_job_and_must_cover_every_unit(self): + from nullius.standard import merge + shards = [self.nop_shard(i, 4, lane) for i, lane in ((1, "a"), (3, "b"))] + whole = merge.merge(shards, os.path.join(self.jobs, "nop-merged"))["out"] + before = sorted(os.listdir(self.jobs)) + got = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole})["kit"] + self.assertTrue(got["nop"]["ok"], got["nop"]) + self.assertEqual((got["nop"]["job"], got["nop"]["units"], got["nop"]["passed"]), ("nop-merged", 3, "0/3")) + self.assertEqual(self.b["proven"]["kit"]["nop"]["job"], "nop-merged") + self.assertEqual(self.b["proven"]["kit"]["nop"]["options"], + [{"lane": "a", "sites": "on", "tool": "none"}, {"lane": "b", "sites": "on", "tool": "none"}]) + self.assertEqual(sorted(set(os.listdir(self.jobs)) - set(before)), ["2026-01-02__03-04-05__agent-task-v1__kit__prove-oracle"]) + self.assertEqual([m["benchmark_options"]["lane"] for m in self.config(whole)["merged_from"]], ["a", "b"]) + half = merge.merge(shards[:1], os.path.join(self.jobs, "nop-half"))["out"] + part = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": half}, write=False, stamp="2026-01-02__03-04-06")["kit"]["nop"] + self.assertFalse(part["ok"]) + self.assertTrue(any("units have no trial" in p for p in part["problems"]), part["problems"]) + wrong = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"oracle": whole}, write=False, stamp="2026-01-02__03-04-07")["kit"]["oracle"] + self.assertIn("nop-merged: its control is 'nop', not 'oracle'", wrong["problems"]) + trial = next(t for t in sorted(os.listdir(whole)) if os.path.isfile(os.path.join(whole, t, "result.json"))) + with open(os.path.join(whole, trial, "result.json")) as f: + res = json.load(f) + res.pop("verifier_result", None) + res.pop("exception_info", None) + with open(os.path.join(whole, trial, "result.json"), "w") as f: + json.dump(res, f) + blank = self.prove(dataset="kit", bench_options={"sites": "on"}, jobs={"nop": whole}, write=False, + stamp="2026-01-02__03-04-08")["kit"]["nop"] + self.assertFalse(blank["ok"]) + self.assertIn("1 unit(s) ended with no reward from 0 to 1 and no error: read the job's trials", blank["problems"]) + for kw in ({"limit": 1}, {"shard": (1, 2)}): + with self.assertRaises(run.Refused): + self.prove(dataset="kit", jobs={"nop": whole}, **kw) + with self.assertRaises(run.Refused): + self.prove(jobs={"nop": os.path.join(self.tmp, "absent")}) + + +NUMERIC = {"key": "numeric", "_ref": "numeric", "label": "Numeric", "family": "agent", "needs": ["agent.task/v1"], + "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "score", "says": "a number from 0 to 1"}, + "controls": {"nop": {"source": "nullius", "from": "Nothing runs."}}, "datasets": [{"id": "all", "units": 3}]} + + +class HeldJob(unittest.TestCase): + """A control's job held to its bound, read from each trial's own record.""" + + def setUp(self): + from nullius.standard import job + self.tmp = tempfile.mkdtemp(); self.addCleanup(shutil.rmtree, self.tmp, True) + self.dir = os.path.join(self.tmp, "nop-job") + os.makedirs(self.dir) + nop = definitions.control(NUMERIC, "nop") + self.units = ["u1", "u2", "u3"] + job.write_config(self.dir, job="nop-job", bench=NUMERIC, other=nop, model="", provider="openai", dataset="all", + units=self.units, concurrency=1, granted=False) + results = [job.write_trial(os.path.join(self.dir, f"{u}__u{i:04d}"), job="nop-job", bench=NUMERIC, other=nop, model="", + unit=u, t0=1_760_000_000.0, t1=1_760_000_001.0, produce=(None, None), score={"reward": 0.0}, + exception=None, calls=[], trajectory=None) for i, u in enumerate(self.units)] + job.write_result(self.dir, "nop-job", 1_760_000_000.0, 1_760_000_002.0, results, 3) + + def held(self): + return prove.control_check(NUMERIC, NUMERIC["datasets"][0], "nop", limit=None, jobs_root=self.tmp, stamp="s", + provider="openai", model="", judge_model=None, env={}, concurrency=1, + say=lambda s: None, job_dir=self.dir, ids=self.units) + + def edit(self, unit, change): + trial = next(t for t in os.listdir(self.dir) if t.startswith(unit + "__")) + path = os.path.join(self.dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + change(res) + with open(path, "w") as f: + json.dump(res, f) + + def test_a_clean_job_passes(self): + r = self.held() + self.assertTrue(r["ok"], r["problems"]) + self.assertEqual((r["passed"], r["value"], r["units"]), ("0/3", 0.0, 3)) + + def test_a_reward_without_a_score_a_score_below_0_and_nan_are_no_score(self): + for value in ({"reward": 1.0}, {"score": -100}, {"score": float("nan")}, {"score": True}, {"score": "0"}): + self.edit("u2", lambda res: res["verifier_result"].update(rewards=value)) + r = self.held() + self.assertFalse(r["ok"], value) + self.assertIn("1 unit(s) ended with no score from 0 to 1 and no error: read the job's trials", r["problems"], value) + self.assertEqual(r["value"], 0.0) + + def test_a_trial_another_run_wrote_is_refused(self): + self.edit("u1", lambda res: res["standard"].update(benchmark="other", control={"name": "oracle"}, plugged="oracle")) + self.edit("u3", lambda res: res["standard"].update(job="another-job")) + r = self.held() + self.assertFalse(r["ok"]) + said = " | ".join(r["problems"]) + for k in ("benchmark", "control", "who ran it", "job"): + self.assertIn(f"whose own record names another {k} than the job", said) + + LEAKY = '''from nullius.standard.jsonl import serve import os ANSWER = "the reference answer is forty-two" diff --git a/app/tests/test_standard_gated.py b/app/tests/test_standard_gated.py new file mode 100644 index 00000000..17506a10 --- /dev/null +++ b/app/tests/test_standard_gated.py @@ -0,0 +1,134 @@ +"""A gated benchmark's failing endpoint is said by its status alone, down to the trial's result.json.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius.standard import definitions, install, relay, run + +FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +CONTROLS = {"nop": {"source": "nullius", "from": "Nothing runs; the answer is empty."}} +WORDS = "the gated task's own sentence" +FAILING = '''from nullius.standard.jsonl import serve +def score(unit, dir, transcript, answer, **_): + raise RuntimeError("the LLM grader's endpoint answered 402, and the task was scored without it") +serve({"units": lambda dataset: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, "score": score}) +''' + + +ANSWERING = '''from nullius.standard.jsonl import serve +def answer(unit, dir, setup, model, base_url, key, **_): + raise RuntimeError("the upstream's grader said: status 402: " + %r) +serve({"units": lambda dataset, setup=None: {"units": ["a"]}, "setup": lambda unit, dir: {"prompt": "hi"}, + "answer": answer, "score": lambda unit, dir, transcript, answer, **_: {"reward": 0.0, "detail": {}}}) +''' % WORDS +SETUPS = {"original": {"summary": "The paper's own pipeline.", "models": ["paper-1"], "default": "paper-1", + "providers": {"custom": ""}}} + + +class Calls: + """The unit's relay as Run.unit reads it: each route's calls, the judge's ending on a refusal with words.""" + def __init__(self, side): + self.side, self.dir = side, os.devnull + + def calls(self, tag): + if tag.endswith("-" + self.side): + return [{"status": 200}, {"status": 402, "error": f"status 402: {WORDS}"}] + return [] + + def url(self, tag, model=None): + return "http://127.0.0.1:9/" + tag + "/v1" + + def stop(self): + pass + + +class Gated(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + here = os.path.join(self.tmp, "failing") + os.makedirs(here) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(FAILING) + self.b = dict(definitions.get("benchmark", "fakebench", FIX), _here=here, controls=CONTROLS, + driver={"command": ["python3", "{here}/driver.py"]}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def saved(self, side, **bench): + b = dict(self.b, **bench) + jobs = os.path.join(self.tmp, "jobs-" + side + str(bench)) + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, definitions.control(b, "nop"), model="", provider="openai", jobs_root=jobs, env={}, + say=lambda s: None) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_refused_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.saved(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_a_benchmark_not_gated_keeps_the_providers_words(self): + said = self.saved("judge") + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn(WORDS, said["exception_message"]) + + def answered(self, side, gated): + """drive_upstream's answer failing on its judge's or simulated user's refusal.""" + here = os.path.join(self.tmp, "paper") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(ANSWERING) + b = dict(self.b, _here=here, setups=SETUPS, **({"gated": True} if gated else {})) + o, why = definitions.setup(b, "original", "paper-1", "custom") + self.assertIsNotNone(o, why) + jobs = os.path.join(self.tmp, f"answer-{side}-{gated}") + with patch.object(run.Run, "unit_relay", lambda r, *a, **k: Calls(side)): + job_dir = run.run(b, o, model="paper-1", provider="custom", jobs_root=jobs, say=lambda s: None, + env={"MODEL_API_URL": "http://127.0.0.1:9/v1", "MODEL_API_KEY": "k"}) + name = next(t for t in os.listdir(job_dir) if "__u" in t) + with open(os.path.join(job_dir, name, "result.json")) as f: + return json.load(f)["exception_info"] + + def test_a_gated_benchmarks_answer_failing_on_its_judge_or_simulator_is_said_by_status_alone(self): + for side in ("judge", "simulator"): + with self.subTest(side): + said = self.answered(side, gated=True) + self.assertEqual(said["exception_type"], "ApiError") + self.assertIn("answered 402", said["exception_message"]) + self.assertNotIn(WORDS, said["exception_message"]) + + def test_an_answer_failing_on_a_benchmark_not_gated_keeps_its_words(self): + said = self.answered("judge", gated=False) + self.assertEqual(said["exception_type"], "NonZeroAgentExitCodeError") + self.assertIn(WORDS, said["exception_message"]) + + def test_the_failure_alone(self): + calls = [{"status": 429, "error": WORDS}] + self.assertEqual(relay.failure(calls), ("RateLimitError", WORDS)) + exc, said = relay.failure(calls, words=False) + self.assertEqual(exc, "RateLimitError") + self.assertNotIn(WORDS, said) + self.assertEqual(relay.failure([{"status": 404, "error": "no such model"}], words=False)[0], "ModelNotFoundError") + + def test_gated_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(self.b, gated=True), "benchmark"), []) + self.assertIn("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept", + definitions.problems(dict(self.b, gated="yes"), "benchmark")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_parts.py b/app/tests/test_standard_parts.py index ea875b87..318af120 100644 --- a/app/tests/test_standard_parts.py +++ b/app/tests/test_standard_parts.py @@ -35,6 +35,15 @@ def trial(self, name, f1s): exception=None, calls=[], trajectory=None) return d + def test_a_stopped_score_says_so_in_the_result(self): + for name, score in (("stopped", {"reward": 0.5, "stopped": "batch cap"}), ("whole", {"reward": 0.5})): + job.write_trial(os.path.join(self.job, name), job="job", bench=BENCH, other={"key": "original", "_kind": "upstream"}, + model="m", unit=name, t0=0, t1=1, produce=(0, 1), score=score, exception=None, calls=[], trajectory=None) + with open(os.path.join(self.job, "stopped", "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["stopped"], "batch cap") + with open(os.path.join(self.job, "whole", "result.json")) as f: + self.assertNotIn("stopped", json.load(f)["verifier_result"]) + def test_the_mean_is_over_every_question(self): self.trial("conv-a", [1.0] * 4) # 4 questions, all right self.trial("conv-b", [0.0] * 12) # 12 questions, all wrong diff --git a/app/tests/test_standard_relay.py b/app/tests/test_standard_relay.py index 1bd8ad9a..d8def297 100644 --- a/app/tests/test_standard_relay.py +++ b/app/tests/test_standard_relay.py @@ -3,6 +3,7 @@ A driver or capability process answers one JSON line per request.""" from __future__ import annotations +import base64 import http.client import json import os @@ -34,6 +35,22 @@ b'{"type":"message","content":[{"type":"output_text","text":"hi"}]}],' b'"usage":{"input_tokens":20,"output_tokens":5,"input_tokens_details":{"cached_tokens":10}}}}\n\n') +RESP_FAILED = json.dumps({"object": "response", "id": "gen-1", "status": "failed", "output": [], "usage": None, + "error": {"code": "server_error", "message": "the request said x"}}).encode() +RESP_FAILED_SSE = (b'event: response.failed\ndata: {"type":"response.failed","response":{"status":"failed","output":[],' + b'"error":{"code":"server_error","message":"the request said x"}}}\n\n') +RESP_INCOMPLETE = json.dumps({"object": "response", "status": "incomplete", "output": [], + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 30, "output_tokens": 8}}).encode() + +HOLLOW = {"hollow": json.dumps({"id": "gen-1", "provider": "Minimax", "model": "m"}).encode(), + "hollow-error": json.dumps({"id": "gen-1", "error": {"code": 502, "message": "the request said x"}}).encode(), + "hollow-429": json.dumps({"id": "gen-1", "error": {"code": 429, "message": "the request said x"}}).encode(), + "hollow-402": json.dumps({"id": "gen-1", "error": {"code": 402, "message": "the request said x"}}).encode(), + "hollow-404": json.dumps({"id": "gen-1", "error": {"code": 404, "message": "the request said x"}}).encode(), + "count": json.dumps({"input_tokens": 12}).encode(), "no-choice": json.dumps({"choices": []}).encode(), + "not-json": b"the request said x"} + class Upstream(BaseHTTPRequestHandler): seen = [] @@ -51,6 +68,23 @@ def do_POST(self): self.reply(401, b'{"error":{"message":"Incorrect API key"}}', "application/json") elif mode == "no-usage": self.reply(200, b'{"choices":[{"message":{"role":"assistant","content":"hello"}}]}', "application/json") + elif mode in ("400", "413", "422"): + self.reply(int(mode), b'{"error":{"message":"could not process image"}}', "application/json") + elif mode in ("cut", "short"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header(*(("Transfer-Encoding", "chunked") if mode == "cut" else ("Content-Length", str(len(CHAT_SSE) + 9)))) + self.end_headers() + self.wfile.write(b"%x\r\n%s\r\n" % (len(CHAT_SSE), CHAT_SSE) if mode == "cut" else CHAT_SSE) + elif mode in HOLLOW: + self.reply(200, HOLLOW[mode], "application/json", *[("X-Generation-Id", "gen-1")] * (mode == "not-json"), + ("Retry-After", "30")) + elif self.path.endswith("/responses") and mode == "failed": + self.reply(200, RESP_FAILED, "application/json") + elif self.path.endswith("/responses") and mode == "failed-sse": + self.reply(200, RESP_FAILED_SSE, "text/event-stream") + elif self.path.endswith("/responses") and mode == "incomplete": + self.reply(200, RESP_INCOMPLETE, "application/json") elif self.path.endswith("/responses"): self.reply(200, RESP_SSE, "text/event-stream") elif body.get("stream"): @@ -58,10 +92,11 @@ def do_POST(self): else: self.reply(200, json.dumps(CHAT).encode(), "application/json") - def reply(self, status, body, kind): + def reply(self, status, body, kind, *headers): try: self.send_response(status) - self.send_header("Content-Type", kind) + for header in (("Content-Type", kind),) + headers: + self.send_header(*header) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) @@ -90,6 +125,7 @@ def post(self, tag, path, body): with urllib.request.urlopen(req) as r: return r.status, r.read() except urllib.error.HTTPError as e: + self.retry_after = e.headers.get("Retry-After") return e.code, e.read() def test_plain_reply_passes_through_and_the_real_key_goes_out(self): @@ -120,6 +156,68 @@ def test_streamed_chat_is_passed_byte_for_byte_and_read_whole(self): self.assertEqual(rec["reply"]["text"], "hello") self.assertEqual(rec["reply"]["usage"]["input"], 7) + def test_a_stream_cut_off_after_its_headers_is_a_failed_call(self): + for mode in ("cut", "short"): + self.post("u" + mode, "/chat/completions", {"model": "m", "stream": True, "mode": mode, "messages": []}) + rec = self.relay.calls("u" + mode)[-1] + self.assertEqual(rec["status"], 502) + self.assertIn("cut off", rec["error"]) + self.assertEqual(rec["reply"]["usage"]["input"], 7) + self.assertEqual(job._usage_total([rec])["cost"], rec["cost_usd"]) + + def test_a_refused_call_keeps_each_inline_image_by_its_type_wholeness_and_size_alone(self): + png = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20 + b"IEND\xaeB`\x82\n").decode() + cut = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * 20).decode() + jpeg = base64.b64encode(b"\xff\xd8\xff" + b"y" * 9 + b"\xff\xd9tail").decode() + webp = base64.b64encode(b"RIFF" + (12).to_bytes(4, "little") + b"WEBPVP8 " + b"z" * 4 + b"\n").decode() + image = lambda kind, data: {"type": "image", "source": {"type": "base64", "media_type": kind, "data": data}} + content = [image("image/png", png), image("image/png", cut), image("image/gif", "not base64!"), + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg}"}}, + {"type": "input_image", "image_url": f"data:image/webp;base64,{webp}"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "grade"}] + sent = [{"type": "image/png", "is": "image/png", "bytes": len(png)}, {"type": "image/png", "is": None, "bytes": len(cut)}, + {"type": "image/gif", "is": None, "bytes": 11}, {"type": "image/jpeg", "is": "image/jpeg", "bytes": len(jpeg)}, + {"type": "image/webp", "is": "image/webp", "bytes": len(webp)}] + for mode in ("400", "413", "422", "429", None): + self.post("img", "/messages", {"model": "m", "mode": mode, "messages": [{"role": "user", "content": content}]}) + rec = self.relay.calls("img")[-1] + with self.subTest(mode): + self.assertEqual(rec.get("images"), sent if mode in ("400", "413", "422") else None) + self.assertFalse(any(data in json.dumps(rec) for data in (png, cut, jpeg, webp))) + + def test_a_200_to_a_model_call_with_no_reply_is_a_502_said_by_its_kind_alone(self): + for mode, path, code, said, kind in ( + ("hollow", "/messages", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/chat/completions", 502, "no reply", "NetworkConnectionError"), + ("hollow", "/responses", 502, "no reply", "NetworkConnectionError"), + ("hollow-error", "/messages", 502, "an error 502", "NetworkConnectionError"), + ("hollow-429", "/messages", 429, "an error 429", "RateLimitError"), + ("hollow-404", "/completions", 502, "an error 404", "NetworkConnectionError"), + ("hollow-402", "/messages", 402, "an error 402", "ApiError"), + ("not-json", "/messages", 502, "no reply", "NetworkConnectionError")): + tag = f"uh-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + rec = self.relay.calls(tag)[-1] + self.assertEqual(self.retry_after, "30") + with self.subTest(mode=mode, path=path): + self.assertEqual((status, rec["status"]), (code, code)) + self.assertEqual(json.loads(body)["error"]["message"], f"the endpoint answered 200 with {said}") + self.assertEqual(rec["error"], f"HTTP {code}: the endpoint answered 200 with {said}") + self.assertEqual(rec["answered"], 200) + self.assertFalse({"reply", "cost_usd"} & set(rec)) + self.assertNotIn("the request said", json.dumps(rec)) + self.assertEqual(relay.failure([rec], agent=True)[0], kind) + self.assertEqual(self.post("uk-asked-stream", "/messages", {"model": "m", "mode": "hollow", "stream": True, + "messages": []}), (200, HOLLOW["hollow"])) + for mode, path in (("count", "/messages/count_tokens"), ("no-choice", "/chat/completions"), ("hollow", "/models"), + ("hollow", "/threads/t1/messages")): + tag = f"uk-{mode}{path.replace('/', '-')}" + status, body = self.post(tag, path, {"model": "m", "mode": mode, "messages": []}) + with self.subTest(path): + self.assertEqual((status, body), (200, HOLLOW[mode])) + self.assertEqual(self.relay.calls(tag)[-1]["status"], 200) + self.assertNotIn("answered", self.relay.calls(tag)[-1]) + def test_streamed_responses_read_from_the_completed_event(self): status, body = self.post("u3", "/responses", {"model": "m", "stream": True, "input": "x"}) self.assertEqual(body, RESP_SSE) @@ -128,6 +226,25 @@ def test_streamed_responses_read_from_the_completed_event(self): self.assertEqual(rec["reply"]["tool_calls"][0]["name"], "shell") self.assertEqual(rec["reply"]["usage"]["cached"], 10) + def test_a_failed_response_with_no_usage_costs_nothing(self): + for tag, mode in (("uf1", "failed"), ("uf2", "failed-sse")): + status, _ = self.post(tag, "/responses", {"model": "m", "mode": mode, "input": "x", "stream": mode == "failed-sse"}) + self.assertEqual(status, 200) + rec = self.relay.calls(tag)[-1] + self.assertEqual((rec["reply"]["end"], rec["reply"]["error"]["code"]), ("failed", "server_error")) + self.assertEqual((rec["cost_usd"], rec["cost_from"]), (0.0, "failed")) + failed = {"status": 200, "cost_usd": 0.0, "cost_from": "failed"} + self.assertEqual(job._usage_total([failed, {"status": 200}])["cost"], None) + self.assertEqual({k: job._usage_total([failed, {"status": 200, "cost_usd": 0.5, "cost_from": "provider"}])[k] + for k in ("cost", "cost_from")}, {"cost": 0.5, "cost_from": "provider"}) + + def test_an_incomplete_response_is_priced_from_its_usage(self): + self.post("uf3", "/responses", {"model": "m", "mode": "incomplete", "input": "x"}) + rec = self.relay.calls("uf3")[-1] + self.assertTrue(rec["reply"]["end"].startswith("incomplete")) + self.assertEqual(rec["cost_from"], "estimate") + self.assertGreater(rec["cost_usd"], 0) + def test_calls_are_recorded_by_their_unit(self): self.post("ua", "/chat/completions", {"model": "m", "messages": []}) self.post("ub", "/chat/completions", {"model": "m", "messages": []}) @@ -230,6 +347,54 @@ def test_a_call_that_fits_is_sent_and_its_booking_becomes_what_it_cost(self): self.assertAlmostEqual(got["spent_usd"], rec["cost_usd"]) self.assertEqual((got["held_usd"], got["unsettled_usd"], got["closed"]), (0, 0.0, False)) + def test_a_stream_cut_off_or_a_failed_response_with_no_usage_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + for tag, path, body in (("uc", "/chat/completions", {"model": "m", "stream": True, "mode": "cut", "messages": []}), + ("uf", "/responses", {"model": "m", "mode": "failed", "input": "x"})): + req = urllib.request.Request(r.url(tag) + path, data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,), (failed,) = r.calls("uc"), r.calls("uf") + self.assertEqual((cut["status"], failed["cost_from"]), (502, "failed")) + self.assertIsNotNone(cut.get("cost_usd")) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"]), (0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], cut["booked_usd"] + failed["booked_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_stream_cut_off_that_already_cost_more_than_its_booking_counts_what_it_cost(self): + r, led = self.capped(1.0) + body = {"model": "m", "stream": True, "mode": "cut", "messages": []} + req = urllib.request.Request(r.url("uc") + "/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + with mock.patch.object(ledger, "bound", lambda *a, **k: 1e-9): + try: + urllib.request.urlopen(req).read() + except (http.client.IncompleteRead, urllib.error.HTTPError): + pass + (cut,) = r.calls("uc") + got = led.state() + self.assertEqual((cut["status"], got["unsettled_usd"]), (502, 0.0)) + self.assertGreater(cut["cost_usd"], cut["booked_usd"]) + self.assertAlmostEqual(got["spent_usd"], cut["cost_usd"]) + self.assertNotIn("m", led._read()["seen"]) + + def test_a_unit_asked_to_stop_books_nothing_and_a_200_with_no_reply_keeps_its_whole_booking(self): + r, led = self.capped(1.0) + with open(os.path.join(r.dir, relay.STOP), "w") as f: + f.write("batch cap\n") + self.assertEqual(self.post(r, "u1", {"model": "m", "messages": []})[0], 400) + got = led.state() + self.assertEqual((got["spent_usd"], got["held_usd"], got["unsettled_usd"]), (0.0, 0, 0.0)) + os.unlink(os.path.join(r.dir, relay.STOP)) + self.assertEqual(self.post(r, "u1", {"model": "m", "mode": "hollow", "messages": []})[0], 502) + (call,) = r.calls("u1") + got = led.state() + self.assertEqual((call["answered"], got["spent_usd"], got["held_usd"]), (200, 0.0, 0)) + self.assertAlmostEqual(got["unsettled_usd"], call["booked_usd"]) + def test_a_call_is_booked_at_what_the_runs_replies_have_been_once_one_came_back(self): """Fifteen calls at the model's limit would fill a cap nothing was spent of: after the first reply (2 output tokens here) a call that names no limit is booked those, at the same price.""" @@ -451,6 +616,11 @@ def test_the_record_says_the_reasoning_effort_a_request_asked_for(self): self.assertEqual(relay._request_tail({"input": [], "reasoning": {"effort": "high"}})["reasoning_effort"], "high") self.assertIsNone(relay._request_tail({"messages": []})["reasoning_effort"]) + def test_reasoning_is_read_as_reasoning_or_thinking_tokens(self): + self.assertEqual(relay._usage({"completion_tokens_details": {"reasoning_tokens": 3}})["reasoning"], 3) + self.assertEqual(relay._usage({"output_tokens_details": {"reasoning_tokens": 5}})["reasoning"], 5) + self.assertEqual(relay._usage({"output_tokens": 9, "output_tokens_details": {"thinking_tokens": 7}})["reasoning"], 7) + class ServeTest(unittest.TestCase): """The relay as a process of its own: the key from its environment only, diff --git a/app/tests/test_standard_relay_pin.py b/app/tests/test_standard_relay_pin.py new file mode 100644 index 00000000..cfe525f2 --- /dev/null +++ b/app/tests/test_standard_relay_pin.py @@ -0,0 +1,263 @@ +"""A pinned route's failure on top of the serving pin (``serving``): a pinned call no endpoint took, or one its reply +says another provider served, fails the unit as its route's. Before a fake OpenRouter.""" +from __future__ import annotations + +import contextlib +import json +import os +import shutil +import tempfile +import threading +import unittest +from unittest import mock +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from nullius.runs import classify +from nullius.standard import definitions, install, relay, run + +M3, SONNET = "minimax/minimax-m3", "anthropic/claude-sonnet-4.6" +PINS = {M3: ["minimax"], SONNET: ["anthropic"]} +SERVES = {M3: ["Parasail", "MiniMax"], SONNET: ["Google", "Anthropic"]} +SECRET = "the words of a gated task" +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") + + +class OpenRouter(BaseHTTPRequestHandler): + """Routes as OpenRouter documents them: ``down`` a 503, no endpoint of the pin a 404.""" + ignore_pin = False + + def log_message(self, *a): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model, pref = body.get("model"), body.get("provider") or {} + allowed = [e for e in SERVES.get(model, ["Someone"]) + if OpenRouter.ignore_pin or not pref.get("order") or e.lower() in pref["order"]] + if body.get("down"): + allowed = [] + if not allowed: + status = 503 if body.get("down") else 404 + return self.reply(status, {"error": {"code": status, "message": f"No allowed providers are available for the selected model: {SECRET}"}}) + if self.path.endswith("/messages"): + return self.reply(200, {"id": "gen-555", "type": "message", "role": "assistant", "content": [{"type": "text", "text": SECRET}], + "usage": {"input_tokens": 3, "output_tokens": 1}}) + self.reply(200, {"id": "gen-123", "provider": allowed[0], "choices": [{"message": {"role": "assistant", "content": SECRET}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "cost": 0.0001}}) + + def reply(self, status, out): + body = json.dumps(out).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url, body): + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY}) + try: + with urllib.request.urlopen(req) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + try: + return e.code, e.read() + finally: + e.close() + + +class Pinned(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), OpenRouter) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/api/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.relay = self.pinned(PINS) + + def pinned(self, serving, model=M3): + r = relay.Relay(self.base, "sk-real", tempfile.mkdtemp(dir=self.dir), model=model, + side_models=["claude-sonnet-4-6=" + SONNET], serving=serving).start() + self.addCleanup(r.stop) + return r + + def test_no_endpoint_of_the_pin_is_the_routes_error_not_an_empty_answer(self): + r = self.pinned({M3: ["fireworks"]}) + status, _ = post(r.url("u3") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertEqual(status, 404) + kind, said = relay.failure(r.calls("u3"), serving={M3: ["fireworks"]}) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to fireworks", said) + self.assertEqual(classify.from_verdict(kind, None), classify.API_ERROR) + + def test_a_pinned_provider_down_is_the_routes_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + (call,) = self.relay.calls("u1") + self.assertEqual(call["status"], 503) + self.assertEqual(relay.failure(self.relay.calls("u1"), serving=PINS)[0], "ApiProviderRouteError") + + def test_a_gated_benchmark_keeps_the_endpoints_words_out_of_a_route_error(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + kind, said = relay.failure(self.relay.calls("u1"), words=False, serving=PINS) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("pinned to minimax", said) + self.assertIn("answered 503", said) + self.assertNotIn(SECRET, said) + self.assertNotIn("No allowed providers", said) + + def test_an_unpinned_404_is_still_what_it_was(self): + calls = [{"status": 404, "model": "other/model", "error": "HTTP 404: model not found"}] + self.assertEqual(relay.failure(calls, serving=PINS)[0], "ModelNotFoundError") + self.assertIsNone(relay.route_failure(calls, PINS)) + + def test_a_side_models_refusal_fails_the_unit_even_when_the_agent_answered_last(self): + unit = self.relay.url("u1") + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + calls = self.relay.calls("u1") + self.assertIsNone(relay.failure(calls, serving=PINS)) + kind, said = relay.route_failure(calls, PINS, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn(SONNET, said) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": []}) + self.assertIsNone(relay.route_failure(self.relay.calls("u1"), PINS)) + + def test_a_routes_calls_are_read_in_the_order_they_began(self): + pin, refused = {M3: ["fireworks"]}, {"model": M3, "status": 404} + self.assertIsNone(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, dict(refused, t0=1)], pin)) + self.assertEqual(relay.route_failure([dict(refused, t0=2), {"model": M3, "status": 200, "t0": 1}], pin)[0], + "ApiProviderRouteError") + self.assertEqual(relay.route_failure([{"model": M3, "status": 200, "t0": 2}, refused], pin)[0], "ApiProviderRouteError") + + def test_a_judges_answer_does_not_cover_the_simulated_users_refusal_of_the_same_model(self): + unit, judge = self.relay.url("u1"), self.relay.url("u1-judge", model=SONNET) + post(unit + "/messages", {"model": "claude-sonnet-4-6", "messages": [], "down": True}) + post(unit + "/messages", {"model": "MiniMax-M3", "messages": []}) + post(judge + "/messages", {"model": SONNET, "messages": []}) + routes = [self.relay.calls("u1"), self.relay.calls("u1-judge"), []] + self.assertEqual(relay.routes_failure(routes, PINS, words=False)[0], "ApiProviderRouteError") + self.assertIsNone(relay.routes_failure([self.relay.calls("u1-judge")], PINS)) + + def test_only_a_route_with_no_endpoint_stops_the_run(self): + post(self.relay.url("u1") + "/messages", {"model": "MiniMax-M3", "messages": [], "down": True}) + self.assertFalse(relay.route_dead(relay.failure(self.relay.calls("u1"), serving=PINS))) + r = self.pinned({M3: ["fireworks"]}) + post(r.url("u5") + "/messages", {"model": "MiniMax-M3", "messages": []}) + self.assertTrue(relay.route_dead(relay.failure(r.calls("u5"), words=False, serving={M3: ["fireworks"]}))) + self.assertFalse(relay.route_dead(("ApiError", "404: x"))) + + def test_an_answer_from_outside_the_pin_fails_the_unit_as_its_route(self): + pin = {"openai/gpt-5-mini": ["minimax"]} + r = self.pinned(pin, model="openai/gpt-5-mini") + SERVES["openai/gpt-5-mini"] = ["Parasail"] + self.addCleanup(SERVES.pop, "openai/gpt-5-mini") + OpenRouter.ignore_pin = True + self.addCleanup(setattr, OpenRouter, "ignore_pin", False) + post(r.url("u4") + "/chat/completions", {"model": "x", "messages": []}) + (call,) = r.calls("u4") + self.assertEqual((call["status"], call["reply"]["served_by"]), (200, "Parasail")) + self.assertIsNone(relay.failure(r.calls("u4"), serving=pin)) + kind, said = relay.route_failure(r.calls("u4"), pin, words=False) + self.assertEqual(kind, "ApiProviderRouteError") + self.assertIn("served by Parasail", said) + self.assertNotIn(SECRET, said) + self.assertIsNone(relay.route_failure(r.calls("u4"), {})) + + def test_a_provider_is_off_its_pin_only_when_its_name_shares_nothing_with_the_pin(self): + self.assertIs(relay.held("MiniMax", ["minimax"]), True) + self.assertIs(relay.held("Anthropic", ["anthropic"]), True) + self.assertIs(relay.held("Google AI Studio", ["google-ai-studio"]), True) + self.assertIs(relay.held("Parasail", ["minimax"]), False) + self.assertIsNone(relay.held("Google", ["google-vertex"])) + self.assertIsNone(relay.route_failure([{"status": 200, "model": "m", "reply": {"served_by": "Google"}}], {"m": ["google-vertex"]})) + + def test_a_name_that_is_not_a_providers_name_fails_nothing_and_is_never_said(self): + odd = [{"status": 200, "model": "m", "reply": {"served_by": "Parasail\n" + SECRET}}] + self.assertIsNone(relay.route_failure(odd, {"m": ["minimax"]}, words=False)) + + +class Definition(unittest.TestCase): + def test_a_benchmark_without_pins_runs_as_before(self): + benches, _ = definitions.load("benchmark") + for b in benches: + with self.subTest(b["key"]): + self.assertEqual(run.pins_of(b, {"_kind": "agent"}, "openrouter", "m"), {}) + for which in (b.get("setups") or {}): + if which == "original": + other, _ = definitions.setup(b, which, "", "openrouter") + self.assertNotIn("pins", other or {}) + + def test_the_definitions_pin_holds_whatever_served_by_says(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + make = lambda **kw: run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", + env={"OPENROUTER_API_KEY": "k"}, unsandboxed=True, jobs_root=tmp, say=lambda s: None, **kw) + self.assertEqual(make().serving, {SONNET: ["anthropic"]}) + r = make(served_by={"m-1": ["prov-a"], SONNET: ["google-vertex"]}) + self.assertEqual((r.serving, r.route_pins), ({"m-1": ["prov-a"], SONNET: ["anthropic"]}, {SONNET: ["anthropic"]})) + + +class Wired(unittest.TestCase): + """A unit of a run whose judge the definition pins and whose own model the user pins (``--served-by``).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(self.tmp, "home") + b = definitions.get("benchmark", "fakebench", ROOT) + b = dict(b, score=dict(b["score"], judge={"model": SONNET, "says": "a verdict", "pin": {"openrouter": ["anthropic"]}})) + self.r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "k"}, + unsandboxed=True, jobs_root=self.tmp, say=lambda s: None, served_by={"m-1": ["prov-a"]}) + + def ended(self, routes): + rr = mock.Mock(mcp_listeners=[], unit_image="", dir=self.tmp) + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 0} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit"): + stack.enter_context(mock.patch.object(self.r, name)) + self.noted = stack.enter_context(mock.patch.object(self.r, "note_refusal")) + stack.enter_context(mock.patch.object(self.r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(self.r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(self.r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = self.r.unit(0, "good") + return (res.get("exception_info") or {}).get("exception_type") + + def test_only_a_route_the_definition_pins_fails_the_unit_as_its_route(self): + ok, no = {"model": "m-1", "status": 200}, {"status": 404, "error": "HTTP 404: no endpoint"} + self.assertNotIn(self.ended({"u0000": [dict(no, model="m-1")]}), (None, "ApiProviderRouteError")) + self.assertIsNone(self.ended({"u0000": [dict(ok, reply={"served_by": "Parasail"})]})) + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertTrue(self.noted.call_args[0][0][1].startswith("404: ")) + self.assertEqual(self.ended({"u0000": [dict(no, model=SONNET)]}), "ApiProviderRouteError") + self.assertEqual(self.ended({"u0000": [ok], "u0000-judge": [{"model": SONNET, "status": 200, "reply": {"served_by": "Parasail"}}]}), + "ApiProviderRouteError") + + def test_three_units_in_a_row_with_no_endpoint_on_a_pinned_route_stop_the_run(self): + self.r._halt, self.r._refused, self.r.halted = threading.Event(), [], None + for _ in range(3): + self.r.note_refusal(("ApiProviderRouteError", "404: no endpoint")) + self.assertTrue(self.r._halt.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec268..4defc5cf 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -624,6 +624,35 @@ def unit(*a): got, _ = self.outcomes(job_dir) self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_benchmark_that_says_so_runs_no_rate_limited_unit_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}], + limits={"run_again": False}) + job_dir = run.run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + + def test_a_unit_a_guard_asked_to_stop_is_not_run_again(self): + Echo.status = 429 + b = dict(definitions.get("benchmark", "fakebench", ROOT), datasets=[{"id": "responding", "units": 3}]) + r = run.Run(b, definitions.get("agent", "fake", ROOT), model="m-1", provider="custom", jobs_root=self.tmp, + concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True, dataset="responding") + real = r.unit + + def unit(*a): + res = real(*a) + if res and res["trial_name"].startswith("model__"): + with open(os.path.join(r.job_dir, res["trial_name"], ".work", "relay", relay.STOP), "w") as f: + f.write("batch cap\n") + return res + r.unit = unit + job_dir = r.start() + self.assertFalse(os.path.exists(os.path.join(job_dir, "retried"))) + got, _ = self.outcomes(job_dir) + self.assertEqual(got["model"].outcome, classify.RATE_LIMITED) + def test_a_feature_run_drives_the_feature_through_its_port(self): job_dir = self.go("fakememo", "fakemem") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py index c895787e..9b26507f 100644 --- a/app/tests/test_standard_served_by.py +++ b/app/tests/test_standard_served_by.py @@ -191,7 +191,8 @@ def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): """An error that echoes the request's credentials, and a routing object holding a token: the record keeps neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" - Upstream.always = (200, "application/json", json.dumps({"error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + Upstream.always = (200, "application/json", json.dumps({"choices": [{"message": {"content": "ok"}}], + "error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) for serving in (None, {"m-run": ["prov-a"]}): with self.subTest(serving=serving): r = self.relay(model="m-run", key=KEY, serving=serving) @@ -260,15 +261,15 @@ def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) - def test_what_the_record_read_before_is_read_as_before(self): - """The facts are read apart: a failed Responses stream's text, tool calls and usage stay as the relay read - them before (it took nothing from a ``response.failed`` event), its error beside them.""" + def test_a_failed_responses_stream_is_read_beside_its_facts(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage are read from its + ``response.failed`` event, its error beside them.""" body = sse({"type": "response.output_text.delta", "delta": "part"}, {"type": "response.failed", "response": {"object": "response", "status": "failed", "error": {"code": "server_error", "message": "failed"}, "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], "usage": {"input_tokens": 5, "output_tokens": 2}}}) - self.assertEqual(relay.read_reply("/responses", body, True), {"text": "", "tool_calls": [], "usage": None}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "x", "tool_calls": [], "usage": relay._usage({"input_tokens": 5, "output_tokens": 2})}) self.assertEqual(relay.reply_facts("/responses", body, True), {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) @@ -276,9 +277,7 @@ def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own is added, and a unit's failure reads it as before (a 200 is no failure).""" r = self.relay(model="m-run") - errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", - {"code": 502, "message": "Provider returned error"}), - (sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + errors = [(sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", {"code": "server_error", "message": "upstream ended"}), diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index c19da339..e83750b0 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -17,7 +17,7 @@ from nullius import workspace from nullius.runs import launches, parse -from nullius.standard import job, merge, shard +from nullius.standard import job, merge, prove, shard BENCH = {"key": "fakebench", "_ref": "fakebench", "family": "agent", "needs": ["agent.workspace/v1"], "units": {"noun": "task", "nouns": "tasks"}, "score": {"kind": "pass", "says": "the tests pass"}} @@ -63,14 +63,16 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", reward=lambda place: float(place % 2)): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", bench_options=None, + reward=lambda place: float(place % 2)): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) os.makedirs(d) planned = list(units) if planned is None else planned job.write_config(d, job=name, bench=bench, other=other, model=model, provider="openai", dataset="all", units=planned, - concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend) + concurrency=2, granted=True, launch=launch, shard=shard.record(index, of, total, planned), backend=backend, + bench_options=bench_options) results = [] for k, u in enumerate(units): place = int(u.split("-")[1]) @@ -82,7 +84,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": reward(place)}, exception=exc, calls=[], trajectory=None)) - job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) + job.write_result(d, name, t0, t0 + 10 * (min(index, 9) + 1), results, len(planned), budget=budget) return d @@ -201,6 +203,36 @@ def test_units_no_shard_ran_are_said(self): j = parse.parse_job(got["out"]) self.assertEqual((j.n_total_trials, len(j.trials)), (len(self.split[1]) + len(planned), len(self.split[1]) + len(planned) - 1)) + def test_a_launch_of_one_unit_per_shard_merges_and_says_only_the_units_it_lacks(self): + n = 2 ** 64 + units = [f"t-{i}" for i in range(10)] + dirs = [write_shard(self.tmp, f"one-{u}", [u], shard.of(u, n), n) for u in units[:-1]] + got = merge.merge(dirs, os.path.join(self.tmp, "out")) + self.assertEqual((got["trials"], got["notes"]), (9, ["1 of the dataset's 10 units are not in the merge"])) + + def test_shards_run_with_different_benchmark_options_keep_each_its_own(self): + same = {"websites": "none", "gitlab": "none"} + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, bench_options=same), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example")), + write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)] + std = read(os.path.join(merge.merge(dirs, os.path.join(self.tmp, "out"))["out"], "config.json"))["standard"] + self.assertNotIn("benchmark_options", std) + self.assertEqual([m["benchmark_options"]["gitlab"] for m in std["merged_from"]], ["none", "gitlab.example", "none"]) + alike = [write_shard(self.tmp, f"a{i}", self.split[i], i, 3, bench_options=same) for i in (1, 2, 3)] + std = read(os.path.join(merge.merge(alike, os.path.join(self.tmp, "alike"))["out"], "config.json"))["standard"] + self.assertEqual(std["benchmark_options"], same) + self.assertNotIn("benchmark_options", std["merged_from"][0]) + + def test_a_merge_of_merges_keeps_the_options_of_its_inner_shards(self): + same = {"websites": "none", "gitlab": "none"} + inner = merge.merge([write_shard(self.tmp, "s1", self.split[1], 1, 3), + write_shard(self.tmp, "s2", self.split[2], 2, 3, bench_options=dict(same, gitlab="gitlab.example"))], + os.path.join(self.tmp, "inner"))["out"] + outer = merge.merge([inner, write_shard(self.tmp, "s3", self.split[3], 3, 3, bench_options=same)], + os.path.join(self.tmp, "outer"), force=True)["out"] + self.assertEqual(prove._options_of(read(os.path.join(outer, "config.json"))), + [dict(same, gitlab="gitlab.example"), same, {}]) + def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): dirs = self.shards() os.makedirs(os.path.join(self.tmp, "taken", "x")) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267d..ea68afcb 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -6,6 +6,7 @@ import os import tempfile import unittest +from unittest import mock from nullius.standard import job from nullius.traces import trace @@ -139,6 +140,31 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) + def test_a_trace_that_fails_leaves_the_result_and_says_its_error(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + with mock.patch.object(job, "trace_from_run", side_effect=KeyError("the task's own words")): + path = self.write(AGENT, traj) + agent = os.path.dirname(path) + self.assertFalse(os.path.exists(path)) + with open(os.path.join(os.path.dirname(agent), "result.json")) as f: + self.assertEqual(json.load(f)["verifier_result"]["rewards"], {"reward": 1.0}) + with open(os.path.join(agent, "trace.error.txt")) as f: + self.assertEqual(f.read(), "KeyError\n") + self.assertTrue(os.path.exists(os.path.join(agent, "trajectory.json"))) + + def test_a_trace_error_that_cannot_be_written_leaves_the_result(self): + traj = job.trajectory_from_events(list(EVENTS), [call(1, 2)], AGENT, "m", "j/u") + write = job._write + + def refusing(path, text): + if os.path.basename(path).startswith("trace"): + raise PermissionError(path) + write(path, text) + + with mock.patch.object(job, "_write", refusing): + path = self.write(AGENT, traj) + self.assertTrue(os.path.exists(os.path.join(os.path.dirname(os.path.dirname(path)), "result.json"))) + class Backfill(unittest.TestCase): """A trial written before traces gets one from the files it kept, and nothing else changes.""" diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index 6ed5e162..594fd769 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -115,7 +115,7 @@ def tree(job_dir): mock.patch.object(install, "upstream", side_effect=install.InstallFailed("git fetch: exit 128")), \ self.assertRaises(run.Refused): preparing.start() - self.assertEqual(tree(preparing.job_dir), [".work", "progress.jsonl"]) + self.assertEqual(tree(preparing.job_dir), [".work", ".work/job.lck", "progress.jsonl"]) self.assertEqual(tree(unknown.job_dir), tree(preparing.job_dir)) self.assertIn("nope", progress.refusal(progress.read(unknown.job_dir))) diff --git a/app/tests/test_standard_versions.py b/app/tests/test_standard_versions.py index 898a0dc4..1c860879 100644 --- a/app/tests/test_standard_versions.py +++ b/app/tests/test_standard_versions.py @@ -217,7 +217,7 @@ def test_new_version_copies_the_default_and_refuses_a_duplicate(self): ''' TOY = dict(BENCH, upstream={"license": "MIT"}, default="1", controls={"nop": {"source": "nullius", "from": "nothing runs"}}, - versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 2}], + versions={"2": {"datasets": [{"id": "all", "units": 3}, {"id": "hard", "units": 3}], "driver": {"command": ["python3", "{here}/driver2.py"]}, "proven": {}}, "1": {"proven": {"all": {"units": {"units": 2, "passed": "2 ids", "on": "2026-10-12"}}}}}) diff --git a/docs/cli/read-jobs.md b/docs/cli/read-jobs.md index 86ca991f..c690a671 100644 --- a/docs/cli/read-jobs.md +++ b/docs/cli/read-jobs.md @@ -56,7 +56,7 @@ nullius standard traces [--dry-run] ## Give past standard runs a trace -A standard run writes `agent/trace.json` as it ends. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. +A standard run writes `agent/trace.json` as it ends; one it cannot build leaves `agent/trace.error.txt`, the error's type, and the trial's result as it is. `nullius standard traces` gives an earlier job the same file, from what its trials kept. It only adds that file: an existing trace is never replaced, and Harbor jobs and controls are left alone. ```bash title="See what it would write, then write it" nullius standard traces ./jobs --dry-run diff --git a/docs/cli/run.md b/docs/cli/run.md index 4ac31f7d..559cc828 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -102,11 +102,11 @@ nullius standard run humaneval reference --provider openrouter --model meta-llam | Rule | Detail | | --- | --- | | A slug | As the model's page on OpenRouter names its provider: `deepinfra`, or a provider and its variant, a quantisation (`deepinfra/fp8`), a speed (`deepinfra/turbo`) or a region (`google-vertex/us-central1`). Several are tried in their order, and no other provider answers | -| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings | +| Without `MODEL=` | The run's model. With it, any model the run calls: its judge, its simulated user, an option's embeddings. A model the benchmark pins for the run ([standard](../standard.md), Pinned routes) keeps the benchmark's providers | | What is sent | `provider: {"order": [...], "allow_fallbacks": false}` on every request sent as that model, on every route, in place of the agent's own routing field. A pinned Chat Completions or Responses request to OpenRouter also asks for its routing facts (`X-OpenRouter-Metadata: enabled`). A Responses reply names the provider that served it there alone. A Messages request is sent without the header, since its reply names its provider. A model with no pin is sent as before | | Refused before any unit | A provider other than `openrouter`; a model the run does not call (a control calls only its judge's or an option's, never a simulated user's); a slug that is empty, malformed or given twice; a model named twice. The run, the command line and the relay's own command make the same check | -| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does | -| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| No provider can serve it | OpenRouter answers 404 (no endpoint found) and nothing else is tried; the call fails as any refused call does, but for a model the benchmark pins ([standard](../standard.md), A refused route) | +| The record | `config.json`: `standard.served_by`, the slugs by model. Each call in `agent/calls.jsonl`: `request.routing` and, when the agent sent its own, `request.routing_asked`; `reply.served_by`, the provider the reply names (for a Responses reply, the endpoint its routing facts mark as selected), `reply.end` (one per choice, by index, when a reply has several) and `reply.error`, an error a reply holds under a 200: the top-level one, else that of the first choice that has one, its code, type and message alone. These are read apart from the rest of the record, but for a failed Responses object's cost, and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a model the benchmark pins and a 200 with no reply ([standard](../standard.md)) | | Who served the run | Each trial's `result.json`, `standard.served`, and the job's `result.json`, `served`: the answered calls of every route (the agent's, the judge's, the simulated user's, the benchmark's own relay's) by serving provider, those that named none, and for each pinned model `held`: `true` when every answered call names a provider that reads as a pinned slug, `unknown` when one names none or another name, `null` with no answered call. A name reads as a slug lowercased with its spaces as dashes (`DeepInfra` is `deepinfra`, the first part of `deepinfra/turbo`), and nothing else is folded. The tokens and the cost stay the agent's own calls' | | What `held` reads | The provider's name alone: a reply names no endpoint variant, and `held` for `deepinfra/fp8` is true for any endpoint of `deepinfra`. A reply gives the provider's display name. Where that name is not the slug once lowercased, `held` reads `unknown` though the provider served the call (`Google` for `google-vertex`, `SambaNova` for `sambanova-turbo`) | | Shards | `merge` refuses shards whose pins differ, unless `--force`; a model pinned differently in the merged shards reads `held: unknown` | diff --git a/docs/standard.md b/docs/standard.md index 0b4bc483..85d7c851 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -141,16 +141,18 @@ record); verifier/reward.txt and score.json | Rule | Detail | | --- | --- | | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | -| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | +| A relay per unit | Its record is the trial's `.work/relay` (an earlier run's under the same job name is moved aside to `.work/earlier.`, with its `result.json`, `verifier/` and `agent/`; a run holds its job name until its process ends, and one under a name another still holds is refused before it writes anything); the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A model call answered 200 with no reply in it, not streamed, reaches its caller and the record as a 502, or the 429 or 402 of an error it held, that says only its kind; the record keeps `answered: 200`. A call refused with 400, 413 or 422 keeps, as `images`, each inline image it sent by its declared type, the type its bytes show whole and its size, never a byte | | Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | -| Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | -| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | +| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; the agent's own last call refused as longer than the model's context window is `ContextWindowExceededError`, the agent's; a unit a guard's stop ended keeps the score its driver gave, its own last call's 429 notwithstanding; a turn past its limit is the agent's timeout. A driver's error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind (a setup error, the check's, a stop, or of no known cause) when `answer` or `score` said it, a stop asked of the unit or not. `"kind": "score"` leaves a score and no pass | +| Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Unless the benchmark's `limits.run_again` is `false`, each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | | The relay | Headers pass through except the credentials, replaced by the relay's key; `agent/calls.jsonl` names them, never their values. A 429 reaches the agent as it came, never retried. Every call goes as the run's model, the one asked recorded as `model_asked`, except `model.side_models`, sent as asked and recorded `side`. The key goes as the agent's protocol says: `x-api-key` and `anthropic-version` for `messages`, else a bearer header. Provider `anthropic` serves `messages` only ([protocols](settings/model-providers.md#protocols)) | | Constraints (`--constraint KEY=VALUE`) | What the machine allows Nullius, from Settings ([constraints](settings/run-hosts.md#constraints)): flags on every container the run starts on the machine, on either backend; a systemd scope for each process the run starts there (its drivers, and its agents and capabilities on the local backend); the most units at a time; the CPU affinity; the driver's `CUDA_VISIBLE_DEVICES`. Before the first unit, the run checks the container caps on the engine and the scope caps on systemd: the local backend refuses a cap that no scope holds, and the container backend runs its drivers without a scope. `config.json` records the values under `standard.constraints` | -| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before | +| Who served a call | Each call's record keeps the routing field it was sent with (`request.routing`), the serving provider its reply names (`reply.served_by`), how the reply ended (`reply.end`: `finish_reason`, `stop_reason` or a Responses status) and an error the reply holds under a 200 (`reply.error`: the top-level one, else that of the first choice that has one), each read apart from the rest of the record and cleaned of credentials. The call keeps its status, and the unit's verdict reads it as before, but for a route the definition pins (A refused route) and a 200 with no reply (A relay per unit) | | Serving providers (`--served-by`) | On OpenRouter, the providers that may answer a model, in order and no other: each relay, a container's included, sends `provider: {order, allow_fallbacks: false}` on every request sent as that model, in place of the caller's own (kept as `request.routing_asked`). `config.json` records `standard.served_by`; each trial's `result.json` and the job's say the calls of every route by serving provider and, for each pinned model, `held`, which reads the provider's name alone ([Run a benchmark](cli/run.md#serving-providers)) | | A spending cap (`--max-usd`, `--max-usd-per-unit`) | Each relay books a call in the job's ledger (`.work/ledger`, `standard/ledger.py`) before sending it, and refuses the one that does not fit the run's cap or its unit's ([Run a benchmark](cli/run.md#a-spending-cap)) | +| Pinned routes | On OpenRouter, a model the definition pins (`setups.original.pin` in a run of the Original, `score.judge.pin` while the official judge grades the run) is pinned as `--served-by` pins it, whatever `--served-by` says of that model. The pin follows the model sent, so a model the agent shares with the judge goes to the judge's providers | +| A refused route | A call the definition pins (Pinned routes) answered 404 or 503 is refused. When a model's last call on a route was refused (each route read on its own), or any such call's `reply.served_by` shares nothing with the pin's slugs (a name that is not its slug, `Google` for `google-vertex`, fails no call on a guess), the unit fails with `ApiProviderRouteError` unless its agent's own last call failed it first, an API error outside the agent's rate, even when its code made nothing of it. A gated benchmark keeps it to the status, the pin and provider names. Three units in a row whose route has no endpoint (404) stop the run, as a refused key does | ## Safety @@ -345,6 +347,8 @@ N machines are N runs of the same command, each with `--shard I/N`, and one merg | One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch` (`--launch`, else the UTC day, benchmark, agent or capability and model). With `--units`, `units_total` counts the listed units. The shards' lists hold the same units; a list's file name, comments and sha256 may differ from shard to shard | | `cli merge JOB_DIR... -o OUT` | One job (`standard/merge.py`): trials hard-linked, `result.json` recomputed, `config.json` with `merged_from`. Shards that differ in launch, version, dataset, agent, model, serving providers, backend or N are refused unless `--force`. Shards whose lists hold different units, and two trials of one unit, are refused always. `merge` compares the units each list holds, in any order; the sha256 is kept in the record only | | `cli prove --shard I/N` | The controls and the leak check on that shard's units; each record says `shard` | +| Different options per shard | Shards of one launch may differ in `--bench-option` (a unit that needs a service and a write share of its own); the merged config keeps each shard's in `merged_from` | +| `cli prove --job nop=MERGED` | Holds a merged launch of a control to its bound instead of running it. The job and every trial's own record must name that benchmark, version and control and this job; each unit of the dataset exactly once, with the field its score kind names (a number from 0 to 1) or an error; and the units check must pass. `--dataset` named, no `--limit` or `--shard`. The record names the job and the options it ran with. The runs pages group a launch's shards by their shared `standard.launch` (`--launch`); a name like `______prove-nop` is only a convention for reading | ```bash title="Two machines, one merge" nullius standard run humaneval oracle --shard 1/2 --limit 6 # machine 1 @@ -412,15 +416,20 @@ use the source folders and repository checks. | `turn` | The transcript so far; `simulator: {model, base_url, key}` when there is one; `judge` for an upstream that grades as the conversation ends; `attempt` with `limits.attempts` | The next message, `{done: true}`, or `restart` | | `produce` | A capability benchmark's recall, `base_url` and `key` | The reading, through the relay | | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | -| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | +| `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by`; `stopped`, the reason of a stop asked while the unit ran, kept as `verifier_result.stopped` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | | Field in `standard.json` | Meaning | | --- | --- | -| `score.judge` | `{model, says}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another | +| `gated` | `true` for a benchmark whose tasks are gated: a unit's failing model endpoint (the agent's, the judge's, the simulated user's) is kept as its status and class, never the provider's words, which can quote the request | +| `score.judge` | `{model, says, pin?}`: the official judge. Its route (`-judge`) is asked as that model, recorded beside the unit's calls, never as the agent's. `cli run --judge-model` asks another. `pin` (`{"openrouter": [provider slug]}`) pins every call sent as the official judge's model while the official judge grades the run, an agent's own included; another judge is pinned only when it is a model the setup pins | | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | +| `limits.turn_seconds` | How long the agent's turn may take (default 900); a unit's `agent_seconds` or a setup's own `turn_seconds` replaces it | | `limits.turns` | At most this many agent turns in a conversation (default 20) | +| `limits.score_seconds` | How long `score` may take (default 600). A driver that runs the unit's environment in `score`, as its null control does, sets it | +| `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | +| `limits.run_again` | `false`: a unit that ended rate-limited is not run once more after the others; its outcome stands | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -584,9 +593,11 @@ already accepts arbitrary models needs only the `original` method. | Rule | Detail | | --- | --- | | `original` driver method | A model of `models` only, by the name the upstream knows; the relay sends it as `served_as` says, with the prefix `providers` gives. A provider not in `providers` is refused. A model the upstream asks besides the answerer is a `side_models` entry. Without `models`, any model: its own agent runs as the run's model | -| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError` | +| `pin` | `{"openrouter": {model: [provider slug]}}`, by the names in `models` and `side_models`: on OpenRouter each goes to those providers only, with no fallback, so the original runs on the paper's endpoint. Other setups and agents are not pinned, but for a call sent as the official judge's model while it grades the run (`score.judge.pin`) | +| `model` driver method | The upstream is told `runs_as` (its window, batching and token budget) while the relay sends the model chosen; one that reasons by default can spend that budget first: `OutputLengthExceededError`, when its last reply (by when it was asked) has neither text nor a tool call; a last Responses reply answered 200 and `failed` is an API error. A runner that fails once a guard's `stop` beside the relay's records asks the unit to stop is `CancelledError` | | `units` | A setup may count its own units; `units` is sent `setup` | | `answer` | Runs the upstream's answerer through the unit's relay; `score` is sent `setup` too. Local processes or the Original setup driver container | +| `comparable` | `false` when no model of `models` is the paper's, so its runs compare with each other only; or the list of those models whose runs compare with the paper's (`paper`), when the setup runs several agents and only some are the paper's: the builder offers those alone as Like the paper, the others as Any of your choice | | Recorded | `standard.setup` in the config and each trial: its name, the model `asked` and `sent`, `comparable_with_paper` | | Attempts | `cli run --repeat K` runs each unit K times; `result.json` carries `pass_hat_k` (C(c, k) / C(n, k) averaged over the units, an errored unit left out) | @@ -629,7 +640,7 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c `cli run KEY nop` and `cli run KEY oracle` need no `--model` and no `--unsandboxed`: no model is called but a judged benchmark's judge. The config and every trial record `standard.control`. -`cli prove KEY [--dataset D] [--limit N]` runs, for each dataset: +`cli prove KEY [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop=JOB_DIR]` runs, for each dataset: | Check | Passes when | | --- | --- | @@ -638,6 +649,8 @@ A benchmark proves itself by its two **controls**, `nop` and `oracle`, and two c | `units` | Two `units` calls give the same ids: unique, plain enough to name a trial, as many as the dataset says | | `leak` | After `setup`, the oracle's answer and the lines it writes are not already in the prompt or the workspace. A reference under 12 characters is not looked for; the record says how many were skipped | +`--bench-option NAME=VALUE` sets the benchmark's own options for the units check's driver and each control's run, resolved and refused as `cli run` does them, the programs a value needs on this machine included, before anything is installed or started; the control job's config records the values. + What passed is written into `standard.json` per dataset, dated; a failing check is taken back, and a run on fewer units does not replace one on more. ```json title="standard.json ยท proven"