Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions app/nullius/standard/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container]
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N]
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] [--bench-option NAME=VALUE ...] [--job nop|oracle=JOB_DIR ...]
PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID]
PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force]
PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed
Expand Down Expand Up @@ -603,11 +603,16 @@ def cmd_fidelity(a):

def cmd_prove(a):
b = _bench(a.benchmark)
jobs = options(a.job)
for c in jobs:
if c not in definitions.CONTROLS:
sys.exit(f"--job {c}=...: a control, {' or '.join(definitions.CONTROLS)}")
held, spill = licences(b, from_run=a.from_run)
try:
got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model,
stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch,
env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images)
env=run_env(a.provider) if a.from_run else None, licences=held,
bench_options=options(a.bench_option), jobs=jobs, remove_images=a.remove_images)
except run.Refused as e:
print("refused:", e)
return 2
Expand Down Expand Up @@ -717,6 +722,10 @@ def main(argv=None):
v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)")
v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only")
v.add_argument("--launch", help="the launch id the control jobs of every shard record")
v.add_argument("--bench-option", action="append", metavar="NAME=VALUE",
help="one of the benchmark's own options (repeatable), for the checks' driver and the controls' runs")
v.add_argument("--job", action="append", metavar="CONTROL=JOB_DIR",
help="hold this job, a control run already over the whole dataset (a merged launch), to its bound instead of running it")
v.add_argument("--remove-images", action="store_true",
help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)")
g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job")
Expand Down
11 changes: 8 additions & 3 deletions app/nullius/standard/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,8 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N
def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str,
model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None,
ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]:
ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None,
timeout: Optional[int] = None) -> List[str]:
"""``docker run`` for a unit's relay: on the default bridge (its way to the
provider), its port published on 127.0.0.1 only. The key is never given:
``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with
Expand Down Expand Up @@ -761,6 +762,8 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi
a += ["--served-by", f"{m}={','.join(slugs)}"]
if key_header != "authorization":
a += ["--key-header", key_header]
if timeout:
a += ["--timeout", str(timeout)]
return a


Expand Down Expand Up @@ -1022,10 +1025,11 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str,
side_models=(), key_header: str = "authorization", servers: Iterable[str] = (),
doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None,
ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True,
serving: Optional[Dict[str, List[str]]] = None):
serving: Optional[Dict[str, List[str]]] = None, timeout: Optional[int] = None):
self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app
self.ledger = ledger # the file of the run's spending cap, when it has one
self.serving = dict(serving or {}) # by model, the serving providers that may answer it
self.timeout = timeout
self.servers = list(servers) # licence servers, host:port each: a forwarder apiece
self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece
self.side_models, self.key_header = tuple(side_models), key_header
Expand All @@ -1051,7 +1055,8 @@ def start(self) -> "Unit":
argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir,
key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes,
prices=self.prices, side_models=self.side_models, key_header=self.key_header,
hosts=self.hosts, ledger=self.ledger, serving=self.serving)
hosts=self.hosts, ledger=self.ledger, serving=self.serving,
timeout=self.timeout)
self.started.append(self.names["relay"])
_docker(argv[1:])
_docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]])
Expand Down
74 changes: 63 additions & 11 deletions app/nullius/standard/definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
import re
from typing import Callable, Dict, List, Optional, Tuple

from nullius.standard import ports
from nullius.standard import ports, relay
from nullius import paths
from nullius.catalogue import storage
from . import names
Expand Down Expand Up @@ -1035,6 +1035,25 @@ def control(bench: dict, which: str) -> Optional[dict]:
"_kind": "control", "_ref": which, "_here": bench.get("_here", "")}


def comparable(orig: dict, model: str) -> bool:
"""Whether a run of the original with ``model`` compares with the paper's: ``comparable`` a bool, or the models that do."""
c = orig.get("comparable", True)
return model in c if isinstance(c, list) else bool(c)


def upstream_name(orig: dict, model: str, provider: str) -> str:
"""``model`` by the name the original's upstream knows, when given as ``provider`` lists it."""
pre = (orig.get("providers") or {}).get(provider, "")
names = {pre + (orig.get("served_as") or {}).get(m, m): m for m in orig.get("models") or []}
return model if model in (orig.get("models") or []) else names.get(model, model)


def paper_models(orig: dict) -> List[str]:
"""The original's models whose runs compare with the paper's."""
c = orig.get("comparable", True)
return [m for m in orig.get("models") or [] if not isinstance(c, list) or m in c]


def _leaderboard(d: dict) -> List[str]:
"""``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the
upstream's leaderboard fixes besides the ranked model, from its own submission
Expand Down Expand Up @@ -1066,12 +1085,13 @@ def _leaderboard(d: dict) -> List[str]:

def _setups(s) -> List[str]:
"""``setups``: {original: {summary, models, default, served_as?, providers,
side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s
side_models?, pin?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s
``models`` are the paper's, by the names the upstream's code knows them;
``side_models`` the ones its code asks besides the answerer (an extraction,
a summary), sent as asked through served_as and the provider's prefix;
``providers`` the ones that serve them, each with the prefix its names
carry there; ``model`` needs ``original`` (its pipeline) and asks as one
carry there; ``pin`` {openrouter: {model: [provider slug]}}, the only
providers of each; ``model`` needs ``original`` (its pipeline) and asks as one
of its models (``runs_as``)."""
if s is None:
return []
Expand All @@ -1090,7 +1110,7 @@ def _setups(s) -> List[str]:
bad.append(f"{w}.summary: what runs, in a sentence")
allowed = {"summary"}
if k == "original":
allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"}
allowed |= {"models", "default", "served_as", "providers", "side_models", "pin", "units", "turn_seconds", "paper", "comparable", "reference_agent"}
if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()):
bad.append(f"{w}.reference_agent: the upstream agent's display name")
models = x.get("models")
Expand All @@ -1114,6 +1134,8 @@ def _setups(s) -> List[str]:
pv = x.get("providers")
if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()):
bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}")
if "pin" in x:
bad += _pin(x["pin"], f"{w}.pin", pv if isinstance(pv, dict) else {}, set(models + side))
u = x.get("units")
if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))):
bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is")
Expand All @@ -1122,8 +1144,11 @@ def _setups(s) -> List[str]:
bad.append(f"{w}.turn_seconds: a whole number of seconds")
if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()):
bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase")
if "comparable" in x and not isinstance(x["comparable"], bool):
bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only")
c = x.get("comparable", True)
if not isinstance(c, bool) and (not isinstance(c, list) or not c or len(set(c)) != len(c) or any(m not in models for m in c)
or x.get("default") not in c):
bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only; "
"or those of its models whose runs compare with the paper's, its default among them")
else:
allowed |= {"runs_as"}
orig = s.get("original") if isinstance(s.get("original"), dict) else None
Expand All @@ -1139,6 +1164,25 @@ def _setups(s) -> List[str]:
return bad


def _pin(pin, w: str, providers: dict, models: Optional[set] = None) -> List[str]:
"""``pin``: {provider: {model: [slug]}} for a setup (``models`` given), {provider: [slug]} for a judge."""
if not isinstance(pin, dict) or not pin:
return [f"{w}: {{openrouter: ...}}, the providers that alone serve it"]
bad = []
for p, v in pin.items():
if p != "openrouter" or (providers and p not in providers):
bad.append(f"{w}: a pin is OpenRouter's provider routing, on a provider the benchmark runs on ({p!r})")
continue
pins = v if models is not None else {"_": v}
if not isinstance(pins, dict) or not pins:
bad.append(f"{w}.{p}: {{model: [provider slug, ...]}}")
continue
if models is not None and not all(m in models for m in pins):
bad.append(f"{w}.{p}: pins only its models and side_models")
bad += [f"{w}.{p}: " + b.removeprefix("_: ") for b in relay.serving_problems(pins)]
return bad


# Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder).
SETTINGS_FILE = "upstream-settings.json"
_DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$")
Expand Down Expand Up @@ -1204,7 +1248,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple
return None, f"{bench['label']} has no {which} setup"
orig, x = s["original"], s[which]
model = (model or "").strip()
side = []
side, pins = [], {}
if which == "original" and "models" not in orig: # any model, sent as the provider lists it
if not model:
return None, f"the original {bench['label']} needs a model: its leaderboard ranks models"
Expand All @@ -1215,7 +1259,7 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple
model = model or orig["default"]
if model not in orig["models"]:
return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): "
f"{model} is not one - choose the model setup for another")
f"{model} is not one" + (" - choose the model setup for another" if "model" in bench.get("setups", {}) else ""))
if provider and provider not in orig["providers"]:
return None, (f"the paper's models are not served by {provider}: the original runs on "
+ " or ".join(orig["providers"]))
Expand All @@ -1225,13 +1269,15 @@ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple
sent = served(model)
# the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own
side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent]
pins = {served(m): list(only) for m, only in ((orig.get("pin") or {}).get(provider) or {}).items()
if m == model or m in (orig.get("side_models") or [])}
else:
if not model:
return None, f"the model setup needs a model: the one to put in {orig['default']}'s place"
asked, sent = x["runs_as"], model
units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"}
return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}",
"summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True),
return {"model": {"side_models": side}, **({"pins": pins} if pins else {}), "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}",
"summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and comparable(x, model),
"turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which,
"_here": bench.get("_here", "")}, ""

Expand Down Expand Up @@ -1313,14 +1359,18 @@ def problems(d: dict, kind: str) -> List[str]:
if not isinstance(lim, dict):
bad.append("limits: an object")
else:
for f in ("turn_seconds", "turns"):
for f in ("turn_seconds", "turns", "score_seconds", "call_seconds"):
v = lim.get(f)
if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000):
bad.append(f"limits.{f}: a whole number from 1")
if lim.get("run_again") is not None and not isinstance(lim["run_again"], bool):
bad.append("limits.run_again: false when a rate-limited unit is not run once more")
a = lim.get("attempts")
if a is not None and (not isinstance(a, str) or not a.strip()):
bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), "
"in a sentence - the driver is then sent attempt with setup, turn and answer")
if "gated" in d and not isinstance(d["gated"], bool):
bad.append("gated: true for a benchmark whose tasks are gated, so no provider's words about them are kept")
s = d.get("score") or {}
if s.get("kind") not in SCORES or not s.get("says"):
bad.append(f"score: {{kind: one of {SCORES}, says}}")
Expand All @@ -1332,6 +1382,8 @@ def problems(d: dict, kind: str) -> List[str]:
j = s.get("judge")
if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")):
bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}")
elif j is not None and "pin" in j:
bad += _pin(j["pin"], "score.judge.pin", {})
sim = d.get("simulator")
if sim is not None:
if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"):
Expand Down
4 changes: 2 additions & 2 deletions app/nullius/standard/fidelity.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def check(agent: dict, *, model: str, provider: str, jobs_root: str = "jobs", pa
# records side by side, with the fields fidelity reads.
r.relay = relay.Relay(r.base, r.key, os.path.join(r.job_dir, ".work", "relay"), prices=price_list.table(), model=model,
side_models=(agent.get("model") or {}).get("side_models") or (),
key_header=r.key_header, fields=True).start()
key_header=r.key_header, fields=True, timeout=r.call_seconds).start()
order = [(s, n) for n in range(1, passes + 1) for s in SIDES]
got: Dict[str, List[dict]] = {s: [] for s in SIDES}
solved: Dict[str, List[str]] = {}
Expand Down Expand Up @@ -230,7 +230,7 @@ def _one(r: "run.Run", side: str, unit: str, i: int, tag: str, cmd: List[str], k
except (run._Ended, jsonl.Failed) as e:
r.say(f" {tag} {unit}: {e}")
try:
score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=600)
score = r.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=r.score_seconds)
except jsonl.Failed:
score = {}
if os.path.isdir(os.path.join(base, "agent")):
Expand Down
Loading
Loading