diff --git a/app/nullius/providers.py b/app/nullius/providers.py index e26eb167..7521faa7 100644 --- a/app/nullius/providers.py +++ b/app/nullius/providers.py @@ -16,6 +16,7 @@ import http.client import json import os +import re import secrets import socket import ssl @@ -444,6 +445,53 @@ def decision_models(env=None, *, fetch: bool = True, opener=None, clock=time.tim return fresh +# A model OpenRouter serves: its maker, a slash, its name (a variant after a colon: ``:free``). +_ROUTED = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._:-]*") +# A serving provider's slug, as ``--served-by`` takes it (``standard.relay.SLUG``). +_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def _endpoints(content: bytes) -> list[dict]: + """The serving providers of OpenRouter's endpoint list (``{"data": {"endpoints": [...]}}``), each + ``{slug, name, quantization}``: the slug its ``tag`` (a provider and its variant, ``deepinfra/fp8``), + else the provider's name lowercased with its spaces as dashes. ValueError when there is no list.""" + data = json.loads(content) + rows = ((data.get("data") or {}).get("endpoints") if isinstance(data, dict) and isinstance(data.get("data"), dict) else None) + if not isinstance(rows, list): + raise ValueError("missing endpoint list") + out = {} + for row in rows: + if not isinstance(row, dict): + continue + name = row.get("provider_name") if isinstance(row.get("provider_name"), str) else "" + tag = row.get("tag") if isinstance(row.get("tag"), str) else "" + slug = tag or re.sub(r"\s+", "-", name.strip().lower()) + if not slug or len(slug) > 128 or not _SLUG.fullmatch(slug) or slug in out: + continue + quant = row.get("quantization") if isinstance(row.get("quantization"), str) else "" + out[slug] = {"slug": slug, "name": (name or slug)[:128], "quantization": quant[:32]} + return sorted(out.values(), key=lambda e: (e["name"].casefold(), e["slug"])) + + +def list_endpoints(provider: str, model: str, env=None, *, opener=None) -> list[dict]: + """The serving providers OpenRouter lists for ``model`` (``GET /models/{author}/{slug}/endpoints``), for the + builder's Served by: asked as the model list is (its key when set, no redirect, nothing of the error echoed).""" + env = os.environ if env is None else env + provider, endpoint = split(provider) + if provider != "openrouter": + raise ValueError("Only OpenRouter routes a model to several providers.") + model = served_as(str(model or "").strip(), provider) + if not _ROUTED.fullmatch(model): + raise ValueError("Choose a model as OpenRouter names it, its maker first: maker/model.") + base, _, headers = _where(provider, endpoint, env) + author, _, name = model.partition("/") + url = f"{base}/models/{urllib.parse.quote(author, safe='')}/{urllib.parse.quote(name, safe=':')}/endpoints" + try: + return _endpoints(_get(url, headers, opener)) + except (OSError, ValueError, TypeError, AttributeError, http.client.HTTPException): + raise ValueError("Provider list unavailable. Enter a provider's slug by hand.") from None + + # Settings' Test, for each run provider (TypeSafe has no free request): only requests that cost nothing. A # model list everywhere; OpenRouter's is public, so its key is asked at its own key-information endpoint. _TESTS_LOCK = threading.Lock() diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 5a8b4dc6..81f8544d 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -7,7 +7,7 @@ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -43,6 +43,9 @@ puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option`` sets one of the agent's or capability's options (``options`` in its definition); the rest keep the vendor's defaults, and the job's config records them all. +``--served-by [MODEL=]SLUG[,SLUG...]`` lets only those serving providers answer +a model on OpenRouter, in that order (``run.served_by``); each call's record +names the provider that served it. A run says each stage before its first unit, and each unit's end, as one ``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl`` (``nullius/progress.py``): what the run's page shows while it goes. @@ -528,6 +531,7 @@ def cmd_run(a): licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), + **({"served_by": run.served_by(a.served_by, a.model or "")} if a.served_by else {}), **({"env": run_env(a.provider)} if a.from_run else {})) except run.Refused as e: print("refused:", e) @@ -697,6 +701,9 @@ def main(argv=None): p.add_argument("--constraint", action="append", metavar="KEY=VALUE", help="what the machine allows the run, repeatable: cpus, memory_mb, processes and cpu_set on its " "containers and, through a systemd scope, on its processes here; concurrency; gpu_ids (empty for none)") + p.add_argument("--served-by", action="append", metavar="[MODEL=]SLUG[,SLUG...]", + help="on OpenRouter, the serving providers that may answer a model, tried in this order and no " + "other (repeatable; without MODEL=, the run's model); each call records which one did") p.add_argument("--max-usd-per-unit", type=float, metavar="USD", help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on") v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed") diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index bafb447c..bacddab5 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -724,7 +724,7 @@ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = N def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str, model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None) -> List[str]: + ledger: Optional[str] = None, serving: Optional[Dict[str, List[str]]] = None) -> List[str]: """``docker run`` for a unit's relay: on the default bridge (its way to the provider), its port published on 127.0.0.1 only. The key is never given: ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with @@ -732,7 +732,9 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi an empty folder). ``hosts``: names it resolves (``--add-host``), the machine's own for an endpoint on its loopback. ``ledger``: the file of the run's spending cap, its folder mounted so - this relay books its calls where every other relay of the job does.""" + this relay books its calls where every other relay of the job does. + ``serving``: by model, the serving providers that may answer it, as the + unit's own relay is given them (``relay.Relay``).""" a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init", *capped(), *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")], @@ -752,6 +754,11 @@ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_fi a += ["--ledger", ledger] for m in side_models: a += ["--side-model", m] + # the pins of the models this relay sends; the run's other models (an option's embeddings) are another relay's + sends = {model} | set(routes.values()) | {str(s).partition("=")[2] or str(s) for s in side_models} + for m, slugs in sorted((serving or {}).items()): + if m in sends: + a += ["--served-by", f"{m}={','.join(slugs)}"] if key_header != "authorization": a += ["--key-header", key_header] return a @@ -1014,9 +1021,11 @@ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None, side_models=(), key_header: str = "authorization", servers: Iterable[str] = (), doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None, - ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True): + ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True, + serving: Optional[Dict[str, List[str]]] = None): self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app self.ledger = ledger # the file of the run's spending cap, when it has one + self.serving = dict(serving or {}) # by model, the serving providers that may answer it self.servers = list(servers) # licence servers, host:port each: a forwarder apiece self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece self.side_models, self.key_header = tuple(side_models), key_header @@ -1042,7 +1051,7 @@ def start(self) -> "Unit": argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir, key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes, prices=self.prices, side_models=self.side_models, key_header=self.key_header, - hosts=self.hosts, ledger=self.ledger) + hosts=self.hosts, ledger=self.ledger, serving=self.serving) self.started.append(self.names["relay"]) _docker(argv[1:]) _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]]) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 683d4e14..6234e4e3 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -400,11 +400,74 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: return "full" if full else "partial" +def _named(name: str) -> str: + """A serving provider's name as its slug reads: lowercased, its words joined by dashes (``DeepInfra`` reads + ``deepinfra``) and nothing else folded: ``Prov/A`` is not ``prov-a``, and a provider whose name differs from its + slug otherwise (``Google`` serving ``google-vertex``) reads as another. A run fetches no list of + providers, and this is all a name is matched by.""" + return re.sub(r"\s+", "-", name.strip().lower()) + + +def served(calls: List[dict], serving: Optional[Dict[str, List[str]]] = None) -> Optional[dict]: + """Who served a unit's answered calls (each record's ``reply.served_by``): the calls by serving provider and + those that named none; and, for each model the run pinned (``serving``), whether the pin held - true when + every answered call sent as that model names a provider that reads as one of its pinned slugs (``_named``, + a slug's first part), ``unknown`` when one names none or another name, None when it has no answered call. + A slug's endpoint variant (``/fp8``) is not in a reply's name: ``held`` says which provider answered, and + cannot tell one of its endpoints from another. None for a unit with no answered call in a run with no pin.""" + answered = [c for c in calls if c.get("status") == 200 and not (c.get("reply") or {}).get("error")] + by: Dict[str, int] = {} + unnamed = 0 + for c in answered: + name = (c.get("reply") or {}).get("served_by") + if isinstance(name, str) and name: + by[name] = by.get(name, 0) + 1 + else: + unnamed += 1 + if not answered and not serving: + return None + out = {"calls": dict(sorted(by.items())), "unnamed": unnamed} + if serving: + held = {} + for m, slugs in sorted(serving.items()): + mine = [c for c in answered if c.get("model") == m] + pinned = {str(s).split("/")[0].lower() for s in slugs} + held[m] = None if not mine else True if all( + _named(str((c.get("reply") or {}).get("served_by") or "")) in pinned for c in mine) else "unknown" + out["held"] = held + return out + + +def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[dict]: + """The job's ``served``: its trials' added up with ``also`` (the benchmark's own relay's, :func:`served`), a + pin held where it held in every one that called its model.""" + parts = [p for p in [(r.get("standard") or {}).get("served") for r in results] + [also] if p] + if not parts: + return None + calls: Dict[str, int] = {} + for p in parts: + for name, n in (p.get("calls") or {}).items(): + calls[name] = calls.get(name, 0) + n + out = {"calls": dict(sorted(calls.items())), "unnamed": sum(p.get("unnamed") or 0 for p in parts)} + models = sorted({m for p in parts for m in (p.get("held") or {})}) + if models: + out["held"] = {} + for m in models: + said = [p["held"][m] for p in parts if (p.get("held") or {}).get(m) is not None] + out["held"][m] = None if not said else True if all(v is True for v in said) else "unknown" + return out + + def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict: - """Write one unit's trial and return its result. ``environment``: the + image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), + serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None) -> dict: + """Write one unit's trial and return its result. ``serving``: the run's + pin of serving providers by model; ``standard.served`` says who served + ``served_calls`` (the unit's calls on every route of its relay: its + agent's, its judge's, its simulated user's; ``calls`` when not given) and + whether the pin held (:func:`served`). ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began @@ -426,7 +489,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]}, "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit, "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other), - **({"environment": environment} if environment else {}), **({"image": image} if image else {})}} + **({"environment": environment} if environment else {}), **({"image": image} if image else {}), + **({"served": who} if (who := served(calls if served_calls is None else served_calls, serving)) else {})}} if score is not None: value = float(score.get("reward") or 0.0) # a pass benchmark leaves a reward, as a verifier does; a score @@ -465,7 +529,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``unsandboxed`` is true for an agent on the local backend only: in a container (``backend``, ``image``) it is not. ``standard.options`` holds every option's value, the defaults included, so two runs read side by side @@ -477,7 +542,9 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str ``simulator``: the models that graded and that played the user, when the benchmark has them. ``prices``: the rows its models' costs are estimated from and where the user's list came from, when it has any. - ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any.""" + ``constraints``: what the machine allowed the run (``nullius.constraints``), when it set any. + ``served_by``: by model, the serving providers the run let answer it + (``--served-by``), said only for a run that pinned them.""" with open(os.path.join(job_dir, "config.json"), "w") as f: json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius", "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, @@ -488,7 +555,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {}), - **({"constraints": constraints} if constraints else {})}, + **({"constraints": constraints} if constraints else {}), + **({"served_by": dict(sorted(served_by.items()))} if served_by else {})}, "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]], "n_concurrent_trials": concurrency, "agents": [{"name": other["key"], "model_name": model}], @@ -544,14 +612,18 @@ def pass_hat(results: List[dict]) -> Optional[dict]: "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} -def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): +def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None, + served: Optional[dict] = None): """``budget``: a run with a spending cap (``run.Run.budget``): the cap, what was spent, whether it stopped the run, the units it cut off and - those never begun.""" + those never begun. ``served``: who served the benchmark's own relay's + calls (:func:`served`); the job's ``served`` adds its trials' to them.""" errored = sum(1 for r in results if r.get("exception_info")) hat = pass_hat(results) + who = served_total(results, served) with open(os.path.join(job_dir, "result.json"), "w") as f: json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned, "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0}, - **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1) + **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {}), + **({"served": who} if who else {})}, f, indent=1) diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index 97cb517f..3bdc33b4 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -9,8 +9,9 @@ the shards' with ``shard`` taken out and ``merged_from`` put in. What is merged must be one launch: the same launch id, benchmark and -version, dataset, agent or capability and model, and the same number of shards -- else it is refused, naming what differs, unless ``force``. Two trials of +version, dataset, agent or capability and model, the same serving providers +pinned for each model (``served_by``) and the same number of shards - else it +is refused, naming what differs, unless ``force``. Two trials of one unit are always refused, with the units named: one of them would be counted twice. So are shards whose lists of units (``run.units_file``) hold different units, or a shard of a list and one of none: the merged job @@ -40,7 +41,8 @@ SAME = (("launch", ("standard", "launch")), ("benchmark", ("standard", "benchmark")), ("benchmark version", ("standard", "benchmark_version")), ("dataset", ("datasets", 0, "name")), ("agent or capability", ("standard", "plugged")), ("model", ("agents", 0, "model_name")), - ("backend", ("standard", "backend")), ("number of shards", ("standard", "shard", "of"))) + ("backend", ("standard", "backend")), ("serving providers", ("standard", "served_by")), + ("number of shards", ("standard", "shard", "of"))) class Refused(Exception): @@ -73,6 +75,28 @@ def _listed(cfg: dict) -> Optional[tuple]: return tuple(sorted(str(u) for u in units)) if isinstance(units, list) else None +def _served(shards: List[dict], results: List[dict]) -> Optional[dict]: + """The merged job's ``served`` (``job.served_total``), made of each shard's whole one - its trials' and its + benchmark relay's calls, an ``unknown`` kept - else, for a shard that has none, of its trials'. A model the + shards pinned differently - merged with ``force`` - reads ``unknown``: the merged config keeps the first + shard's pin, which the others' calls were never held to.""" + parts = [] + for s in shards: + whole = (s.get("result") or {}).get("served") + if isinstance(whole, dict): + parts.append(whole) + else: + parts.append(job.served_total([r for r in results if (r.get("standard") or {}).get("shard_job") == s.get("name")])) + who = job.served_total([{"standard": {"served": p}} for p in parts if p]) + pins = [_at(s["config"], ("standard", "served_by")) or {} for s in shards] + pins = [p if isinstance(p, dict) else {} for p in pins] + differ = sorted(m for m in {m for p in pins for m in p} if len({json.dumps(p.get(m)) for p in pins}) > 1) + if differ: + who = who or {"calls": {}, "unnamed": 0} + who["held"] = dict(who.get("held") or {}, **{m: "unknown" for m in differ}) + return who + + def _trials(job_dir: str) -> List[str]: return sorted(n for n in os.listdir(job_dir) if not n.startswith(".") and os.path.isfile(os.path.join(job_dir, n, "result.json"))) @@ -123,7 +147,7 @@ def check(job_dirs: List[str], force: bool = False) -> dict: + " (--force does not merge them)") differ = [] for label, path in SAME: - seen = {json.dumps(_at(s["config"], path)) for s in shards} + seen = {json.dumps(_at(s["config"], path), sort_keys=True) for s in shards} # a mapping by its content if len(seen) > 1: differ.append(f"{label}: " + ", ".join(f"{s['name']} {_at(s['config'], path)!r}" for s in shards)) if differ and not force: @@ -205,8 +229,9 @@ def _budget(budgets: list) -> Optional[dict]: def result_of(shards: list, results: List[dict]) -> dict: - """The merged job's ``result.json``, but its ``id``: ``shards`` as ``check`` reads them, ``results`` every - trial's record. The runs page folds a launch's shards with it (``runs.launches.fold``).""" + """The merged job's ``result.json``, but its ``id`` and its ``served`` (``_served``, which ``merge`` adds): + ``shards`` as ``check`` reads them, ``results`` every trial's record. The runs page folds a launch's shards + with it (``runs.launches.fold``).""" planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards) starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")] ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")] @@ -250,7 +275,7 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: summary = result_of(shards, results) planned = summary["n_total_trials"] with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f: - json.dump({"id": name, **summary}, f, indent=1) + json.dump({"id": name, **summary, **({"served": who} if (who := _served(shards, results)) else {})}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 4a5d7e10..5e5d759c 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -40,7 +40,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove +from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -50,6 +50,8 @@ MAX_CONCURRENT = 32 BACKENDS = definitions.names.BACKENDS _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") +# A pin's field, as the Review's Model row names it (served_by, --served-by) +SERVED_LABEL = {"model": "model", "simulator": "user simulator", "judge": "judge"} _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$") @@ -282,11 +284,17 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, whole to every job that calls a model: a unit is a unit wherever it runs. ``unpriced``: asked, for a recipe with a cap, which of the models it names have no price in the user's list (``nullius.prices``): a run - with a cap is refused without one, and the Limits step says so first.""" + with a cap is refused without one, and the Limits step says so first. + + ``served_by``: on OpenRouter, the serving providers each model field + pins, ``{model|simulator|judge: [slug, ...]}``, checked as ``--served-by`` + is and compiled into it for each job that calls that model, by the name + the run sends it as.""" d = defs or _all() errors, notes = [], [] add = lambda step, message: errors.append({"step": STEP[step], "message": message}) - out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""} + out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": "", + "served_models": {}} name = str(recipe.get("name") or "") if not _NAME.match(name): add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.") @@ -446,6 +454,20 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model + # the serving providers each model field pins, on OpenRouter (served_by: {model|simulator|judge: [slug, ...]}), + # checked as --served-by is (relay.serving_problems); an empty list pins nothing + roles = {"model": "the model", "simulator": "the user simulator's model", "judge": "the judge's model"} + pins = recipe.get("served_by") + served: Dict[str, List[str]] = {} + if pins not in (None, {}): + if not isinstance(pins, dict) or any(k not in roles for k in pins): + add("model", "Served by names the serving providers of the model, the user simulator's or the judge's.") + else: + served = {k: v for k, v in pins.items() if v not in (None, [], "")} + for problem in relay.serving_problems({roles[k]: v for k, v in served.items()}): + add("model", "Served by " + problem + ".") + if served and provider_ != "openrouter": + add("model", "Served by needs OpenRouter, the provider that routes a model to several: choose it, or leave Served by empty.") # the run's spending cap: an amount in USD, or none cap = recipe.get("max_usd") if cap in ("", None): @@ -504,6 +526,42 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, limits.of(t) except ValueError as e: add("where", f"{e}: fix it in Settings.") + judge_def, sim_def = (b.get("score") or {}).get("judge"), b.get("simulator") + + def sent_of(p: dict) -> Dict[str, Optional[str]]: + """Each model field's model as job ``p`` sends it; None where it has none (a control has no model of its own).""" + return {"model": None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model, + "judge": (judge_model or providers.served_as(judge_def["model"], provider)) if judge_def else None, + "simulator": (sim_model or providers.served_as(sim_def["model"], provider)) if sim_def else None} + + def called_of(p: dict) -> List[str]: + s = sent_of(p) + return ledger.models(p["o"], s["model"], s["judge"], s["simulator"], None, control=p["control"]) + + # the pins by model, as the run sends each: a field's pin is its model's, on every job that calls that model + # (a control's judge job too, when the judge's model is the one the model field pinned) + by_model: Dict[str, List[str]] = {} + for p in picks: + for role, slugs in served.items(): + m = sent_of(p).get(role) + if not m: + continue + if m in by_model and by_model[m] != list(slugs): + add("model", f"Served by pins {m} twice, with different providers: give its fields the same ones.") + by_model[m] = list(slugs) + # each field's model as the run sends it, for the builder to ask its providers under that name + out["served_models"] = {role: next((sent_of(p)[role] for p in sorted(picks, key=lambda q: q["control"]) if sent_of(p)[role]), None) + for role in roles} + + def pinned(p: dict) -> Dict[str, List[str]]: + """The --served-by of one job: the pins of the models it calls.""" + called = called_of(p) + return {m: s for m, s in by_model.items() if m in called} + if served and not errors: + everything = {m for p in picks for m in called_of(p)} + for role in served: + if not {sent_of(p)[role] for p in picks} & everything: + add("model", f"Served by: no job of this recipe calls {roles[role]}; leave it empty.") if errors: return out nouns = (b.get("units") or {}).get("nouns", "units") @@ -545,7 +603,8 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--backend", "container"] if backend == "container" else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch_of(d, p)] if of > 1 else []), - *limits.cli_words(limits.of(targets[index - 1])), "--from-run"] + *limits.cli_words(limits.of(targets[index - 1])), + *[w for m_, slugs in pinned(p).items() for w in ("--served-by", f"{m_}={','.join(slugs)}")], "--from-run"] return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"' def script_for(d: dict, p: dict, command: str, index: int) -> str: @@ -627,7 +686,8 @@ def row(p: dict) -> list: [STEP["model"], "Model", "none: a control calls no model" if out["keyless"] else f"{providers.label(providers.choice(provider, str(recipe.get('endpoint') or '')))} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only") + (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "") - + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")], + + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + + "".join(f" · {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ") + (" of each dataset" if many and isinstance(limit, int) else "") diff --git a/app/nullius/standard/relay.py b/app/nullius/standard/relay.py index e3f97845..13d28ba0 100644 --- a/app/nullius/standard/relay.py +++ b/app/nullius/standard/relay.py @@ -9,7 +9,11 @@ byte for byte - and recorded once it is complete, one line per call in ``/.jsonl``: when it started and ended, the model, the status, the tokens, and an abridged request and reply. Three API shapes are read, -Chat Completions, Responses and Anthropic's Messages, streamed or not. The +Chat Completions, Responses and Anthropic's Messages, streamed or not. Read +apart from the rest (:func:`reply_facts`), the reply also says, when it does, +the serving provider that answered (``served_by``), how it ended (``end``) +and an error it holds though its status was a success (``error``); reading +them never changes what the caller receives, nor the rest of the record. The cost is the provider's when it says it (OpenRouter's ``usage.cost``), else an estimate from ``prices`` when one is given for exactly that model (the user's price list, ``nullius.prices``), marked as an estimate. @@ -24,6 +28,12 @@ limits). It never answers for the provider: a 429 reaches the caller and the record as it was sent, which is what makes rate limits measurable. +A run may pin, for a model, the serving providers that may answer it +(``serving``, OpenRouter's routing): the relay then puts +``provider: {order, allow_fallbacks: false}`` on every request it sends as +that model, in place of the caller's own, and records both, cleaned of +credentials. Without a pin a request's body goes on as it did. + One answer is the relay's own: a run with a spending cap (``ledger``) books each call before it goes out, and a call that does not fit is not sent. Its caller gets a 402 that names the relay and the cap, and the record says @@ -211,6 +221,202 @@ def read_reply(path: str, body: bytes, streamed: bool) -> dict: "usage": _usage(told) if told is not None else usage} +# -- serving providers: the pin a run sets, and what a reply says of who served it -- +SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*") + + +def routing(slugs) -> dict: + """OpenRouter's routing field for a strict pin: these serving providers, + tried in this order, and no other.""" + return {"order": list(slugs), "allow_fallbacks": False} + + +def serving_problems(serving) -> List[str]: + """What is wrong with a pin by model (``{model: [slug, ...]}``): the one + check the run, its command line and the relay's own command share.""" + if not isinstance(serving, dict): + return ["the serving providers are given by model: {MODEL: [SLUG, ...]}"] + bad = [] + for model, slugs in serving.items(): + if not isinstance(model, str) or not model.strip(): + bad.append("a pin names no model") + continue + if not isinstance(slugs, (list, tuple)) or not slugs: + bad.append(f"{model}: no serving provider") + continue + wrong = [s for s in slugs if not isinstance(s, str) or not SLUG.fullmatch(s)] + if wrong: + bad.append(f"{model}: {', '.join(repr(s) for s in wrong)} is not a serving provider's slug " + "(as a model's page names it: deepinfra, deepinfra/fp8)") + elif len(set(slugs)) != len(slugs): + bad.append(f"{model}: a serving provider is named twice") + return bad + + +def parse_served_by(given, model: str) -> tuple: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable, as ``({model: [slug, ...]}, problems)``. Without + ``MODEL=`` it is ``model``, the run's. A model named twice is a problem; so is each of :func:`serving_problems`.""" + out: Dict[str, List[str]] = {} + bad = [] + for g in given or []: + name, eq, listed = str(g).rpartition("=") + name = name.strip() if eq else (model or "") + if not name: + bad.append(f"--served-by {g!r}: MODEL=SLUG[,SLUG...], or SLUG[,SLUG...] for the run's model") + continue + if name in out: + bad.append(f"--served-by names {name} twice: list its serving providers once, in the order they are tried") + continue + out[name] = [s.strip() for s in listed.split(",")] + return out, bad + serving_problems(out) + + +def _clean(value, secrets=()): + """A recorded copy, never what is sent: a value under a credential's name + replaced, the relay's own key cut out of every string - a key of a + mapping as much as a value - then ``redact``.""" + if isinstance(value, dict): + return {_clean(k, secrets): "[REDACTED]" if isinstance(k, str) and _CREDENTIAL.search(k) else _clean(v, secrets) + for k, v in value.items()} + if isinstance(value, list): + return [_clean(v, secrets) for v in value] + if isinstance(value, str): + for s in secrets: + if s: + value = value.replace(s, "[REDACTED]") + return redact(value) + return value + + +def _name(v) -> Optional[str]: + return v if isinstance(v, str) and v else None + + +def _an_error(e) -> Optional[dict]: + """An error object, in one shape: its code (a word or a number), type and + message (words), as given; None when it has none of them of that kind. + Nothing else of the object is kept.""" + if not isinstance(e, dict) or not e: + return None + got = {k: e[k] for k in ("code", "type") if isinstance(e.get(k), (str, int)) and not isinstance(e.get(k), bool)} + if isinstance(e.get("message"), str): + got["message"] = _cut(e["message"], 500) + return got or None + + +def _routing_facts(obj) -> Optional[list]: + """The serving endpoints OpenRouter's routing facts (``openrouter_metadata``) + list as available, in an object of a reply, its ``message`` or its + ``response``; None when none of them holds such a list.""" + if not isinstance(obj, dict): + return None + for o in (obj, obj.get("message"), obj.get("response")): + meta = o.get("openrouter_metadata") if isinstance(o, dict) else None + endpoints = meta.get("endpoints") if isinstance(meta, dict) else None + available = endpoints.get("available") if isinstance(endpoints, dict) else None + if isinstance(available, list): + return available + return None + + +def _provider(obj) -> Optional[str]: + """The serving provider an object of a reply names: its own ``provider``, + its ``message``'s or its ``response``'s, else the one endpoint OpenRouter's + routing facts mark ``selected`` among the available ones (asked for with + ``X-OpenRouter-Metadata``; a Responses reply names its provider there + alone). Facts that mark no single named endpoint give none.""" + if not isinstance(obj, dict): + return None + for o in [obj] + [o for o in (obj.get("message"), obj.get("response")) if isinstance(o, dict)]: + if _name(o.get("provider")): + return o["provider"] + chosen = [a for a in _routing_facts(obj) or () if isinstance(a, dict) and a.get("selected") is True] + return _name(chosen[0].get("provider")) if len(chosen) == 1 else None + + +def _choices(choices, ends: dict, errors: dict) -> None: + """Each choice's end and error, by its index.""" + if not isinstance(choices, list): + return + for n, ch in enumerate(choices): + if not isinstance(ch, dict): + continue + i = ch.get("index") if isinstance(ch.get("index"), int) and not isinstance(ch.get("index"), bool) else n + if _name(ch.get("finish_reason")): + ends[i] = ch["finish_reason"] + e = _an_error(ch.get("error")) + if e and i not in errors: + errors[i] = e + + +def _status(resp) -> Optional[str]: + """A Responses object's end: its status, and why when it is incomplete.""" + if not isinstance(resp, dict) or not _name(resp.get("status")): + return None + why = resp.get("incomplete_details") + why = _name(why.get("reason")) if isinstance(why, dict) else None + return f"{resp['status']}: {why}" if why else resp["status"] + + +def _facts(path: str, body: bytes, streamed: bool) -> dict: + served, end, error, ends, errors = None, None, None, {}, {} + if not streamed: + got = json.loads(body) + if not isinstance(got, dict): + return {} + served, error = _provider(got), _an_error(got.get("error")) + _choices(got.get("choices"), ends, errors) + if got.get("type") == "message": + end = _name(got.get("stop_reason")) + elif got.get("object") == "response" or "output" in got: + end = _status(got) + else: + for line in body.decode("utf-8", "replace").splitlines(): + if not line.startswith("data:"): + continue + try: + ev = json.loads(line[5:].strip()) + except ValueError: + continue + if not isinstance(ev, dict): + continue + # the latest event that names one; routing facts that mark no single named endpoint leave none + served = _provider(ev) if _routing_facts(ev) is not None else (_provider(ev) or served) + kind = ev.get("type") + if kind == "error": # Messages' error event, or a Responses one + error = error or _an_error(ev.get("error")) or _an_error({k: v for k, v in ev.items() if k != "type"}) + elif isinstance(ev.get("error"), dict): # an error chunk under a 200 + error = error or _an_error(ev["error"]) + if kind == "message_delta" and isinstance(ev.get("delta"), dict): + end = _name(ev["delta"].get("stop_reason")) or end + if kind in ("response.completed", "response.done", "response.incomplete", "response.failed"): + resp = ev.get("response") + end = _status(resp) or end + error = error or _an_error(resp.get("error") if isinstance(resp, dict) else None) + _choices(ev.get("choices"), ends, errors) + if ends: + end = ends[min(ends)] if len(ends) == 1 else {str(i): ends[i] for i in sorted(ends)} + if error is None and errors: + i = min(errors) + error = dict(errors[i], choice=i) + return {k: v for k, v in (("served_by", served), ("end", end), ("error", error)) if v is not None} + + +def reply_facts(path: str, body: bytes, streamed: bool) -> dict: + """What a reply says besides what :func:`read_reply` reads, read apart from + it: the serving provider that answered (``served_by``), how it ended + (``end``: ``finish_reason``, ``stop_reason`` or a Responses status; one per + choice, by index, when there are several) and an error it holds though its + status was a success (``error``: code, type and message; a top-level one + whatever else the reply holds, else a choice's). A fact of the wrong type + is left out, and a reply that cannot be read gives none: nothing here can + stop a reply reaching its caller.""" + try: + return _facts(path, body, streamed) + except Exception: + return {} + + def _request_tail(req: dict) -> dict: """The part of a request a reader needs: the model, how many messages, the last two, abridged, and the reasoning effort it asked for (chat's @@ -355,7 +561,8 @@ def close(self): class Relay: def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[str, dict]] = None, model: Optional[str] = None, side_models=(), key_header: str = "authorization", fields: bool = False, - ledger: Optional[str] = None, unit_caps: bool = True): + ledger: Optional[str] = None, unit_caps: bool = True, serving: Optional[Dict[str, List[str]]] = None, + facts: Optional[bool] = None): """``base``: the provider's endpoint (``https://api.openai.com/v1``). ``model``: when given, every call is sent as this model, whatever the caller named - the run's model is the recipe's, not the agent's. A @@ -373,8 +580,16 @@ def __init__(self, base: str, key: str, record_dir: str, prices: Optional[Dict[s (:func:`request_fields`) - ``fidelity``'s, never a normal run's. ``ledger``: the file of the run's spending cap (``ledger``), shared by every relay of the job: a call is booked there before it is sent, - and not sent when it does not fit.""" + and not sent when it does not fit. ``serving``: by model, the serving + providers that may answer a request sent as that model (:func:`routing`), + put on every such request in place of the caller's own. ``facts``: a + pinned Chat Completions or Responses request asks OpenRouter for its + routing facts, where its reply names who served it; by default when + ``base`` is OpenRouter's.""" self.fields = fields + self.serving = {m: list(s) for m, s in (serving or {}).items()} + host = (urllib.parse.urlsplit(base).hostname or "").lower() + self.facts = (host == "openrouter.ai" or host.endswith(".openrouter.ai")) if facts is None else bool(facts) self.ledger = ledgers.Ledger(ledger) if ledger else None # False for the benchmark's own relay: its calls (an option's embeddings) are no one unit's, so only the run's cap holds them self.unit_caps = unit_caps @@ -437,10 +652,16 @@ def _said(self, e: Exception) -> str: text = text.replace(k, "[REDACTED]") return redact(text)[:200] - def _headers(self, caller, tag: str = "") -> Dict[str, str]: + def _headers(self, caller, tag: str = "", facts: bool = False) -> Dict[str, str]: """The headers sent on: the key as ``key_header`` says on the agent's own routes; a route pinned - to a model of the benchmark's (a judge, a simulator, embeddings) speaks chat, so a bearer header.""" + to a model of the benchmark's (a judge, a simulator, embeddings) speaks chat, so a bearer header. + ``facts``: a pinned Chat Completions or Responses request to + OpenRouter, sent asking for its routing facts whatever the caller + said of them.""" out = forwarded(caller) + if facts: + out = {k: v for k, v in out.items() if k.lower() != "x-openrouter-metadata"} + out["X-OpenRouter-Metadata"] = "enabled" has = {k.lower() for k in out} # a name in any case is the same header: never send it twice x_api_key = self.key_header == "x-api-key" and tag not in self.models and tag not in self.embedding_models for name, value in (("Content-Type", "application/json"), ("Accept", "*/*")) + \ @@ -468,6 +689,7 @@ def _forward(self, h: BaseHTTPRequestHandler): req = {} asked = req.get("model") if isinstance(req, dict) else None side = False + routing_asked, pinned = None, None if isinstance(req, dict) and req: if rest.split("?")[0] == "/embeddings" and self.embedding_models.get(tag): req["model"] = self.embedding_models[tag] @@ -480,6 +702,10 @@ def _forward(self, h: BaseHTTPRequestHandler): req["model"] = self.model if req.get("stream") and rest.split("?")[0].endswith("/chat/completions"): req.setdefault("stream_options", {})["include_usage"] = True + pinned = self.serving.get(req["model"]) if isinstance(req.get("model"), str) else None + if pinned: + routing_asked = req.get("provider") + req["provider"] = routing(pinned) raw = json.dumps(req).encode() booking = booked = None # a call that names its model is booked, whatever its route, but a token count, which costs nothing and @@ -502,7 +728,9 @@ def _forward(self, h: BaseHTTPRequestHandler): conn, r, back, body = None, None, [], b"" cut = False # the reply was cut off: the call may have been billed try: - conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, tag)) + # a Responses reply names its provider in the routing facts alone; a Messages reply names it of itself + facts = bool(pinned) and self.facts and rest.split("?")[0].endswith(("/chat/completions", "/responses")) + conn, r = self.pool.send(h.command, rest, raw if h.command == "POST" else None, self._headers(h.headers, tag, facts)) status, kind, back = r.status, r.getheader("Content-Type") or "application/json", returned(r.headers) except Exception as e: status, kind = 502, "application/json" @@ -526,11 +754,22 @@ def record(body): "headers": sorted(k.lower() for k in forwarded(h.headers))} # the names sent on, never a value if side: rec["side"] = True + if isinstance(req, dict) and req.get("provider") is not None: + # the routing as sent, and the caller's own the pin replaced: recorded copies, cleaned + rec["request"]["routing"] = _clean(req["provider"], (self.key, self.key.strip())) + if pinned and routing_asked is not None: + rec["request"]["routing_asked"] = _clean(routing_asked, (self.key, self.key.strip())) if self.fields and isinstance(req, dict) and req: rec["fields"] = request_fields(req) cost = None if status == 200: rec["reply"] = read_reply(rest, body, streamed) + # who served it, how it ended, an error under the 200: beside what was read, never in its place + try: + rec["reply"].update({k: v for k, v in _clean(reply_facts(rest, body, streamed), (self.key, self.key.strip())).items() + if k not in rec["reply"]}) + except Exception: + pass cost, how = self._cost(rec["model"], rec["reply"].get("usage")) if cost is not None: rec["cost_usd"], rec["cost_from"] = cost, how @@ -728,7 +967,7 @@ def failure(calls: List[dict]) -> Optional[tuple]: def serve(argv=None) -> int: """``python3 -m nullius.standard.relay serve --listen HOST:PORT --base URL --record DIR --key-file PATH [--model M] [--route TAG=MODEL ...] [--side-model M ...] [--key-header NAME] - [--public URL] [--prices JSON] [--ledger FILE]``. + [--public URL] [--prices JSON] [--ledger FILE] [--served-by MODEL=SLUG[,SLUG...] ...]``. ``--base`` is the provider's endpoint, ``--route`` a route sent as another model (a benchmark's judge), ``--public`` the address ``url`` hands out. @@ -750,7 +989,25 @@ def serve(argv=None) -> int: ap.add_argument("--public", help="the address url() hands out, when not the listening one") ap.add_argument("--prices", help="a JSON object: model -> USD per million tokens, for a cost estimate") ap.add_argument("--ledger", help="the file of the run's spending cap: a call is booked there before it is sent") + ap.add_argument("--served-by", action="append", default=[], metavar="MODEL=SLUG[,SLUG...]", + help="the serving providers that may answer a request sent as MODEL, in order, and no other; " + "MODEL one this relay sends (--model, a --route's, a --side-model's), and --base OpenRouter's " + "or --routing given") + ap.add_argument("--routing", action="store_true", + help="the endpoint at --base routes a model to several serving providers, as OpenRouter does") a = ap.parse_args(argv) + serving, bad = parse_served_by(a.served_by, "") + if serving and not bad: + host = (urllib.parse.urlsplit(a.base).hostname or "").lower() + if not (a.routing or host == "openrouter.ai" or host.endswith(".openrouter.ai")): + bad.append(f"{a.base} routes no model to several serving providers (OpenRouter's does; --routing says another does)") + sent = {a.model} | {r.partition("=")[2] for r in a.route} | {(s.partition("=")[2] or s) for s in a.side_model} + stray = [m for m in serving if a.model and m not in sent] + if stray: + bad.append(f"{', '.join(stray)}: a model this relay does not send (it sends {', '.join(sorted(x for x in sent if x))})") + if bad: + print(json.dumps({"error": "--served-by: " + "; ".join(bad)}), flush=True) + return 2 try: with open(a.key_file) as f: key = f.read().strip() @@ -762,7 +1019,7 @@ def serve(argv=None) -> int: return 2 host, _, port = a.listen.rpartition(":") r = Relay(a.base, key, a.record, prices=json.loads(a.prices) if a.prices else None, model=a.model, - side_models=a.side_model, key_header=a.key_header, ledger=a.ledger) + side_models=a.side_model, key_header=a.key_header, ledger=a.ledger, serving=serving) for route in a.route: tag, _, model = route.partition("=") if not _TAG.match(tag) or not model: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 2ea72822..20fbcc56 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -167,6 +167,19 @@ def key_header(provider: str, other: dict, env=os.environ) -> str: return providers.PROTOCOLS[providers.protocol_for(provider, apis, env)]["auth"] +def served_by(given, model: str) -> Dict[str, List[str]]: + """``--served-by [MODEL=]SLUG[,SLUG...]``, repeatable: by model, the + serving providers that may answer it, in the order they are tried, and no + other (OpenRouter's slugs, as a model's page gives them: ``deepinfra``, + ``deepinfra/fp8``). Without ``MODEL=`` it is the run's model. Refused as + ``relay.parse_served_by`` finds it wrong: the check the run and the + relay's own command make too.""" + out, bad = relay.parse_served_by(given, model) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + return out + + def _slug(s: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit" @@ -378,7 +391,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, - constraints: Optional[dict] = None): + constraints: Optional[dict] = None, + served_by: Optional[Dict[str, List[str]]] = None): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for @@ -424,7 +438,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas agents on the local backend). Before the first unit the container constraints are checked on the engine (where those containers run), and the scope's on systemd: on the local backend a cap no scope holds is refused, on the container - backend its drivers then run without one.""" + backend its drivers then run without one. + ``served_by``: by model, the serving providers that may answer it + (:func:`served_by`), put on every request sent as that model by the + run's relays (``relay.routing``) and recorded in the job's config. + Refused when the provider routes no model to several (only + ``openrouter`` does), or when it names a model the run does not call.""" backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") @@ -548,6 +567,22 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) + self.serving = dict(served_by or {}) + if self.serving: + bad = relay.serving_problems(self.serving) + if bad: + raise Refused("--served-by: " + "; ".join(bad)) + self.serving = {m: list(s) for m, s in self.serving.items()} + if provider != "openrouter": + raise Refused(f"--served-by: {provider} serves each model itself, with no serving providers to choose " + "among; the option needs a provider that routes a model to several (openrouter)") + # a control calls no model of its own and plays no user: only its judge and an option's models + called = ([m for m in (self.judge_model, self.embed_model, self.rerank_model) if m] if other.get("_kind") == "control" + else self.models_called()) + stray = [m for m in self.serving if m not in called] + if stray: + raise Refused(f"--served-by names {', '.join(stray)}, which this run does not call: it calls " + + (", ".join(called) or "no model")) self.bench_relay = None keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env) @@ -705,7 +740,7 @@ def start(self) -> str: # a shard of a list splits the listed units: they are its whole shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) if self.shard else None, units_file=self.units_file, - repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, + repeat=self.repeat, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "") # every unit once, then every unit again: a repeat cut short still has each unit's first run work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine] @@ -782,7 +817,8 @@ def work_unit(iur): self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) self.capped = bool(budget and budget["stopped"]) - job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget) + job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget, + served=job.served(self.benchmark_calls(), getattr(self, "serving", None))) errored = sum(1 for x in results if x.get("exception_info")) self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results)) # the units' stage says why it ended early: a line of its own would be a stage that never starts @@ -944,7 +980,7 @@ def ready(self) -> None: if (self.embed_model or self.rerank_model) and self.bench_relay is None: self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"), prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), - unit_caps=False).start() # embeddings and rerank: a bearer key + unit_caps=False, serving=self.serving).start() # embeddings and rerank: a bearer key def runtimes_needed(self) -> List[str]: """What the agent needs in an own image that the image may not have: @@ -1120,7 +1156,10 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, setup_end=p0 or time.time(), checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), - environment=environment or "", + environment=environment or "", serving=getattr(self, "serving", None), + # who served the unit: its agent's calls and its judge's and simulated user's routes + served_calls=calls + ([c for side in ("judge", "simulator") for c in rl.calls(f"{tag}-{side}")] + if rl is not None else []), image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}" @@ -1305,6 +1344,16 @@ def ledger_file(self) -> Optional[str]: led = getattr(self, "ledger", None) return led.path if led is not None else None + def benchmark_calls(self) -> List[dict]: + """The calls of the benchmark's own relay (an option's embeddings or reranking), every route of it.""" + folder = os.path.join(self.job_dir, ".work", "relay-benchmark") + calls = [] + for name in sorted(os.listdir(folder)) if os.path.isdir(folder) else []: + if name.endswith(".jsonl"): + with open(os.path.join(folder, name)) as f: + calls += [json.loads(line) for line in f if line.strip()] + return calls + def models_called(self) -> List[str]: """Every model this run's relays may send a call to, as sent: the run's, the judge's, the simulator's, an option's embeddings and the @@ -1329,7 +1378,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] key_header = self.key_header if self.backend == "local": rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, - key_header=key_header, ledger=self.ledger_file()).start() + key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() rl.unit_url = rl.url(tag) return rl routes = {tag + "-judge": self.judge_model} if self.judge_model else {} @@ -1358,7 +1407,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] model=self.model, routes=routes, side_models=side, key_header=key_header, prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None, - ledger=self.ledger_file(), + ledger=self.ledger_file(), serving=getattr(self, "serving", None), network_probe=probe, probe_now=bool(probe) and self.first_probe()) rl.mcp_listeners = listeners try: diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 5ed1d5ba..1d53e123 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -69,6 +69,7 @@ r.model_choice = choice; if (paperListed(r)) r.model = ''; if (choice !== 'any') { delete r.judge_model; delete r.simulator_model; } + if (choice !== was) delete r.served_by; // its models change with it: a pin belongs to the model it was set for if (lb && was === 'leaderboard' && choice !== 'leaderboard') { r.bench_options = {...(r.bench_options || {})}; for (const k of Object.keys(lb.bench_options || {})) delete r.bench_options[k]; @@ -79,6 +80,57 @@ } return r; } + // Served by (recipe.py, served_by): on OpenRouter, the serving providers each model field pins, in the order they are + // tried; a field per model the run calls - the model, the user simulator's and the judge's when the benchmark has them. + const SERVED_ROLES = ['model', 'simulator', 'judge']; + function servedRoles(r) { + const b = standardBench(r); + if (!b || r.provider !== 'openrouter' || standardKeyless(r)) return []; + return SERVED_ROLES.filter(k => k === 'model' || (k === 'simulator' && b.simulator) || (k === 'judge' && b.judge)); + } + // A field's text, "deepinfra, together", as the pin's list, every part kept for the server to check ("a,,b" is + // refused in sight, as the option refuses it); a field with nothing in it pins nothing. + const servedList = text => { const parts = String(text || '').split(',').map(s => s.trim()); return parts.some(Boolean) ? parts : []; }; + // A model field's role, and its model changed: the pin set for the model before it goes with it. + const SERVED_FIELD = {model: 'model', simulator_model: 'simulator', judge_model: 'judge'}; + function setModelOf(r, field, value) { + if (SERVED_FIELD[field] && r[field] !== value && r.served_by) setServed(r, SERVED_FIELD[field], []); + r[field] = value; + return r; + } + // The model a field's Served by lists the providers of: the one the run sends, as the server compiled it + // (recipe.py, served_models: a paper model's served name, an official one's with its maker), else the one shown. + const servedModelOf = (role, shown, compiled) => ((compiled || {}).served_models || {})[role] || shown || ''; + // What decides the models the run sends: the recipe but its pins. A pin added or taken out changes none of them. + const servedKey = r => JSON.stringify({...r, served_by: undefined}); + // The models the server last named, and what decided them ({out, key}): a compile that names them (`out`, the + // compile route's answer for the recipe `recipeKey`, its JSON) replaces them; one that failed keeps the last ones. + const servedCompiled = (last, out, recipeKey) => out && out.served_models && typeof out.served_models === 'object' + ? {out, key: servedKey(JSON.parse(recipeKey))} : last; + // A Served by row as the recipe holds it: the model its providers are listed for, the one the run sends as the + // server compiled it for this recipe (`compiled`; none while not known, and never the name shown: the list is not + // asked under a name the run may not send, and the row waits), its slugs in order, and its placeholder. Without a + // pin it waits for the list too, then hides only for one entry. A failed or empty list still allows a typed slug. + function servedRow(r, role, shown, compiled, providers) { + const chosen = (r.served_by || {})[role] || [], model = servedModelOf(role, '', compiled); + const visible = !!chosen.length || !!providers && (!providers.ok || providers.models.length !== 1); + return {shown, model, pending: !model && !!shown, chosen, visible, placeholder: chosen.length ? 'Add a provider' : 'any provider'}; + } + function setServed(r, role, slugs) { + const list = Array.isArray(slugs) ? slugs : servedList(slugs); + r.served_by = {...(r.served_by || {})}; + if (list.length) r.served_by[role] = list; else delete r.served_by[role]; + if (!Object.keys(r.served_by).length) delete r.served_by; + return r; + } + // A pick, or slugs typed ("a, b"), join the field's list after the ones there, in order, each once; a removal takes + // out the one at its place, and the last one gone leaves the field empty: any provider. + function addServed(r, role, slugs) { + const list = [...((r.served_by || {})[role] || [])]; + for (const s of (Array.isArray(slugs) ? slugs : servedList(slugs)).map(x => String(x).trim())) if (s && !list.includes(s)) list.push(s); + return setServed(r, role, list); + } + const removeServed = (r, role, at) => setServed(r, role, ((r.served_by || {})[role] || []).filter((_, i) => i !== Number(at))); function createStandard(benchmark = '', other = '') { const st = standard(), b = st.benchmarks.find(x => x.key === benchmark) || st.benchmarks[0]; const r = {version: 1, harness: 'standard', name: 'standard-experiment', title: 'Standard experiment', benchmark: '', datasets: [], others: [], @@ -94,7 +146,7 @@ const b = standardBench({benchmark: key}); if (!b) return r; const was = standardBench(r); - if (was !== b) r.model_choice = 'any'; + if (was !== b) { r.model_choice = 'any'; delete r.served_by; } r.benchmark = key; const kept = was === b ? datasetsOf(r).filter(id => b.datasets.some(d => d.id === id)) : []; r.datasets = kept.length ? kept : [b.datasets[0].id]; @@ -141,6 +193,11 @@ // and each unit's own: a unit that reaches it is cut off there, the run going on if (typeof raw.max_usd_per_unit === 'number' && Number.isFinite(raw.max_usd_per_unit) && raw.max_usd_per_unit > 0) r.max_usd_per_unit = raw.max_usd_per_unit; for (const key of ['judge_model', 'simulator_model']) if (typeof raw[key] === 'string' && raw[key].trim()) r[key] = raw[key].trim(); + // Served by: each field's serving providers, slugs in their order (the server checks them as --served-by) + if (raw.served_by && typeof raw.served_by === 'object' && !Array.isArray(raw.served_by)) + for (const role of SERVED_ROLES) { const v = raw.served_by[role]; + // kept as saved, a slug the server refuses included: a pin is refused in sight, never dropped in silence + if (Array.isArray(v) && v.length) setServed(r, role, v.map(s => typeof s === 'string' ? s.trim() : String(s))); } r.unsandboxed = raw.unsandboxed === true; if(raw.modelFormat === 'provider-id')r.modelFormat='provider-id'; else r.model = bareModel(r.model); @@ -230,7 +287,8 @@ const recipeProvider = r => isStandard(r) ? r.provider : providerOf(r, r.model || r.userModel); const choiceOf = r => recipeProvider(r) === 'custom' && r.endpoint ? 'custom:' + r.endpoint : recipeProvider(r); // The Model step's choice into the recipe: its provider, and for a named endpoint its id. - function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); } + // a provider that routes no model to several takes no Served by: a pin made for OpenRouter is dropped with it + function setChoice(r, c) { r.provider = kindOf(c); keepEndpoint(r, {endpoint: endpointOf(c)}); if (r.provider !== 'openrouter') delete r.served_by; } // Why Run cannot start this recipe here, or '': its provider is not one of those. A recipe naming one keeps it, // said, until another is chosen - never switched behind the person's back. The server refuses the same // (web/execute.py, provider_problem). @@ -774,5 +832,5 @@ at: String(d.at || '')}; } catch (e) { return e && e.message === RUNNER_GONE ? {recipe: null, gone: RUNNER_GONE} : null; } } - return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; + return {STEPS,STEP,runners,RUNNER_GONE,isStandard,standard,standardBench,standardOthers,standardKeyless,paperModels,paperListed,paperOffered,leaderboard,setModelChoice,servedRoles,servedList,setServed,addServed,removeServed,setModelOf,servedModelOf,servedKey,servedCompiled,servedRow,stepShown,chosenOf,datasetsOf,whereList,machineLabel,createStandard,setStandardBenchmark,openingStep,runnersOf,runnerFor,benchKey,whereLabel,create,benchmarks,routeBlocker,dataset,sources,setBenchmark,parameters,scopes,validate,compile,runnable,fromJSON,fromJob,prefill,review,copy,packDraft,unpackDraft,adapterPath,taskCount,PROVIDERS,PROVIDER_LABEL,PROVIDER_NEEDS,available,labelOf,needsOf,choiceOf,setChoice,providersOf,recipeProvider,providerBlocker,providerOf,bareModel,settleModel,agentModel,quote:q,path,defaults,needsCredentials,credentialBindings,declaredTimeouts,effectiveTimeouts,setPhaseMinutes,PHASES,harborExecutable,harborPin,harborProblem,harborKey,harborRepoUrl,harborLabel,repoLabel,agents,mergeAgents,setScanned,COMMON,agentById}; }); diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index e37e0166..a64956ce 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -246,3 +246,13 @@ #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +/* Served by: a row after its model's row, as that row is: the label left, the picker right, the providers chosen + under the picker in their order */ +#nullius-builder .std-served-list { grid-column: 2; min-width: 0; } +#nullius-builder .std-served-list:empty { display: none; } +#nullius-builder label.std-served:has(.std-served-list:not(:empty)) > .ui-field-copy { grid-row: span 2; align-self: start; margin-top: 6px; } +#nullius-builder .std-served-pick { cursor: default; gap: 2px; padding: 3px 3px 3px 11px; font: 12.5px/1.4 var(--mono); border-color: var(--nb-blue); background: var(--nb-tint); max-width: 100%; } +/* a slug is never cut: a long one wraps inside its chip, which stays inside its row */ +#nullius-builder .std-served-slug { min-width: 0; overflow-wrap: anywhere; } +#nullius-builder .std-served-pick .nb-remove { flex: none; padding: 0 6px; font-size: 14px; line-height: 1; } +@media (max-width: 650px) { #nullius-builder .std-served-list { grid-column: 1; } #nullius-builder label.std-served:has(.std-served-list:not(:empty)) > .ui-field-copy { grid-row: auto; margin-top: 0; } } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 1f242b0b..a25326b6 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -1,6 +1,57 @@ /* Builder page controller: edit recipes, resolve their dependencies, and start an explicit Run. */ (() => { 'use strict'; + // Served by: a row after each model field's, on OpenRouter alone: the serving providers that may answer that model, + // in order (recipe.py, served_by). Its picker is the Model row's (model-picker.js, list="nb-served"): typing filters + // the model's providers (POST /new-run/endpoints), a pick or a slug typed and Enter joins the list under it, each + // removable. Empty: any provider. What a row shows is C.servedRow's; `compiled()` is the server's last compile and + // the key of the recipe it was made for (C.servedKey). The tests run this under Node, with stand-ins for the page. + function servedRows({C, root, r, compiled, esc, button, providers}) { + const NAME = {model: 'model', simulator: 'user simulator', judge: 'judge'}; + // the model a Served by is for, as the step shows it: the field's, else the leaderboard's or the official one + const shown = (rec, role) => { + if (role === 'model') return rec.model || ''; + const b = C.standardBench(rec) || {}, lb = rec.model_choice === 'leaderboard' ? C.leaderboard(rec) : null, paper = rec.model_choice === 'paper' || !!lb; + return (!paper && rec[role + '_model']) || lb && lb[role] || b[role] || ''; + }; + // the server's compile, while what decides the models it names has not changed since + const known = rec => { const c = compiled(); return c && c.out && c.key === C.servedKey(rec) ? c.out : null; }; + providers.subscribe(model => { + // A late list for a previous model never draws its row. Shared models update every field still using it, + // including a list refreshed by opening the picker after the cache expires. + const current = r(), out = known(current); + if (C.servedRoles(current).some(role => C.servedModelOf(role, '', out) === model)) sync(); + }); + // A compile's models can still apply after a pin edit; the rest of its answer needs the whole recipe's key. + const acceptCompile = (rec, last, out, key, currentKey) => ({ + current: key === currentKey, + served: C.servedKey(JSON.parse(key)) === C.servedKey(rec) ? C.servedCompiled(last, out, key) : last, + }); + const row = (rec, role) => { + const out = known(rec), model = C.servedModelOf(role, '', out); + const entry = C.servedRoles(rec).includes(role) && model ? providers(model) : null; + return C.servedRow(rec, role, shown(rec, role), out, entry && entry.result); + }; + const chips = (role, chosen) => chosen.map((slug, i) => `${esc(slug) || '“”'}` + + button('×', `data-served-remove="${i}" data-served-role="${role}" aria-label="Remove ${esc(slug)} from Served by, ${NAME[role]}"`, 'nb-remove') + '').join(''); + const field = (rec, role) => { if (!C.servedRoles(rec).includes(role)) return ''; const s = row(rec, role); + return ``; }; + // each Served by drawn again in place, as the recipe holds it now: its list, and the model its providers are listed + // for. The rest of the form stays as it is: the keyboard stays where it is, and an open menu stays open. + function sync() { + const rec = r(); + root.querySelectorAll('input[data-served-by]').forEach(input => { + const role = input.dataset.servedBy, s = row(rec, role), list = root.querySelector(`[data-served-list="${role}"]`); + input.closest('.std-served').hidden = !C.servedRoles(rec).includes(role) || !s.visible; + Object.assign(input.dataset, {servedModel: s.model, servedPending: String(s.pending), chosen: s.chosen.join(',')}); + input.placeholder = s.placeholder; + if (list && list.dataset.of !== JSON.stringify(s.chosen)) { list.dataset.of = JSON.stringify(s.chosen); list.innerHTML = chips(role, s.chosen); } + }); + } + return {field, sync, acceptCompile}; + } + if (typeof module === 'object' && module.exports) { module.exports = {servedRows}; return; } const root = document.getElementById('nullius-builder'); if (!root) return; const schema = JSON.parse(document.getElementById('nb-schema').textContent); @@ -114,7 +165,9 @@ }; // The standard's recipe as the server compiles it from the definitions (POST // /new-run/runner). Nothing typed here is a command; Run sends the choices. - let runnerOut = null, runnerKey = '', runnerTimer; + let runnerOut = null, runnerKey = '', runnerTimer, runnerServed = null; + // Served by's rows (servedRows above), given the models the server last named and what decided them (C.servedCompiled) + const {field: servedField, sync: syncServed, acceptCompile} = servedRows({C, root, r, esc, button, compiled: () => runnerServed, providers: window.NulliusServingProviders}); // Switching runner keeps the recipe being edited, one per runner, for as long as the page is open: switching back finds it as it was. const keptByRunner = {}; // Nullius's own mark for the standard; Harbor's for Harbor @@ -602,12 +655,13 @@ return `

Choose the model.

` + modelChoiceCards(rec) + providerField(rec, C.providersOf(rec)) + field('Model', 'model', rec.model, {list:'nb-models', placeholder:paper && C.paperListed(rec) ? 'Choose a paper model' : 'Search or enter a model ID'}) + + servedField(rec, 'model') // the benchmark's other models: the user simulator its conversations need, the judge that grades - each its official one // unless named; like the paper, the official one; like the leaderboard, its own and the options it fixes + (b.simulator ? (paper ? fixed('Model for the user simulator', lb && lb.simulator || b.simulator, lb && lb.simulator ? 'leaderboard' : 'official') : field('Model for the user simulator', 'simulator_model', rec.simulator_model || '', {list: 'nb-models', placeholder: b.simulator + ' (official)', - hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) : '') + hint: 'Plays the user in every conversation. Empty: ' + esc(b.label) + "'s official one; another compares only with runs that used it."})) + servedField(rec, 'simulator') : '') + (b.judge ? (paper ? fixed('Model for the judge', lb && lb.judge || b.judge, lb && lb.judge ? 'leaderboard' : 'official') : field('Model for the judge', 'judge_model', rec.judge_model || '', {list: 'nb-models', placeholder: b.judge + ' (official)', - hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) : '') + hint: 'Grades the answers. Empty: ' + esc(b.label) + "'s official judge; another compares only with runs that used it."})) + servedField(rec, 'judge') : '') + (lb ? Object.entries(lb.bench_options || {}).map(([k, v]) => fixed((b.options[k] || {}).label || k, v, 'leaderboard')).join('') : '') + keyLine(rec); } @@ -636,7 +690,10 @@ const res = await nulliusFetch('/new-run/runner', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': schema.csrf}, body: JSON.stringify({recipe: rec})}); out = await res.json(); } catch (e) { out = {errors: [{step: REVIEW, message: 'The recipe could not be compiled here: ' + e}], notes: [], review: []}; } - if (key !== runnerKey) return; + const accepted = acceptCompile(r(), runnerServed, out, key, runnerKey); + runnerServed = accepted.served; + if (step === S.model) syncServed(); + if (!accepted.current) return; runnerOut = out; if (step === REVIEW) renderForm(); refreshPlan(); @@ -646,7 +703,6 @@ const select = (label,key,value,options) => ``; const fixed = (label,value,whose='official') => ``; const note = (title,text) => `
${title}${text}
`; - function customRows(agent = '') { // '' lists the shared entries (every agent, and the other owners); an id lists that agent's own --ak. const rows = r().custom.map((p, i) => [p, i]).filter(([p]) => agent ? p.scope === 'agent' && p.agent === agent : !p.agent); @@ -891,6 +947,9 @@ root.addEventListener('click',async event=>{ const b=event.target.closest('button');if(!b)return;const d=b.dataset; if(d.modelChoice){C.setModelChoice(r(),d.modelChoice);renderForm();refreshPlan();return;} + // a provider taken out of a Served by. Its row's label does not then focus the field (its menu would open); by + // key, the keyboard goes back to that field + if(d.servedRemove!==undefined){event.preventDefault();C.removeServed(r(),d.servedRole,d.servedRemove);syncServed();refreshPlan();if(event.detail===0)root.querySelector(`input[data-served-by="${d.servedRole}"]`)?.focus();return;} if(d.benchOpt!==undefined){const rec=r(),o=((C.standardBench(rec)||{}).options||{})[d.benchOpt]||{};rec.bench_options={...(rec.bench_options||{})};if(d.benchVal===o.default)delete rec.bench_options[d.benchOpt];else rec.bench_options[d.benchOpt]=d.benchVal;renderForm();refreshPlan();return;} if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check');return;} if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install');return;} @@ -949,6 +1008,7 @@ const el=event.target,d=el.dataset,rec=r(); if(el.id==='nb-recipe-title'){updateSavePreview();return;} if(d.pickFind!==undefined){pickOf().q=el.value;applyPick();return;} + if(d.servedBy!==undefined)return; // what is typed in a Served by filters its menu: a pick, or Enter, adds it (nb-pick) if(d.providerModel && rec.modelFormat !== 'provider-id'){for(const key of ['model','userModel','evalModel'])if(typeof rec[key]==='string')rec[key]=C.bareModel(rec[key]);rec.modelFormat='provider-id';} if(d.field){const p=d.field.startsWith('param:')?C.parameters().find(p=>p.key===d.field.slice(6)):null; const value=el.type==='number'?(el.value===''?'':Number(el.value)):el.value; @@ -958,6 +1018,7 @@ else if(!d.providerModel&&['model','userModel','evalModel'].includes(d.field)&&/^(openrouter|openai)\//.test(String(value))){rec.provider=C.providerOf(rec,value);rec[d.field]=C.bareModel(value); renderForm();refreshPlan();const f=root.querySelector(`[data-field="${d.field}"]`);if(f){f.focus();f.setSelectionRange(f.value.length,f.value.length);}return;} // the provider the prefix named, and whether it is set else if(d.field==='provider'){C.setChoice(rec,value);renderForm();refreshPlan();return;} // whether it is set in Settings, and the names a runner's models are sent as, follow the provider + else if(['model','simulator_model','judge_model'].includes(d.field)){C.setModelOf(rec,d.field,value);syncServed();} // the field's pin goes with its model; its Served by follows, in place else rec[d.field]=value; if(el.list?.id==='nb-paths')completeDirectory(el.value); }else if(d.minutes){C.setPhaseMinutes(rec,d.minutes,el.value===''?'':Number(el.value));const row=el.closest('tr');if(row){const m=row.querySelector('.nb-mono');const t=C.effectiveTimeouts(rec);const p=t&&t.phases.find(x=>x.key===d.minutes);if(m&&p)m.textContent='×'+p.multiplier;}refreshPlan();return;} @@ -975,6 +1036,8 @@ const el=event.target; if(el.type==='number'&&el.dataset.field&&el.validity&&el.validity.badInput){el.value='';window.nulliusNotify?.(`${el.getAttribute('aria-label')}: enter a number.`,{error:true,key:'bad-number'});} }); + // a provider picked in a Served by, or slugs typed there and Enter: they join its list, after the ones there + root.addEventListener('nb-pick',event=>{const role=event.target.dataset.servedBy;if(!role)return;C.addServed(r(),role,event.detail);syncServed();refreshPlan();}); root.addEventListener('change',async event=>{ const el=event.target,d=el.dataset; if(d.custom?.endsWith(':type')){$('#nb-parameters').innerHTML=customRows(!['common','extra'].includes(tab)?tab:'');} diff --git a/app/static/builder/model-picker.js b/app/static/builder/model-picker.js index 0dd6e1ac..25c64740 100644 --- a/app/static/builder/model-picker.js +++ b/app/static/builder/model-picker.js @@ -1,9 +1,37 @@ // Provider model IDs stay verbatim; browsing a catalogue never chooses a model. +// The same picker serves a model's Served by (list="nb-served"): its rows are the providers of the model the field +// names (data-served-model), and a field of several takes each pick, or a slug typed as is, into its list. (() => { + // the rows a typed query keeps: the query in the id or the name, case aside; the ones already chosen left out + const matching = (items, query, chosen = []) => items.filter(m => !chosen.includes(m.id) && (m.id + ' ' + m.name).toLowerCase().includes(query.toLowerCase())); + // what Enter takes: the highlighted row, else, in a field of several, the text as typed (a slug the list may lack) + const entered = (items, index, text, several) => items[index] || (several && text.trim() ? {id: text.trim(), typed: true} : null); + // The row and its menu share one request per model for five minutes, including failures. An in-flight request + // stays shared until it settles; result is absent while the row is waiting to learn how many entries there are. + function servingProviders(request, csrf) { + const cache = new Map(), listeners = new Set(); + const providers = model => { + let entry = cache.get(model); + if (!entry || (entry.result && Date.now() - entry.at > 300000)) { + entry = {at: Date.now(), result: null}; + entry.promise = request('/new-run/endpoints', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-CSRF-Token': csrf}, body: JSON.stringify({provider: 'openrouter', model})}) + .then(r => r.json()).then(r => ({...r, models: (Array.isArray(r.endpoints) ? r.endpoints : []).map(e => ({id: e.slug, name: e.name + (e.quantization ? ' · ' + e.quantization : '')}))})) + .catch(() => ({ok: false, models: [], message: "Provider list unavailable. Enter a provider's slug by hand."})) + .then(result => { entry.at = Date.now(); entry.result = result; listeners.forEach(listener => listener(model)); return result; }); + cache.set(model, entry); + } + return entry; + }; + providers.subscribe = listener => listeners.add(listener); + return providers; + } + if (typeof module === 'object' && module.exports) { module.exports = {matching, entered, servingProviders}; return; } const root = document.getElementById('nullius-builder'); if (!root) return; const schema = JSON.parse(document.getElementById('nb-schema').textContent); + const providers = window.NulliusServingProviders = servingProviders(nulliusFetch, schema.csrf); const cache = new Map(); let active = null, serial = 0, choosing = false; + const severalOf = input => input.dataset.pickerSeveral === 'true'; function close() { if (!active) return; active.input.setAttribute('aria-expanded', 'false'); @@ -28,25 +56,33 @@ const row = rows[active.index]; if (row) { active.input.setAttribute('aria-activedescendant', row.id); row.scrollIntoView({block: 'nearest'}); } } - function choose(model) { - const input = active.input; - close(); choosing = true; input.value = model.id; - input.dispatchEvent(new Event('input', {bubbles: true})); - input.dispatchEvent(new Event('change', {bubbles: true})); + function choose(model, input = active.input) { + close(); choosing = true; + // a field of several hands the pick to the page, which lists it, and is left empty for the next one + if (severalOf(input)) { input.value = ''; input.dispatchEvent(new CustomEvent('nb-pick', {bubbles: true, detail: model.id})); } + else { + input.value = model.id; + input.dispatchEvent(new Event('input', {bubbles: true})); + input.dispatchEvent(new Event('change', {bubbles: true})); + } input.focus(); choosing = false; } function paint() { if (!active) return; - const {menu, input, result, query} = active; + const {menu, input, result, query, several} = active; menu.replaceChildren(); active.items = []; active.index = -1; const status = document.createElement('div'); status.className = 'nb-model-status'; status.setAttribute('role', 'status'); - if (!result) status.textContent = 'Loading models…'; + // no model yet, or the one the run sends not yet known (the server compiles it): no list is asked + if (several && !input.dataset.servedModel) status.textContent = input.dataset.servedPending === 'true' ? 'Loading providers…' : 'Choose the model first'; + else if (!result) status.textContent = several ? 'Loading providers…' : 'Loading models…'; else if (result.ok && input.dataset.field === 'model' && JSON.parse(root.dataset.paperModels || 'null') !== null && !result.models.length) status.textContent = 'No paper models available from this provider.'; - else if (!result.ok) status.textContent = result.message || 'Model list unavailable. Enter a model ID manually.'; + else if (!result.ok) status.textContent = result.message || (several ? "Provider list unavailable. Enter a provider's slug by hand." : 'Model list unavailable. Enter a model ID manually.'); else { - const matches = result.models.filter(m => (m.id + ' ' + m.name).toLowerCase().includes(query.toLowerCase())); + const matches = matching(result.models, query, several ? (input.dataset.chosen || '').split(',').filter(Boolean) : []); active.items = matches.slice(0, 100); - status.textContent = matches.length ? `${matches.length} model${matches.length === 1 ? '' : 's'}${matches.length > 100 ? ' · Type to narrow the list' : ''}` : 'No matching models. You can enter an exact model ID.'; + const noun = several ? 'provider' : 'model'; + status.textContent = matches.length ? `${matches.length} ${noun}${matches.length === 1 ? '' : 's'}${matches.length > 100 ? ' · Type to narrow the list' : ''}` + : several ? (result.models.length ? 'No matching provider. Enter takes the slug as typed.' : 'No provider listed: type a slug') : 'No matching models. You can enter an exact model ID.'; } menu.append(status); active.items.forEach((model, i) => { @@ -75,11 +111,18 @@ async function open(input, all = false) { close(); const menu = document.createElement('div'); menu.className = 'ui-select-menu nb-model-menu'; menu.id = input.getAttribute('aria-controls'); - menu.setAttribute('popover', 'manual'); menu.setAttribute('role', 'listbox'); menu.setAttribute('aria-label', 'Provider models'); + const several = severalOf(input); + menu.setAttribute('popover', 'manual'); menu.setAttribute('role', 'listbox'); menu.setAttribute('aria-label', several ? 'Serving providers' : 'Provider models'); document.body.append(menu); - const state = {input, menu, wrap: input.parentElement, provider: root.dataset.modelProvider, query: all ? '' : input.value, result: null, items: [], index: -1}; + const state = {input, menu, wrap: input.parentElement, several, provider: root.dataset.modelProvider, model: input.dataset.servedModel || '', query: all ? '' : input.value, result: null, items: [], index: -1}; active = state; input.setAttribute('aria-expanded', 'true'); paint(); if (menu.showPopover) menu.showPopover(); + if (several) { + if (!state.model) return; + const result = await providers(state.model).promise; + if (active === state) { state.result = result; paint(); } + return; + } const paper = input.dataset.field === 'model' ? JSON.parse(root.dataset.paperModels || 'null') : null; const result = await models(state.provider); state.result = paper === null || !result.ok ? result : {...result, models:Object.entries(paper) @@ -88,24 +131,34 @@ if (active === state) paint(); } function scan() { - if (active && (!active.input.isConnected || active.provider !== root.dataset.modelProvider)) close(); - root.querySelectorAll('input[list="nb-models"]').forEach(input => { - input.removeAttribute('list'); input.dataset.providerModel = 'true'; input.autocomplete = 'off'; + if (active && (!active.input.isConnected || active.input.closest('[hidden]') || active.provider !== root.dataset.modelProvider)) close(); + // the model a Served by lists the providers of has moved (the server named the one the run sends): its rows follow + else if (active && active.several && active.model !== (active.input.dataset.servedModel || '')) void open(active.input, !active.query); + root.querySelectorAll('input[list="nb-models"], input[list="nb-served"]').forEach(input => { + if (input.getAttribute('list') === 'nb-served') input.dataset.pickerSeveral = 'true'; else input.dataset.providerModel = 'true'; + input.removeAttribute('list'); input.autocomplete = 'off'; input.setAttribute('role', 'combobox'); input.setAttribute('aria-autocomplete', 'list'); input.setAttribute('aria-expanded', 'false'); input.setAttribute('aria-controls', 'provider-models-' + (++serial)); const wrap = document.createElement('span'); wrap.className = 'nb-model-control'; input.before(wrap); wrap.append(input); - const button = document.createElement('button'); button.type = 'button'; button.className = 'nb-model-open'; button.textContent = '⌄'; button.setAttribute('aria-label', 'Browse provider models'); + const button = document.createElement('button'); button.type = 'button'; button.className = 'nb-model-open'; button.textContent = '⌄'; + button.setAttribute('aria-label', severalOf(input) ? 'Browse the serving providers' : 'Browse provider models'); button.addEventListener('click', e => { e.preventDefault(); input.focus(); void open(input, true); }); wrap.append(button); input.addEventListener('focus', () => { if(!choosing)void open(input); }); + // a field of several keeps the keyboard after a pick: a click on it opens its menu again for the next one + if (severalOf(input)) input.addEventListener('click', () => { if(active?.input !== input)void open(input); }); input.addEventListener('input', () => { if(choosing)return; if (active?.input === input) { active.query = input.value; paint(); } else void open(input); }); input.addEventListener('keydown', e => { if (e.key === 'Escape' || e.key === 'Tab') { if(e.key === 'Escape' && active){e.preventDefault();e.stopPropagation();} close(); return; } if (e.key === 'ArrowDown' || e.key === 'ArrowUp') { e.preventDefault(); if(active?.input !== input)void open(input, true); else highlight(active.index + (e.key === 'ArrowDown' ? 1 : -1)); } - if (e.key === 'Enter' && active?.input === input && active.items[active.index]) { e.preventDefault(); choose(active.items[active.index]); } + if (e.key === 'Enter') { + const pick = entered(active?.input === input ? active.items : [], active?.input === input ? active.index : -1, input.value, severalOf(input)); + if (!pick) return; + e.preventDefault(); choose(pick, input); + } }); }); } - new MutationObserver(scan).observe(root, {childList: true, subtree: true, attributes: true, attributeFilter: ['data-model-provider']}); + new MutationObserver(scan).observe(root, {childList: true, subtree: true, attributes: true, attributeFilter: ['data-model-provider', 'data-served-model']}); document.addEventListener('pointerdown', e => { if(active && !active.wrap.contains(e.target) && !active.menu.contains(e.target))close(); }); document.addEventListener('scroll', e => { if(active && !active.menu.contains(e.target))close(); }, true); window.addEventListener('resize', close); diff --git a/app/templates/builder/new_run.html b/app/templates/builder/new_run.html index 8b4fd664..39075d68 100644 --- a/app/templates/builder/new_run.html +++ b/app/templates/builder/new_run.html @@ -50,6 +50,6 @@

Give this recipe a name.

{% endblock %} {% block scripts %} - + {% endblock %} diff --git a/app/tests/test_provider_pins.py b/app/tests/test_provider_pins.py new file mode 100644 index 00000000..c0f2c7d6 --- /dev/null +++ b/app/tests/test_provider_pins.py @@ -0,0 +1,107 @@ +"""Serving pins alongside the protocol registry and the relay's route authentication.""" +import io +import json +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest.mock import Mock, patch + +from nullius import providers +from nullius.standard import cli, recipe, relay, run +from tests import test_standard_recipe_served_by as recipes +from tests import test_standard_served_by as calls + + +class RegistryPins(unittest.TestCase): + def setUp(self): + self.tmp = self.enterContext(tempfile.TemporaryDirectory()) + self.defs = recipes.defs() + self.bench = recipes.fakebench() + self.agent = next(a for a in self.defs["agent"] if a["key"] == "fake") + self.agent = dict(self.agent, model={**self.agent.get("model", {}), "apis": ["messages"]}) + self.defs["agent"] = [self.agent if a["key"] == "fake" else a for a in self.defs["agent"]] + + def test_openrouter_protocol_checks_and_serving_pins_both_apply(self): + env = {"OPENROUTER_API_KEY": "fixture-key"} + options = dict(model="m", provider="openrouter", served_by={"m": ["prov-a"]}, + env=env, unsandboxed=True, jobs_root=self.tmp) + with self.assertRaisesRegex(run.Refused, "does not serve"): + run.Run(self.bench, self.agent, **options) + env["OPENROUTER"] = json.dumps({"endpoints": {"messages": "/messages"}}) + started = run.Run(self.bench, self.agent, **options) + self.assertEqual((started.base, started.key_header, started.serving), + (providers.PRESETS["openrouter"]["base"], "x-api-key", {"m": ["prov-a"]})) + with patch.dict(os.environ, env): + compiled = recipe.compile(recipes.a_recipe(model="m", served_by={"model": ["prov-a"]}), self.defs) + self.assertEqual(compiled["errors"], []) + self.assertEqual(compiled["served_models"]["model"], "m") + self.assertIn("--served-by m=prov-a", compiled["script"]) + + def test_a_named_custom_protocol_provider_still_cannot_take_a_pin(self): + for base in ("https://gateway.example/v1", "https://openrouter.ai/api/v1"): + env = {"MODEL_GATEWAY": json.dumps({"base": base, "endpoints": {"messages": "/messages"}}), + "MODEL_GATEWAY_API_KEY": "fixture-key"} + self.assertEqual(providers.protocol_for("custom:GATEWAY", ["chat", "messages"], env), "messages") + handed = cli.run_env("custom", {"OPENAI_BASE_URL": providers.base_of("MODEL_GATEWAY_API_URL", env), + "OPENAI_API_KEY": env["MODEL_GATEWAY_API_KEY"]}) + with self.subTest(base=base), self.assertRaisesRegex(run.Refused, "no serving providers"): + run.Run(self.bench, self.agent, model="m", provider="custom", env=handed, + served_by={"m": ["prov-a"]}, unsandboxed=True, jobs_root=self.tmp) + with patch.dict(os.environ, env): + compiled = recipe.compile(recipes.a_recipe(provider="custom", endpoint="GATEWAY", model="m", + served_by={"model": ["prov-a"]}), self.defs) + self.assertTrue(any("Served by needs OpenRouter" in e["message"] for e in compiled["errors"])) + out = io.StringIO() + with patch.object(cli, "_bench", return_value=self.bench), patch.object(cli, "_other", return_value=self.agent), \ + patch.object(cli, "_exit_on_term"), patch.object(cli.settings, "load"), \ + patch.dict(os.environ, {"OPENAI_BASE_URL": base, "OPENAI_API_KEY": "fixture-key"}), redirect_stdout(out): + code = cli.main(["run", "fakebench", "fake", "--provider", "custom", "--from-run", "--model", "m", + "--served-by", "prov-a", "--unsandboxed", "--jobs", self.tmp]) + self.assertEqual(code, 2) + self.assertIn("no serving providers", out.getvalue()) + + +class ProtocolRouting(unittest.TestCase): + def test_pinned_agent_and_benchmark_routes_keep_their_own_authentication(self): + pins = {m: ["prov-a"] for m in ("m", "judge", "simulator", "embedding")} + folder = self.enterContext(tempfile.TemporaryDirectory()) + r = relay.Relay("https://openrouter.ai/api/v1", "fixture-key", folder, + model="m", key_header="x-api-key", serving=pins) + self.addCleanup(r.stop) + r.public = "http://relay" + pool = self.enterContext(patch.object(r, "pool")) + routes = [("unit", None, None, "/messages"), ("unit-judge", "judge", None, "/chat/completions"), + ("unit-simulator", "simulator", None, "/responses"), ("search", None, "embedding", "/embeddings")] + for tag, model, embedding, path in routes: + with self.subTest(path=path): + r.url(tag, model=model, embeddings=embedding) + raw = json.dumps(dict(calls.body_for(path, False), provider={"sort": "price"})).encode() + handler = Mock(path=f"/{tag}/v1{path}", command="POST", headers={"Content-Length": str(len(raw))}, + rfile=io.BytesIO(raw), wfile=io.BytesIO()) + body = json.dumps(calls.SHAPES.get((path, False), (calls.CHAT,))[0]).encode() + response = Mock(status=200, headers={"Content-Type": "application/json"}) + response.getheader.return_value = "application/json" + response.read.return_value = body + pool.send.return_value = (Mock(), response) + r._forward(handler) + headers = {k.lower(): v for k, v in pool.send.call_args.args[3].items()} + if tag == "unit": + self.assertEqual(headers["x-api-key"], "fixture-key") + self.assertEqual(headers["anthropic-version"], relay.ANTHROPIC_VERSION) + self.assertNotIn("authorization", headers) + else: + self.assertEqual(headers["authorization"], "Bearer fixture-key") + self.assertNotIn("x-api-key", headers) + self.assertNotIn("anthropic-version", headers) + self.assertEqual(headers.get("x-openrouter-metadata"), + "enabled" if path in ("/chat/completions", "/responses") else None) + record = r.calls(tag)[-1] + self.assertEqual(record["request"]["routing"], relay.routing(["prov-a"])) + self.assertEqual(record["request"]["routing_asked"], {"sort": "price"}) + if path != "/embeddings": + self.assertTrue(record["reply"].get("served_by")) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_recipe_served_by.py b/app/tests/test_standard_recipe_served_by.py new file mode 100644 index 00000000..a24a99e6 --- /dev/null +++ b/app/tests/test_standard_recipe_served_by.py @@ -0,0 +1,180 @@ +"""The builder's Served by: a recipe's ``served_by`` (the serving providers +each model field pins, on OpenRouter) is checked as ``--served-by`` is and +compiled into it for each job that calls that model, by the name the run +sends it as; a recipe with none compiles to the command it did. The model's +providers come from OpenRouter's endpoint list, asked as the model list is.""" +from __future__ import annotations + +import io +import json +import os +import unittest +import urllib.error + +from nullius import providers +from nullius.standard import definitions, recipe + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +NOP = {"nop": {"source": "nullius", "from": "nothing said"}} + + +def defs(bench=None): + d = {k: definitions.load(k, ROOT)[0] for k in ("benchmark", "agent", "capability")} + if bench is not None: + d["benchmark"] = [bench if x["key"] == bench["key"] else x for x in d["benchmark"]] + return d + + +def fakebench(**kw): + return dict(definitions.find(definitions.load("benchmark", ROOT)[0], "fakebench"), **kw) + + +def judged(): + b = fakebench(controls=NOP) + return dict(b, score=dict(b["score"], judge={"model": "gpt-4o-mini", "says": "a judge"})) + + +def a_recipe(**kw): + r = {"version": 1, "harness": "standard", "name": "fake-0101-120000", "title": "t", "benchmark": "fakebench", "dataset": "all", + "other": "fake", "provider": "openrouter", "model": "openai/gpt-5-mini", "limit": 2, "concurrency": 2, "unsandboxed": True, + "where": "local"} + r.update(kw) + return r + + +class TheRecipe(unittest.TestCase): + def test_a_pin_compiles_to_the_option_and_shows_in_the_review(self): + out = recipe.compile(a_recipe(served_by={"model": ["deepinfra", "together/fp8"]}), defs()) + self.assertEqual(out["errors"], []) + self.assertIn("--served-by openai/gpt-5-mini=deepinfra,together/fp8", out["script"]) + model_row = next(row[2] for row in out["review"] if row[1] == "Model") + self.assertIn("model served by deepinfra, together/fp8", model_row) + + def test_each_job_is_pinned_where_it_calls_the_model(self): + """A control beside the agent calls no model of its own: its job takes no pin for the run's model; a judge + pin goes on every job of a judged benchmark, the control's included, by the name OpenRouter serves it as.""" + d = defs(judged()) + out = recipe.compile(a_recipe(others=["fake", "nop"], served_by={"model": ["deepinfra"], "judge": ["prov-j"]}), d) + self.assertEqual(out["errors"], []) + scripts = [p["script"] for p in out["parts"]] + self.assertIn("--served-by openai/gpt-5-mini=deepinfra", scripts[0]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[0]) + self.assertNotIn("openai/gpt-5-mini=deepinfra", scripts[1]) + self.assertIn("--served-by openai/gpt-4o-mini=prov-j", scripts[1]) + row = next(r for r in out["review"] if r[1] == "Model")[2] + self.assertIn("judge served by prov-j", row) + + def test_a_pin_is_refused_as_the_option_is(self): + for kw, said in (({"provider": "openai", "model": "gpt-5-mini", "served_by": {"model": ["deepinfra"]}}, "needs OpenRouter"), + ({"served_by": {"model": ["bad slug"]}}, "not a serving provider's slug"), + ({"served_by": {"model": ["deepinfra", "deepinfra"]}}, "named twice"), + ({"served_by": {"agent": ["deepinfra"]}}, "Served by names"), + ({"served_by": ["deepinfra"]}, "Served by names"), + ({"served_by": {"judge": ["prov-j"]}}, "no job of this recipe calls the judge's model"), + ({"served_by": {"model": "deepinfra"}}, "Served by")): + with self.subTest(kw=kw): + out = recipe.compile(a_recipe(**kw), defs()) + self.assertTrue(out["errors"], out["errors"]) + self.assertTrue(all(e["step"] == recipe.STEP["model"] for e in out["errors"]), out["errors"]) + self.assertTrue(any(said in e["message"] for e in out["errors"]), out["errors"]) + self.assertEqual(out["script"], "") + + def test_one_model_in_two_fields_takes_one_list(self): + d = defs(judged()) + same = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["a"]}), d) + self.assertEqual(same["errors"], []) + lines = [line for line in same["script"].splitlines() if "--served-by" in line] + self.assertTrue(lines and all(line.count("--served-by") == 1 for line in lines)) # each way of starting it: once + other = recipe.compile(a_recipe(model="openai/gpt-4o-mini", served_by={"model": ["a"], "judge": ["b"]}), d) + self.assertIn("pins openai/gpt-4o-mini twice", other["errors"][0]["message"]) + + def test_a_pin_is_its_models_on_every_job_that_calls_it(self): + """The model field's model is also the judge's: the control beside the agent calls it as its judge, and its job + takes the pin too.""" + out = recipe.compile(a_recipe(others=["fake", "nop"], model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), defs(judged())) + self.assertEqual(out["errors"], []) + for part in out["parts"]: + self.assertIn("--served-by openai/gpt-4o-mini=prov-a", part["script"]) + + def test_a_list_with_an_empty_or_a_bad_slug_is_refused_in_sight(self): + for slugs in (["deepinfra", ""], ["deepinfra", "bad slug"]): + with self.subTest(slugs=slugs): + out = recipe.compile(a_recipe(served_by={"model": slugs}), defs()) + self.assertTrue(any("not a serving provider's slug" in e["message"] for e in out["errors"]), out["errors"]) + + def test_each_fields_model_is_given_as_the_run_sends_it(self): + """Like the paper, LongMemEval's gpt-4o is sent as openai/gpt-4o-2024-08-06 on OpenRouter: its providers are that + model's, and the pin is put on that name.""" + sets = definitions.find(definitions.load("benchmark")[0], "longmemeval")["datasets"] + r = a_recipe(benchmark="longmemeval", dataset=sets[0]["id"], other="", others=["original"], model_choice="paper", model="gpt-4o", + served_by={"model": ["openai"]}) + out = recipe.compile(r) + sent = out["served_models"]["model"] + self.assertEqual(sent, "openai/gpt-4o-2024-08-06") + self.assertIn(f"--served-by {sent}=openai", out["script"]) + self.assertEqual(recipe.compile(a_recipe(), defs())["served_models"]["model"], "openai/gpt-5-mini") + + def test_a_recipe_with_no_pin_compiles_to_the_command_it_did(self): + plain = recipe.compile(a_recipe(), defs()) + self.assertEqual(plain["errors"], []) + self.assertNotIn("--served-by", plain["script"]) + for none in ({}, {"model": []}, None): + with self.subTest(none=none): + self.assertEqual(recipe.compile(a_recipe(served_by=none), defs())["script"], plain["script"]) + openai = recipe.compile(a_recipe(provider="openai", model="gpt-5-mini", served_by={}), defs()) + self.assertEqual(openai["errors"], []) + + +class TheProviders(unittest.TestCase): + """``providers.list_endpoints``: OpenRouter's endpoint list, a fake opener in the place of the network.""" + + def opener(self, body=None, error=None): + seen = [] + + def opener(req, timeout=None): + seen.append(req) + if error: + raise error + return io.BytesIO(json.dumps(body).encode()) + return opener, seen + + def test_a_models_providers_by_slug_and_name(self): + body = {"data": {"id": "meta-llama/llama-3.3-70b-instruct", "endpoints": [ + {"provider_name": "DeepInfra", "tag": "deepinfra/fp8", "quantization": "fp8"}, + {"provider_name": "Together", "tag": "together"}, + {"provider_name": "Google AI Studio"}, # no tag: its name as a slug reads + {"provider_name": "Bad", "tag": "bad slug"}, "not a row", {"tag": 7}]}} + opener, seen = self.opener(body) + got = providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + self.assertEqual(got, [{"slug": "deepinfra/fp8", "name": "DeepInfra", "quantization": "fp8"}, + {"slug": "google-ai-studio", "name": "Google AI Studio", "quantization": ""}, + {"slug": "together", "name": "Together", "quantization": ""}]) + (req,) = seen + self.assertEqual(req.full_url, "https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints") + self.assertIsNone(req.get_header("Authorization")) # no key set: the list is asked without one + self.assertEqual(req.get_method(), "GET") + + def test_the_key_goes_as_for_the_model_list_and_a_bare_name_gets_its_maker(self): + opener, seen = self.opener({"data": {"endpoints": []}}) + providers.list_endpoints("openrouter", "gpt-4o-mini", env={"OPENROUTER_API_KEY": "or-test"}, opener=opener) + self.assertEqual(seen[0].full_url, "https://openrouter.ai/api/v1/models/openai/gpt-4o-mini/endpoints") + self.assertEqual(seen[0].get_header("Authorization"), "Bearer or-test") + + def test_what_cannot_be_asked_or_read_is_said_in_fixed_words(self): + for kw in ({"error": urllib.error.URLError("refused at https://openrouter.ai?key=x")}, {"body": {"error": "no"}}, + {"body": {"data": {"endpoints": "none"}}}): + with self.subTest(kw=kw): + opener, _ = self.opener(**kw) + with self.assertRaisesRegex(ValueError, r"^Provider list unavailable\. Enter a provider's slug by hand\.$"): + providers.list_endpoints("openrouter", "meta-llama/llama-3.3-70b-instruct", env={}, opener=opener) + opener, seen = self.opener({}) + with self.assertRaisesRegex(ValueError, "Only OpenRouter"): + providers.list_endpoints("openai", "gpt-4o", env={"OPENAI_API_KEY": "x"}, opener=opener) + for model in ("no-maker", "a/b/../../c", "maker/model?x=1", ""): + with self.subTest(model=model), self.assertRaisesRegex(ValueError, "maker/model"): + providers.list_endpoints("openrouter", model, env={}, opener=opener) + self.assertEqual(seen, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_served_by.py b/app/tests/test_standard_served_by.py new file mode 100644 index 00000000..c895787e --- /dev/null +++ b/app/tests/test_standard_served_by.py @@ -0,0 +1,769 @@ +"""A pin of the serving providers that may answer a model (``--served-by``): +the relay puts OpenRouter's routing field on every request it sends as that +model and on no other, in place of the caller's own; each call records the +routing sent, the provider that served the reply, how the reply ended and an +error inside a success, as cleaned copies, read apart from the rest of the +record; the job and each trial say who served the calls of every route and +whether the pin held; ``merge`` refuses shards of different pins; a +container's relay is given the pin as the unit's own relay is. A run without +the option sends the same requests and ends its units as it did. Every +upstream here is a fake on this machine's loopback: no request leaves it.""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import select +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +import urllib.request +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest import mock + +from nullius import progress +from nullius.standard import cli, container, definitions, install, job, merge, relay, run + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIN = {"order": ["prov-a", "prov-b/fp8"], "allow_fallbacks": False} +KEY = "or-test-relay-key-0123456789" +NEW = ("served_by", "end", "error") # what the record's reply gains, beside what it held + + +def sse(*events) -> bytes: + return b"".join(b"data: " + json.dumps(e).encode() + b"\n\n" for e in events) + b"data: [DONE]\n\n" + + +def routing_facts(selected, *others) -> dict: + """OpenRouter's routing facts (``openrouter_metadata``) as a reply holds them when asked: the endpoint that + served it is the one marked ``selected`` among the available ones.""" + available = [{"provider": o, "model": "asked", "selected": False} for o in others] + if selected is not None: + available.insert(len(available) // 2, {"provider": selected, "model": "asked", "selected": True}) + return {"requested": "asked", "attempt": 1, "summary": f"available={len(available)}, selected={selected}", + "endpoints": {"total": len(available) + 2, "available": available}} + + +# Each reply names its serving provider and how it ended, as OpenRouter's do: Chat Completions and Messages of +# themselves; Responses in the routing facts alone, given when the request asks for them (``ASKED``). +CHAT = {"id": "c1", "provider": "Prov A", "choices": [{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}} +CHAT_SSE = sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "h"}, "finish_reason": None}]}, + {"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "i"}, "finish_reason": "length"}]}, + {"id": "c1", "provider": "Prov A", "choices": [], "usage": {"prompt_tokens": 3, "completion_tokens": 1}}) +MESSAGES = {"type": "message", "provider": "Prov B", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 1}} +MESSAGES_SSE = sse({"type": "message_start", "message": {"type": "message", "provider": "Prov B", "usage": {"input_tokens": 3}}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 1}}, + {"type": "message_stop"}) +RESPONSES = {"object": "response", "status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "hi"}]}], + "usage": {"input_tokens": 3, "output_tokens": 1}} +COMPLETED = {"object": "response", "status": "completed", "output": [], "usage": {"input_tokens": 3, "output_tokens": 1}} +RESPONSES_SSE = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": COMPLETED}) +RESPONSES_ASKED = dict(RESPONSES, openrouter_metadata=routing_facts("Prov C", "Prov D", "Prov E")) +RESPONSES_SSE_ASKED = sse({"type": "response.created", "response": {"object": "response", "status": "in_progress"}}, + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov C", "Prov D"))}) +UNASKED = {False: RESPONSES, True: RESPONSES_SSE} +SHAPES = {("/chat/completions", False): (CHAT, "Prov A", "stop"), ("/chat/completions", True): (CHAT_SSE, "Prov A", "length"), + ("/messages", False): (MESSAGES, "Prov B", "end_turn"), ("/messages", True): (MESSAGES_SSE, "Prov B", "max_tokens"), + ("/responses", False): (RESPONSES_ASKED, "Prov C", "incomplete: max_output_tokens"), + ("/responses", True): (RESPONSES_SSE_ASKED, "Prov C", "completed")} + + +def body_for(path: str, stream: bool) -> dict: + """A caller's request of that shape.""" + if path == "/responses": + return {"model": "asked", "input": "hello", "stream": stream} + return {"model": "asked", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5, "stream": stream} + + +def without_new(reply: dict) -> dict: + return {k: v for k, v in reply.items() if k not in NEW} + + +class Upstream(BaseHTTPRequestHandler): + """A provider on the loopback: keeps every request as it came, bytes and headers, and answers it in its shape; + ``Upstream.next`` replaces the next reply (status, content type, body), ``Upstream.always`` every one.""" + seen = [] + next = None + always = None + + def log_message(self, *a): + pass + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + Upstream.seen.append((self.path, {k.lower(): v for k, v in self.headers.items()}, raw)) + req = json.loads(raw) + if Upstream.next is not None or Upstream.always is not None: + (status, kind, body), Upstream.next = Upstream.next or Upstream.always, None + else: + path = "/" + self.path.split("/v1/", 1)[-1] + reply = SHAPES.get((path, bool(req.get("stream"))), (CHAT,))[0] + if path == "/responses" and self.headers.get("X-OpenRouter-Metadata") != "enabled": + reply = UNASKED[bool(req.get("stream"))] + status, kind, body = 200, "text/event-stream" if req.get("stream") else "application/json", \ + reply if isinstance(reply, bytes) else json.dumps(reply).encode() + self.send_response(status) + self.send_header("Content-Type", kind) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def post(url: str, body: dict, headers=None) -> bytes: + req = urllib.request.Request(url, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer " + relay.PLACEHOLDER_KEY, + **(headers or {})}) + with urllib.request.urlopen(req, timeout=10) as r: + return r.read() + + +class WithUpstream(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.base = f"http://127.0.0.1:{cls.up.server_address[1]}/v1" + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + cls.up.server_close() + + def setUp(self): + self.tmp = tempfile.mkdtemp() + Upstream.seen, Upstream.next, Upstream.always = [], None, None + self.relays = [] + self.addCleanup(setattr, Upstream, "always", None) + + def tearDown(self): + for r in self.relays: + r.stop() + shutil.rmtree(self.tmp, ignore_errors=True) + + def relay(self, **kw): + r = relay.Relay(self.base, kw.pop("key", "or-test"), os.path.join(self.tmp, f"rec{len(self.relays)}"), **kw).start() + self.relays.append(r) + return r + + def sent(self, i=-1) -> dict: + return json.loads(Upstream.seen[i][2]) + + +class TheRelay(WithUpstream): + def test_the_pin_goes_on_every_request_sent_as_the_pinned_model_and_on_no_other(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0-judge", model="m-judge") + "/chat/completions", body_for("/chat/completions", False)) + post(r.url("u0") + "/messages", body_for("/messages", True)) + self.assertEqual([(json.loads(raw)["model"], json.loads(raw).get("provider")) for _, _, raw in Upstream.seen], + [("m-run", PIN), ("m-judge", None), ("m-run", PIN)]) + self.assertEqual([c["request"].get("routing") for c in r.calls("u0")], [PIN, PIN]) + self.assertNotIn("routing", r.calls("u0-judge")[0]["request"]) + + def test_a_callers_own_routing_is_replaced_and_both_are_recorded(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a", "prov-b/fp8"]}) + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"sort": "price"})) + self.assertEqual(self.sent()["provider"], PIN) + (c,) = r.calls("u0") + self.assertEqual((c["request"]["routing"], c["request"]["routing_asked"]), (PIN, {"sort": "price"})) + + def test_without_a_pin_a_callers_routing_goes_on_as_it_came_and_is_recorded(self): + r = self.relay(model="m-run") + post(r.url("u0") + "/chat/completions", dict(body_for("/chat/completions", False), provider={"only": ["prov-z"]})) + self.assertEqual(self.sent()["provider"], {"only": ["prov-z"]}) + (c,) = r.calls("u0") + self.assertEqual(c["request"]["routing"], {"only": ["prov-z"]}) + self.assertNotIn("routing_asked", c["request"]) + + def test_the_recorded_copies_are_cleaned_and_what_is_sent_is_not(self): + """An error that echoes the request's credentials, and a routing object holding a token: the record keeps + neither the relay's key nor the token; the provider receives the caller's bytes as they were.""" + Upstream.always = (200, "application/json", json.dumps({"error": {"code": 401, "message": f"Authorization: Bearer {KEY}"}}).encode()) + for serving in (None, {"m-run": ["prov-a"]}): + with self.subTest(serving=serving): + r = self.relay(model="m-run", key=KEY, serving=serving) + asked = dict(body_for("/chat/completions", False), provider={"order": ["prov-a"], "token": KEY, "note": f"key {KEY}", KEY: "as a key"}) + post(r.url("u0") + "/chat/completions", asked) + c = r.calls("u0")[-1] + self.assertNotIn(KEY, json.dumps(c)) + self.assertEqual(c["reply"]["error"]["code"], 401) + if serving is None: # sent as the caller sent it: the record alone is cleaned + self.assertEqual(self.sent()["provider"], asked["provider"]) + self.assertEqual(c["request"]["routing"]["token"], "[REDACTED]") + else: + self.assertEqual(c["request"]["routing_asked"]["token"], "[REDACTED]") + + def test_each_shape_streamed_or_not_says_who_served_it_and_how_it_ended(self): + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + for (path, stream), (_, who, end) in SHAPES.items(): + with self.subTest(path=path, stream=stream): + post(r.url("u0") + path, body_for(path, stream)) + self.assertEqual(self.sent()["provider"], {"order": ["prov-a"], "allow_fallbacks": False}) + reply = r.calls("u0")[-1]["reply"] + self.assertEqual((reply.get("served_by"), reply.get("end"), reply.get("error")), (who, end, None)) + # what was read before is read as before, beside the new facts + self.assertEqual(without_new(reply), relay.read_reply(path, SHAPES[(path, stream)][0] if stream + else json.dumps(SHAPES[(path, stream)][0]).encode(), stream)) + + def test_several_choices_keep_their_endings_and_errors_by_index(self): + body = {"choices": [{"index": 0, "message": {"content": "ok"}, "finish_reason": "stop"}, + {"index": 1, "message": {}, "finish_reason": "error", "error": {"code": 429, "message": "limited"}}]} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(body).encode(), False), + {"end": {"0": "stop", "1": "error"}, "error": {"code": 429, "message": "limited", "choice": 1}}) + stream = sse({"choices": [{"index": 0, "delta": {}, "finish_reason": "length"}, {"index": 1, "delta": {}, "finish_reason": None}]}, + {"choices": [{"index": 1, "delta": {}, "finish_reason": "stop"}]}) + self.assertEqual(relay.reply_facts("/chat/completions", stream, True)["end"], {"0": "length", "1": "stop"}) + + def test_an_error_beside_a_whole_reply_is_read_in_every_shape(self): + error = {"code": 502, "message": "failed"} + for path, body in (("/messages", {"type": "message", "content": [], "error": error}), + ("/embeddings", {"object": "list", "data": [], "error": error}), + ("/chat/completions", {"choices": [{"message": {"content": "ok"}}], "error": error}), + ("/responses", {"object": "response", "status": "failed", "output": [], "error": error})): + with self.subTest(path=path): + self.assertEqual(relay.reply_facts(path, json.dumps(body).encode(), False)["error"], error) + + def test_a_completed_responses_events_routing_facts_name_who_served_it(self): + body = sse({"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov A", "Prov X"))}) + self.assertEqual(relay.reply_facts("/responses", body, True), {"served_by": "Prov A", "end": "completed"}) + whole = json.dumps(dict(RESPONSES, openrouter_metadata=routing_facts("Prov A", "Prov X", "Prov Y"))).encode() + self.assertEqual(relay.reply_facts("/responses", whole, False)["served_by"], "Prov A") + + def test_facts_of_the_wrong_type_are_left_out_and_never_stop_a_reply(self): + odd = [{"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": {"endpoints": {"available": 1}}}, + {"type": "message", "content": [], "provider": 7, "error": "a string", "stop_reason": ["x"]}, + {"choices": [{"index": "0", "finish_reason": 3, "error": []}], "openrouter_metadata": [1]}, + {"choices": [], "openrouter_metadata": {"endpoints": [1]}}, + {"choices": [], "openrouter_metadata": {"endpoints": {"available": [7, {"provider": 7, "selected": True}, + {"provider": "Prov A", "selected": "true"}]}}}] + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}) + for body in odd: + with self.subTest(body=body): + raw = json.dumps(body).encode() + Upstream.next = (200, "application/json", raw) + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False)), raw) + self.assertEqual({k: v for k, v in r.calls("u0")[-1]["reply"].items() if k in NEW}, {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"[1, 2]", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"not json", False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b"data: [1]\n\ndata: {\"error\": 5}\n\n", True), {}) + + def test_what_the_record_read_before_is_read_as_before(self): + """The facts are read apart: a failed Responses stream's text, tool calls and usage stay as the relay read + them before (it took nothing from a ``response.failed`` event), its error beside them.""" + body = sse({"type": "response.output_text.delta", "delta": "part"}, + {"type": "response.failed", "response": {"object": "response", "status": "failed", + "error": {"code": "server_error", "message": "failed"}, + "output": [{"type": "message", "content": [{"type": "output_text", "text": "x"}]}], + "usage": {"input_tokens": 5, "output_tokens": 2}}}) + self.assertEqual(relay.read_reply("/responses", body, True), {"text": "", "tool_calls": [], "usage": None}) + self.assertEqual(relay.reply_facts("/responses", body, True), + {"end": "failed", "error": {"code": "server_error", "message": "failed"}}) + + def test_an_error_inside_a_200_is_recorded_and_changes_nothing_else(self): + """Recorded beside the reply as it was read; the call keeps its status, no ``error`` of the record's own + is added, and a unit's failure reads it as before (a 200 is no failure).""" + r = self.relay(model="m-run") + errors = [(b'{"error":{"code":502,"message":"Provider returned error"}}', "application/json", + {"code": 502, "message": "Provider returned error"}), + (sse({"id": "c1", "provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "part"}}]}, + {"id": "c1", "error": {"code": "server_error", "message": "upstream ended"}, + "choices": [{"index": 0, "delta": {}, "finish_reason": "error"}]}), "text/event-stream", + {"code": "server_error", "message": "upstream ended"}), + (sse({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), "text/event-stream", + {"type": "overloaded_error", "message": "Overloaded"})] + for body, kind, said in errors: + with self.subTest(said=said): + Upstream.next = (200, kind, body) + streamed = kind == "text/event-stream" + self.assertEqual(post(r.url("u0") + "/chat/completions", body_for("/chat/completions", streamed)), body) + c = r.calls("u0")[-1] + self.assertEqual((c["status"], c["reply"]["error"]), (200, said)) + self.assertNotIn("error", c) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", body, streamed)) + self.assertIsNone(relay.failure(r.calls("u0"))) + + def test_the_routing_facts_are_asked_of_openrouter_alone_and_always_as_enabled(self): + meta = json.dumps(dict({k: v for k, v in CHAT.items() if k != "provider"}, + openrouter_metadata=routing_facts("Prov A", "Prov X"))).encode() + r = self.relay(model="m-run", serving={"m-run": ["prov-a"]}, facts=True) + Upstream.next = (200, "application/json", meta) + post(r.url("u0") + "/chat/completions", body_for("/chat/completions", False), {"X-OpenRouter-Metadata": "disabled"}) + got = [v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"] + self.assertEqual(got, ["enabled"]) + self.assertEqual(r.calls("u0")[-1]["reply"]["served_by"], "Prov A") + for stream in (False, True): # a Responses request: asked too + post(r.url("u0") + "/responses", body_for("/responses", stream), {"X-OpenRouter-Metadata": "disabled"}) + self.assertEqual([v for k, v in Upstream.seen[-1][1].items() if k == "x-openrouter-metadata"], ["enabled"]) + post(r.url("u0") + "/messages", body_for("/messages", False)) # a Messages reply names its provider: nothing asked + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + for kw in ({"serving": {"m-run": ["prov-a"]}}, {"facts": True}): # another provider's address; no pin + with self.subTest(kw=kw): + other = self.relay(model="m-run", **kw) + post(other.url("u1") + "/chat/completions", body_for("/chat/completions", False)) + self.assertNotIn("x-openrouter-metadata", Upstream.seen[-1][1]) + self.assertTrue(relay.Relay("https://openrouter.ai/api/v1", "k", os.path.join(self.tmp, "or")).facts) + self.assertFalse(relay.Relay("https://api.openai.com/v1", "k", os.path.join(self.tmp, "oa")).facts) + + def test_without_a_pin_every_request_goes_on_byte_for_byte_as_before(self): + """What a relay sent before pins existed: the caller's body with the model set (and a streamed Chat + Completions request's usage flag), serialised once, and the caller's headers but its credentials. The + three shapes, streamed or not, with and without a routing field of the caller's, as the run's model, a + side model and a judge's route's model; with no pin, or a pin on another model.""" + for serving in (None, {"m-other": ["prov-a"]}): + r = self.relay(model="m-run", side_models=("m-side",), serving=serving) + for (path, stream) in SHAPES: + for extra in ({}, {"provider": {"order": ["prov-z"]}}): + for url, asked, model in ((r.url("u0"), "asked", "m-run"), (r.url("u0"), "m-side", "m-side"), + (r.url("u0-judge", model="m-judge"), "asked", "m-judge")): + with self.subTest(serving=serving, path=path, stream=stream, extra=extra, model=model): + req = dict(body_for(path, stream), model=asked, **extra) + post(url + path, req, {"X-Caller": "kept"}) + expected = dict(req, model=model) + if stream and path == "/chat/completions": + expected.setdefault("stream_options", {})["include_usage"] = True + sent_path, headers, raw = Upstream.seen[-1] + self.assertEqual(raw, json.dumps(expected).encode()) + self.assertEqual(sent_path, "/v1" + path) + self.assertEqual({k: v for k, v in headers.items() if k not in ("host", "content-length")}, + {"content-type": "application/json", "x-caller": "kept", "accept": "*/*", + "accept-encoding": "identity", "authorization": "Bearer or-test", + "user-agent": headers["user-agent"]}) + + +class TheSidecar(WithUpstream): + def setUp(self): + super().setUp() + self.enterContext(mock.patch.object(container, "_security", "")) # a default rootful engine, never asked + + def test_the_containers_relay_is_given_the_pin_on_its_command_line(self): + argv = container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", base="http://x/v1", + model="m-run", routes={"u0-judge": "m-judge"}, + serving={"m-run": ["prov-a", "prov-b/fp8"], "m-judge": ["prov-c"]}) + got = [argv[i + 1] for i, a in enumerate(argv) if a == "--served-by"] + self.assertEqual(got, ["m-judge=prov-c", "m-run=prov-a,prov-b/fp8"]) + self.assertNotIn("--served-by", container.relay_argv("img", name="r", job="j", app="/app", record="/rec", key_file="/k", + base="http://x/v1", model="m-run", routes={})) + + def test_the_unit_hands_its_pin_to_the_sidecars_command(self): + unit = container.Unit("job", "u0", os.path.join(self.tmp, "rec"), image="img", app=APP, base=self.base, key="or-test", + model="m-run", serving={"m-run": ["prov-a"]}) + ran = [] + + def docker(args, **kw): + ran.append(args) + if "run" in args[:1]: + raise container.Unavailable("no engine in this test") + return subprocess.CompletedProcess(args, 0, "", "") + with mock.patch.object(container, "_docker", docker), self.assertRaises(container.Unavailable): + unit.start() + (started,) = [a for a in ran if a[:1] == ["run"]] + self.assertIn("m-run=prov-a", started[started.index("--served-by") + 1:]) + + def test_the_relays_own_command_refuses_a_pin_it_cannot_take(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + for given in (["m=bad slug"], ["m=prov-a,prov-a"], ["m="], ["m=prov-a", "m=prov-b"], ["prov-a"]): + with self.subTest(given=given): + # a process of its own, bounded: a relay that took the pin would serve until stopped + got = subprocess.run([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", + "--base", self.base, "--record", os.path.join(self.tmp, "rec"), "--key-file", key, + *[w for g in given for w in ("--served-by", g)]], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), capture_output=True, text=True, timeout=20) + self.assertEqual(got.returncode, 2, got.stdout) + self.assertIn("error", json.loads(got.stdout.splitlines()[0])) + + def test_the_relay_as_a_process_of_its_own_puts_the_pin_on_its_requests(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("or-test") + p = subprocess.Popen([sys.executable, "-m", "nullius.standard.relay", "serve", "--listen", "127.0.0.1:0", "--base", self.base, + "--record", os.path.join(self.tmp, "rec"), "--key-file", key, "--model", "m-run", + "--served-by", "m-run=prov-a,prov-b/fp8", "--routing"], + cwd=APP, env=dict(os.environ, PYTHONPATH=APP), stdout=subprocess.PIPE, text=True) + try: + ready, _, _ = select.select([p.stdout], [], [], 20) # its one line, or no more than 20 s + self.assertTrue(ready, "the relay did not say it listens within 20 s") + url = json.loads(p.stdout.readline())["listening"] + post(url + "/u0/v1/chat/completions", body_for("/chat/completions", False)) + self.assertEqual(self.sent()["provider"], PIN) + finally: + p.terminate() + try: + p.wait(10) + except subprocess.TimeoutExpired: + p.kill() + p.wait(10) + p.stdout.close() + + +class TheRun(WithUpstream): + def setUp(self): + super().setUp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = definitions.get("benchmark", "fakebench", ROOT) + self.a = definitions.get("agent", "fake", ROOT) + + def tearDown(self): + install.HOME = self._home + super().tearDown() + + def make(self, **kw): + kw.setdefault("other", self.a) + return run.Run(kw.pop("bench", self.b), kw.pop("other"), model=kw.pop("model", "m-1"), provider=kw.pop("provider", "openrouter"), + env=kw.pop("env", {"OPENROUTER_API_KEY": "or-test"}), unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, **kw) + + def test_the_option_reads_model_and_slugs_and_refuses_what_it_cannot_take(self): + self.assertEqual(run.served_by(["prov-a", "m-judge=prov-b/fp8, prov-c"], "m-run"), + {"m-run": ["prov-a"], "m-judge": ["prov-b/fp8", "prov-c"]}) + for given, said in ((["m-run=prov-a,"], "not a serving provider's slug"), (["m-run=a b"], "not a serving provider's slug"), + (["prov-a,prov-a"], "named twice"), (["prov-a", "m-run=prov-b"], "m-run twice"), + (["m-run="], "not a serving provider's slug")): + with self.subTest(given=given), self.assertRaisesRegex(run.Refused, said): + run.served_by(given, "m-run") + with self.assertRaisesRegex(run.Refused, "MODEL=SLUG"): + run.served_by(["prov-a"], "") + + def test_a_run_refuses_a_pin_it_cannot_take_however_it_is_given(self): + for pin in ({"m-1": []}, {"m-1": ["bad slug"]}, {"m-1": ["prov-a", "prov-a"]}, {"": ["prov-a"]}, {"m-1": "prov-a"}): + with self.subTest(pin=pin), self.assertRaisesRegex(run.Refused, "--served-by"): + self.make(served_by=pin) + + def test_a_provider_that_routes_no_model_to_several_is_refused_before_any_unit(self): + for provider, env in (("openai", {"OPENAI_API_KEY": "sk-x"}), ("anthropic", {"ANTHROPIC_API_KEY": "x"}), + ("custom", {"MODEL_API_URL": self.base})): + with self.subTest(provider=provider), self.assertRaisesRegex(run.Refused, "no serving providers to choose among"): + self.make(provider=provider, env=env, served_by={"m-1": ["prov-a"]}) + + def test_a_model_the_run_does_not_call_is_refused_before_any_unit(self): + with self.assertRaisesRegex(run.Refused, "names m-other, which this run does not call: it calls m-1"): + self.make(served_by={"m-1": ["prov-a"], "m-other": ["prov-b"]}) + nop = definitions.control(dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), "nop") + with self.assertRaisesRegex(run.Refused, "names m-1, which this run does not call: it calls no model"): + self.make(other=nop, bench=dict(self.b, controls={"nop": {"source": "nullius", "from": "nothing said"}}), + served_by={"m-1": ["prov-a"]}) + + def test_the_command_line_refuses_each_with_exit_2(self): + for patch in (mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_other", return_value=self.a), + mock.patch.object(cli, "_exit_on_term"), mock.patch.object(cli.settings, "load"), + mock.patch.object(progress, "ECHO", None)): + patch.start() + self.addCleanup(patch.stop) + for extra, said in ((["--provider", "openai", "--served-by", "prov-a"], "no serving providers"), + (["--served-by", "m-2=prov-a"], "does not call"), + (["--served-by", "m-1=bad slug"], "not a serving provider's slug"), + (["--served-by", "m-1="], "not a serving provider's slug"), + (["--served-by", "prov-a,prov-a"], "named twice"), + (["--served-by", "prov-a", "--served-by", "m-1=prov-b"], "m-1 twice")): + with self.subTest(extra=extra), mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-x", "OPENROUTER_API_KEY": "or-test"}): + out = io.StringIO() + with redirect_stdout(out): + code = cli.main(["run", "fakebench", "fake", "--model", "m-1", "--unsandboxed", "--provider", "openrouter", + "--jobs", os.path.join(self.tmp, "jobs"), *extra]) + self.assertEqual(code, 2) + self.assertTrue(out.getvalue().strip().splitlines()[-1].startswith("refused:"), out.getvalue()) + self.assertIn(said, out.getvalue()) + + def test_a_pinned_run_records_its_pin_and_says_who_served_each_unit(self): + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + with mock.patch.dict(run.providers.PRESETS, {"openrouter": dict(run.providers.PRESETS["openrouter"], base=self.base)}): + job_dir = run.run(b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, unsandboxed=True, + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding", + served_by={"m-1": ["prov-a"]}) + self.assertTrue(Upstream.seen) + self.assertTrue(all(json.loads(raw).get("provider") == {"order": ["prov-a"], "allow_fallbacks": False} + for _, _, raw in Upstream.seen)) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + trials = {} + for d in os.listdir(job_dir): + if "__u" in d: + with open(os.path.join(job_dir, d, "result.json")) as f: + trials[d.split("__")[0]] = json.load(f)["standard"] + n = len(Upstream.seen) + self.assertEqual(trials["model"]["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + self.assertEqual(trials["good"]["served"], {"calls": {}, "unnamed": 0, "held": {"m-1": None}}) # it called no model + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + self.assertTrue(all(c["request"]["routing"]["order"] == ["prov-a"] and c["reply"]["served_by"] == "Prov A" for c in calls)) + with open(os.path.join(job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": n}, "unnamed": 0, "held": {"m-1": True}}) + + def test_a_run_without_the_option_sends_records_and_ends_as_before(self): + """Every request as before, the record's earlier fields as they were read, and the same verdicts, even + when the provider answers a 200 that holds an error: the relay reads that error beside the reply and the + unit ends as it did.""" + b = dict(self.b, datasets=[{"id": "responding", "units": 3}]) + reply = dict({k: v for k, v in CHAT.items() if k != "provider"}, error={"code": 502, "message": "Provider returned error"}) + Upstream.always = (200, "application/json", json.dumps(reply).encode()) + job_dir = run.run(b, self.a, model="m-1", provider="custom", env={"MODEL_API_URL": self.base, "MODEL_API_KEY": "k"}, + unsandboxed=True, jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, dataset="responding") + self.assertTrue(Upstream.seen) + for _, headers, raw in Upstream.seen: + self.assertNotIn("provider", json.loads(raw)) + self.assertNotIn("x-openrouter-metadata", headers) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertNotIn("served_by", json.load(f)["standard"]) + with open(os.path.join(job_dir, "model__u0002", "result.json")) as f: + res = json.load(f) + self.assertNotIn("exception_info", res) # a 200 is no failure: the unit's verdict stands + self.assertEqual(res["verifier_result"]["rewards"]["reward"], 0.0) + self.assertNotIn("served", res["standard"]) # no call answered, no pin: nothing to say + with open(os.path.join(job_dir, "model__u0002", "agent", "calls.jsonl")) as f: + calls = [json.loads(line) for line in f] + for c in calls: + self.assertEqual((c["status"], "error" in c), (200, False)) + self.assertEqual(without_new(c["reply"]), relay.read_reply("/chat/completions", json.dumps(reply).encode(), False)) + self.assertEqual(c["reply"]["error"], {"code": 502, "message": "Provider returned error"}) + self.assertNotIn("routing", c["request"]) + + def test_the_summary_counts_every_route_of_the_unit(self): + """A judge on the pinned model, served by another provider than the agent's: the pin does not hold.""" + r = self.make(judge_model="m-1", served_by={"m-1": ["prov-a"]}) + call = lambda who: {"model": "m-1", "status": 200, "reply": {"served_by": who}} + routes = {"u0000": [call("Prov A")], "u0000-judge": [call("Prov Z")], "u0000-simulator": [call("Prov A")]} + rr = mock.Mock(mcp_listeners=[], unit_image="") + rr.calls.side_effect = lambda tag: routes.get(tag, []) + driver = mock.Mock() + driver.request.side_effect = lambda op, **kw: {} if op == "setup" else {"reward": 1} + with contextlib.ExitStack() as stack: + for name in ("phase", "said_unit", "note_refusal"): + stack.enter_context(mock.patch.object(r, name)) + stack.enter_context(mock.patch.object(r, "driver", return_value=driver)) + stack.enter_context(mock.patch.object(r, "unit_relay", return_value=rr)) + stack.enter_context(mock.patch.object(r, "drive_agent", return_value=("answer", None, {}))) + stack.enter_context(mock.patch.object(run.subprocess, "run")) + stack.enter_context(mock.patch.object(run.progress, "ECHO", None)) + res = r.unit(0, "good") + self.assertEqual(res["standard"]["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + self.assertEqual(res["agent_result"]["n_model_calls"], 1) # the tokens and cost: the agent's calls, as before + + def test_the_benchmarks_own_relay_counts_in_the_job(self): + r = self.make(served_by={"m-1": ["prov-a"]}) + folder = os.path.join(r.job_dir, ".work", "relay-benchmark") + os.makedirs(folder) + with open(os.path.join(folder, "embed-1.jsonl"), "w") as f: + f.write(json.dumps({"model": "m-1", "status": 200, "reply": {"served_by": "Prov Z"}}) + "\n") + bench = job.served(r.benchmark_calls(), r.serving) + job.write_result(r.job_dir, r.name, 0, 1, [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 0, "held": {"m-1": True}}}}], + 1, served=bench) + with open(os.path.join(r.job_dir, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 2, "Prov Z": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + def test_the_containers_unit_is_given_the_runs_pin(self): + with mock.patch.object(container, "unusable", return_value=None): + r = run.Run(self.b, self.a, model="m-1", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, backend="container", + jobs_root=os.path.join(self.tmp, "jobs"), say=lambda s: None, served_by={"m-1": ["prov-a"]}) + r.image, r.setup_driver, r.licences = "img", None, run.Licences(self.b, given={}) + trial = os.path.join(self.tmp, "trial") + with mock.patch.object(container, "Unit") as unit, mock.patch.object(run.Run, "unit_image", return_value="img"): + r.unit_relay("u0000", trial, {}) + self.assertEqual(unit.call_args.kwargs["serving"], {"m-1": ["prov-a"]}) + + +class TheRecord(unittest.TestCase): + def call(self, model, who=None, status=200, error=None): + reply = {**({"served_by": who} if who else {}), **({"error": error} if error else {})} + return {"model": model, "status": status, "reply": reply} + + def test_who_served_a_units_calls_and_whether_the_pin_held(self): + pin = {"m-1": ["google-ai-studio", "prov-b/fp8"], "m-judge": ["prov-c"]} + self.assertEqual(job.served([self.call("m-1", "Google AI Studio"), self.call("m-1", "Prov B"), self.call("m-1", status=429)], pin), + {"calls": {"Google AI Studio": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": True, "m-judge": None}}) + self.assertEqual(job.served([self.call("m-1", "Prov Z"), self.call("m-1")], pin)["held"]["m-1"], "unknown") + self.assertEqual(job.served([self.call("m-1", "Prov B", error={"message": "x"})], pin)["calls"], {}) # failed: not answered + self.assertEqual(job.served([self.call("m-1", "Prov A")], None), {"calls": {"Prov A": 1}, "unnamed": 0}) + self.assertIsNone(job.served([self.call("m-1", status=429)], None)) + + def test_a_name_reads_as_a_slug_by_its_case_and_spaces_alone(self): + pin = {"m-1": ["prov-a"]} + for name, held in (("Prov A", True), ("prov-a", True), ("PROV A", True), ("Prov/A", "unknown"), ("Prov_A", "unknown"), + ("Prov.A", "unknown"), ("Prov Ab", "unknown")): + with self.subTest(name=name): + self.assertEqual(job.served([self.call("m-1", name)], pin)["held"]["m-1"], held) + + def test_answered_calls_that_name_no_provider_are_counted(self): + known, unknown = job.served([self.call("m-1", "Prov A")]), job.served([self.call("m-1")]) + self.assertEqual(unknown, {"calls": {}, "unnamed": 1}) + self.assertEqual(job.served_total([{"standard": {"served": known}}, {"standard": {"served": unknown}}, {"standard": {}}]), + {"calls": {"Prov A": 1}, "unnamed": 1}) + parts = [{"standard": {"served": {"calls": {"Prov A": 2}, "unnamed": 1, "held": {"m-1": True}}}}, + {"standard": {"served": {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}}}] + self.assertEqual(job.served_total(parts), {"calls": {"Prov A": 3, "Prov B": 1}, "unnamed": 1, "held": {"m-1": "unknown"}}) + + +class TheReview(unittest.TestCase): + """What a review of the pin found, each a rule of its own.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def test_the_selected_endpoint_of_the_replys_end_is_recorded_never_another(self): + """The routing facts list the endpoints available: the one marked selected served the reply, never the + first listed nor one an earlier event's facts marked.""" + events = [{"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=routing_facts("Prov B", "Prov A", "Prov C"))}] + facts = relay.reply_facts("/responses", sse(*events), True) + held = job.served([{"status": 200, "model": "m", "reply": facts}], {"m": ["prov-a"]})["held"]["m"] + self.assertEqual((facts["served_by"], held), ("Prov B", "unknown")) + body = dict(RESPONSES, openrouter_metadata=routing_facts("Prov B", "Prov A")) + self.assertEqual(relay.reply_facts("/responses", json.dumps(body).encode(), False)["served_by"], "Prov B") + + def test_facts_that_mark_no_single_named_endpoint_take_no_earlier_name(self): + both = routing_facts("Prov A", "Prov X") + for e in both["endpoints"]["available"]: + e["selected"] = True + for meta in (routing_facts(None, "Prov A", "Prov X"), routing_facts(7, "Prov A"), both): + with self.subTest(meta=meta): + body = {"choices": [{"message": {"content": "ok"}}], "openrouter_metadata": meta} + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", json.dumps(body).encode(), False)) + stream = sse({"type": "response.created", "response": {"status": "in_progress", "openrouter_metadata": routing_facts("Prov A")}}, + {"type": "response.completed", "response": dict(COMPLETED, openrouter_metadata=meta)}) + self.assertNotIn("served_by", relay.reply_facts("/responses", stream, True)) + chunks = sse({"provider": "Prov A", "choices": [{"index": 0, "delta": {"content": "o"}}]}, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "openrouter_metadata": meta}) + self.assertNotIn("served_by", relay.reply_facts("/chat/completions", chunks, True)) + + def test_a_pinned_responses_call_is_recorded_with_its_provider_and_the_pin_holds(self): + """A Responses reply names its provider in the routing facts alone: the relay asks for them on a pinned + request to OpenRouter, records the endpoint they mark, and the pin reads held, streamed or not. Without + them the reply names no provider and the pin reads unknown.""" + up = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) + threading.Thread(target=up.serve_forever, daemon=True).start() + self.addCleanup(up.server_close) + self.addCleanup(up.shutdown) + Upstream.seen, Upstream.next, Upstream.always = [], None, None + base, pin = f"http://127.0.0.1:{up.server_address[1]}/v1", {"m-run": ["prov-c"]} + for facts, who, held in ((True, "Prov C", True), (False, None, "unknown")): + r = relay.Relay(base, "or-test", os.path.join(self.tmp, f"rec-{facts}"), model="m-run", serving=pin, facts=facts).start() + self.addCleanup(r.stop) + for stream in (False, True): + with self.subTest(facts=facts, stream=stream): + post(r.url("u0") + "/responses", body_for("/responses", stream)) + self.assertEqual(Upstream.seen[-1][1].get("x-openrouter-metadata"), "enabled" if facts else None) + self.assertEqual(json.loads(Upstream.seen[-1][2])["provider"], {"order": ["prov-c"], "allow_fallbacks": False}) + self.assertEqual(r.calls("u0")[-1]["reply"].get("served_by"), who) + self.assertEqual(job.served(r.calls("u0"), pin)["held"], {"m-run": held}) + + def test_an_error_keeps_its_stated_fields_alone(self): + nested = relay.reply_facts("/chat/completions", json.dumps({"error": {"api_key": ["review-only-credential"]}}).encode(), False) + self.assertNotIn("review-only-credential", json.dumps(nested)) + self.assertEqual(relay.reply_facts("/messages", b'{"type":"message","content":[],"error":{"code":[],"type":{},"message":17}}', False), {}) + self.assertEqual(relay.reply_facts("/chat/completions", b'{"error":{"code":502,"message":"m","detail":{"key":"x"}}}', False), + {"error": {"code": 502, "message": "m"}}) + + def test_of_several_choices_errors_the_top_level_one_else_the_first_is_kept(self): + two = [{"index": 0, "finish_reason": "error", "error": {"code": 429, "message": "rate limited"}}, + {"index": 1, "finish_reason": "error", "error": {"code": 503, "message": "unavailable"}}] + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps({"choices": two}).encode(), False)["error"], + {"code": 429, "message": "rate limited", "choice": 0}) + top = {"choices": two, "error": {"code": 500, "message": "whole"}} + self.assertEqual(relay.reply_facts("/chat/completions", json.dumps(top).encode(), False)["error"], {"code": 500, "message": "whole"}) + + def test_a_merged_job_keeps_each_shards_whole_summary(self): + """A shard's summary holds its benchmark relay's calls too: a merge made of its trials alone would drop + them, and an ``unknown`` would read true.""" + pin = {"m": ["prov-a"]} + trial = {"standard": {"served": job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov A"}}], pin), "shard_job": "s1"}} + bench = job.served([{"status": 200, "model": "m", "reply": {"served_by": "Prov B"}}], pin) + shard = {"name": "s1", "config": {"standard": {"served_by": pin}}, "result": {"served": job.served_total([trial], bench)}} + self.assertEqual(merge._served([shard], [trial]), shard["result"]["served"]) + old = {"name": "s2", "config": {"standard": {"served_by": pin}}, "result": {}} # a shard written before: its trials + self.assertEqual(merge._served([old], [dict(trial, standard=dict(trial["standard"], shard_job="s2"))])["held"], {"m": True}) + + def test_a_control_is_refused_a_pin_on_its_simulated_users_model(self): + b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing"}}, + simulator={"model": "sim"}) + with self.assertRaisesRegex(run.Refused, "names sim, which this run does not call"): + run.Run(b, definitions.control(b, "nop"), model="sim", provider="openrouter", env={}, + jobs_root=self.tmp, say=lambda _: None, served_by={"sim": ["prov-a"]}) + + def test_the_relays_own_command_refuses_another_provider_and_a_model_it_does_not_send(self): + key = os.path.join(self.tmp, "key") + with open(key, "w") as f: + f.write("review-test-key") + for base, model, extra in (("https://api.openai.com/v1", "m", []), ("https://openrouter.ai/api/v1", "unused", []), + ("http://127.0.0.1:9/v1", "m", [])): + with self.subTest(base=base, model=model): + out = io.StringIO() + with mock.patch.object(relay, "Relay") as fake, redirect_stdout(out): + code = relay.serve(["--base", base, "--model", "m", "--key-file", key, "--record", self.tmp, + "--served-by", model + "=prov-a", *extra]) + self.assertEqual(code, 2) + self.assertIn("--served-by", json.loads(out.getvalue())["error"]) + fake.assert_not_called() + + +class TheMerge(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.b = dict(definitions.get("benchmark", "fakebench", ROOT), controls={"nop": {"source": "nullius", "from": "nothing said"}}) + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + def shards(self, *pins): + jobs = [run.run(self.b, definitions.control(self.b, "nop"), model="", provider="openai", jobs_root=os.path.join(self.tmp, "jobs"), + env={}, say=lambda s: None, shard=(i, len(pins)), launch="L", name=f"run-shard-{i}of{len(pins)}") + for i in range(1, len(pins) + 1)] + for job_dir, pin in zip(jobs, pins): + path = os.path.join(job_dir, "config.json") + with open(path) as f: + cfg = json.load(f) + cfg["standard"]["served_by"] = pin + with open(path, "w") as f: + json.dump(cfg, f) + return jobs + + def test_shards_of_different_pins_are_refused(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + self.assertEqual(merge.check(jobs, force=True)["differ"][0].split(":")[0], "serving providers") + jobs = self.shards({"m-1": ["prov-a", "prov-b"]}, {"m-1": ["prov-b", "prov-a"]}) # the order is the pin's + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(jobs) + + def test_equal_pins_merge_whatever_the_order_of_their_models(self): + jobs = self.shards({"a": ["prov-a"], "b": ["prov-b"]}, {"b": ["prov-b"], "a": ["prov-a"]}) + self.assertEqual(merge.check(jobs)["differ"], []) + + def test_a_forced_merge_of_different_pins_holds_no_pin(self): + jobs = self.shards({"m-1": ["prov-a"]}, {"m-1": ["prov-b"]}) + for job_dir, who in zip(jobs, ("Prov A", "Prov B")): + trial = sorted(d for d in os.listdir(job_dir) if "__u" in d)[0] + path = os.path.join(job_dir, trial, "result.json") + with open(path) as f: + res = json.load(f) + res["standard"]["served"] = {"calls": {who: 1}, "unnamed": 0, "held": {"m-1": True}} + with open(path, "w") as f: + json.dump(res, f) + out = os.path.join(self.tmp, "merged") + merge.merge(jobs, out, force=True) + with open(os.path.join(out, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["served_by"], {"m-1": ["prov-a"]}) + with open(os.path.join(out, "result.json")) as f: + self.assertEqual(json.load(f)["served"], {"calls": {"Prov A": 1, "Prov B": 1}, "unnamed": 0, "held": {"m-1": "unknown"}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_served_by_constraints.py b/app/tests/test_standard_served_by_constraints.py new file mode 100644 index 00000000..b606c49e --- /dev/null +++ b/app/tests/test_standard_served_by_constraints.py @@ -0,0 +1,172 @@ +"""Provider pins and machine constraints on the same run and recipe. + +These intersections need both features: neither side's tests exercise them +together. The fixture driver scores locally; no endpoint or engine is used. +""" +from __future__ import annotations + +import io +import json +import os +from pathlib import Path +import shlex +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from nullius import constraints, progress +from nullius.standard import cli, container, definitions, install, job, merge, recipe, run +from tests import test_standard_recipe_served_by as recipes + + +class Together(unittest.TestCase): + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(mock.patch.object(install, "HOME", str(self.tmp / "install"))) + self.addCleanup(container.cap, {}) + self.b = recipes.judged() + self.a = definitions.get("agent", "fake", recipes.ROOT) + self.pin = {"m-judge": ["prov-a"]} + + def make(self, **kw): + values = dict(model="m", provider="openrouter", env={"OPENROUTER_API_KEY": "or-test"}, + judge_model="m-judge", served_by=self.pin, unsandboxed=True, + jobs_root=str(self.tmp), say=lambda _: None) + values.update(kw) + return run.Run(self.b, self.a, **values) + + def test_the_cli_records_units_constraints_and_pins_together(self): + units = self.tmp / "units.txt" + units.write_text("good\n") + self.enterContext(mock.patch.object(cli, "_bench", return_value=self.b)) + self.enterContext(mock.patch.object(cli, "_exit_on_term")) + self.enterContext(mock.patch.object(cli.settings, "load")) + self.enterContext(mock.patch.object(progress, "ECHO", None)) + self.enterContext(mock.patch.object(run.relay.Relay, "start", lambda r: r)) + self.enterContext(mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9")) + self.enterContext(mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"})) + seen = [] + proc = run.jsonl.Proc + + def driver(cmd, cwd, env, log, **kw): + seen.append(env) + return proc(cmd, cwd, env, log, **kw) + + self.enterContext(mock.patch.object(run.jsonl, "Proc", driver)) + for i, extra in enumerate(([], ["--from-run"])): + out = io.StringIO() + with self.subTest(extra=extra), redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--jobs", str(self.tmp), "--name", f"job-{i}", + "--units", str(units), "-j", "4", "--constraint", "concurrency=1", + "--constraint", "gpu_ids=", "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", *extra]) + self.assertEqual(code, 0, out.getvalue()) + path = Path(out.getvalue().strip().splitlines()[-1]) + cfg = json.loads((path / "config.json").read_text()) + self.assertEqual(cfg["standard"]["units_file"], run.units_file(str(units))) + self.assertEqual(cfg["standard"]["constraints"], {"concurrency": 1, "gpu_ids": []}) + self.assertEqual(cfg["standard"]["served_by"], self.pin) + self.assertEqual(cfg["n_concurrent_trials"], 1) + self.assertEqual(sorted(p.name for p in path.iterdir() if "__u" in p.name), ["good__u0000"]) + self.assertEqual(json.loads((path / "result.json").read_text())["served"]["held"], {"m-judge": None}) + self.assertTrue(seen) + self.assertTrue(all(env["CUDA_VISIBLE_DEVICES"] == "" for env in seen)) + + def test_constraint_and_host_refusals_precede_pin_and_endpoint_checks(self): + with mock.patch.object(run, "provider_endpoint") as endpoint: + with self.assertRaisesRegex(run.Refused, "constraints:.*at least 1"): + self.make(constraints={"concurrency": 0}, served_by={"stray": ["prov-a"]}) + self.b = dict(self.b, host={"gpu": {"check": ["nvidia-smi"]}}, + options={"compute": {"default": "gpu", "values": ["gpu"], "needs": {"gpu": {"host": ["gpu"]}}}}) + with self.assertRaisesRegex(run.Refused, "allows Nullius no GPU"): + self.make(constraints={"gpu_ids": []}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + with self.assertRaisesRegex(run.Refused, "--served-by.*does not call"): + self.make(constraints={"gpu_ids": [0]}, served_by={"stray": ["prov-a"]}, which=lambda _: "/bin/true") + endpoint.assert_not_called() + out = io.StringIO() + with mock.patch.object(cli, "_bench", return_value=self.b), mock.patch.object(cli, "_exit_on_term"), \ + mock.patch.object(cli.settings, "load"), mock.patch.object(run, "run_and_report") as start, redirect_stdout(out): + code = cli.main(["run", "fakebench", "nop", "--constraint", "concurrency=0", "--served-by", "bad slug"]) + self.assertEqual(code, 2) + self.assertIn("refused: constraints:", out.getvalue()) + start.assert_not_called() + + def test_a_valid_pin_still_requires_enforced_machine_caps(self): + r = self.make(constraints={"cpus": 1}) + with mock.patch.object(constraints, "scope_check", return_value={"why": "unavailable", "enforced": [], "refused": {}}), \ + mock.patch.object(install, "packages") as setup, self.assertRaisesRegex(run.Refused, "systemd does not enforce"): + r.ready() + setup.assert_not_called() + + def test_the_container_relay_takes_both_caps_and_pins(self): + self.enterContext(mock.patch.object(container, "_security", "")) + with mock.patch.object(container, "unusable", return_value=None): + r = self.make(backend="container", constraints={"cpus": 1, "memory_mb": 512}) + argv = container.relay_argv("img", name="relay", job="job", app="/app", record="/rec", key_file="/key", + base="https://openrouter.ai/api/v1", model="m", routes={"u0-judge": "m-judge"}, serving=r.serving) + self.assertEqual(argv[argv.index("--cpus") + 1], "1") + self.assertEqual(argv[argv.index("--memory") + 1], "512m") + self.assertEqual(argv[argv.index("--served-by") + 1], "m-judge=prov-a") + with mock.patch.object(r, "scope_ready", return_value={}), \ + mock.patch.object(container, "base_tag", return_value="img"), \ + mock.patch.object(container, "present", return_value=True), \ + mock.patch.object(container, "base_image", return_value="img"), \ + mock.patch.object(container, "unenforced", return_value={"cpus": "max 100000"}), \ + self.assertRaisesRegex(run.Refused, "engine does not apply"): + r.ready() + + def test_each_compiled_shard_has_its_constraints_and_each_called_models_pin(self): + given = {"local": {"concurrency": 1}, "127.0.0.1": {"cpus": 2, "gpu_ids": []}} + with mock.patch.dict(os.environ, {constraints.SETTING: json.dumps(given)}): + out = recipe.compile(recipes.a_recipe(where=list(given), others=["fake", "nop"], + model="openai/gpt-4o-mini", served_by={"model": ["prov-a"]}), recipes.defs(self.b)) + self.assertEqual(out["errors"], []) + self.assertEqual(out["served_models"]["model"], "openai/gpt-4o-mini") + for part in out["shards"]: + for line in part["command"].splitlines(): + words = shlex.split(line) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--constraint"], + constraints.cli_words(given[part["where"]])[1::2]) + self.assertEqual([words[i + 1] for i, w in enumerate(words) if w == "--served-by"], + ["openai/gpt-4o-mini=prov-a"]) + self.assertLess(words.index("--constraint"), words.index("--served-by")) + + def test_builder_keeps_both_validation_errors_in_main_first_order(self): + with mock.patch.dict(os.environ, {constraints.SETTING: '{"local": {"cpus": 0}}'}): + out = recipe.compile(recipes.a_recipe(model="openai/gpt-4o-mini", + served_by={"model": ["prov-a"], "judge": ["prov-b"]}), recipes.defs(self.b)) + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"], recipe.STEP["model"]]) + self.assertIn("HOST_CONSTRAINTS", out["errors"][0]["message"]) + self.assertIn("pins openai/gpt-4o-mini twice", out["errors"][1]["message"]) + self.assertEqual((out["script"], out["parts"]), ("", [])) + + def test_merge_allows_machine_limits_to_differ_but_still_checks_pins(self): + paths = [] + for i in (1, 2): + path = self.tmp / f"shard-{i}" + path.mkdir() + job.write_config(str(path), job=path.name, bench=self.b, other=self.a, model="m", provider="openrouter", + dataset="all", units=[], concurrency=i, granted=True, launch="launch", + shard={"index": i, "of": 2, "units_total": 0, "planned": []}, + constraints={"concurrency": i}, served_by=self.pin) + (path / "result.json").write_text(json.dumps({"served": {"calls": {"Prov A": 1}, "unnamed": 0, + "held": {"m-judge": True}}})) + paths.append(str(path)) + self.assertEqual(merge.check(paths)["differ"], []) + merge.merge(paths, str(self.tmp / "joined")) + cfg = json.loads((self.tmp / "joined/config.json").read_text())["standard"] + self.assertEqual((cfg["constraints"], cfg["served_by"]), ({"concurrency": 1}, self.pin)) + self.assertEqual(json.loads((self.tmp / "joined/result.json").read_text())["served"]["held"], {"m-judge": True}) + path = Path(paths[1]) / "config.json" + cfg = json.loads(path.read_text()) + cfg["standard"]["served_by"] = {"m-judge": ["prov-b"]} + path.write_text(json.dumps(cfg)) + with self.assertRaisesRegex(merge.Refused, "serving providers"): + merge.check(paths) + merge.merge(paths, str(self.tmp / "forced"), force=True) + self.assertEqual(json.loads((self.tmp / "forced/result.json").read_text())["served"]["held"], {"m-judge": "unknown"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_units.py b/app/tests/test_standard_units.py index d7c93daa..6ed5e162 100644 --- a/app/tests/test_standard_units.py +++ b/app/tests/test_standard_units.py @@ -347,6 +347,22 @@ def test_the_listed_units_run_under_the_machine_s_constraints(self): cfg = read(os.path.join(job, "config.json")) self.assertEqual((cfg["standard"]["units_file"]["units"], cfg["standard"]["constraints"], cfg["n_concurrent_trials"]), (["slow", "good"], {"concurrency": 1}, 1)) + def test_the_command_records_both_the_units_file_and_the_serving_pin(self): + path = self.file("good\n") + self.b["score"] = dict(self.b["score"], judge={"model": "m-judge", "says": "a judge"}) + # The fixture scores locally; no relay listener is needed. + for extra in ([], ["--from-run"]): + with self.subTest(extra=extra), mock.patch.object(run.relay.Relay, "start", lambda r: r), \ + mock.patch.object(run.relay.Relay, "public", "http://127.0.0.1:9"), \ + mock.patch.dict(os.environ, {"OPENROUTER_API_KEY": "or-test", "OPENAI_API_KEY": "or-test"}): + code, out = self.cli("--units", path, "--provider", "openrouter", "--judge-model", "m-judge", + "--served-by", "m-judge=prov-a", "--name", "pinned" + "".join(extra), *extra) + self.assertEqual(code, 0, out) + job = out.strip().splitlines()[-1] + standard = read(os.path.join(job, "config.json"))["standard"] + self.assertEqual(standard["units_file"], run.units_file(path)) + self.assertEqual(standard["served_by"], {"m-judge": ["prov-a"]}) + self.assertEqual(sorted(d for d in os.listdir(job) if "__u" in d), ["good__u0000"]) def test_a_list_the_run_cannot_take_is_refused_with_exit_2(self): for text, said in (("good\ngood\n", "more than once"), ("# none\n", "lists no unit"), ("good\nnope\n", "nope")): diff --git a/app/ui_tests/test_served_by_builder.py b/app/ui_tests/test_served_by_builder.py new file mode 100644 index 00000000..2ef92437 --- /dev/null +++ b/app/ui_tests/test_served_by_builder.py @@ -0,0 +1,390 @@ +"""The Model step's Served by: the route that lists a model's providers on +OpenRouter (local, form-token guarded, asked as the model list is), the row +and its picker (the Model row's, taking several providers in order), and the +recipe's ``served_by`` in the builder's core script (its fields on +OpenRouter alone, kept by a saved recipe, dropped with another provider).""" +import json +import os +import shutil +import subprocess +import unittest +from pathlib import Path +from unittest.mock import patch + +from .builder_support import CompilerCase +from .test_routes import web + +LIST = [{'slug': 'deepinfra/fp8', 'name': 'DeepInfra', 'quantization': 'fp8'}, {'slug': 'together', 'name': 'Together', 'quantization': ''}] + + +class ProviderList(unittest.TestCase): + def setUp(self): + self.client = web.app.test_client() + with self.client.session_transaction() as session: + session['csrf'] = 'served-test-token' + self.headers = {'X-CSRF-Token': 'served-test-token'} + + def ask(self, body, **kw): + return self.client.post('/new-run/endpoints', json=body, headers=kw.pop('headers', self.headers), **kw) + + def test_a_models_providers_on_openrouter(self): + with patch('web.builder.providers.list_endpoints', return_value=LIST) as fetch: + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json, {'ok': True, 'endpoints': LIST}) + fetch.assert_called_once_with('openrouter', 'meta-llama/llama-3.3-70b-instruct') + + def test_a_list_that_cannot_be_had_is_no_empty_success(self): + with patch('web.builder.providers.list_endpoints', side_effect=ValueError("Provider list unavailable. Enter a provider's slug by hand.")): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.status_code, 502) + self.assertEqual(response.json, {'ok': False, 'endpoints': [], 'message': "Provider list unavailable. Enter a provider's slug by hand."}) + + def test_no_key_asks_without_one(self): + """OpenRouter's endpoint list is public: with no key set, it is asked with no Authorization header.""" + sent = [] + + def get(url, headers, opener=None): + sent.append((url, headers)) + return json.dumps({'data': {'endpoints': [{'provider_name': 'Together', 'tag': 'together'}]}}).encode() + env = {k: v for k, v in os.environ.items() if k != 'OPENROUTER_API_KEY'} + with patch.dict(os.environ, env, clear=True), patch('web.builder.providers._get', side_effect=get): + response = self.ask({'provider': 'openrouter', 'model': 'meta-llama/llama-3.3-70b-instruct'}) + self.assertEqual(response.json['endpoints'], [{'slug': 'together', 'name': 'Together', 'quantization': ''}]) + self.assertEqual(sent, [('https://openrouter.ai/api/v1/models/meta-llama/llama-3.3-70b-instruct/endpoints', {})]) + + def test_guards_and_another_provider_do_not_fetch(self): + with patch('web.builder.providers.list_endpoints') as fetch: + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, headers={}).status_code, 403) + self.assertEqual(self.ask({'provider': 'openrouter', 'model': 'a/b'}, environ_overrides={'REMOTE_ADDR': '192.0.2.4'}).status_code, 403) + for body in ({'provider': 'openai', 'model': 'gpt-4o'}, {'provider': 'openrouter'}, {'provider': 'openrouter', 'model': ' '}, []): + with self.subTest(body=body): + self.assertEqual(self.ask(body).status_code, 400) + fetch.assert_not_called() + + +class TheStep(unittest.TestCase): + """What the Model step's script draws and what the compile route hands it.""" + STATIC = Path(__file__).resolve().parents[1] / 'static' + SCRIPT = (STATIC / 'builder' / 'command-builder.js').read_text() + PICKER = (STATIC / 'builder' / 'model-picker.js').read_text() + STYLE = (STATIC / 'builder' / 'command-builder.css').read_text() + + def test_served_by_holds_no_sentence_and_its_help_is_a_tooltip(self): + field = self.SCRIPT[self.SCRIPT.index('const field = (rec, role)'):self.SCRIPT.index('function sync()')] + self.assertNotIn('', field) + self.assertIn('title="', field) + + def test_served_by_is_a_row_with_the_model_rows_picker(self): + """A row as the Model row is (a ``label.nb-field``, which the page sets as a settings row), its input one the + Model row's picker takes (``list="nb-served"``), with no block set in under the model's row.""" + field = self.SCRIPT[self.SCRIPT.index('const field = (rec, role)'):self.SCRIPT.index('function sync()')] + self.assertIn('