From 7c3a40472a63504c27c7cd90a6e2d708c924cf34 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Fri, 13 Nov 2026 23:52:21 +0100 Subject: [PATCH 1/9] feat(standard): run each unit's agent in a Boat VM of its own Adds the boat backend (app/nullius/standard/boat.py): one boat.dev VM per unit, the agent as a user locked to the VM's loopback, its model calls through a reverse tunnel to the unit's relay on this machine, the workspace copied back without links that leave it, the VM deleted after. Concurrency is capped to the VMs the account runs at once. The builder's Where step offers Boat; Settings > Sandbox keeps its API key with Edit and Remove. --- AGENTS.md | 2 +- app/nullius/standard/boat.py | 403 +++++++++++++++++++++ app/nullius/standard/cli.py | 11 +- app/nullius/standard/names.py | 2 +- app/nullius/standard/recipe.py | 32 +- app/nullius/standard/run.py | 52 ++- app/static/brand/README.md | 2 + app/static/brand/boat.svg | 3 + app/static/builder/command-builder-core.js | 2 +- app/static/builder/command-builder.js | 9 +- app/static/settings/sandbox.js | 48 ++- app/templates/settings/settings.html | 14 +- app/tests/test_standard_boat.py | 359 ++++++++++++++++++ app/ui_tests/test_agents.py | 8 +- app/ui_tests/test_settings_page.py | 36 +- app/web/builder.py | 6 +- app/web/execute.py | 4 +- app/web/settings.py | 30 +- docs/builder.md | 2 +- docs/cli/run.md | 4 +- docs/settings/README.md | 2 +- docs/settings/sandbox.md | 5 +- docs/standard.md | 17 +- 23 files changed, 1003 insertions(+), 50 deletions(-) create mode 100644 app/nullius/standard/boat.py create mode 100644 app/static/brand/boat.svg create mode 100644 app/tests/test_standard_boat.py diff --git a/AGENTS.md b/AGENTS.md index 9aeb60b9..69e93cb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ and its resources. The source layout stays under `app/`. `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), `standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`, - `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its + `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its `payload` snapshots the running backend and effective catalogue for a remote host; conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: diff --git a/app/nullius/standard/boat.py b/app/nullius/standard/boat.py new file mode 100644 index 00000000..d6424be6 --- /dev/null +++ b/app/nullius/standard/boat.py @@ -0,0 +1,403 @@ +"""The ``boat`` backend: each unit's agent or capability in a boat.dev VM of its own. + +A boat VM is a cloud Ubuntu machine created by its API (``API``) and reached +over plain OpenSSH. Everything that holds an answer or a key stays on this +machine: the driver, the unit's relay, the spending cap, the scoring. Only +the process of the agent (or the capability) runs in the VM, as ``USER``, a +user of its own whose only way out is the VM's loopback (``LOCK``): no +public address, no DNS. Its model calls reach the unit's relay through a +reverse tunnel on the same port (``ssh -R``), so the address it is given is +the one the local backend gives, and the relay records every call here. + +One unit, one VM (:class:`VM`): created when the unit's agent is about to +start, deleted once its workspace is copied back. A run's concurrency is +therefore the number of VMs at once, never more than the account allows at +once (:func:`slots`); a VM the account cannot start yet (its limit, its rate +of starts) is waited for, and the wait ends with the run's Stop. Every VM +carries the job's name as a label, so whatever a crash left behind is +deleted by :func:`sweep` when the job ends. + +What the VM is given, each at the same absolute path it has here so every +path in the agent's configuration still holds: the agent's install, its +definition, the runtimes the install links to outside itself (a ``uv`` +Python), its home and the unit's workspace. The workspace comes back with +links that leave it dropped (``rsync --safe-links``): the driver reads it to +score, and a link the agent left must not reach a file of this machine. +Nothing of this machine's environment goes with it but its locale. + +The API key is kept in the owner-only store (``nullius.secrets``, the folder +``STORE``), set in Settings → Sandbox; ``BOAT_API_KEY`` in the environment +wins. It is sent to boat.dev only, never to the VM. Standard library only. +""" +from __future__ import annotations + +import json +import os +import shlex +import shutil +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from contextlib import contextmanager +from typing import Callable, Dict, Iterable, List, Optional, Sequence + +from nullius import secrets +from nullius.standard import install + +API = "https://boat.dev/api/v1" +STORE, NAME = "sandbox-provider-boat", "BOAT_API_KEY" +SIZE = "small" # 2 vCPU, 4 GB: an agent's process, not its model +TTL = 2 * 3600 # the longest a VM lives, whatever happens here (the trial's maximum) +USER = "agent" # the VM's user the agent runs as +LABEL = "nullius-job" +VM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +KEEP = ("LANG", "LC_ALL", "TERM") # of the local environment, what makes sense on another machine +WAIT_SECONDS = 1800 # how long a unit waits for the account to start its VM +READY_SECONDS = 300 +SSH_SECONDS = 900 + +# The agent's user reaches the VM's loopback and nothing else: no public address, no DNS. Inserted +# first in OUTPUT, before the chains boat's own firewall accepts in. +LOCK = ("U=$(id -u {user}) && for t in iptables ip6tables; do " + "sudo -n $t -I OUTPUT 1 -m owner --uid-owner $U -j REJECT && " + "sudo -n $t -I OUTPUT 1 -o lo -p udp --dport 53 -m owner --uid-owner $U -j REJECT && " + "sudo -n $t -I OUTPUT 1 -o lo -p tcp --dport 53 -m owner --uid-owner $U -j REJECT && " + "sudo -n $t -I OUTPUT 1 -o lo -m owner --uid-owner $U -j ACCEPT; done") +_SYSTEM = ("/usr/", "/bin/", "/lib", "/sbin/", "/etc/", "/opt/") + + +class Unavailable(Exception): + """boat cannot run this unit: no key, a refusal of its API, a VM that never came up.""" + + +def key(env=os.environ) -> Optional[str]: + """The account's API key: ``BOAT_API_KEY``, else the store's; None when neither is set.""" + if env.get(NAME): + return env[NAME].strip() + v = secrets.value(STORE, NAME) + return v.decode().strip() if v else None + + +def unusable(env=os.environ, which: Callable = shutil.which) -> Optional[str]: + """Why the boat backend cannot run from this machine, or None.""" + if not key(env): + return "no Boat API key: set it in Settings → Sandbox" + lacking = [p for p in ("ssh", "ssh-keygen", "rsync", "tar") if not which(p)] + if lacking: + return "missing on this machine: " + ", ".join(lacking) + return None + + +class Client: + """boat.dev's API, with one account's key.""" + + def __init__(self, api_key: str, base: str = API, opener: Callable = urllib.request.urlopen): + self.key, self.base, self.opener = api_key, base, opener + + def call(self, method: str, path: str, body: Optional[dict] = None, headers: Optional[dict] = None) -> dict: + req = urllib.request.Request(self.base + path, method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", + "Accept": "application/json", **(headers or {})}) + try: + with self.opener(req, timeout=60) as r: + return json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: + try: + d = json.loads(e.read() or b"{}") + except ValueError: + d = {} + return {"ok": False, "status": e.code, **d} if isinstance(d, dict) else {"ok": False, "status": e.code} + except (urllib.error.URLError, OSError, ValueError) as e: + return {"ok": False, "code": "unreachable", "message": f"Boat did not answer: {e}"} + + def limits(self) -> dict: + return self.call("GET", "/limits") + + def create(self, labels: dict, size: str = SIZE, ttl: int = TTL) -> dict: + return self.call("POST", "/sandboxes", {"type": size, "ttlSeconds": ttl, "name": "nullius-unit", "labels": labels}) + + def get(self, vm: str) -> dict: + return self.call("GET", f"/sandboxes/{urllib.parse.quote(vm)}") + + def ssh_key(self, vm: str, public: str) -> dict: + return self.call("POST", f"/sandboxes/{urllib.parse.quote(vm)}/sshkey", {"key": public}) + + def delete(self, vm: str) -> dict: + return self.call("DELETE", f"/sandboxes/{urllib.parse.quote(vm)}", headers={"X-Ascii-Confirm-Delete": vm}) + + def labelled(self, label: str, value: str) -> List[dict]: + d = self.call("GET", "/sandboxes?" + urllib.parse.urlencode({"label": f"{label}={value}", "limit": 100})) + return [s for s in d.get("sandboxes") or [] if (s.get("labels") or {}).get(label) == value] + + +def said(d: dict) -> str: + return str(d.get("message") or d.get("code") or d.get("status") or "no answer") + + +def slots(client: Client) -> Optional[int]: + """How many VMs the account may run at once; None when its API does not say.""" + n = client.limits().get("maxActiveSandboxes") + return n if isinstance(n, int) and n > 0 else None + + +def sweep(job: str, client: Optional[Client] = None) -> int: + """Deletes every VM labelled with ``job``: what a crash or a kill left. The number deleted.""" + if client is None: + k = key() + if not k: + return 0 + client = Client(k) + gone = 0 + for s in client.labelled(LABEL, job): + if client.delete(s["id"]).get("ok"): + gone += 1 + return gone + + +def outside_roots(where: str) -> List[str]: + """The folders an install links to outside itself and outside the system's own: a ``uv`` + environment's Python, kept in uv's folder of this user. Each must reach the VM too.""" + roots = set() + real_where = os.path.realpath(where) + for top, dirs, files in os.walk(where): + for n in dirs + files: + p = os.path.join(top, n) + if not os.path.islink(p): + continue + t = os.path.realpath(p) + if t == real_where or t.startswith(real_where + os.sep) or t.startswith(_SYSTEM) or not os.path.exists(t): + continue + d = os.path.dirname(t) + roots.add(os.path.dirname(d) if os.path.basename(d) == "bin" else d) + out: List[str] = [] + for r in sorted(roots, key=len): # a root inside another is sent with it + if not any(r == o or r.startswith(o + os.sep) for o in out): + out.append(r) + return out + + +class Keys: + """A job's SSH key (made for it, never this user's own) and the host keys it pinned.""" + + def __init__(self): + self.dir = tempfile.mkdtemp(prefix="nullius-boat-") + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "nullius-boat", "-f", os.path.join(self.dir, "id")], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + with open(os.path.join(self.dir, "id.pub")) as f: + self.public = f.read().strip() + + def close(self) -> None: + shutil.rmtree(self.dir, ignore_errors=True) + + +class VM: + """One unit's VM, ready: created, its host key pinned, the agent's user made and locked in. + ``stopped``: the run's Stop, which ends a wait for the account to start it.""" + + def __init__(self, client: Client, keys: Keys, job: str, *, stopped: Optional[threading.Event] = None, + wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep): + self.client, self.keys, self.id = client, keys, None + t0 = time.time() + pause = 5.0 + while True: + d = client.create({LABEL: job}) + if d.get("ok"): + break + # the account's limit of VMs at once, or of starts a minute: another unit's VM ends, a minute passes + if d.get("status") != 429 and d.get("code") not in ("limit_reached", "rate_limited"): + raise Unavailable("Boat refused a VM: " + said(d)) + if (stopped is not None and stopped.is_set()) or time.time() - t0 > wait: + raise Unavailable("Boat could not start a VM in time: " + said(d)) + sleep(pause) + pause = min(pause * 1.5, 30.0) + self.id = d["sandbox"]["id"] + try: + self._ready(sleep) + self.sh(f"sudo -n useradd -m -s /bin/bash {USER} && " + LOCK.format(user=USER)) + except BaseException: + self.delete() + raise + self.seconds = round(time.time() - t0, 1) + + def _ready(self, sleep) -> None: + t0 = time.time() + while True: + s = self.client.get(self.id).get("sandbox") or {} + if s.get("state") in ("ready", "idle", "running"): + break + if s.get("state") in ("archived", "deleted", "failed", "error") or time.time() - t0 > READY_SECONDS: + raise Unavailable(f"Boat VM {self.id} did not come up (state {s.get('state')})") + sleep(2) + k = self.client.ssh_key(self.id, self.keys.public) + if not k.get("ok") or not k.get("hostKey"): + raise Unavailable(f"Boat VM {self.id} took no SSH key: " + said(k)) + ep = k.get("sshEndpoint") or f"{k['machineIp']}:22" + self.host, self.port = ep.rsplit(":", 1) + self.user = k.get("sshUser") or "user" + known = os.path.join(self.keys.dir, f"known-{self.id}") + with open(known, "w") as f: + f.write(f"{self.id} {k['hostKey']}\n") + self.ssh = ["ssh", "-F", "/dev/null", "-i", os.path.join(self.keys.dir, "id"), "-p", self.port, "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", f"UserKnownHostsFile={known}", + "-o", f"HostKeyAlias={self.id}", "-o", "ServerAliveInterval=30", "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=20"] + self.target = f"{self.user}@{self.host}" + for _ in range(30): # sshd answers a moment after the key is in + if subprocess.run(self.ssh + [self.target, "true"], capture_output=True, timeout=60).returncode == 0: + return + sleep(2) + raise Unavailable(f"Boat VM {self.id} did not answer over SSH") + + def sh(self, cmd: str, stdin=None) -> str: + r = subprocess.run(self.ssh + [self.target, cmd], stdin=stdin, capture_output=True, timeout=SSH_SECONDS) + if r.returncode: + raise Unavailable(f"Boat VM {self.id}: {cmd.split(' ', 3)[:3]}: " + + r.stderr.decode(errors="replace").strip()[-300:]) + return r.stdout.decode(errors="replace") + + def send(self, read: Iterable[str], owned: Iterable[str] = ()) -> None: + """Each folder at its own path in the VM, in one compressed stream: ``read`` the agent's user + may read, ``owned`` it owns (its home, its workspace).""" + read, owned = [p.rstrip("/") for p in read if p], [p.rstrip("/") for p in owned if p] + rel = [os.path.relpath(p, "/") for p in dict.fromkeys(read + owned)] + tar = subprocess.Popen(["tar", "-C", "/", "-czf", "-", "--", *rel], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + # every parent of each folder made traversable, the agent's own folders its own + q = " ".join(shlex.quote("/" + r) for r in rel) + mine = " ".join(shlex.quote(p) for p in owned) + self.sh("sudo -n tar -C / -xzpf - --no-same-owner && " + f"for p in {q}; do while [ \"$p\" != / ]; do sudo -n chmod o+rx \"$p\"; p=$(dirname \"$p\"); done; done" + + (f" && sudo -n chown -R {USER}:{USER} {mine}" if mine else "") + + f" && sudo -n chmod -R o+rX {q}", stdin=tar.stdout) + finally: + tar.stdout.close() + err = tar.stderr.read().decode(errors="replace") + if tar.wait(): + raise Unavailable(f"could not pack what Boat VM {self.id} needs: {err.strip()[-300:]}") + + def fetch(self, path: str) -> None: + """The folder back from the VM, as it is there: what the agent removed is removed here, a link + leaving the folder is not copied, nor a device or a pipe.""" + path = path.rstrip("/") + rsh = " ".join(shlex.quote(w) for w in self.ssh) + r = subprocess.run(["rsync", "-rlt", "--delete", "--safe-links", "--no-devices", "--no-specials", + "--rsync-path=sudo -n rsync", "-e", rsh, f"{self.target}:{path}/", path + "/"], + capture_output=True, timeout=SSH_SECONDS) + if r.returncode: + raise Unavailable(f"the workspace did not come back from Boat VM {self.id}: " + + r.stderr.decode(errors="replace").strip()[-300:]) + + def argv(self, cmd: List[str], env: Dict[str, str], workdir: str, port: Optional[int]) -> List[str]: + """``cmd`` run in the VM as the agent's user, in ``workdir``, with ``env`` alone; its stdin and + stdout this process's (ACP, JSON lines). ``port``: the relay's, the same port in the VM.""" + keep = " ".join(shlex.quote(f"{k}={v}") for k, v in env.items()) + remote = (f"cd {shlex.quote(workdir)} && exec sudo -n -u {USER} env -i {keep} " + + " ".join(shlex.quote(w) for w in cmd)) + tunnel = ["-o", "ExitOnForwardFailure=yes", "-R", f"127.0.0.1:{port}:127.0.0.1:{port}"] if port else [] + return self.ssh + ["-T", *tunnel, self.target, remote] + + def delete(self) -> None: + if self.id: + self.client.delete(self.id) + try: + os.remove(os.path.join(self.keys.dir, f"known-{self.id}")) + except OSError: + pass + + +def remote_env(env: Dict[str, str], bin_dirs: List[str], local: Sequence[str] = ()) -> Dict[str, str]: + """A process's environment as the VM gives it: ``local``, the variables taken of this machine's + environment, dropped but for its locale; its ``PATH`` the install's programs, then the VM's.""" + out = {k: v for k, v in env.items() if k not in local or k in KEEP} + out["PATH"] = os.pathsep.join(bin_dirs + [VM_PATH]) + return out + + +MCP_REFUSAL = ("the benchmark's own MCP servers run on this machine, which an agent in a Boat VM cannot start " + "(--backend local or container)") + + +def refusal(env_def: Optional[dict]) -> Optional[str]: + """Why an environment's agent cannot run in a Boat VM, or None: the public web it is granted + (``network``), a licence file or server of this machine it must reach.""" + if (env_def or {}).get("network"): + return "grants its agent the public web, which a Boat VM does not yet (--backend container)" + for s in (env_def or {}).get("needs_secrets") or []: + if s.get("to") == "agent" and s.get("kind") in ("file", "server"): + return f"gives its agent the licence {s['name']} of this machine, which a Boat VM cannot reach (--backend container)" + return None + + +class Backend: + """The boat backend for one run: its account, the label its VMs carry (this run's alone, though a + job's name may be used again), its SSH key, made with its first VM, and the VM each unit's thread + holds while its agent runs (:meth:`unit`, :meth:`place`). :meth:`close` deletes what is left.""" + + def __init__(self, job: str, env=os.environ, client: Optional[Client] = None): + self.client = client or Client(key(env) or "") + self.label = f"{job}.{os.urandom(4).hex()}" + self._keys: Optional[Keys] = None + self._roots: Dict[str, List[str]] = {} + self._lock = threading.Lock() + self._held = threading.local() + + def most(self) -> Optional[int]: + """How many units may run at once: the VMs the account runs at once.""" + return slots(self.client) + + @contextmanager + def unit(self, trial: str, ws: str, home: str, stopped: Optional[threading.Event] = None): + """The unit's VM while its agent or capability runs (``agent/boat.json`` says which, and how long + it took to come up); its workspace and home brought back from it, then the VM deleted. A + failure to bring them back after the agent ended is the unit's ``Unavailable``; after the + agent failed, its own failure stands.""" + with self._lock: + if self._keys is None: + self._keys = Keys() + vm = VM(self.client, self._keys, self.label, stopped=stopped) + try: + os.makedirs(os.path.join(trial, "agent"), exist_ok=True) + with open(os.path.join(trial, "agent", "boat.json"), "w") as f: + json.dump({"vm": vm.id, "size": SIZE, "ready_seconds": vm.seconds}, f) + self._held.vm = vm + try: + yield vm + except BaseException: + for d in (ws, home): # what it left, for the evidence + try: + vm.fetch(d) + except Unavailable: + pass + raise + for d in (ws, home): + vm.fetch(d) + finally: + self._held.vm = None + vm.delete() + + def place(self, cmd: List[str], env: Dict[str, str], workdir: str, *, inst: Optional[str], here: Optional[str], + owned: List[str], relay_url: Optional[str], local: Sequence[str] = ()) -> tuple: + """``cmd`` and the environment to start it with here: ``ssh`` to this thread's VM, where it runs as + its locked user. Sent first, each at its own path: the install (``inst``) and the runtimes it + links to outside itself, the definition (``here``), and ``owned`` - its home and workspace.""" + vm = getattr(self._held, "vm", None) + if vm is None: + raise Unavailable("the unit has no Boat VM") + if inst and inst not in self._roots: + with self._lock: + self._roots[inst] = outside_roots(inst) if os.path.isdir(inst) else [] + read = [p for p in [inst, here, *self._roots.get(inst or "", [])] if p and os.path.isdir(p)] + vm.send(read, owned) + port = urllib.parse.urlsplit(relay_url).port if relay_url else None + env = remote_env(env, install.bin_dirs(inst) if inst else [], local) + return vm.argv(list(cmd), env, workdir, port), {k: os.environ[k] for k in ("PATH", "HOME", "LANG") if k in os.environ} + + def close(self) -> None: + sweep(self.label, self.client) + if self._keys is not None: + self._keys.close() diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 81f8544d..c0e94ff5 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,8 +6,8 @@ PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION - PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container|boat] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container|boat] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -19,6 +19,9 @@ ``--backend container`` (``docker``, its earlier name) runs each unit's agent or capability in a container whose only way out is the unit's relay, or a public web proxy when its environment declares ``network`` (``standard/container.py``), and needs no such yes. +``--backend boat`` runs each unit's agent or capability in a boat.dev VM of its +own, whose only way out is a tunnel to the unit's relay on this machine +(``standard/boat.py``); its API key is Settings → Sandbox's, or ``BOAT_API_KEY``. A benchmark's control (``nop``, ``oracle``: its ``controls``) runs no agent and calls no model - only a judged benchmark's judge - so it asks for neither. ``prove`` runs a benchmark's controls and checks its units and that nothing @@ -663,8 +666,8 @@ def main(argv=None): p.add_argument("--unsandboxed", action="store_true", help="allow an agent, or a setup check without Docker, to execute code on this machine as you") p.add_argument("--backend", default="local", type=definitions.names.backend, choices=list(run.BACKENDS), - help="local processes (the default), or container: each unit in a container, its relay beside it " - "(docker: its earlier name)") + help="local processes (the default); container: each unit in a container, its relay beside it " + "(docker: its earlier name); boat: each unit's agent in a boat.dev VM of its own") p.add_argument("--from-run", action="store_true", help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read") p.add_argument("--option", action="append", metavar="NAME=VALUE", diff --git a/app/nullius/standard/names.py b/app/nullius/standard/names.py index 4ff6cf5e..d54fa174 100644 --- a/app/nullius/standard/names.py +++ b/app/nullius/standard/names.py @@ -1,7 +1,7 @@ """Canonical names, with read compatibility for earlier catalogues, jobs and recipes.""" from __future__ import annotations -BACKENDS = ("local", "container") +BACKENDS = ("local", "container", "boat") def backend(value): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 5e5d759c..a410ed5f 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -28,8 +28,10 @@ ``limit`` is its own. ``backend``: ``local`` processes, or ``container`` - each unit's agent or capability in a container, ``docker`` its earlier name - which a host must have Docker or Podman for (``docker`` in the result: the run checks it -before anything is sent). Each machine's constraints in Settings -(``nullius.constraints``) go on its command. +before anything is sent), or ``boat`` - each unit's agent or capability in a +boat.dev VM of its own (``boat.py``), started from this machine with its key. +Each machine's constraints in Settings (``nullius.constraints``) go on its +command. """ from __future__ import annotations @@ -262,6 +264,7 @@ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]: def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, docker_unusable: Optional[Callable[[], Optional[str]]] = None, + boat_unusable: Optional[Callable[[], Optional[str]]] = None, unpriced: Optional[Callable[[List[str]], List[str]]] = None) -> dict: """{errors, notes, review, jobs, confirm, script, parts, model} for one recipe; errors name the step @@ -274,7 +277,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, machine, why docker cannot run here (``container.unusable``) - a reason is the Where step's error. Not given, it is not asked: compiling stays a function of the definitions alone, and the routes that preview or start - a run pass it. + a run pass it. ``boat_unusable``: the same for the boat backend + (``boat.unusable``): its key, and the programs that reach a VM. ``max_usd``: the run's spending cap (``standard.ledger``), one amount for the whole run: its jobs that call a model - each dataset with each @@ -318,7 +322,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # Several may be chosen: each is its own job of the one run, one after the other. backend = definitions.names.backend(recipe.get("backend") or "local") if backend not in BACKENDS: - add("where", "The backend is local processes or a container.") + add("where", "The backend is local processes, a container or Boat.") backend = "local" what = "an agent" if side == "agent" else f"a {_bench_page(b)['kind'] or 'capability'}" refs = chosen_of(recipe) @@ -498,6 +502,12 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, why = docker_unusable() if why: add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.") + elif backend == "boat" and targets != ["local"]: + add("where", "Boat VMs are started from this machine, with its key: choose This machine alone.") + elif backend == "boat" and boat_unusable is not None: + why = boat_unusable() + if why: + add("where", f"Boat cannot run from this machine: {why}.") # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) @@ -600,7 +610,7 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], - *(["--backend", "container"] if backend == "container" else []), + *(["--backend", backend] if backend != "local" else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch_of(d, p)] if of > 1 else []), *limits.cli_words(limits.of(targets[index - 1])), @@ -707,9 +717,13 @@ def row(p: dict) -> list: notes.append(f"Each machine runs the units whose shard is its own, by their ids' hash: {of} jobs under one launch, " f"{name}, put together by cli merge.") docker_only = [n for n, e in (b.get("environments") or {}).items() if not e.get("local_ok")] - if docker_only and not only_controls and backend == "local": + if docker_only and not only_controls and backend in ("local", "boat"): notes.append(f"Units in {b['label']}'s environment{'s' if len(docker_only) > 1 else ''} {', '.join(docker_only)} " - "need the container backend: on local processes they are recorded as not run.") + f"need the container backend: on {'local processes' if backend == 'local' else 'Boat'} they are recorded as not run.") + web_only = [n for n, e in (b.get("environments") or {}).items() if e.get("network") and e.get("local_ok")] + if web_only and side == "agent" and not only_controls and backend == "boat": + notes.append(f"Units in {b['label']}'s environment{'s' if len(web_only) > 1 else ''} {', '.join(web_only)} " + "give the agent the public web, which a Boat VM does not yet: they are recorded as not run.") if any(p["control"] for p in picks) and any(definitions.own_image(e) for e in (b.get("environments") or {}).values()): notes.append(f"Each {(b.get('units') or {}).get('noun', 'unit')} of {b['label']} is its own image: the control " "runs in it, so docker is needed where it runs, whatever the backend.") @@ -759,7 +773,7 @@ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: errors.append("Choose a model or enter its ID.") backend = definitions.names.backend(backend) if backend not in BACKENDS: - errors.append("Choose local processes or a container for the checks.") + errors.append("Choose local processes, a container or Boat for the checks.") if kind == "agent" and backend == "local" and unsandboxed is not True: errors.append(f"{o['label']} runs on this machine as you while its kits run: the local backend is not a sandbox. Say yes to that to check it.") if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""): @@ -771,7 +785,7 @@ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: if errors: return {"errors": errors, "script": "", "jobs": [], "label": o["label"]} words = ["nullius", "standard", "conform", o["_ref"], "--model", model, "--provider", provider, - "--stamp", stamp, *(["--backend", "container"] if backend == "container" else []), + "--stamp", stamp, *(["--backend", backend] if backend != "local" else []), *(["--unsandboxed"] if kind == "agent" and backend == "local" else []), "--from-run"] command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"' script = ("#!/usr/bin/env bash\n" diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 20fbcc56..bb027dcf 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -13,13 +13,16 @@ unit backs off on its own, as it always did. Its record sits in the trial's ``.work/relay``. -Two backends (``backend``). ``local``: processes on this machine, which are +Backends (``backend``). ``local``: processes on this machine, which are not a sandbox - what an agent runs, it runs as the user (``docs/standard.md``, "Safety"). ``docker``: the agent or the capability in a container of its own on an internal network with the unit's relay beside it (``container``); an own-image task may grant a public-address proxy with ``internet: true``, and an agent whose environment declares ``network`` gets a public web proxy. On docker the agent's or capability's packages are installed in the base image, on the libc and tools its container runs on. +A third, ``boat``: the agent's or capability's process alone in a Boat VM of +its own, its relay still here, reached through the VM's tunnel (``boat.Backend``); +a check or a setup on it runs on local processes. The driver, its environment installed here, stays on this machine either way: it owns the answers. So do a benchmark's own MCP servers (``mcp_servers``), in the driver's environment: on the local backend the @@ -71,7 +74,7 @@ from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed -from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards +from nullius.standard import acp, boat, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards # the totals a unit's end carries onto the live board, from its trial's agent_result FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd") @@ -447,6 +450,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") + if backend == "boat" and other.get("_kind") not in ("agent", "capability"): + backend = "local" # a check or a setup: nothing of an agent's runs, so nothing goes to a VM + if backend == "boat": + why = boat.unusable(env, which) + if why: + raise Refused(f"the Boat backend cannot run here: {why}") if backend == "local" and other.get("_kind") == "agent" and not unsandboxed: raise Refused(f"{other['label']} would run commands as you: the local backend is not a sandbox. " "Say so explicitly to go on (--unsandboxed), or run it in a container (--backend container).") @@ -610,6 +619,13 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # each stage said as it goes, on the cli's stderr and in the job's progress.jsonl (nullius/progress.py) self.progress = progress.Reporter(self.job_dir) self._said = threading.Lock() + self.boat = boat.Backend(self.name, env) if backend == "boat" else None + if self.boat is not None: + # one VM per unit: never more units at once than the account runs VMs + most = self.boat.most() + if most is not None and self.concurrency > most: + say(f"Boat runs {most} VMs at once on this account: {most} units at a time, not {self.concurrency}") + self.concurrency = most # the job page's live board (nullius/runs/live.py): written once the units are known self.feed: Optional[live_feed.Writer] = None self._ended = 0 # units ended so far: the i of the next unit's line @@ -813,6 +829,8 @@ def work_unit(iur): self.bench_relay.stop() if self.backend == "container": container.sweep(self.name) # whatever an interrupted unit left + if getattr(self, "boat", None) is not None: + self.boat.close() # whatever an interrupted unit left if self.loopback is not None: self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) @@ -1045,6 +1063,8 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: own = definitions.own_image(env_def) if "internet" in setup and (not isinstance(setup["internet"], bool) or not own): raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image") + if self.other["_kind"] == "agent" and self.backend == "boat" and mcp_machine.wanted(setup): + raise _Ended("EnvironmentSetupError", "not run: " + boat.MCP_REFUSAL) if self.other["_kind"] == "agent" and self.backend == "local" and mcp_machine.wanted(setup): try: # the benchmark's own servers: the agent starts them from the driver's environment setup = dict(setup, mcp=mcp_machine.entries(self.bench, setup, install.driver_dir(self.bench), @@ -1072,7 +1092,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" else DRIVER_SECONDS if self.other["key"] == "oracle" else None) try: - with self.agent_internet(setup, box, rl, trial): + with self.agent_internet(setup, box, rl, trial), self.unit_vm(ws, home, trial): if self.other["_kind"] == "control": answer = self.control_answer(unit, ws, box, name) elif self.upstream: @@ -1271,9 +1291,13 @@ def not_run(self, name: Optional[str], env_def: Optional[dict]) -> Optional[tupl refused the same way, for what would run in it (not a control).""" if name is None: return None - if self.backend == "local" and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"): + if self.backend in ("local", "boat") and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"): return ("EnvironmentSetupError", f"not run: the environment {name} needs the container backend " "(--backend container); it does not say local_ok") + if self.backend == "boat": + why = boat.refusal(env_def) + if why: + return ("EnvironmentSetupError", f"not run: the environment {name} {why}") missing = definitions.missing_secrets(env_def, self.licences) if missing: return ("EnvironmentSecretMissingError", "not run: needs " + ", ".join(missing)) @@ -1376,7 +1400,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] record = os.path.join(trial, ".work", "relay") side = tuple((self.other.get("model") or {}).get("side_models") or ()) key_header = self.key_header - if self.backend == "local": + if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() rl.unit_url = rl.url(tag) @@ -1506,6 +1530,12 @@ def place(self, rl, cmd: List[str], env: dict, workdir: str, home: str, interact if self.backend == "local": return limits.wrap(cmd, env, getattr(self, "scope_limits", {})) ws = os.path.join(os.path.dirname(home), "workspace") + if self.backend == "boat": + try: + return self.boat.place(cmd, env, workdir, inst=self.inst, here=self.other.get("_here"), owned=[ws, home], + relay_url=rl.unit_url if rl is not None else None, local=KEEP) + except boat.Unavailable as e: + raise _Ended("EnvironmentSetupError", str(e)) # What the agent's container sees: its workspace and home, its own # install and definition, and the files its MCP servers need - never # this application's folder (its .env holds the keys) nor the @@ -1546,6 +1576,18 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: return str(reply.get("answer") or "") # -- an own image: the unit's container ------------------------------------ + @contextmanager + def unit_vm(self, ws: str, home: str, trial: str): + """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``).""" + if getattr(self, "boat", None) is None: + yield + return + try: + with self.boat.unit(trial, ws, home, stopped=self._stopped): + yield + except boat.Unavailable as e: + raise _Ended("EnvironmentSetupError", str(e)) + @contextmanager def agent_internet(self, setup: dict, box, rl, trial: str): """The benchmark's task policy, never a run option: a public-address proxy diff --git a/app/static/brand/README.md b/app/static/brand/README.md index 38c01b5d..d4708c49 100644 --- a/app/static/brand/README.md +++ b/app/static/brand/README.md @@ -18,3 +18,5 @@ third-party brand asset, not a Nullius mark or part of Nullius's font licensing. `docker.svg` is the Docker mark from [Simple Icons](https://github.com/simple-icons/simple-icons/blob/develop/icons/docker.svg), distributed under CC0, rendered in Docker blue. The Docker trademark belongs to Docker, Inc. It identifies Docker Hub under Data providers. `huggingface.svg` is Hugging Face’s official mark, copied from https://huggingface.co/front/assets/huggingface_logo-noborder.svg. It identifies the data provider; its trademark belongs to Hugging Face. + +`boat.svg` is Boat's official mark, copied unchanged from https://boat.dev/icon.svg. It identifies the Boat sandbox provider in Settings and the builder's Where step; its trademark belongs to Boat. diff --git a/app/static/brand/boat.svg b/app/static/brand/boat.svg new file mode 100644 index 00000000..815ef042 --- /dev/null +++ b/app/static/brand/boat.svg @@ -0,0 +1,3 @@ + + + diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 1d53e123..3fe0e7cb 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -172,7 +172,7 @@ const picked = Array.isArray(raw.others) ? raw.others.filter(k => typeof k === 'string') : typeof raw.other === 'string' && raw.other ? [raw.other] : null; if (picked) r.others = [...new Set(picked)]; if (Array.isArray(raw.where) && raw.where.length && raw.where.every(w => typeof w === 'string')) r.where = raw.where.length > 1 ? [...raw.where] : raw.where[0]; - if (['local', 'container', 'docker'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name + if (['local', 'container', 'docker', 'boat'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name if (raw.bench_options && typeof raw.bench_options === 'object' && !Array.isArray(raw.bench_options)) // the benchmark's own options, a word each r.bench_options = ownOptions(standardBench(r), Object.fromEntries(Object.entries(raw.bench_options).filter(([, v]) => typeof v === 'string'))); // Options per choice, {ref: {name: word}}; a single agent also accepts flat {name: word} options. diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index a25326b6..45e2726a 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -427,9 +427,11 @@ // Review's rule (nullius/standard/recipe.py): a setup's image installs what has an install; nothing else reaches its container const cannot = selected.some(o => o.setup) ? optionNeeds(rec).host.filter(n => (b.host || {})[n] && !b.host[n].install.length) : []; const containerWhy = dockerWhy || (cannot.length ? cannot.join(', ') + ' cannot be provided in a container: choose local processes.' : ''); + // boat VMs are started from this machine, with its key (Settings → Sandbox) + const boatWhy = list.some(w => w !== 'local') ? 'Started from this machine only' : here.boat ? here.boat : ''; const agentRun = b && b.family === 'agent' && selected.some(o => !o.control && !o.setup); const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : '▣', title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : v === 'boat' ? '' : '▣', title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy return '

Where does it run?

' @@ -438,9 +440,10 @@ + '

How it runs there

' + `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' - : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '
' + : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + + how('boat', 'Boat', 'Sandboxed: each unit in a Boat VM of its own, whose only way out is its relay on this machine.', boatWhy) + '' + (agentRun && backend === 'local' ? consentBox(rec) : '') - + (dockerOnly.length && backend === 'local' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on local processes their units are recorded as not run.

` : '') + + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.

` : '') + hostSetup(rec); } // What the chosen options of the benchmark need: an embedding model, requirements of the machine (standard.json host). diff --git a/app/static/settings/sandbox.js b/app/static/settings/sandbox.js index a199f4e2..c7559c01 100644 --- a/app/static/settings/sandbox.js +++ b/app/static/settings/sandbox.js @@ -1,17 +1,55 @@ -// The Sandbox section: docker's state, asked once when the section is first shown. +// The Sandbox section: docker's state and whether a Boat key is saved, asked once when the section is first shown. (() => { const section = document.getElementById('sandbox'); if (!section) return; + const $ = s => section.querySelector(s); + const state = $('[data-boat-state]'), form = $('[data-boat-form]'); + const edit = $('[data-boat-edit]'), remove = $('[data-boat-remove]'), cancel = $('[data-boat-cancel]'); + const notify = (message, error = false) => window.nulliusNotify?.(message, {error}); + let boat = {set: false, env: false}, editing = false; + + const render = () => { + state.textContent = boat.env ? 'Key from BOAT_API_KEY' : boat.set ? 'Key saved' : 'No key'; + if (form) form.hidden = !editing; + if (edit) { edit.hidden = editing; edit.textContent = boat.set ? 'Edit' : 'Add key'; } + if (remove) remove.hidden = editing || !boat.set; + }; let asked = false; const load = async () => { asked = true; let j; - try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {docker: {ok: false, error: String(err)}}; } - const state = section.querySelector('[data-docker-state]'); - state.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error; - state.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad'); + try { j = await (await nulliusFetch(section.dataset.sandbox)).json(); } catch (err) { j = {docker: {ok: false, error: String(err)}, boat: {set: false, env: false}}; } + const docker = $('[data-docker-state]'); + docker.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error; + docker.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad'); + boat = j.boat || {set: false, env: false}; + render(); + }; + + const change = async action => { + const fields = new FormData(form); + fields.set('action', action); + if (action === 'remove') fields.delete('key'); + section.querySelectorAll('button').forEach(b => { b.disabled = true; }); + try { + const response = await nulliusFetch(section.dataset.boat, {method: 'POST', headers: {'X-CSRF-Token': section.dataset.csrf}, body: fields}); + const result = await response.json(); + if (!response.ok || !result.ok) { notify(result.error || 'Could not save the key.', true); return; } + form.reset(); + editing = false; + notify(action === 'remove' ? 'Boat key removed.' : 'Boat key saved.'); + await load(); + } catch (err) { + notify(String(err), true); + } finally { + section.querySelectorAll('button').forEach(b => { b.disabled = false; }); + } }; + if (edit) edit.addEventListener('click', () => { editing = true; render(); form.querySelector('input').focus(); }); + if (cancel) cancel.addEventListener('click', () => { form.reset(); editing = false; render(); edit.focus(); }); + if (form) form.addEventListener('submit', e => { e.preventDefault(); change('save'); }); + if (remove) remove.addEventListener('click', () => change('remove')); // Asked the first time the section is shown, not on every visit to Settings. const ask = () => { if (!asked && !section.hidden) load(); }; diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index d321c70f..38612c4d 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -86,7 +86,7 @@

Settings

Execution
{{ section_link('hosts', 'Hosts', 'server', 'ssh remote machines server hosts') }} - {{ section_link('sandbox', 'Sandbox', 'kit', 'docker container') }} + {{ section_link('sandbox', 'Sandbox', 'kit', 'docker container boat vm') }}
Application
@@ -392,11 +392,21 @@

Judges

{% endif %} {% include 'settings/data_providers.html' %} {# the container backend's engine on this machine, outside the form: asked when the section opens (sandbox.js) #} -
+

Sandbox

Where agents run, apart from this machine.

EngineChecking…
+ {# Boat: a VM per unit (standard/boat.py); its key kept in the owner-only store, never shown again #} +
+

Boat

Checking…

{% if writable %}{% endif %}
+ {% if writable %} + + {% endif %} +
{# the user's price list (nullius.prices), outside the form: prices.js draws the state and answers the buttons #} diff --git a/app/tests/test_standard_boat.py b/app/tests/test_standard_boat.py new file mode 100644 index 00000000..f5d31b7a --- /dev/null +++ b/app/tests/test_standard_boat.py @@ -0,0 +1,359 @@ +"""The boat backend (``standard/boat.py``): its API client, a unit's VM, what it is sent, and a run +whose agent goes to a VM - here a stand-in that runs it on this machine, so nothing reaches boat.dev.""" +from __future__ import annotations + +import io +import json +import os +import shutil +import subprocess +import tempfile +import threading +import unittest +import urllib.error +from unittest import mock + +from nullius.standard import boat, definitions, recipe, run + +from .test_standard_recipe import a_recipe, defs +from .test_standard_run import ROOT, Echo +from http.server import ThreadingHTTPServer +from nullius.runs import parse +from nullius.standard import install + + +class Answer: + def __init__(self, body): + self.body = json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return self.body + + +class FakeAPI: + """boat.dev's API as a list of answers to POST /sandboxes, and a VM ready at once.""" + + def __init__(self, creates): + self.creates, self.calls = list(creates), [] + + def __call__(self, req, timeout=None): + body = json.loads(req.data) if req.data else None + self.calls.append((req.get_method(), req.full_url.split("/api/v1", 1)[1], body, dict(req.header_items()))) + method, path = self.calls[-1][:2] + if method == "POST" and path == "/sandboxes": + status, d = self.creates.pop(0) + if status != 200: + raise urllib.error.HTTPError(req.full_url, status, "no", {}, io.BytesIO(json.dumps(d).encode())) + return Answer(d) + if method == "GET" and path.startswith("/sandboxes/"): + return Answer({"ok": True, "sandbox": {"state": "ready"}}) + if path.endswith("/sshkey"): + return Answer({"ok": True, "sshEndpoint": "vm.example:2201", "hostKey": "ssh-ed25519 AAAA", "sshUser": "user"}) + if method == "GET" and path.startswith("/sandboxes?"): + return Answer({"ok": True, "sandboxes": [{"id": "bx_1", "labels": {boat.LABEL: "job.1"}}, + {"id": "bx_2", "labels": {boat.LABEL: "other"}}]}) + if method == "GET" and path == "/limits": + return Answer({"ok": True, "maxActiveSandboxes": 2}) + return Answer({"ok": True}) + + +class FakeKeys: + def __init__(self): + self.dir, self.public = tempfile.mkdtemp(), "ssh-ed25519 AAAA nullius-boat" + + def close(self): + shutil.rmtree(self.dir, ignore_errors=True) + + +SLOT_FULL = (429, {"ok": False, "code": "limit_reached", "message": "2 active"}) +MADE = (200, {"ok": True, "sandbox": {"id": "bx_9"}}) + + +class Client(unittest.TestCase): + def test_the_key_is_sent_to_boat_alone_and_never_in_a_body(self): + api = FakeAPI([MADE]) + boat.Client("boat_secret", opener=api).create({boat.LABEL: "j"}) + method, path, body, headers = api.calls[0] + self.assertEqual((method, path), ("POST", "/sandboxes")) + self.assertEqual(headers["Authorization"], "Bearer boat_secret") + self.assertNotIn("boat_secret", json.dumps(body)) + self.assertEqual(body["labels"], {boat.LABEL: "j"}) + + def test_a_refusal_is_an_answer_not_an_exception(self): + d = boat.Client("k", opener=FakeAPI([SLOT_FULL])).create({}) + self.assertEqual((d["ok"], d["status"], d["code"]), (False, 429, "limit_reached")) + + def test_slots_is_the_accounts_limit_of_vms_at_once(self): + self.assertEqual(boat.slots(boat.Client("k", opener=FakeAPI([]))), 2) + + def test_sweep_deletes_only_this_jobs_vms(self): + api = FakeAPI([]) + self.assertEqual(boat.sweep("job.1", boat.Client("k", opener=api)), 1) + deletes = [c for c in api.calls if c[0] == "DELETE"] + self.assertEqual([c[1] for c in deletes], ["/sandboxes/bx_1"]) + self.assertEqual(deletes[0][3]["X-ascii-confirm-delete"], "bx_1") + + def test_the_key_comes_from_the_environment_first_then_the_store(self): + with mock.patch.object(boat.secrets, "value", return_value=b"boat_stored\n"): + self.assertEqual(boat.key({}), "boat_stored") + self.assertEqual(boat.key({boat.NAME: "boat_env"}), "boat_env") + with mock.patch.object(boat.secrets, "value", return_value=None): + self.assertIn("Settings", boat.unusable({})) + self.assertIsNone(boat.unusable({boat.NAME: "k"}, which=lambda p: "/usr/bin/" + p)) + self.assertIn("rsync", boat.unusable({boat.NAME: "k"}, which=lambda p: None if p == "rsync" else p)) + + +class VM(unittest.TestCase): + def setUp(self): + self.keys = FakeKeys() + self.addCleanup(self.keys.close) + self.ssh = [] + ok = lambda argv, **kw: self.ssh.append(argv) or subprocess.CompletedProcess(argv, 0, b"", b"") + patcher = mock.patch.object(boat.subprocess, "run", side_effect=ok) + patcher.start() + self.addCleanup(patcher.stop) + + def test_a_full_account_is_waited_for_then_the_vm_is_locked_in(self): + api = FakeAPI([SLOT_FULL, SLOT_FULL, MADE]) + slept = [] + vm = boat.VM(boat.Client("k", opener=api), self.keys, "job.1", sleep=slept.append) + self.assertEqual(vm.id, "bx_9") + self.assertEqual(len([c for c in api.calls if c[1] == "/sandboxes"]), 3) + self.assertEqual(slept[:2], [5.0, 7.5]) + # the host key pinned, no config of this user's read + self.assertIn("StrictHostKeyChecking=yes", vm.ssh) + self.assertEqual(vm.ssh[1:3], ["-F", "/dev/null"]) + with open(os.path.join(self.keys.dir, "known-bx_9")) as f: + self.assertEqual(f.read(), "bx_9 ssh-ed25519 AAAA\n") + setup = self.ssh[-1][-1] + self.assertIn(f"useradd -m -s /bin/bash {boat.USER}", setup) + self.assertIn("-I OUTPUT 1 -m owner --uid-owner $U -j REJECT", setup) + self.assertIn("--dport 53", setup) + + def test_another_refusal_or_a_stop_ends_the_wait(self): + with self.assertRaisesRegex(boat.Unavailable, "refused"): + boat.VM(boat.Client("k", opener=FakeAPI([(402, {"code": "no_credit", "message": "no credit"})])), self.keys, "j", + sleep=lambda s: None) + stopped = threading.Event() + stopped.set() + with self.assertRaisesRegex(boat.Unavailable, "in time"): + boat.VM(boat.Client("k", opener=FakeAPI([SLOT_FULL])), self.keys, "j", stopped=stopped, sleep=lambda s: None) + + def test_a_vm_that_fails_its_setup_is_deleted(self): + api = FakeAPI([MADE]) + fail = lambda argv, **kw: subprocess.CompletedProcess(argv, 1 if "useradd" in argv[-1] else 0, b"", b"no") + with mock.patch.object(boat.subprocess, "run", side_effect=fail), self.assertRaises(boat.Unavailable): + boat.VM(boat.Client("k", opener=api), self.keys, "j", sleep=lambda s: None) + self.assertEqual([c[1] for c in api.calls if c[0] == "DELETE"], ["/sandboxes/bx_9"]) + + def test_the_agent_runs_as_its_user_with_its_environment_alone_and_the_relay_tunnelled(self): + vm = boat.VM(boat.Client("k", opener=FakeAPI([MADE])), self.keys, "j", sleep=lambda s: None) + argv = vm.argv(["/inst/bin/agent", "--acp"], {"PATH": "/inst/bin:" + boat.VM_PATH, "HOME": "/h"}, "/w s", 41234) + self.assertEqual(argv[argv.index("-R") + 1], "127.0.0.1:41234:127.0.0.1:41234") + self.assertIn("ExitOnForwardFailure=yes", argv) + self.assertEqual(argv[-1], f"cd '/w s' && exec sudo -n -u {boat.USER} env -i PATH=/inst/bin:{boat.VM_PATH} HOME=/h /inst/bin/agent --acp") + + def test_the_workspace_comes_back_without_links_that_leave_it(self): + vm = boat.VM(boat.Client("k", opener=FakeAPI([MADE])), self.keys, "j", sleep=lambda s: None) + vm.fetch("/jobs/t/workspace/") + argv = self.ssh[-1] + self.assertEqual(argv[0], "rsync") + for flag in ("--safe-links", "--no-devices", "--no-specials", "--delete"): + self.assertIn(flag, argv) + self.assertNotIn("-a", argv) # no owner, group or permissions of the VM's + self.assertEqual(argv[-2:], [f"user@vm.example:/jobs/t/workspace/", "/jobs/t/workspace/"]) + + +class Backend(unittest.TestCase): + """One run's boat backend: a unit's VM held by its thread, brought back and always deleted.""" + + def setUp(self): + self.trial = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.trial, True) + StandInVM.made.clear(); StandInVM.deleted.clear(); StandInVM.fetched.clear() + for name, value in (("VM", StandInVM), ("Keys", FakeKeys)): + p = mock.patch.object(boat, name, value) + p.start() + self.addCleanup(p.stop) + self.backend = boat.Backend("job", client=boat.Client("k", opener=FakeAPI([]))) + + def test_the_label_is_this_runs_and_the_vm_comes_back_then_goes(self): + self.assertTrue(self.backend.label.startswith("job.") and self.backend.label != boat.Backend("job").label) + with self.backend.unit(self.trial, "/t/workspace", "/t/.home"): + self.assertIsNotNone(self.backend._held.vm) + self.assertEqual(StandInVM.made, [self.backend.label]) + self.assertEqual(StandInVM.fetched, ["/t/workspace", "/t/.home"]) + self.assertEqual(StandInVM.deleted, ["bx_0"]) + self.assertIsNone(self.backend._held.vm) + with self.assertRaisesRegex(boat.Unavailable, "no Boat VM"): # no VM held outside a unit + self.backend.place(["a"], {}, "/t", inst=None, here=None, owned=[], relay_url=None) + + def test_the_agents_own_failure_stands_and_a_failed_copy_back_is_the_units(self): + failing = lambda self, path: (_ for _ in ()).throw(boat.Unavailable("no way back")) + with mock.patch.object(StandInVM, "fetch", failing): + with self.assertRaisesRegex(RuntimeError, "the agent's"): + with self.backend.unit(self.trial, "/t/workspace", "/t/.home"): + raise RuntimeError("the agent's") + with self.assertRaisesRegex(boat.Unavailable, "no way back"): + with self.backend.unit(self.trial, "/t/workspace", "/t/.home"): + pass + self.assertEqual(StandInVM.deleted, ["bx_0", "bx_1"]) + + +class Sent(unittest.TestCase): + def test_a_runtime_an_install_links_to_goes_with_it_and_the_systems_does_not(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp) + py = os.path.join(tmp, "uv", "python", "cpython-3.12", "bin") + os.makedirs(py) + open(os.path.join(py, "python3.12"), "w").close() + venv = os.path.join(tmp, "inst", "venv", "bin") + os.makedirs(venv) + os.symlink(os.path.join(py, "python3.12"), os.path.join(venv, "python")) + os.symlink("/bin/sh", os.path.join(venv, "sh")) + os.symlink("python", os.path.join(venv, "python3")) # inside the install + self.assertEqual(boat.outside_roots(os.path.join(tmp, "inst")), [os.path.join(tmp, "uv", "python", "cpython-3.12")]) + + def test_the_vm_is_given_its_own_paths_and_none_of_this_machines(self): + env = {"PATH": "/home/me/.nvm/bin:/usr/bin", "HOME": "/jobs/t/.home", "LANG": "C.UTF-8", "TMPDIR": "/home/me/tmp", + "NVM_DIR": "/home/me/.nvm", "OPENAI_BASE_URL": "http://127.0.0.1:5000/u0000/v1"} + got = boat.remote_env(env, ["/inst/node_modules/.bin"], run.KEEP) + self.assertEqual(got, {"PATH": "/inst/node_modules/.bin:" + boat.VM_PATH, "HOME": "/jobs/t/.home", "LANG": "C.UTF-8", + "OPENAI_BASE_URL": "http://127.0.0.1:5000/u0000/v1"}) + + +class StandInVM: + """A unit's VM that runs its command on this machine: what the run sends, starts and takes back.""" + made, deleted, sent, fetched = [], [], [], [] + + def __init__(self, client, keys, job, stopped=None): + self.id, self.seconds = f"bx_{len(StandInVM.made)}", 0.1 + StandInVM.made.append(job) + + def send(self, read, owned=()): + StandInVM.sent.append((list(read), list(owned))) + + def fetch(self, path): + StandInVM.fetched.append(path) + + def argv(self, cmd, env, workdir, port): + assert port, "the relay's port is tunnelled" + return ["env", "-i", *[f"{k}={v}" for k, v in env.items() if k != "PATH"], f"PATH={os.environ['PATH']}", *cmd] + + def delete(self): + StandInVM.deleted.append(self.id) + + +class BoatRun(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.up = ThreadingHTTPServer(("127.0.0.1", 0), Echo) + threading.Thread(target=cls.up.serve_forever, daemon=True).start() + cls.env = {"MODEL_API_URL": f"http://127.0.0.1:{cls.up.server_address[1]}/v1", "MODEL_API_KEY": "sk-real"} + + @classmethod + def tearDownClass(cls): + cls.up.shutdown() + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.addCleanup(setattr, install, "HOME", install.HOME) + install.HOME = os.path.join(self.tmp, "home") + Echo.status, Echo.refused, Echo.limited = 200, None, 0 + for v in (StandInVM.made, StandInVM.deleted, StandInVM.sent, StandInVM.fetched): + v.clear() + self.swept = [] + for name, value in (("unusable", lambda env=None, which=None: None), ("slots", lambda client: 1), + ("VM", StandInVM), ("Keys", FakeKeys), ("sweep", lambda job, client: self.swept.append(job))): + p = mock.patch.object(boat, name, value) + p.start() + self.addCleanup(p.stop) + + def test_each_unit_runs_in_a_vm_of_its_own_and_the_rest_stays_here(self): + said = [] + b = definitions.get("benchmark", "fakebench", ROOT) + a = definitions.get("agent", "fake", ROOT) + job_dir = run.run(b, a, model="m-1", provider="custom", jobs_root=self.tmp, concurrency=2, env=self.env, + say=said.append, backend="boat") + trials = {t.trial_name.split("__")[0]: t for t in parse.parse_job(job_dir).trials} + # the units end as their scripts say, but for one: no file of this machine is served to an agent in a VM + # (acp fs), so the fake agent's write through it is refused and its answer never lands, as in a container + self.assertEqual({k: (t.reward, t.exception_type) for k, t in trials.items()}, + {"good": (0.0, ""), "wrong": (0.0, ""), "model": (0.0, ""), "slow": (None, "AgentTimeoutError")}) + with open(os.path.join(job_dir, next(t for t in os.listdir(job_dir) if t.startswith("good__")), "agent", "acp.jsonl")) as f: + self.assertNotIn('"fs": {"readTextFile": true', f.read()) + # no yes asked, one VM per unit, each deleted, the account's limit of one at a time said + self.assertEqual(len(StandInVM.made), len(trials)) + self.assertEqual(sorted(StandInVM.deleted), sorted(f"bx_{i}" for i in range(len(trials)))) + self.assertIn("Boat runs 1 VMs at once on this account: 1 units at a time, not 2", said) + # the run's label on every VM, swept at the end + self.assertEqual(len(set(StandInVM.made)), 1) + self.assertEqual(self.swept, StandInVM.made[:1]) + # what goes: the agent's definition (and install), its home and workspace; never the benchmark's folder + for read, owned in StandInVM.sent: + self.assertIn(a["_here"], read) + self.assertNotIn(b["_here"], read) + self.assertEqual([os.path.basename(p) for p in owned], ["workspace", ".home"]) + self.assertEqual(sorted(os.path.basename(p) for p in StandInVM.fetched), sorted(["workspace", ".home"] * len(trials))) + for t in os.listdir(job_dir): + if "__u" in t: + with open(os.path.join(job_dir, t, "agent", "boat.json")) as f: + self.assertEqual(json.load(f)["size"], boat.SIZE) + with open(os.path.join(job_dir, "config.json")) as f: + self.assertEqual(json.load(f)["standard"]["backend"], "boat") + for root, _, files in os.walk(job_dir): + for n in files: + with open(os.path.join(root, n), errors="replace") as f: + self.assertNotIn("sk-real", f.read(), os.path.join(root, n)) + + def test_a_check_runs_here_and_no_key_means_no_run(self): + b = definitions.get("benchmark", "fakebench", ROOT) + b["controls"] = {"nop": {"source": "nullius", "from": "nothing said"}} + nop = definitions.control(b, "nop") + self.assertEqual(run.Run(b, nop, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat").backend, "local") + a = definitions.get("agent", "fake", ROOT) + with mock.patch.object(boat, "unusable", lambda env=None, which=None: "no Boat API key: set it in Settings → Sandbox"): + with self.assertRaisesRegex(run.Refused, "Boat backend cannot run here: no Boat API key"): + run.Run(b, a, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat") + + def test_an_environment_a_vm_cannot_give_is_not_run(self): + self.assertIn("public web", boat.refusal({"network": {"probe": "https://example.org"}})) + self.assertIn("LICENSE", boat.refusal({"needs_secrets": [{"name": "LICENSE", "to": "agent", "kind": "file"}]})) + self.assertIsNone(boat.refusal({"needs_secrets": [{"name": "KEY", "to": "agent", "kind": "value"}]})) + self.assertIsNone(boat.refusal({"local_ok": True})) + + +class BoatRecipe(unittest.TestCase): + def test_boat_needs_no_yes_and_runs_from_this_machine(self): + out = recipe.compile(a_recipe(backend="boat", unsandboxed=False), defs(), boat_unusable=lambda: None) + self.assertEqual(out["errors"], []) + self.assertIn("--backend boat", out["script"]); self.assertNotIn("--unsandboxed", out["script"]) + self.assertFalse(out["docker"]) + self.assertIn("boat backend", out["review"][-1][2]) + out = recipe.compile(a_recipe(backend="boat", where="me@bench"), defs()) + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) + self.assertIn("This machine", out["errors"][0]["message"]) + + def test_without_its_key_the_where_step_says_so(self): + out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda: "no Boat API key: set it in Settings → Sandbox") + self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) + self.assertIn("Settings → Sandbox", out["errors"][0]["message"]) + + def test_a_conformance_check_on_boat_says_its_backend(self): + agent = definitions.load("agent", ROOT)[0][0] + with mock.patch.object(definitions, "get", return_value=agent): + got = recipe.conformance("agent", agent["key"], model="m", provider="openai", unsandboxed=False, + stamp="2026-10-10__12-00-00", backend="boat") + self.assertEqual(got["errors"], []) + self.assertIn("--backend boat", got["script"]); self.assertNotIn("--unsandboxed", got["script"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index ae4d363e..62832823 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -487,13 +487,17 @@ def test_the_builder_gets_the_skills_and_a_benchmark_needing_one_says_who_lacks_ self.assertIn('.filter(s => !s.shown).map(s => `(.*?)', html, re.S).group(1)) ds = schema["standard"]["benchmarks"][0]["datasets"][0] self.assertEqual([(c["key"], c["source"], c["proven"], c["failed"]) for c in ds["controls"]], [("nop", "nullius", None, None), ("oracle", "adapter", None, None)]) - self.assertEqual(schema["standardHere"], {"docker": None, "licences": {}, "host": {}}) # no benchmark here needs a machine setup + # no benchmark here needs a machine setup; boat says why it cannot run, never its key + self.assertEqual(schema["standardHere"], {"docker": None, "boat": "no Boat API key: set it in Settings → Sandbox", + "licences": {}, "host": {}}) def test_docker_is_refused_on_the_where_step_with_this_machines_reason(self): self.fixture() diff --git a/app/ui_tests/test_settings_page.py b/app/ui_tests/test_settings_page.py index 05b36bb6..9fbd1346 100644 --- a/app/ui_tests/test_settings_page.py +++ b/app/ui_tests/test_settings_page.py @@ -1091,8 +1091,16 @@ def test_the_sandbox_is_asked_when_shown_and_a_login_goes_to_docker_only(self): self.assertIn('data-sandbox="/settings/sandbox"', section) self.assertNotIn("data-images", section) # no Images card self.assertNotIn('name="value:', section) # outside Save: nothing of it lands in the file - for login in ("Docker Hub", 'type="password"', "data-login", "data-logout"): # signed in once, under Data providers + for login in ("Docker Hub", "data-login", "data-logout"): # signed in once, under Data providers self.assertNotIn(login, section) + # the one secret asked here: boat.dev's key, outside Save + self.assertEqual(section.count('type="password"'), 1) + self.assertIn('name="key" type="password"', section) + # a provider card like Data providers': its mark, its name, Edit and Remove, the key asked only on Edit + self.assertIn('

Boat

', section) + self.assertIn('brand/boat.svg', section) + for control in ("data-boat-edit", "data-boat-remove", "data-boat-cancel", "data-boat-form autocomplete=\"off\" hidden"): + self.assertIn(control, section) self.assertNotIn('data-data-form', section) providers = html[html.index('id="data-providers"'):html.index('id="sandbox"')] self.assertIn('data-data-search', providers) @@ -1102,7 +1110,7 @@ def test_the_sandbox_is_asked_when_shown_and_a_login_goes_to_docker_only(self): with patch.object(sandbox, "docker_state", lambda: {"ok": True, "version": "29.0"}), \ patch.object(sandbox, "login_user", lambda: self.fail("asked")): r = self.client.get("/settings/sandbox"); self.addCleanup(r.close) - self.assertEqual(r.json, {"docker": {"ok": True, "version": "29.0"}}) + self.assertEqual(r.json, {"docker": {"ok": True, "version": "29.0"}, "boat": {"set": False, "env": False}}) seen = [] headers = {"Origin": "http://localhost", "X-CSRF-Token": self.token()} with patch.object(sandbox, "login", lambda u, t: seen.append((u, t)) or {"ok": True, "user": u}): @@ -1115,6 +1123,30 @@ def test_the_sandbox_is_asked_when_shown_and_a_login_goes_to_docker_only(self): self.assertEqual(seen, [("someone", "dckr_pat_synthetic")]) self.assertIsNone(self.file()) + def test_the_boat_key_is_saved_from_this_machines_browser_and_never_echoed(self): + from nullius import secrets + from nullius.standard import boat + with tempfile.TemporaryDirectory() as folder, patch.dict(os.environ, {"SECRETS_DIR": folder}): + os.environ.pop(boat.NAME, None) + headers = {"Origin": "http://localhost", "X-CSRF-Token": self.token()} + url = "/settings/sandbox/boat" + for kwargs in ({"headers": {"Origin": "http://localhost"}}, + {"headers": headers, "environ_base": {"REMOTE_ADDR": "10.0.0.7"}}): + self.assertEqual(self.client.post(url, data={"key": "boat_synthetic"}, **kwargs).status_code, 403) + self.assertEqual(self.client.post(url, data={"key": " "}, headers=headers).status_code, 400) + response = self.client.post(url, data={"key": "boat_synthetic"}, headers=headers) + self.assertTrue(response.json["ok"]) + self.assertNotIn("boat_synthetic", response.get_data(as_text=True)) + self.assertEqual(secrets.value(boat.STORE, boat.NAME), b"boat_synthetic") + self.assertEqual(boat.key({}), "boat_synthetic") + with patch.object(sandbox, "docker_state", lambda: {"ok": True, "version": "29.0"}): + state = self.client.get("/settings/sandbox").json + self.assertEqual(state["boat"], {"set": True, "env": False}) + self.assertNotIn("boat_synthetic", json.dumps(state)) + self.assertIsNone(self.file()) # never in the settings file + self.assertTrue(self.client.post(url, data={"action": "remove"}, headers=headers).json["ok"]) + self.assertIsNone(boat.key({})) + def test_data_provider_save_is_local_guarded_and_never_echoes_token(self): from nullius import secrets from nullius.standard import definitions, run diff --git a/app/web/builder.py b/app/web/builder.py index d09d0806..2dfcfef9 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -13,7 +13,7 @@ from nullius.harbor import source as harbor_source from nullius.catalogue import catalog from nullius.recipes import launch as command_builder -from nullius.standard import definitions, hostreq +from nullius.standard import boat, definitions, hostreq from nullius.standard import recipe as standard_recipe from .core import app @@ -54,7 +54,7 @@ def new_run(): gpus = constraints.of(constraints.LOCAL, strict=False).get("gpu_ids") host = {b["key"]: {n: [why] for n, why in hostreq.problems(b, list(b["host"]), gpu_ids=gpus).items()} for b in definitions.load("benchmark", every=True)[0] if b.get("host")} - data["standardHere"] = {"docker": execute.docker_problem(), "licences": _licence_states(), "host": host} + data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat.unusable(), "licences": _licence_states(), "host": host} data["csrf"] = shared._csrf_token() data["local"] = shared_local_client() # ?run=: the recipe a run of this application was started from, @@ -92,7 +92,7 @@ def new_run_runner(): if len(json.dumps(recipe)) > 200_000: abort(413) if recipe.get("harness") == standard_recipe.HARNESS: - out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, unpriced=execute.unpriced) + out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, boat_unusable=boat.unusable, unpriced=execute.unpriced) # served_models: each model field's model as the run sends it, which Served by lists the providers of return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm", "served_models")} return {"errors": [{"step": providers.STEP["runner"], "message": providers.GONE}], diff --git a/app/web/execute.py b/app/web/execute.py index 8ff2665e..8e7af585 100644 --- a/app/web/execute.py +++ b/app/web/execute.py @@ -8,7 +8,7 @@ from flask import abort, jsonify, request, url_for from nullius import execution, hosts, prerequisites, prices, providers, settings from nullius.harbor import source as harbor_source -from nullius.standard import recipe as standard_recipe +from nullius.standard import boat, recipe as standard_recipe from .core import app from . import shared @@ -127,7 +127,7 @@ def _start_standard(recipe: dict): one checked before any starts.""" if len(json.dumps(recipe)) > 200_000: return jsonify(ok=False, error="the recipe is too large to keep with the run"), 400 - out = standard_recipe.compile(recipe, docker_unusable=docker_problem, unpriced=unpriced) + out = standard_recipe.compile(recipe, docker_unusable=docker_problem, boat_unusable=boat.unusable, unpriced=unpriced) if out["errors"]: return jsonify(ok=False, error=" ".join(e["message"] for e in out["errors"])), 400 # a run of controls calls no model: it is handed no key, so none need be set diff --git a/app/web/settings.py b/app/web/settings.py index 00f3409c..b5844aed 100644 --- a/app/web/settings.py +++ b/app/web/settings.py @@ -7,7 +7,7 @@ import socket from flask import abort, flash, jsonify, redirect, render_template, request, url_for from nullius import constraints, execution, hosts, paths, prerequisites, prices, providers, sandbox, secrets, settings -from nullius.standard import container +from nullius.standard import boat, container from nullius.standard import recipe as standard_recipe from nullius.judge import client @@ -249,8 +249,32 @@ def settings_host_enrol(): @app.route("/settings/sandbox") def settings_sandbox(): - """The Sandbox section's state, asked when the section opens: docker's.""" - return jsonify(docker=sandbox.docker_state()) + """The Sandbox section's state, asked when the section opens: docker's, and whether a boat key is + set (never the key).""" + return jsonify(docker=sandbox.docker_state(), boat={"set": bool(secrets.status(boat.STORE).get(boat.NAME, {}).get("set")), + "env": bool(os.environ.get(boat.NAME))}) + + +@app.route("/settings/sandbox/boat", methods=["POST"]) +def settings_sandbox_boat(): + """The boat.dev API key, saved to the owner-only store or removed. Same guards as Save.""" + _sandbox_guard("Sandbox providers are changed") + action = request.form.get("action", "save") + try: + if action == "remove": + secrets.delete(boat.STORE, boat.NAME) + elif action == "save": + value = request.form.get("key", "").strip() + if not value: + raise ValueError("Enter the API key.") + secrets.set(boat.STORE, boat.NAME, value) + else: + return jsonify(ok=False, error="Unknown action."), 400 + except ValueError as e: + return jsonify(ok=False, error=str(e)), 400 + except OSError: + return jsonify(ok=False, error="Could not update the credential store."), 500 + return jsonify(ok=True) @app.route("/settings/data-providers") diff --git a/docs/builder.md b/docs/builder.md index 8112ba0e..3bc63d4d 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -28,7 +28,7 @@ Then [read its results](results.md). | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit; a [list of units](cli/run.md#a-list-of-units) is set only on the command line | -| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | +| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, each unit's agent in a VM of its own, started from this machine with the key from Settings → Sandbox. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | Original setups run on local processes or in Docker; Docker changes isolation only. diff --git a/docs/cli/run.md b/docs/cli/run.md index 4ac31f7d..7502fd5a 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -4,7 +4,7 @@ One benchmark and one agent, capability, control or setup, into a job. ```bash title="Run" nullius standard run [@version] [--model M] [--dataset D] [--units FILE] [--limit N] [-j N] \ - [--unsandboxed | --backend container] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ + [--unsandboxed | --backend container|boat] [--option NAME=VALUE]... [--bench-option NAME=VALUE]... \ [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]]... [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE]... \ [--served-by [MODEL=]SLUG[,SLUG...]]... \ [--repeat K] [--remove-images] [--judge-model M] [--simulator-model M] [--name NAME] [--jobs DIR] [--provider P] @@ -27,7 +27,7 @@ nullius standard run locomo model --model gpt-4o-mini | `--dataset`, `--limit`, `-j` | One dataset, the first N units, N at a time | | `--units FILE` | Only the units the file lists: [below](#a-list-of-units) | | `--unsandboxed` | An agent runs as you; else `--backend container` | -| `--backend` | `local` (default) or `container`, each unit in a container with its relay | +| `--backend` | `local` (default); `container`, each unit in a container with its relay; or `boat`, each unit's agent in a boat.dev VM of its own ([The boat backend](../standard.md#the-boat-backend)) | | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | | `--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]` | This run's price for a model in USD per million tokens (repeatable), over your [price list](../settings/model-prices.md) | diff --git a/docs/settings/README.md b/docs/settings/README.md index 6e9f598b..ecfefed8 100644 --- a/docs/settings/README.md +++ b/docs/settings/README.md @@ -13,7 +13,7 @@ changes save automatically; data providers and benchmark credentials use **Save* | [Model prices](model-prices.md) | What a run's cost is estimated from | | [Hosts](run-hosts.md) | SSH machines | | [Data providers](data-providers.md) | Hugging Face and Docker Hub connections | -| [Sandbox](sandbox.md) | Docker on this machine | +| [Sandbox](sandbox.md) | Docker on this machine, the Boat key | | [General](other.md) | Tour and checklist; start-up options remain in the environment | ## Variables diff --git a/docs/settings/sandbox.md b/docs/settings/sandbox.md index bbc21930..1267a28f 100644 --- a/docs/settings/sandbox.md +++ b/docs/settings/sandbox.md @@ -1,12 +1,13 @@ # Sandbox -Docker or Podman on this machine, where agents run apart from it. Under **Execution**. +Where agents run apart from this machine: Docker or Podman here, and Boat VMs. Under **Execution**. | Card | Does | | --- | --- | | Engine | Docker or Podman: whether it answers, its name and version; why not when it does not | +| Boat | The API key of the [Boat backend](../standard.md#the-boat-backend): Edit saves it to the owner-only store, Remove deletes it; it is never shown again. `BOAT_API_KEY` in the environment wins | | Rule | | | --- | --- | | Hosts | This machine only; a host keeps its own login and images | -| Code | `app/nullius/sandbox.py` | +| Code | `app/nullius/sandbox.py`, `app/nullius/standard/boat.py` | diff --git a/docs/standard.md b/docs/standard.md index 0b4bc483..c93d6426 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -142,7 +142,7 @@ record); verifier/reward.txt and score.json | --- | --- | | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | | A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost | -| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image | +| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited`, a refused key an API error, both outside the agent's rate, unless the unit passed; a turn past its limit is the agent's timeout. `"kind": "score"` leaves a score and no pass | | Rate limits | After a unit ends `rate_limited`, no unit starts for 5 s, doubling while units keep ending so (up to 300 s) and reset by any other end; `stage=units state=progress` says the pause and how many run. Each `rate_limited` unit is run once more after the others, one at a time: its first attempt moves whole to `retried//`, the second is the trial and counts. Not after a Stop, a refused key or the spending cap, which counts both attempts | @@ -175,6 +175,21 @@ Through Podman's Docker API (a docker CLI with `DOCKER_HOST` at `podman system s A rootful Docker with `userns-remap` maps the uids of each container to sub-uids. There, the containers that use your files (the agent's, the relay, an install, the check, a door and an own image) run with `--userns=host`, as on a default Docker. The licence forwarder and the check's proxy keep the remap. +### The boat backend + +`--backend boat` runs each unit's agent or capability in a [boat.dev](https://boat.dev) VM of its own, a cloud Ubuntu machine reached over OpenSSH, and needs no yes to running unsandboxed. The driver, the unit's relay, the spending cap and the scoring stay on this machine. A check or an Original setup runs nothing of an agent's, so it runs on local processes. Per unit: + +| Piece | Rule | +| --- | --- | +| The key | boat's API key, from Settings → Sandbox (the owner-only store) or `BOAT_API_KEY`. Sent to boat.dev only, never to a VM | +| One VM per unit | Created when the unit's agent is about to start (size `small`, 2 vCPU, 4 GB, auto-stopped after 2 hours at most) and deleted once its workspace is back, after an error too. Every VM carries the run's label (`nullius-job`); the run deletes whatever carries it when it ends. `agent/boat.json` records the VM and how long it took to come up | +| Units at a time | The run's concurrency, never more than the VMs the account runs at once (the run says so and lowers it). A VM the account cannot start yet, at its limit or its rate of starts, is waited for, up to 30 minutes; a Stop ends the wait | +| What the VM gets | Each at its own path: the agent's install, its definition, the runtimes the install links to outside itself (a uv Python), its home and the workspace, sent as one compressed stream. Of this machine's environment, only the locale; its `PATH` is the install's programs, then the VM's. Never `app/`, the benchmark's folder, its upstream or its data | +| The agent's user | A user of its own (`agent`) whose only way out is the VM's loopback: no public address, no DNS (`iptables` rules inserted first, before boat's own firewall). The host key boat gives is pinned; a job's SSH key is made for it and removed when it ends; no SSH configuration of yours is read | +| Model calls | Through a reverse tunnel (`ssh -R`) to the unit's relay here, on the same port, so the agent is given the address the local backend gives. No file of this machine is served to the agent (`acp` fs) | +| The workspace back | Copied back as the VM has it, after the agent ends and before `score`: what it removed is removed, and a link that leaves the workspace, a device or a pipe is not copied. Its home comes back too | +| Where it cannot run | An environment that is not `local_ok`, one that grants the public web (`network`), one that gives the agent a licence file or server of this machine, and a benchmark's own MCP servers run on the machine: such units are recorded as not run. A run starts boat VMs from this machine only, not from a host. boat has no GPUs | + ### A benchmark's MCP server on the machine A benchmark whose tools are its upstream's own code declares `mcp_servers`, run from the driver's environment **on this machine**; a unit's `setup` asks for one by name in its `mcp` list. From 08d600aeb4bdb3503503e7e2efc488c95ee089a2 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Mon, 16 Nov 2026 14:03:52 +0100 Subject: [PATCH 2/9] fix(standard): harden the Boat backend and warn when the account cannot start every unit Send the install before the agent's time and the home and workspace as it starts; follow the unit's time with the VM's auto-stop, within a trial's 2 hours; refuse links that would send this user's own files; delete a VM whatever ends the unit, and skip the fetch on a Stop. Review refuses a size the account cannot start, Remove asks first, the run says when the account can start fewer VMs this hour or today than it has units, and the Boat mark has its own version on a dark background. --- app/nullius/standard/boat.py | 218 +++++++++++++++------ app/nullius/standard/cli.py | 11 +- app/nullius/standard/job.py | 5 +- app/nullius/standard/recipe.py | 22 ++- app/nullius/standard/run.py | 68 ++++--- app/static/brand/README.md | 2 +- app/static/brand/boat-dark.svg | 3 + app/static/builder/command-builder-core.js | 1 + app/static/builder/command-builder.css | 9 +- app/static/builder/command-builder.js | 13 +- app/static/settings/sandbox.js | 14 +- app/static/style.css | 2 + app/templates/settings/settings.html | 8 +- app/tests/test_standard_boat.py | 188 ++++++++++++++++-- app/ui_tests/test_agents.py | 3 +- app/ui_tests/test_settings_page.py | 7 +- app/web/builder.py | 7 +- app/web/execute.py | 22 ++- docs/builder.md | 2 +- docs/cli/run.md | 1 + docs/standard.md | 6 +- 21 files changed, 471 insertions(+), 141 deletions(-) create mode 100644 app/static/brand/boat-dark.svg diff --git a/app/nullius/standard/boat.py b/app/nullius/standard/boat.py index d6424be6..6a3c7e2c 100644 --- a/app/nullius/standard/boat.py +++ b/app/nullius/standard/boat.py @@ -9,18 +9,23 @@ reverse tunnel on the same port (``ssh -R``), so the address it is given is the one the local backend gives, and the relay records every call here. -One unit, one VM (:class:`VM`): created when the unit's agent is about to +One unit, one VM (:class:`VM`) of the run's size (``SIZES``, ``small`` by +default; :func:`size_problem` says when the account cannot start it): created when the unit's agent is about to start, deleted once its workspace is copied back. A run's concurrency is therefore the number of VMs at once, never more than the account allows at once (:func:`slots`); a VM the account cannot start yet (its limit, its rate of starts) is waited for, and the wait ends with the run's Stop. Every VM -carries the job's name as a label, so whatever a crash left behind is -deleted by :func:`sweep` when the job ends. +carries the run's label, so whatever an interrupted unit left is deleted by +:func:`sweep` when the run ends; a VM of a run whose process was killed stops +at its auto-stop: as long as its unit may take, at least ``TTL``, and ``TTL`` +at most on a free trial, which allows no more. What the VM is given, each at the same absolute path it has here so every path in the agent's configuration still holds: the agent's install, its definition, the runtimes the install links to outside itself (a ``uv`` -Python), its home and the unit's workspace. The workspace comes back with +Python, never a folder holding this user's home, keys or Nullius's own +storage), sent before the agent's time starts; then, as the agent starts, its +home and the unit's workspace, which its setup writes up to then. The workspace comes back with links that leave it dropped (``rsync --safe-links``): the driver reads it to score, and a link the agent left must not reach a file of this machine. Nothing of this machine's environment goes with it but its locale. @@ -45,20 +50,23 @@ from contextlib import contextmanager from typing import Callable, Dict, Iterable, List, Optional, Sequence -from nullius import secrets +from nullius import paths, secrets from nullius.standard import install API = "https://boat.dev/api/v1" STORE, NAME = "sandbox-provider-boat", "BOAT_API_KEY" -SIZE = "small" # 2 vCPU, 4 GB: an agent's process, not its model -TTL = 2 * 3600 # the longest a VM lives, whatever happens here (the trial's maximum) -USER = "agent" # the VM's user the agent runs as +# the VM runs the agent's process, never its model: the smallest is enough +SIZES = {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB", "large": "", "xlarge": ""} +SIZE = "small" +TTL = 2 * 3600 # a VM's auto-stop at least, and a free trial's at most +USER = "agent" LABEL = "nullius-job" VM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" KEEP = ("LANG", "LC_ALL", "TERM") # of the local environment, what makes sense on another machine -WAIT_SECONDS = 1800 # how long a unit waits for the account to start its VM +WAIT_SECONDS = 3600 # an account's starts an hour come back within one READY_SECONDS = 300 SSH_SECONDS = 900 +DRIVER = 1800 # a unit's time beyond its agent's: sending, bringing back # The agent's user reaches the VM's loopback and nothing else: no public address, no DNS. Inserted # first in OUTPUT, before the chains boat's own firewall accepts in. @@ -95,8 +103,8 @@ def unusable(env=os.environ, which: Callable = shutil.which) -> Optional[str]: class Client: """boat.dev's API, with one account's key.""" - def __init__(self, api_key: str, base: str = API, opener: Callable = urllib.request.urlopen): - self.key, self.base, self.opener = api_key, base, opener + def __init__(self, api_key: str, base: str = API, opener: Callable = urllib.request.urlopen, timeout: float = 60): + self.key, self.base, self.opener, self.timeout = api_key, base, opener, timeout def call(self, method: str, path: str, body: Optional[dict] = None, headers: Optional[dict] = None) -> dict: req = urllib.request.Request(self.base + path, method=method, @@ -104,7 +112,7 @@ def call(self, method: str, path: str, body: Optional[dict] = None, headers: Opt headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Accept": "application/json", **(headers or {})}) try: - with self.opener(req, timeout=60) as r: + with self.opener(req, timeout=self.timeout) as r: return json.loads(r.read() or b"{}") except urllib.error.HTTPError as e: try: @@ -145,6 +153,36 @@ def slots(client: Client) -> Optional[int]: return n if isinstance(n, int) and n > 0 else None +def starts_problem(client: Client, n: int) -> Optional[str]: + """What the account's starts left say of a run of ``n`` VMs, one per unit, or None.""" + left = client.limits().get("starts") or {} + day, hour = ((left.get(k) or {}).get("remaining") for k in ("day", "hour")) + if isinstance(day, int) and n > day: + return (f"Boat can start {day} more VMs today on this account: {n - day} of the {n} units " + "will end without running") + if isinstance(hour, int) and n > hour: + return (f"Boat can start {hour} more VMs this hour on this account: the other units wait " + "for the next hour") + return None + + +def machine_types(client: Client) -> Dict[str, Optional[str]]: + """Each machine type the account knows: None when it may start one, else boat's reason why not + ("not during the free trial"). Empty when its API does not say.""" + out = {} + for t in client.limits().get("machineTypes") or []: + if isinstance(t, dict) and isinstance(t.get("type"), str): + out[t["type"]] = None if t.get("available") else str(t.get("reason") or "not available on this account") + return out + + +def size_problem(client: Client, size: str) -> Optional[str]: + """Why this account cannot start a VM of ``size``, or None.""" + if size not in SIZES: + return f"{size!r} is not one of Boat's sizes: {', '.join(SIZES)}" + return machine_types(client).get(size) + + def sweep(job: str, client: Optional[Client] = None) -> int: """Deletes every VM labelled with ``job``: what a crash or a kill left. The number deleted.""" if client is None: @@ -159,10 +197,19 @@ def sweep(job: str, client: Optional[Client] = None) -> int: return gone -def outside_roots(where: str) -> List[str]: +def _private(env=os.environ) -> tuple: + """What a folder sent to a VM may not hold, and what it may not be inside of.""" + home = os.path.realpath(os.path.expanduser("~")) + keys = [os.path.join(home, d) for d in (".ssh", ".gnupg", ".aws")] + [os.path.realpath(paths.config_dir(env))] + return [home, os.path.realpath(paths.data_dir(env)), os.path.realpath(paths.cache_dir(env)), *keys], keys + + +def outside_roots(where: str, env=os.environ) -> List[str]: """The folders an install links to outside itself and outside the system's own: a ``uv`` - environment's Python, kept in uv's folder of this user. Each must reach the VM too.""" - roots = set() + environment's Python, kept in uv's folder of this user. Each must reach the VM too. A link + whose folder would hold this user's home, keys or Nullius's storage is the install's + ``Unavailable``: none of it may leave this machine.""" + roots = {} real_where = os.path.realpath(where) for top, dirs, files in os.walk(where): for n in dirs + files: @@ -172,8 +219,16 @@ def outside_roots(where: str) -> List[str]: t = os.path.realpath(p) if t == real_where or t.startswith(real_where + os.sep) or t.startswith(_SYSTEM) or not os.path.exists(t): continue - d = os.path.dirname(t) - roots.add(os.path.dirname(d) if os.path.basename(d) == "bin" else d) + if os.path.isdir(t): + r = t + else: + d = os.path.dirname(t) + r = os.path.dirname(d) if os.path.basename(d) == "bin" else d + roots.setdefault(r, p) + holds, inside = _private(env) + for r, link in roots.items(): + if any(r == s or s.startswith(r.rstrip(os.sep) + os.sep) for s in holds) or any(r.startswith(s + os.sep) for s in inside): + raise Unavailable(f"{link} links to {r}, which would send this user's own files to a Boat VM") out: List[str] = [] for r in sorted(roots, key=len): # a root inside another is sent with it if not any(r == o or r.startswith(o + os.sep) for o in out): @@ -197,17 +252,21 @@ def close(self) -> None: class VM: """One unit's VM, ready: created, its host key pinned, the agent's user made and locked in. - ``stopped``: the run's Stop, which ends a wait for the account to start it.""" + ``stopped``: the run's Stop, which ends a wait for the account to start it. ``ttl``: its + auto-stop, asked for; ``self.ttl`` the one it has, ``TTL`` on a free trial.""" - def __init__(self, client: Client, keys: Keys, job: str, *, stopped: Optional[threading.Event] = None, - wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep): - self.client, self.keys, self.id = client, keys, None + def __init__(self, client: Client, keys: Keys, job: str, *, size: str = SIZE, stopped: Optional[threading.Event] = None, + wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep, ttl: int = TTL): + self.client, self.keys, self.id, self.size, self.ttl = client, keys, None, size, max(int(ttl), TTL) t0 = time.time() pause = 5.0 while True: - d = client.create({LABEL: job}) + d = client.create({LABEL: job}, size=size, ttl=self.ttl) if d.get("ok"): break + if d.get("code") == "trial_auto_stop_required" and self.ttl > TTL: + self.ttl = TTL + continue # the account's limit of VMs at once, or of starts a minute: another unit's VM ends, a minute passes if d.get("status") != 429 and d.get("code") not in ("limit_reached", "rate_limited"): raise Unavailable("Boat refused a VM: " + said(d)) @@ -247,14 +306,20 @@ def _ready(self, sleep) -> None: "-o", f"HostKeyAlias={self.id}", "-o", "ServerAliveInterval=30", "-o", "LogLevel=ERROR", "-o", "ConnectTimeout=20"] self.target = f"{self.user}@{self.host}" - for _ in range(30): # sshd answers a moment after the key is in - if subprocess.run(self.ssh + [self.target, "true"], capture_output=True, timeout=60).returncode == 0: - return + for _ in range(30): # sshd answers a moment after the key is in, or hangs while it boots + try: + if subprocess.run(self.ssh + [self.target, "true"], capture_output=True, timeout=60).returncode == 0: + return + except subprocess.TimeoutExpired: + pass sleep(2) raise Unavailable(f"Boat VM {self.id} did not answer over SSH") def sh(self, cmd: str, stdin=None) -> str: - r = subprocess.run(self.ssh + [self.target, cmd], stdin=stdin, capture_output=True, timeout=SSH_SECONDS) + try: + r = subprocess.run(self.ssh + [self.target, cmd], stdin=stdin, capture_output=True, timeout=SSH_SECONDS) + except subprocess.TimeoutExpired: + raise Unavailable(f"Boat VM {self.id}: {cmd.split(' ', 3)[:3]}: no answer in {SSH_SECONDS} s") from None if r.returncode: raise Unavailable(f"Boat VM {self.id}: {cmd.split(' ', 3)[:3]}: " + r.stderr.decode(errors="replace").strip()[-300:]) @@ -265,29 +330,42 @@ def send(self, read: Iterable[str], owned: Iterable[str] = ()) -> None: may read, ``owned`` it owns (its home, its workspace).""" read, owned = [p.rstrip("/") for p in read if p], [p.rstrip("/") for p in owned if p] rel = [os.path.relpath(p, "/") for p in dict.fromkeys(read + owned)] - tar = subprocess.Popen(["tar", "-C", "/", "-czf", "-", "--", *rel], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - # every parent of each folder made traversable, the agent's own folders its own - q = " ".join(shlex.quote("/" + r) for r in rel) - mine = " ".join(shlex.quote(p) for p in owned) - self.sh("sudo -n tar -C / -xzpf - --no-same-owner && " - f"for p in {q}; do while [ \"$p\" != / ]; do sudo -n chmod o+rx \"$p\"; p=$(dirname \"$p\"); done; done" - + (f" && sudo -n chown -R {USER}:{USER} {mine}" if mine else "") - + f" && sudo -n chmod -R o+rX {q}", stdin=tar.stdout) - finally: - tar.stdout.close() - err = tar.stderr.read().decode(errors="replace") - if tar.wait(): - raise Unavailable(f"could not pack what Boat VM {self.id} needs: {err.strip()[-300:]}") + q = " ".join(shlex.quote("/" + r) for r in rel) + mine = " ".join(shlex.quote(p) for p in owned) + failed = None + with tempfile.TemporaryFile() as err: # a file, never a pipe: tar's warnings cannot stall the stream + tar = subprocess.Popen(["tar", "-C", "/", "-czf", "-", "--", *rel], stdout=subprocess.PIPE, stderr=err) + try: + self.sh("sudo -n tar -C / -xzpf - --no-same-owner && " + f"for p in {q}; do while [ \"$p\" != / ]; do sudo -n chmod o+rx \"$p\"; p=$(dirname \"$p\"); done; done" + + (f" && sudo -n chown -R {USER}:{USER} {mine}" if mine else "") + + f" && sudo -n chmod -R o+rX {q}", stdin=tar.stdout) + except Unavailable as e: + failed = e + finally: + tar.stdout.close() + code = tar.wait() + err.seek(0) + said = err.read().decode(errors="replace").strip()[-300:] + # tar's 1 is a file that changed as it was read: what the VM received still stands; a stream the + # VM stopped reading is the VM's failure, said by the VM + cut = "status 141" in said or "Broken pipe" in said + if (code > 1 and not (failed and cut)) or (code and failed and not cut): + raise Unavailable(f"could not pack what Boat VM {self.id} needs: {said}") + if failed: + raise failed def fetch(self, path: str) -> None: """The folder back from the VM, as it is there: what the agent removed is removed here, a link leaving the folder is not copied, nor a device or a pipe.""" path = path.rstrip("/") rsh = " ".join(shlex.quote(w) for w in self.ssh) - r = subprocess.run(["rsync", "-rlt", "--delete", "--safe-links", "--no-devices", "--no-specials", - "--rsync-path=sudo -n rsync", "-e", rsh, f"{self.target}:{path}/", path + "/"], - capture_output=True, timeout=SSH_SECONDS) + try: + r = subprocess.run(["rsync", "-rlt", "--delete", "--safe-links", "--no-devices", "--no-specials", + "--rsync-path=sudo -n rsync", "-e", rsh, f"{self.target}:{path}/", path + "/"], + capture_output=True, timeout=SSH_SECONDS) + except subprocess.TimeoutExpired: + raise Unavailable(f"the workspace did not come back from Boat VM {self.id} in {SSH_SECONDS} s") from None if r.returncode: raise Unavailable(f"the workspace did not come back from Boat VM {self.id}: " + r.stderr.decode(errors="replace").strip()[-300:]) @@ -338,37 +416,57 @@ class Backend: job's name may be used again), its SSH key, made with its first VM, and the VM each unit's thread holds while its agent runs (:meth:`unit`, :meth:`place`). :meth:`close` deletes what is left.""" - def __init__(self, job: str, env=os.environ, client: Optional[Client] = None): + def __init__(self, job: str, env=os.environ, client: Optional[Client] = None, size: str = SIZE, + say: Callable[[str], None] = lambda s: None): self.client = client or Client(key(env) or "") + self.size, self.say, self._cut = size, say, False self.label = f"{job}.{os.urandom(4).hex()}" self._keys: Optional[Keys] = None self._roots: Dict[str, List[str]] = {} self._lock = threading.Lock() self._held = threading.local() + def starts(self, n: int) -> Optional[str]: + return starts_problem(self.client, n) + def most(self) -> Optional[int]: """How many units may run at once: the VMs the account runs at once.""" return slots(self.client) @contextmanager - def unit(self, trial: str, ws: str, home: str, stopped: Optional[threading.Event] = None): - """The unit's VM while its agent or capability runs (``agent/boat.json`` says which, and how long - it took to come up); its workspace and home brought back from it, then the VM deleted. A - failure to bring them back after the agent ended is the unit's ``Unavailable``; after the - agent failed, its own failure stands.""" + def unit(self, trial: str, ws: str, home: str, stopped: Optional[threading.Event] = None, *, + inst: Optional[str] = None, here: Optional[str] = None, seconds: float = 0): + """The unit's VM while its agent or capability runs, given before it starts, each at its own + path: the install (``inst``) and the runtimes it links to outside itself, and the definition + (``here``). ``agent/boat.json`` says which VM, and how long it took to come up and be sent + all that. Its workspace and home (sent by :meth:`place`) are brought back + from it, then the VM deleted. A failure to bring them back after the agent ended is the + unit's ``Unavailable``; after the agent failed, its own failure stands, and after a Stop + nothing is brought back. ``seconds``: the longest the unit may take, its VM's auto-stop + with the sending and the bringing back (``DRIVER``); a trial's shorter one is said once.""" with self._lock: if self._keys is None: self._keys = Keys() - vm = VM(self.client, self._keys, self.label, stopped=stopped) + if inst and inst not in self._roots: + self._roots[inst] = outside_roots(inst) if os.path.isdir(inst) else [] + read = [p for p in [inst, here, *self._roots.get(inst or "", [])] if p and os.path.isdir(p)] + t0 = time.time() + vm = VM(self.client, self._keys, self.label, size=self.size, stopped=stopped, ttl=seconds + DRIVER) try: + if vm.ttl < seconds + DRIVER and not self._cut: + self._cut = True + self.say(f"Boat stops a free trial's VMs after {TTL // 3600} hours: a unit still running then ends there") + vm.send(read) os.makedirs(os.path.join(trial, "agent"), exist_ok=True) with open(os.path.join(trial, "agent", "boat.json"), "w") as f: - json.dump({"vm": vm.id, "size": SIZE, "ready_seconds": vm.seconds}, f) + json.dump({"vm": vm.id, "size": self.size, "ttl_seconds": vm.ttl, "ready_seconds": vm.seconds, + "sent_seconds": round(time.time() - t0, 1)}, f) self._held.vm = vm try: yield vm - except BaseException: - for d in (ws, home): # what it left, for the evidence + except BaseException as e: + stop = isinstance(e, KeyboardInterrupt) or (stopped is not None and stopped.is_set()) + for d in () if stop else (ws, home): # what it left, for the evidence try: vm.fetch(d) except Unavailable: @@ -380,19 +478,15 @@ def unit(self, trial: str, ws: str, home: str, stopped: Optional[threading.Event self._held.vm = None vm.delete() - def place(self, cmd: List[str], env: Dict[str, str], workdir: str, *, inst: Optional[str], here: Optional[str], - owned: List[str], relay_url: Optional[str], local: Sequence[str] = ()) -> tuple: + def place(self, cmd: List[str], env: Dict[str, str], workdir: str, *, inst: Optional[str], owned: List[str], + relay_url: Optional[str], local: Sequence[str] = ()) -> tuple: """``cmd`` and the environment to start it with here: ``ssh`` to this thread's VM, where it runs as - its locked user. Sent first, each at its own path: the install (``inst``) and the runtimes it - links to outside itself, the definition (``here``), and ``owned`` - its home and workspace.""" + its locked user, the install's programs (``inst``) first on its ``PATH``. ``owned``, its home and + workspace as its setup left them, are sent first and become its user's.""" vm = getattr(self._held, "vm", None) if vm is None: raise Unavailable("the unit has no Boat VM") - if inst and inst not in self._roots: - with self._lock: - self._roots[inst] = outside_roots(inst) if os.path.isdir(inst) else [] - read = [p for p in [inst, here, *self._roots.get(inst or "", [])] if p and os.path.isdir(p)] - vm.send(read, owned) + vm.send([], owned) port = urllib.parse.urlsplit(relay_url).port if relay_url else None env = remote_env(env, install.bin_dirs(inst) if inst else [], local) return vm.argv(list(cmd), env, workdir, port), {k: os.environ[k] for k in ("PATH", "HOME", "LANG") if k in os.environ} diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index c0e94ff5..c01c7c7b 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,8 +6,8 @@ PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION - PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container|boat] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container|boat] + PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -80,7 +80,7 @@ if __name__ == "__main__" and "--from-run" not in sys.argv[1:]: settings.load(paths.settings_file()) -from nullius.standard import conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard +from nullius.standard import boat, conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard ENV_FILE = paths.settings_file() @@ -530,7 +530,7 @@ def cmd_run(a): try: job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit, concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, - simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, + boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), @@ -668,6 +668,9 @@ def main(argv=None): p.add_argument("--backend", default="local", type=definitions.names.backend, choices=list(run.BACKENDS), help="local processes (the default); container: each unit in a container, its relay beside it " "(docker: its earlier name); boat: each unit's agent in a boat.dev VM of its own") + if name == "run": + p.add_argument("--boat-size", default=boat.SIZE, choices=list(boat.SIZES), + help="on the boat backend, its VMs' machine type (default small: 2 vCPU, 4 GB)") p.add_argument("--from-run", action="store_true", help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read") p.add_argument("--option", action="append", metavar="NAME=VALUE", diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 6234e4e3..51a74390 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -525,7 +525,7 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str, provider: str, dataset: str, units: List[str], - concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", + concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", boat_size: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, @@ -550,7 +550,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, "needs": bench["needs"], "permissions_granted": granted, "unsandboxed": granted and backend == "local", "backend": backend, - **({"image": image} if image else {}), **({"judge": judge} if judge else {}), + **({"image": image} if image else {}), **({"boat_size": boat_size} if boat_size else {}), + **({"judge": judge} if judge else {}), **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index a410ed5f..e7f4e3a4 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -42,7 +42,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay +from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -265,6 +265,7 @@ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]: def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, docker_unusable: Optional[Callable[[], Optional[str]]] = None, boat_unusable: Optional[Callable[[], Optional[str]]] = None, + boat_size_problem: Optional[Callable[[str], Optional[str]]] = None, unpriced: Optional[Callable[[List[str]], List[str]]] = None) -> dict: """{errors, notes, review, jobs, confirm, script, parts, model} for one recipe; errors name the step @@ -278,7 +279,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, is the Where step's error. Not given, it is not asked: compiling stays a function of the definitions alone, and the routes that preview or start a run pass it. ``boat_unusable``: the same for the boat backend - (``boat.unusable``): its key, and the programs that reach a VM. + (``boat.unusable``): its key, and the programs that reach a VM; + ``boat_size_problem``, why the account cannot start a VM of the size. ``max_usd``: the run's spending cap (``standard.ledger``), one amount for the whole run: its jobs that call a model - each dataset with each @@ -504,10 +506,16 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.") elif backend == "boat" and targets != ["local"]: add("where", "Boat VMs are started from this machine, with its key: choose This machine alone.") - elif backend == "boat" and boat_unusable is not None: - why = boat_unusable() + elif backend == "boat" and boat_unusable is not None and (why := boat_unusable()): + add("where", f"Boat cannot run from this machine: {why}.") + elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES: + why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE)) if why: - add("where", f"Boat cannot run from this machine: {why}.") + add("where", f"Boat cannot start a {recipe.get('boat_size') or boat.SIZE} VM on this account: {why}. Choose another size.") + boat_size = str(recipe.get("boat_size") or boat.SIZE) + if backend == "boat" and boat_size not in boat.SIZES: + add("where", f"Choose a Boat size: {', '.join(boat.SIZES)}.") + boat_size = boat.SIZE # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) @@ -611,6 +619,7 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], *(["--backend", backend] if backend != "local" else []), + *(["--boat-size", boat_size] if backend == "boat" and not control and not p.get("setup") else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch_of(d, p)] if of > 1 else []), *limits.cli_words(limits.of(targets[index - 1])), @@ -709,7 +718,8 @@ def row(p: dict) -> list: + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []), *([[STEP["limits"], f"Spending cap per {noun}", f"${ledger.usd(per_unit)}"]] if per_unit else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") - + ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]] + + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" + + (f" · {boat_size} VMs" if backend == "boat" else "")]] if agents_local: notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: " "the local backend is no sandbox.") diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index bb027dcf..3a851ed7 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -395,13 +395,15 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None): + served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for local agents, and for setup checks that run code without Docker. - ``backend``: ``local`` or ``container`` (``docker``, its earlier name), which needs - no such yes and is refused when docker is missing or not usable. + ``backend``: ``local``, ``container`` (``docker``, its earlier name), which needs + no such yes and is refused when docker is missing or not usable, or ``boat``; + ``boat_size``: its VMs' machine type (``boat.SIZES``), refused when the + account cannot start it. ``options``: the agent's or capability's options set for this run, by name; the rest keep their defaults, and the job's config records every value. @@ -619,9 +621,11 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # each stage said as it goes, on the cli's stderr and in the job's progress.jsonl (nullius/progress.py) self.progress = progress.Reporter(self.job_dir) self._said = threading.Lock() - self.boat = boat.Backend(self.name, env) if backend == "boat" else None + self.boat = boat.Backend(self.name, env, size=boat_size, say=say) if backend == "boat" else None if self.boat is not None: - # one VM per unit: never more units at once than the account runs VMs + why = boat.size_problem(self.boat.client, boat_size) + if why: + raise Refused(f"Boat cannot start a {boat_size} VM: {why}") most = self.boat.most() if most is not None and self.concurrency > most: say(f"Boat runs {most} VMs at once on this account: {most} units at a time, not {self.concurrency}") @@ -747,10 +751,15 @@ def start(self) -> str: mine = mine[: self.limit] if self.limit else mine units = [u for _, u in mine] self.n_units = len(units) + if getattr(self, "boat", None) is not None: + why = self.boat.starts(len(units) * self.repeat) + if why: + self.say(why) p.emit("units", "start", n=len(units), of=len(everything), noun=self.bench["units"]["nouns"]) job.write_config(self.job_dir, job=self.name, bench=self.bench, other=self.other, model=self.model, provider=self.provider, dataset=self.ds["id"], units=units, concurrency=self.concurrency, granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend, + boat_size=self.boat.size if getattr(self, "boat", None) is not None else "", image=self.setup_driver.image if self.setup_driver else self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, # a shard of a list splits the listed units: they are its whole @@ -830,7 +839,7 @@ def work_unit(iur): if self.backend == "container": container.sweep(self.name) # whatever an interrupted unit left if getattr(self, "boat", None) is not None: - self.boat.close() # whatever an interrupted unit left + self.boat.close() if self.loopback is not None: self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) @@ -1088,23 +1097,24 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: box = self.own_box(tag, setup, rl, home) if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) - p0 = time.time() - self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" - else DRIVER_SECONDS if self.other["key"] == "oracle" else None) - try: - with self.agent_internet(setup, box, rl, trial), self.unit_vm(ws, home, trial): - if self.other["_kind"] == "control": - answer = self.control_answer(unit, ws, box, name) - elif self.upstream: - answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) - elif self.other["_kind"] == "agent": - answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) - else: - answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) - finally: - produce = (p0, time.time()) - if box is not None: # what the agent (or the control) left in the container, before the check - changes = self.box_changes(box) + with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts + p0 = time.time() + self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" + else DRIVER_SECONDS if self.other["key"] == "oracle" else None) + try: + with self.agent_internet(setup, box, rl, trial): + if self.other["_kind"] == "control": + answer = self.control_answer(unit, ws, box, name) + elif self.upstream: + answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) + elif self.other["_kind"] == "agent": + answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) + else: + answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + finally: + produce = (p0, time.time()) + if box is not None: # what the agent (or the control) left in the container, before the check + changes = self.box_changes(box) cut = self.cap_reached(rl, tag) if cut: # cut off at a spending cap: nothing of it is checked, and its judge is asked nothing raise _Ended(cut[0], cut[1], traj) @@ -1532,8 +1542,7 @@ def place(self, rl, cmd: List[str], env: dict, workdir: str, home: str, interact ws = os.path.join(os.path.dirname(home), "workspace") if self.backend == "boat": try: - return self.boat.place(cmd, env, workdir, inst=self.inst, here=self.other.get("_here"), owned=[ws, home], - relay_url=rl.unit_url if rl is not None else None, local=KEEP) + return self.boat.place(cmd, env, workdir, inst=self.inst, owned=[ws, home], relay_url=rl.unit_url if rl is not None else None, local=KEEP) except boat.Unavailable as e: raise _Ended("EnvironmentSetupError", str(e)) # What the agent's container sees: its workspace and home, its own @@ -1577,13 +1586,16 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: # -- an own image: the unit's container ------------------------------------ @contextmanager - def unit_vm(self, ws: str, home: str, trial: str): - """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``).""" + def unit_vm(self, ws: str, home: str, trial: str, setup: dict): + """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``), + living as long as its every turn may take.""" if getattr(self, "boat", None) is None: yield return + turns = self.max_turns if "agent.conversation/v1" in self.bench["needs"] else 1 try: - with self.boat.unit(trial, ws, home, stopped=self._stopped): + with self.boat.unit(trial, ws, home, stopped=self._stopped, inst=self.inst, here=self.other.get("_here"), + seconds=self.turn_limit(setup) * turns): yield except boat.Unavailable as e: raise _Ended("EnvironmentSetupError", str(e)) diff --git a/app/static/brand/README.md b/app/static/brand/README.md index d4708c49..d32c31c7 100644 --- a/app/static/brand/README.md +++ b/app/static/brand/README.md @@ -19,4 +19,4 @@ third-party brand asset, not a Nullius mark or part of Nullius's font licensing. `huggingface.svg` is Hugging Face’s official mark, copied from https://huggingface.co/front/assets/huggingface_logo-noborder.svg. It identifies the data provider; its trademark belongs to Hugging Face. -`boat.svg` is Boat's official mark, copied unchanged from https://boat.dev/icon.svg. It identifies the Boat sandbox provider in Settings and the builder's Where step; its trademark belongs to Boat. +`boat.svg` is Boat's official mark, copied unchanged from https://boat.dev/icon.svg, and `boat-dark.svg` its version for dark backgrounds, from https://boat.dev/icon-dark.svg. They identify the Boat sandbox provider in Settings and the builder's Where step; its trademark belongs to Boat. diff --git a/app/static/brand/boat-dark.svg b/app/static/brand/boat-dark.svg new file mode 100644 index 00000000..16c3dcd4 --- /dev/null +++ b/app/static/brand/boat-dark.svg @@ -0,0 +1,3 @@ + + + diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 3fe0e7cb..a58e76e4 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -173,6 +173,7 @@ if (picked) r.others = [...new Set(picked)]; if (Array.isArray(raw.where) && raw.where.length && raw.where.every(w => typeof w === 'string')) r.where = raw.where.length > 1 ? [...raw.where] : raw.where[0]; if (['local', 'container', 'docker', 'boat'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name + if (['small', 'default', 'large', 'xlarge'].includes(raw.boat_size)) r.boat_size = raw.boat_size; if (raw.bench_options && typeof raw.bench_options === 'object' && !Array.isArray(raw.bench_options)) // the benchmark's own options, a word each r.bench_options = ownOptions(standardBench(r), Object.fromEntries(Object.entries(raw.bench_options).filter(([, v]) => typeof v === 'string'))); // Options per choice, {ref: {name: word}}; a single agent also accepts flat {name: word} options. diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index a64956ce..82cf5f45 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -240,12 +240,17 @@ /* Compact settings rows use one shared control column, including paired fields. */ #nullius-builder .nb-two:has(> .ui-field-row) { grid-template-columns: minmax(0, 1fr); gap: 0; } -#nullius-builder .std-backends { grid-template-columns: repeat(2, minmax(0, 240px)); gap: 8px; } +#nullius-builder .std-backends { grid-template-columns: repeat(3, minmax(0, 240px)); gap: 8px; } +@media (max-width: 860px) { #nullius-builder .std-backends { grid-template-columns: repeat(2, minmax(0, 240px)); } } #nullius-builder .std-backends .nb-tile { flex-direction: row; align-items: center; gap: 8px; padding: 10px 12px; } #nullius-builder .std-backends .nb-tile-head { flex: 1; } #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } #nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } -@media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +#nullius-builder .std-boat-sizes { grid-template-columns: repeat(4, minmax(0, 160px)); grid-auto-rows: auto; } +#nullius-builder .std-boat-sizes .nb-tile { flex-direction: column; align-items: stretch; gap: 2px; } +#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; } +@media (max-width: 760px) { #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 180px)); } } +@media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 1fr)); } } /* Served by: a row after its model's row, as that row is: the label left, the picker right, the providers chosen under the picker in their order */ #nullius-builder .std-served-list { grid-column: 2; min-width: 0; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 45e2726a..2ef65f7a 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -427,11 +427,10 @@ // Review's rule (nullius/standard/recipe.py): a setup's image installs what has an install; nothing else reaches its container const cannot = selected.some(o => o.setup) ? optionNeeds(rec).host.filter(n => (b.host || {})[n] && !b.host[n].install.length) : []; const containerWhy = dockerWhy || (cannot.length ? cannot.join(', ') + ' cannot be provided in a container: choose local processes.' : ''); - // boat VMs are started from this machine, with its key (Settings → Sandbox) const boatWhy = list.some(w => w !== 'local') ? 'Started from this machine only' : here.boat ? here.boat : ''; const agentRun = b && b.family === 'agent' && selected.some(o => !o.control && !o.setup); const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : v === 'boat' ? '' : '▣', title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : v === 'boat' ? '' : '▣', title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy return '

Where does it run?

' @@ -442,10 +441,19 @@ + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + how('boat', 'Boat', 'Sandboxed: each unit in a Boat VM of its own, whose only way out is its relay on this machine.', boatWhy) + '' + + (backend === 'boat' && !boatWhy ? boatSizes(rec) : '') + (agentRun && backend === 'local' ? consentBox(rec) : '') + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.

` : '') + hostSetup(rec); } + const BOAT_SIZES = [['small', 'Small'], ['default', 'Default'], ['large', 'Large'], ['xlarge', 'XLarge']]; + function boatSizes(rec) { + const here = schema.standardHere || {}, why = here.boatSizes || {}, spec = here.boatSpecs || {}, size = rec.boat_size || 'small'; + const card = ([v, t]) => tile({attr: 'std-boat-size', key: v, on: size === v && !why[v], disabled: !!why[v], + note: why[v] ? 'Not on this account' : spec[v] || '', title: t}); + return '

VM size

' + + `
${BOAT_SIZES.map(card).join('')}
`; + } // What the chosen options of the benchmark need: an embedding model, requirements of the machine (standard.json host). function optionNeeds(rec) { const b = C.standardBench(rec), out = {embeddings: [], host: []}; @@ -976,6 +984,7 @@ // the standard's machines: each a toggle, one at least; several split the run into shards else if(d.stdWhere){const rec=r(),list=C.whereList(rec),on=list.includes(d.stdWhere),next=on?list.filter(w=>w!==d.stdWhere):[...list,d.stdWhere];if(!next.length)return;rec.where=next.length>1?next:next[0];} else if(d.stdBackend){r().backend=d.stdBackend;} + else if(d.stdBoatSize){r().boat_size=d.stdBoatSize;} else if(d.harborSource){r().source=d.harborSource;} else if(d.benchmark){C.setBenchmark(r(),d.benchmark);} else if(d.where!==undefined){r().where=d.where;} diff --git a/app/static/settings/sandbox.js b/app/static/settings/sandbox.js index c7559c01..465a1993 100644 --- a/app/static/settings/sandbox.js +++ b/app/static/settings/sandbox.js @@ -4,7 +4,7 @@ if (!section) return; const $ = s => section.querySelector(s); const state = $('[data-boat-state]'), form = $('[data-boat-form]'); - const edit = $('[data-boat-edit]'), remove = $('[data-boat-remove]'), cancel = $('[data-boat-cancel]'); + const edit = $('[data-boat-edit]'), remove = $('[data-boat-remove]'), cancel = $('[data-boat-cancel]'), removeAsk = $('[data-boat-ask]'); const notify = (message, error = false) => window.nulliusNotify?.(message, {error}); let boat = {set: false, env: false}, editing = false; @@ -13,6 +13,13 @@ if (form) form.hidden = !editing; if (edit) { edit.hidden = editing; edit.textContent = boat.set ? 'Edit' : 'Add key'; } if (remove) remove.hidden = editing || !boat.set; + if (remove?.hidden) asking(false, false); + }; + const asking = (open, focus = true) => { + if (!removeAsk) return; + removeAsk.hidden = !open; + remove.setAttribute('aria-expanded', String(open)); + if (focus) (open ? $('[data-boat-remove-keep]') : remove).focus({preventScroll: true}); }; let asked = false; @@ -49,7 +56,10 @@ if (edit) edit.addEventListener('click', () => { editing = true; render(); form.querySelector('input').focus(); }); if (cancel) cancel.addEventListener('click', () => { form.reset(); editing = false; render(); edit.focus(); }); if (form) form.addEventListener('submit', e => { e.preventDefault(); change('save'); }); - if (remove) remove.addEventListener('click', () => change('remove')); + if (remove) remove.addEventListener('click', () => asking(removeAsk.hidden)); + $('[data-boat-remove-keep]')?.addEventListener('click', () => asking(false)); + $('[data-boat-remove-yes]')?.addEventListener('click', () => { asking(false, false); change('remove'); }); + removeAsk?.addEventListener('keydown', e => { if (e.key === 'Escape') { e.preventDefault(); e.stopPropagation(); asking(false); } }); // Asked the first time the section is shown, not on every visit to Settings. const ask = () => { if (!asked && !section.hidden) load(); }; diff --git a/app/static/style.css b/app/static/style.css index 0b4d1b6b..0a611b98 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1759,6 +1759,8 @@ select.ui-select-native { display: none !important; } /* Bundled upstream agent marks, shared by catalogue pages and the builder. */ .agent-logo { display: block; width: 100%; height: 100%; object-fit: contain; flex: none; } .nav-agent-logo { width: 18px; height: 18px; } +/* A mark with its own version for dark backgrounds: the one that suits the theme. */ +:root[data-theme="dark"] .on-light-bg, :root:not([data-theme="dark"]) .on-dark-bg { display: none; } /* An expanded section is context; its selected child owns the filled highlight. */ .nav-tree:not(.folded):has(.tree-item.on) > .nav-row .nav-item.on { background: transparent; } diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index 38612c4d..042e6418 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -397,10 +397,14 @@

Judges

EngineChecking…
- {# Boat: a VM per unit (standard/boat.py); its key kept in the owner-only store, never shown again #}
-

Boat

Checking…

{% if writable %}{% endif %}
+

Boat

Checking…

{% if writable %}{% endif %}
{% if writable %} +
' + (backend === 'boat' && !boatWhy ? boatSizes(rec) : '') + (agentRun && backend === 'local' ? consentBox(rec) : '') + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.

` : '') diff --git a/app/tests/test_execution.py b/app/tests/test_execution.py index e841bdc3..3831a02a 100644 --- a/app/tests/test_execution.py +++ b/app/tests/test_execution.py @@ -416,11 +416,13 @@ def test_a_harbor_run_gets_the_agent_keys_and_the_standard_none(self): def test_only_a_standard_run_on_boat_is_handed_the_boat_key(self): # the Boat key starts VMs: over stdin to the standard run that uses it, never inherited by any other secret = "boat_0123456789abcdef" - for runner, backend in (("standard", "boat"), ("standard", "container"), ("harbor", "")): - with self.subTest(runner=runner, backend=backend): - name = f"boat-{runner}-{backend or 'none'}"; root = self.tmp / name; marker = root / "seen" + for target, runner, backend in (("here", "standard", "boat"), ("host", "standard", "boat"), + ("here", "standard", "container"), ("here", "harbor", "")): + with self.subTest(target, runner=runner, backend=backend): + name = f"boat-{target}-{runner}-{backend or 'none'}"; root = self.tmp / name; marker = root / "seen" parts = [{"id": "a", "label": "a", "script": f'echo "boat=${{BOAT_API_KEY:-none}}" > {marker}; echo "$BOAT_API_KEY"'}] - plan = execution.Plan(name=name, script="", parts=parts, runner=runner, recipe={"backend": backend} if backend else {}) + plan = execution.Plan(name=name, script="", parts=parts) if target == "here" else self.plan(parts, name) + plan.runner, plan.recipe = runner, ({"backend": backend} if backend else {}) with patch.dict(os.environ, {"BOAT_API_KEY": secret}): ex = self.wait(execution.Execution(plan, root=str(root), key=self.KEY, url=self.URL).start()) self.assertEqual(ex.state, "done", ex.lines) diff --git a/app/tests/test_prerequisites.py b/app/tests/test_prerequisites.py index e1c60b0c..3b2c90bf 100644 --- a/app/tests/test_prerequisites.py +++ b/app/tests/test_prerequisites.py @@ -107,6 +107,17 @@ def test(host, programs=None): self.assertEqual((rows["python3"]["state"], rows["git"]["state"], rows["setup:sandbox"]["state"]), ("ok", "missing", "missing")) self.assertEqual(prerequisites.check_host(self.HOST, ["docker"], test=self.probe())[1]["state"], "ok") + def test_a_boat_run_s_programs_are_asked_of_its_host(self): + # a run on Boat starts and fills its VMs from the host: ssh, ssh-keygen, rsync and tar there + asked = {} + + def test(host, programs=None): + asked["programs"] = programs + return self.probe(missing=["rsync"])(host) + rows = by_name(prerequisites.check_host(self.HOST, ["python3", "git", "ssh", "ssh-keygen", "rsync", "tar"], test=test)) + self.assertEqual(sorted(asked["programs"]), ["git", "python3", "rsync", "ssh", "ssh-keygen", "tar"]) + self.assertEqual((rows["ssh"]["state"], rows["rsync"]["state"]), ("ok", "missing")) + def test_a_host_s_docker_with_userns_remap_is_said_for_the_standard_alone(self): remapped = self.probe(docker_userns=True) rows = by_name(prerequisites.check_host(self.HOST, ["docker"], standard=True, test=remapped)) diff --git a/app/tests/test_standard_boat.py b/app/tests/test_standard_boat.py index 582e7cfe..54fcba34 100644 --- a/app/tests/test_standard_boat.py +++ b/app/tests/test_standard_boat.py @@ -131,9 +131,9 @@ def test_the_key_comes_from_the_environment_first_then_the_store(self): self.assertEqual(boat.key({}), "boat_stored") self.assertEqual(boat.key({boat.NAME: "boat_env"}), "boat_env") with mock.patch.object(boat.secrets, "value", return_value=None): - self.assertIn("Settings", boat.unusable({})) - self.assertIsNone(boat.unusable({boat.NAME: "k"}, which=lambda p: "/usr/bin/" + p)) - self.assertIn("rsync", boat.unusable({boat.NAME: "k"}, which=lambda p: None if p == "rsync" else p)) + self.assertIn("Settings", boat.unusable(env={})) + self.assertIsNone(boat.unusable(env={boat.NAME: "k"}, which=lambda p: "/usr/bin/" + p)) + self.assertIn("rsync", boat.unusable(env={boat.NAME: "k"}, which=lambda p: None if p == "rsync" else p)) class VM(unittest.TestCase): @@ -532,31 +532,34 @@ def test_an_environment_a_vm_cannot_give_is_not_run(self): class BoatRecipe(unittest.TestCase): - def test_boat_needs_no_yes_and_runs_from_this_machine(self): - out = recipe.compile(a_recipe(backend="boat", unsandboxed=False), defs(), boat_unusable=lambda: None) + def test_boat_needs_no_yes_and_runs_here_or_on_a_host(self): + out = recipe.compile(a_recipe(backend="boat", unsandboxed=False), defs(), boat_unusable=lambda here=True: None) self.assertEqual(out["errors"], []) self.assertIn("--backend boat --boat-size small", out["script"]); self.assertNotIn("--unsandboxed", out["script"]) self.assertFalse(out["docker"]) - sized = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda: None) + sized = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None) self.assertIn("--boat-size default", sized["script"]) self.assertIn("default VMs", sized["review"][-1][2]) self.assertEqual([e["step"] for e in recipe.compile(a_recipe(backend="boat", boat_size="huge"), defs())["errors"]], [recipe.STEP["where"]]) self.assertIn("boat backend", out["review"][-1][2]) - out = recipe.compile(a_recipe(backend="boat", where="me@bench"), defs()) - self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) - self.assertIn("This machine", out["errors"][0]["message"]) + # on a host: the key is this machine's, the programs the host's (its prerequisites) + asked = [] + out = recipe.compile(a_recipe(backend="boat", where="me@bench"), defs(), boat_unusable=lambda here=True: asked.append(here)) + self.assertEqual(out["errors"], []) + self.assertEqual(asked, [False]) + self.assertIn("--backend boat", out["script"]) def test_a_size_the_account_cannot_start_is_the_where_steps_error(self): problem = lambda size: "not during the free trial" if size == "large" else None - out = recipe.compile(a_recipe(backend="boat", boat_size="large"), defs(), boat_unusable=lambda: None, boat_size_problem=problem) + out = recipe.compile(a_recipe(backend="boat", boat_size="large"), defs(), boat_unusable=lambda here=True: None, boat_size_problem=problem) self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) self.assertIn("large VM on this account: not during the free trial", out["errors"][0]["message"]) - out = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda: None, boat_size_problem=problem) + out = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None, boat_size_problem=problem) self.assertEqual(out["errors"], []) def test_without_its_key_the_where_step_says_so(self): - out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda: "no Boat API key: set it in Settings → Sandbox") + out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda here=True: "no Boat API key: set it in Settings → Sandbox") self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) self.assertIn("Settings → Sandbox", out["errors"][0]["message"]) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index edb68edd..50277f1b 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -489,7 +489,7 @@ def test_the_builder_gets_the_skills_and_a_benchmark_needing_one_says_who_lacks_ def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_here(self): from nullius.standard import boat self.fixture() - with patch.object(boat, "unusable", lambda: "no Boat API key: set it in Settings → Sandbox"): + with patch.object(boat, "unusable", lambda here=True: "no Boat API key: set it in Settings → Sandbox"): html = self.get("/new-run").get_data(as_text=True) schema = json.loads(re.search(r'', html, re.S).group(1)) ds = schema["standard"]["benchmarks"][0]["datasets"][0] @@ -497,7 +497,7 @@ def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_ [("nop", "nullius", None, None), ("oracle", "adapter", None, None)]) # no benchmark here needs a machine setup self.assertEqual(schema["standardHere"], {"docker": None, "boat": "no Boat API key: set it in Settings → Sandbox", - "boatSizes": {}, "boatSpecs": {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB"}, + "boatKey": "no Boat API key: set it in Settings → Sandbox", "boatSizes": {}, "boatSpecs": {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB"}, "licences": {}, "host": {}}) def test_docker_is_refused_on_the_where_step_with_this_machines_reason(self): diff --git a/app/web/builder.py b/app/web/builder.py index 47364750..f2edc8dc 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -54,8 +54,10 @@ def new_run(): gpus = constraints.of(constraints.LOCAL, strict=False).get("gpu_ids") host = {b["key"]: {n: [why] for n, why in hostreq.problems(b, list(b["host"]), gpu_ids=gpus).items()} for b in definitions.load("benchmark", every=True)[0] if b.get("host")} - boat_why = boat.unusable() - data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat_why, "boatSizes": {} if boat_why else execute.boat_sizes(), + # boat: why it cannot run here (its key, this machine's programs); boatKey: from a host, only the key matters here + boat_why, boat_key = boat.unusable(), boat.unusable(here=False) + data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat_why, "boatKey": boat_key, + "boatSizes": {} if boat_key else execute.boat_sizes(), "boatSpecs": {k: v for k, v in boat.SIZES.items() if v}, "licences": _licence_states(), "host": host} data["csrf"] = shared._csrf_token() data["local"] = shared_local_client() @@ -123,7 +125,7 @@ def new_run_prerequisites(): if out["errors"]: return jsonify(machines=[], ready=False, errors=[e["message"] for e in out["errors"]]) wheres = [sh["where"] for sh in out["shards"]] - needs = ["engine"] if out["docker"] else [] + needs = (["engine"] if out["docker"] else []) + (list(boat.PROGRAMS) if recipe.get("backend") == "boat" else []) h = out.get("host") or {} b = definitions.get("benchmark", h.get("benchmark", "")) if h.get("needs") else None setups = {n["name"]: {**n, "install": hostreq.install_lines(b, [n["name"]])} diff --git a/docs/builder.md b/docs/builder.md index b98dd0f3..6d4ea85a 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -28,7 +28,7 @@ Then [read its results](results.md). | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, retries, attempts per unit, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit; a [list of units](cli/run.md#a-list-of-units) is set only on the command line | -| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit, from this machine only. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | +| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | | Review | One row per decision and the prerequisites. **Run** | Original setups run on local processes or in Docker; Docker changes isolation only. diff --git a/docs/standard.md b/docs/standard.md index e2fce100..64a30b42 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -181,14 +181,14 @@ A rootful Docker with `userns-remap` maps the uids of each container to sub-uids | Piece | Rule | | --- | --- | -| Key | Settings → Sandbox, or `BOAT_API_KEY`. Goes to boat.dev only | +| Key | Settings → Sandbox, or `BOAT_API_KEY`. Goes to boat.dev, and to a host that runs the run, over stdin. The machine that runs it needs `ssh` and `rsync` | | VM | One per unit, sized by `--boat-size` (`small` by default). Deleted whatever ends the unit; a failed delete is said. Auto-stops after the unit's time, 2 hours at most on a free trial | | Units at a time | No more than the account's VMs at once. The run warns when the account cannot start every unit this hour or today | | Agent time | Starts once the VM is up and holds the install | | Isolation | The agent reaches only the VM's loopback: no internet, no DNS. Model calls go through an SSH tunnel to the relay here, so no key reaches the VM | | Sent, brought back | The install and its runtimes, the home, the workspace. A link into your home or keys refuses the unit. The workspace comes back without links that leave it; nothing after a Stop | | Code checks | A check that runs the agent's code (`runs_code`) runs it here, in the check container: without Docker, the run is refused | -| Not run | Units that need the public web, their own image, a licence file or server of this machine, or MCP servers. Boat runs from this machine only, without GPUs | +| Not run | Units that need the public web, their own image, a licence file or server of this machine, or MCP servers. Boat has no GPUs | ### A benchmark's MCP server on the machine From 004bdd6e3b9cdda03c33fa6149c9d831756ff169 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 17 Nov 2026 16:58:54 +0100 Subject: [PATCH 8/9] docs: keep docs/ terse, and say how in AGENTS.md The Boat section drops what is mechanism (how the key travels, what a host is checked for) and keeps what a user decides on. AGENTS.md gains a Documentation convention so later changes keep the same standard. --- AGENTS.md | 12 ++++++++++++ docs/standard.md | 18 +++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 69e93cb1..916a83f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -293,6 +293,18 @@ one assistant or one benchmark. Use `nullius.thinkflowlab@proton.me` for Nullius-owned contact email addresses, including security reports. +### Documentation + +`docs/` is terse: fragments and tables, the fewest words that change what a +reader chooses, does or concludes. + +- Keep guarantees, requirements, refusals and their fix. Drop the mechanism + (flags, files, protocols, how a key travels), interface narration, history + and asides. +- A table cell is a line, not a paragraph. One fact lives on one page; others link. +- A prerequisite Review already checks is not repeated. +- `docs/standard.md` headings are never renamed: the import skills cite them. + ### Attribution **No machine attribution in commit messages or pull request descriptions** — no diff --git a/docs/standard.md b/docs/standard.md index 64a30b42..fbbfae5b 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -177,18 +177,18 @@ A rootful Docker with `userns-remap` maps the uids of each container to sub-uids ### The boat backend -`--backend boat` runs each unit's agent or capability in a [boat.dev](https://boat.dev) VM of its own. The driver, the relay, the spending cap and the scoring stay here; checks and Original setups run locally. +`--backend boat`: each unit's agent or capability in its own [boat.dev](https://boat.dev) VM. Driver, relay, spending cap and scoring stay with the run. | Piece | Rule | | --- | --- | -| Key | Settings → Sandbox, or `BOAT_API_KEY`. Goes to boat.dev, and to a host that runs the run, over stdin. The machine that runs it needs `ssh` and `rsync` | -| VM | One per unit, sized by `--boat-size` (`small` by default). Deleted whatever ends the unit; a failed delete is said. Auto-stops after the unit's time, 2 hours at most on a free trial | -| Units at a time | No more than the account's VMs at once. The run warns when the account cannot start every unit this hour or today | -| Agent time | Starts once the VM is up and holds the install | -| Isolation | The agent reaches only the VM's loopback: no internet, no DNS. Model calls go through an SSH tunnel to the relay here, so no key reaches the VM | -| Sent, brought back | The install and its runtimes, the home, the workspace. A link into your home or keys refuses the unit. The workspace comes back without links that leave it; nothing after a Stop | -| Code checks | A check that runs the agent's code (`runs_code`) runs it here, in the check container: without Docker, the run is refused | -| Not run | Units that need the public web, their own image, a licence file or server of this machine, or MCP servers. Boat has no GPUs | +| Key | Settings → Sandbox, or `BOAT_API_KEY` | +| VM | One per unit, `--boat-size` (`small` by default). Always deleted. Free trial: 2 hours at most | +| Units at a time | Capped by the account's VM limit. The run warns when it cannot start every unit | +| Agent time | Excludes the VM's start-up | +| Isolation | The agent reaches loopback only: no internet, no DNS, no model key | +| Files | A link into your home or keys refuses the unit. Links out of the workspace are not copied back | +| Code checks | A `runs_code` check needs Docker where the run runs | +| Not run | Units needing the public web, their own image, a licence file or server of this machine, or MCP servers. No GPUs | ### A benchmark's MCP server on the machine From 08a7bac0851d9ecf5aa35716b879e93a160d7425 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 17 Nov 2026 17:04:47 +0100 Subject: [PATCH 9/9] fix(standard): keep the Boat key a run on a host is handed A run started by Nullius keeps only its provider's key and endpoint of the environment its launcher gives it (run_env), which dropped the Boat key handed over stdin. Here the run fell back to the key saved in Settings; a host has none, so a run on Boat there was refused for want of a key. --- app/nullius/standard/cli.py | 6 ++++-- app/tests/test_standard_recipe.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index c01c7c7b..9a803b58 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -104,8 +104,10 @@ def run_env(provider: str, env=os.environ) -> dict: """The environment a run started by Nullius reads its provider from: ``execution``'s launcher hands every run its one key as ``OPENAI_API_KEY`` and its endpoint as ``OPENAI_BASE_URL`` (empty for OpenAI's own), so they - are named here as the provider's own settings and nothing else is.""" - out = {k: v for k, v in env.items() if k in run.KEEP} + are named here as the provider's own settings and nothing else is - but + the Boat key, handed the same way to a run on Boat: on a host, the only + way it has one.""" + out = {k: v for k, v in env.items() if k in run.KEEP or k == boat.NAME} if env.get("OPENAI_API_KEY"): out[providers.key_setting(provider)] = env["OPENAI_API_KEY"] if provider == "custom" and env.get("OPENAI_BASE_URL"): diff --git a/app/tests/test_standard_recipe.py b/app/tests/test_standard_recipe.py index fd77857f..4d086c3a 100644 --- a/app/tests/test_standard_recipe.py +++ b/app/tests/test_standard_recipe.py @@ -418,6 +418,14 @@ def test_the_handed_key_is_named_as_the_providers_own_and_nothing_else_is_kept(s self.assertEqual(cli.run_env("custom", env), {"MODEL_API_KEY": "sk-handed", "MODEL_API_URL": "http://127.0.0.1:9/v1", "PATH": "/bin"}) self.assertEqual(cli.run_env("openai", env), {"OPENAI_API_KEY": "sk-handed", "PATH": "/bin"}) + def test_the_boat_key_handed_to_a_run_on_boat_is_kept(self): + # on a host, the key handed over stdin is the run's only one: no store there holds it + env = {"OPENAI_API_KEY": "sk-handed", "BOAT_API_KEY": "boat_handed", "PATH": "/bin"} + self.assertEqual(cli.run_env("openai", env), {"OPENAI_API_KEY": "sk-handed", "BOAT_API_KEY": "boat_handed", "PATH": "/bin"}) + from nullius.standard import boat + with patch.object(boat.secrets, "value", return_value=None): + self.assertEqual(boat.key(cli.run_env("openai", env)), "boat_handed") + def test_a_run_started_by_nullius_does_not_read_app_env(self): loaded = [] with patch.object(cli.settings, "load", lambda path: loaded.append(path)), patch.object(cli, "cmd_run", lambda a: 0):