diff --git a/AGENTS.md b/AGENTS.md index 9aeb60b9..916a83f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ and its resources. The source layout stays under `app/`. `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), `standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`, - `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its + `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `boat` (the boat backend: each unit's agent in a boat.dev VM), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its `payload` snapshots the running backend and effective catalogue for a remote host; conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: @@ -293,6 +293,18 @@ one assistant or one benchmark. Use `nullius.thinkflowlab@proton.me` for Nullius-owned contact email addresses, including security reports. +### Documentation + +`docs/` is terse: fragments and tables, the fewest words that change what a +reader chooses, does or concludes. + +- Keep guarantees, requirements, refusals and their fix. Drop the mechanism + (flags, files, protocols, how a key travels), interface narration, history + and asides. +- A table cell is a line, not a paragraph. One fact lives on one page; others link. +- A prerequisite Review already checks is not repeated. +- `docs/standard.md` headings are never renamed: the import skills cite them. + ### Attribution **No machine attribution in commit messages or pull request descriptions** — no diff --git a/app/nullius/execution.py b/app/nullius/execution.py index cbbade16..aa8991ab 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -336,7 +336,8 @@ def withheld() -> List[str]: names other tools give them or this application gave them before (``settings.INSTEAD``, those that stand for a secret), the launcher's own names for the run's key and endpoint, the agent keys (a Harbor run is - sent them by its launcher, as the key), the names a standard benchmark's + sent them by its launcher, as the key), the Boat key (a standard run on + Boat is sent it the same way), the names a standard benchmark's own licences go by - kept in its own store (``nullius.secrets``), although imported saved settings may still hold a value until it is moved in Custom keys - and GitHub tokens from saved settings or the @@ -344,7 +345,7 @@ def withheld() -> List[str]: run gets its own endpoint's key alone. A data provider's token name is not one.""" secret = {v.name for v in settings.VARIABLES if v.secret or v.kind in ("endpoint", "provider")} | set(settings.endpoint_names()) return sorted(secret | {old for old, new in settings.INSTEAD.items() if new in secret} - | set(KEY_NAMES + URL_NAMES) | {"GITHUB_TOKEN", "GH_TOKEN"} | set(settings.agent_key_names()) + | set(KEY_NAMES + URL_NAMES) | {"GITHUB_TOKEN", "GH_TOKEN", "BOAT_API_KEY"} | set(settings.agent_key_names()) | set(_licence_names())) @@ -398,6 +399,10 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl self._key, self._url = "", "" # the agent keys Settings holds (CURSOR_API_KEY, ...): a Harbor run's agent may need its own vendor's; the standard's never gets one self._agent_keys = dict(agent_keys) if agent_keys is not None else (settings.agent_key_values(env) if plan.runner == "harbor" else {}) + if plan.runner == "standard" and (plan.recipe or {}).get("backend") == "boat": + from nullius.standard import boat # the Boat key, to the standard run that starts VMs with it, and to no other command + if boat.key(env): + self._agent_keys[boat.NAME] = boat.key(env) # the benchmark's licences a standard run on a host is sent: read from its store at each launch, unless given self._licences_given = licences self._redact: List[str] = [] # the licences' values sent, as text: never in a log line diff --git a/app/nullius/prerequisites.py b/app/nullius/prerequisites.py index daaf3598..e52b6a7b 100644 --- a/app/nullius/prerequisites.py +++ b/app/nullius/prerequisites.py @@ -98,7 +98,7 @@ def check_host(host: hosts.Host, needs: List[str], setups: Optional[Dict[str, di """The rows for a host, from one ssh probe. Unreachable: one row, and nothing else is known.""" test = test or hosts.test programs = [p for s in (setups or {}).values() for p in s.get("check") or []] - programs += [n for n in needs if n in ("python3", "git")] + programs += [n for n in needs if n in ("python3", "git", "ssh", "ssh-keygen", "rsync", "tar")] got = test(host, programs=programs) if not got["ok"]: return [row("connect", "missing", got.get("error") or "no answer", label="Connection")] diff --git a/app/nullius/standard/boat.py b/app/nullius/standard/boat.py new file mode 100644 index 00000000..1eb6fe8a --- /dev/null +++ b/app/nullius/standard/boat.py @@ -0,0 +1,551 @@ +"""The ``boat`` backend: each unit's agent or capability in a boat.dev VM of its own. + +A boat VM is a cloud Ubuntu machine created by its API (``API``) and reached +over plain OpenSSH. Everything that holds an answer or a key stays on this +machine: the driver, the unit's relay, the spending cap, the scoring. Only +the process of the agent (or the capability) runs in the VM, as ``USER``, a +user of its own whose only way out is the VM's loopback (``LOCK``): no +public address, no DNS. Its model calls reach the unit's relay through a +reverse tunnel on the same port (``ssh -R``), so the address it is given is +the one the local backend gives, and the relay records every call here. + +One unit, one VM (:class:`VM`) of the run's size (``SIZES``, ``small`` by +default; :func:`size_problem` says when the account cannot start it): created when the unit's agent is about to +start, deleted once its workspace is copied back. A run's concurrency is +therefore the number of VMs at once, never more than the account allows at +once (:func:`slots`); a VM the account cannot start yet (its limit, its rate +of starts) is waited for, and the wait ends with the run's Stop. Every VM +carries the run's label, so whatever an interrupted unit left is deleted by +:func:`sweep` when the run ends; a VM of a run whose process was killed stops +at its auto-stop: as long as its unit may take, at least ``TTL``, and ``TTL`` +at most on a free trial, which allows no more. + +What the VM is given, each at the same absolute path it has here so every +path in the agent's configuration still holds: the agent's install, its +definition, the runtimes the install links to outside itself (a ``uv`` +Python, never a folder holding this user's home, keys or Nullius's own +storage), sent before the agent's time starts; then, as the agent starts, its +home and the unit's workspace, which its setup writes up to then. The workspace comes back with +links that leave it dropped (``rsync --safe-links``): the driver reads it to +score, and a link the agent left must not reach a file of this machine. +Nothing of this machine's environment goes with it but its locale. + +The API key is kept in the owner-only store (``nullius.secrets``, the folder +``STORE``), set in Settings → Sandbox; ``BOAT_API_KEY`` in the environment +wins. It is sent to boat.dev only, never to the VM. Standard library only. +""" +from __future__ import annotations + +import json +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from contextlib import contextmanager +from typing import Callable, Dict, Iterable, List, Optional, Sequence + +from nullius import paths, secrets +from nullius.standard import install + +API = "https://boat.dev/api/v1" +STORE, NAME = "sandbox-provider-boat", "BOAT_API_KEY" +# the VM runs the agent's process, never its model: the smallest is enough +SIZES = {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB", "large": "", "xlarge": ""} +SIZE = "small" +TTL = 2 * 3600 # a VM's auto-stop at least, and a free trial's at most +USER = "agent" +LABEL = "nullius-job" +VM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +KEEP = ("LANG", "LC_ALL", "TERM") # of the local environment, what makes sense on another machine +WAIT_SECONDS = 3600 # an account's starts an hour come back within one +READY_SECONDS = 300 +SSH_SECONDS = 900 +DRIVER = 1800 # a unit's time beyond its agent's: sending, bringing back + +# The agent's user reaches the VM's loopback and nothing else: no public address, no DNS. Inserted +# first in OUTPUT, before the chains boat's own firewall accepts in; each at 1, so the last inserted +# is matched first: DNS on loopback refused, then loopback accepted, then everything else refused. +LOCK = ("U=$(id -u {user}) && for t in iptables ip6tables; do " + "sudo -n $t -I OUTPUT 1 -m owner --uid-owner $U -j REJECT && " + "sudo -n $t -I OUTPUT 1 -o lo -m owner --uid-owner $U -j ACCEPT && " + "sudo -n $t -I OUTPUT 1 -o lo -p tcp --dport 53 -m owner --uid-owner $U -j REJECT && " + "sudo -n $t -I OUTPUT 1 -o lo -p udp --dport 53 -m owner --uid-owner $U -j REJECT; done") +_SYSTEM = ("/usr/", "/bin/", "/lib", "/sbin/", "/etc/", "/opt/") +_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") # a variable a shell can export + + +class Unavailable(Exception): + """boat cannot run this unit: no key, a refusal of its API, a VM that never came up.""" + + +def key(env=os.environ) -> Optional[str]: + """The account's API key: ``BOAT_API_KEY``, else the store's; None when neither is set.""" + if env.get(NAME): + return env[NAME].strip() + v = secrets.value(STORE, NAME) + return v.decode().strip() if v else None + + +PROGRAMS = ("ssh", "ssh-keygen", "rsync", "tar") # what starts, reaches and fills a VM, where the run runs + + +def unusable(here: bool = True, env=os.environ, which: Callable = shutil.which) -> Optional[str]: + """Why the boat backend cannot run, or None: its key, and with ``here`` this machine's programs.""" + if not key(env): + return "no Boat API key: set it in Settings → Sandbox" + lacking = [p for p in PROGRAMS if not which(p)] if here else [] + if lacking: + return "missing on this machine: " + ", ".join(lacking) + return None + + +class Client: + """boat.dev's API, with one account's key.""" + + def __init__(self, api_key: str, base: str = API, opener: Callable = urllib.request.urlopen, timeout: float = 60): + self.key, self.base, self.opener, self.timeout = api_key, base, opener, timeout + + def call(self, method: str, path: str, body: Optional[dict] = None, headers: Optional[dict] = None) -> dict: + req = urllib.request.Request(self.base + path, method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", + "Accept": "application/json", **(headers or {})}) + try: + with self.opener(req, timeout=self.timeout) as r: + return json.loads(r.read() or b"{}") + except urllib.error.HTTPError as e: + try: + d = json.loads(e.read() or b"{}") + except ValueError: + d = {} + return {"ok": False, "status": e.code, **d} if isinstance(d, dict) else {"ok": False, "status": e.code} + except (urllib.error.URLError, OSError, ValueError) as e: + return {"ok": False, "code": "unreachable", "message": f"Boat did not answer: {e}"} + + def limits(self) -> dict: + return self.call("GET", "/limits") + + def create(self, labels: dict, size: str = SIZE, ttl: int = TTL) -> dict: + return self.call("POST", "/sandboxes", {"type": size, "ttlSeconds": ttl, "name": "nullius-unit", "labels": labels}) + + def get(self, vm: str) -> dict: + return self.call("GET", f"/sandboxes/{urllib.parse.quote(vm)}") + + def ssh_key(self, vm: str, public: str) -> dict: + return self.call("POST", f"/sandboxes/{urllib.parse.quote(vm)}/sshkey", {"key": public}) + + def delete(self, vm: str) -> dict: + return self.call("DELETE", f"/sandboxes/{urllib.parse.quote(vm)}", headers={"X-Ascii-Confirm-Delete": vm}) + + def labelled(self, label: str, value: str) -> List[dict]: + d = self.call("GET", "/sandboxes?" + urllib.parse.urlencode({"label": f"{label}={value}", "limit": 100})) + return [s for s in d.get("sandboxes") or [] if (s.get("labels") or {}).get(label) == value] + + +def said(d: dict) -> str: + return str(d.get("message") or d.get("code") or d.get("status") or "no answer") + + +def slots(client: Client) -> Optional[int]: + """How many VMs the account may run at once; None when its API does not say.""" + n = client.limits().get("maxActiveSandboxes") + return n if isinstance(n, int) and n > 0 else None + + +def starts_problem(client: Client, n: int) -> Optional[str]: + """What the account's starts left say of a run of ``n`` VMs, one per unit, or None.""" + left = client.limits().get("starts") or {} + day, hour = ((left.get(k) or {}).get("remaining") for k in ("day", "hour")) + if isinstance(day, int) and n > day: + return (f"Boat can start {day} more VMs today on this account: {n - day} of the {n} units " + "will end without running") + if isinstance(hour, int) and n > hour: + return (f"Boat can start {hour} more VMs this hour on this account: the other units wait " + "for the next hour") + return None + + +def machine_types(client: Client) -> Dict[str, Optional[str]]: + """Each machine type the account knows: None when it may start one, else boat's reason why not + ("not during the free trial"). Empty when its API does not say.""" + out = {} + for t in client.limits().get("machineTypes") or []: + if isinstance(t, dict) and isinstance(t.get("type"), str): + out[t["type"]] = None if t.get("available") else str(t.get("reason") or "not available on this account") + return out + + +def size_problem(client: Client, size: str) -> Optional[str]: + """Why this account cannot start a VM of ``size``, or None.""" + if size not in SIZES: + return f"{size!r} is not one of Boat's sizes: {', '.join(SIZES)}" + return machine_types(client).get(size) + + +def gone(d: dict) -> bool: + """A delete's answer: deleted, or not there to delete.""" + return bool(d.get("ok")) or d.get("status") == 404 + + +def sweep(job: str, client: Optional[Client] = None, say: Optional[Callable[[str], None]] = None) -> int: + """Deletes every VM labelled with ``job``: what a crash or a kill left. The number deleted; + each one Boat would not delete is said.""" + if client is None: + k = key() + if not k: + return 0 + client = Client(k) + n = 0 + for s in client.labelled(LABEL, job): + d = client.delete(s["id"]) + if gone(d): + n += 1 + elif say is not None: + say(f"Boat did not delete VM {s['id']}: {said(d)}. It stops by itself at its auto-stop; delete it at boat.dev") + return n + + +def _private(env=os.environ) -> tuple: + """What a folder sent to a VM may not hold, and what it may not be inside of.""" + home = os.path.realpath(os.path.expanduser("~")) + # where this user's tools keep their logins: never sent, nor a folder holding one (``~/.local`` holds keyrings) + keys = [os.path.join(home, d) for d in (".ssh", ".gnupg", ".aws", ".docker", ".kube", ".password-store", ".config")] \ + + [os.path.realpath(paths.config_dir(env))] + holds = [os.path.join(home, d) for d in (".local", ".cache", ".netrc", ".git-credentials")] + return [home, *holds, os.path.realpath(paths.data_dir(env)), os.path.realpath(paths.cache_dir(env)), *keys], keys + + +def outside_roots(where: str, env=os.environ) -> List[str]: + """The folders an install links to outside itself and outside the system's own: a ``uv`` + environment's Python, kept in uv's folder of this user. Each must reach the VM too. A link + whose folder would hold this user's home, keys or Nullius's storage is the install's + ``Unavailable``: none of it may leave this machine.""" + roots = {} + real_where = os.path.realpath(where) + for top, dirs, files in os.walk(where): + for n in dirs + files: + p = os.path.join(top, n) + if not os.path.islink(p): + continue + t = os.path.realpath(p) + if t == real_where or t.startswith(real_where + os.sep) or t.startswith(_SYSTEM) or not os.path.exists(t): + continue + if os.path.isdir(t): + r = t + else: + d = os.path.dirname(t) + r = os.path.dirname(d) if os.path.basename(d) == "bin" else d + roots.setdefault(r, p) + holds, inside = _private(env) + for r, link in roots.items(): + if any(r == s or s.startswith(r.rstrip(os.sep) + os.sep) for s in holds) or any(r.startswith(s + os.sep) for s in inside): + raise Unavailable(f"{link} links to {r}, which would send this user's own files to a Boat VM") + out: List[str] = [] + for r in sorted(roots, key=len): # a root inside another is sent with it + if not any(r == o or r.startswith(o + os.sep) for o in out): + out.append(r) + return out + + +class Keys: + """A job's SSH key (made for it, never this user's own) and the host keys it pinned.""" + + def __init__(self): + self.dir = tempfile.mkdtemp(prefix="nullius-boat-") + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", "nullius-boat", "-f", os.path.join(self.dir, "id")], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + with open(os.path.join(self.dir, "id.pub")) as f: + self.public = f.read().strip() + + def close(self) -> None: + shutil.rmtree(self.dir, ignore_errors=True) + + +class VM: + """One unit's VM, ready: created, its host key pinned, the agent's user made and locked in. + ``stopped``: the run's Stop, which ends a wait for the account to start it, or for it to come up. ``ttl``: its + auto-stop, asked for; ``self.ttl`` the one it has, ``TTL`` on a free trial.""" + + def __init__(self, client: Client, keys: Keys, job: str, *, size: str = SIZE, stopped: Optional[threading.Event] = None, + wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep, ttl: int = TTL): + self.client, self.keys, self.id, self.size, self.ttl = client, keys, None, size, max(int(ttl), TTL) + self.stopped = stopped + t0 = time.time() + pause = 5.0 + while True: + d = client.create({LABEL: job}, size=size, ttl=self.ttl) + if d.get("ok"): + break + if d.get("code") == "trial_auto_stop_required" and self.ttl > TTL: + self.ttl = TTL + continue + # the account's limit of VMs at once, or of starts a minute: another unit's VM ends, a minute passes + if d.get("status") != 429 and d.get("code") not in ("limit_reached", "rate_limited"): + raise Unavailable("Boat refused a VM: " + said(d)) + if (stopped is not None and stopped.is_set()) or time.time() - t0 > wait: + raise Unavailable("Boat could not start a VM in time: " + said(d)) + sleep(pause) + pause = min(pause * 1.5, 30.0) + self.id = d["sandbox"]["id"] + try: + self._ready(sleep) + self.sh(f"sudo -n useradd -m -s /bin/bash {USER} && " + LOCK.format(user=USER)) + except BaseException: + self.delete() + raise + self.seconds = round(time.time() - t0, 1) + + def _halt(self) -> None: + if self.stopped is not None and self.stopped.is_set(): + raise Unavailable(f"Boat VM {self.id}: the run was stopped") + + def _ready(self, sleep) -> None: + t0 = time.time() + while True: + self._halt() + s = self.client.get(self.id).get("sandbox") or {} + if s.get("state") in ("ready", "idle", "running"): + break + if s.get("state") in ("archived", "deleted", "failed", "error") or time.time() - t0 > READY_SECONDS: + raise Unavailable(f"Boat VM {self.id} did not come up (state {s.get('state')})") + sleep(2) + k = self.client.ssh_key(self.id, self.keys.public) + if not k.get("ok") or not k.get("hostKey"): + raise Unavailable(f"Boat VM {self.id} took no SSH key: " + said(k)) + ep = k.get("sshEndpoint") or f"{k['machineIp']}:22" + self.host, self.port = ep.rsplit(":", 1) + self.user = k.get("sshUser") or "user" + known = os.path.join(self.keys.dir, f"known-{self.id}") + with open(known, "w") as f: + f.write(f"{self.id} {k['hostKey']}\n") + self.ssh = ["ssh", "-F", "/dev/null", "-i", os.path.join(self.keys.dir, "id"), "-p", self.port, "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=yes", "-o", f"UserKnownHostsFile={known}", + "-o", f"HostKeyAlias={self.id}", "-o", "ServerAliveInterval=30", "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=20"] + self.target = f"{self.user}@{self.host}" + for _ in range(30): # sshd answers a moment after the key is in, or hangs while it boots + self._halt() + try: + if subprocess.run(self.ssh + [self.target, "true"], capture_output=True, timeout=60).returncode == 0: + return + except subprocess.TimeoutExpired: + pass + sleep(2) + raise Unavailable(f"Boat VM {self.id} did not answer over SSH") + + def sh(self, cmd: str, stdin=None) -> str: + try: + r = subprocess.run(self.ssh + [self.target, cmd], stdin=stdin, capture_output=True, timeout=SSH_SECONDS) + except subprocess.TimeoutExpired: + raise Unavailable(f"Boat VM {self.id}: {cmd.split(' ', 3)[:3]}: no answer in {SSH_SECONDS} s") from None + if r.returncode: + raise Unavailable(f"Boat VM {self.id}: {cmd.split(' ', 3)[:3]}: " + + r.stderr.decode(errors="replace").strip()[-300:]) + return r.stdout.decode(errors="replace") + + def send(self, read: Iterable[str], owned: Iterable[str] = ()) -> None: + """Each folder at its own path in the VM, in one compressed stream: ``read`` the agent's user + may read, ``owned`` it owns (its home, its workspace).""" + read, owned = [p.rstrip("/") for p in read if p], [p.rstrip("/") for p in owned if p] + rel = [os.path.relpath(p, "/") for p in dict.fromkeys(read + owned)] + q = " ".join(shlex.quote("/" + r) for r in rel) + mine = " ".join(shlex.quote(p) for p in owned) + failed = None + with tempfile.TemporaryFile() as err: # a file, never a pipe: tar's warnings cannot stall the stream + tar = subprocess.Popen(["tar", "-C", "/", "-czf", "-", "--", *rel], stdout=subprocess.PIPE, stderr=err) + try: + self.sh("sudo -n tar -C / -xzpf - --no-same-owner && " + f"for p in {q}; do while [ \"$p\" != / ]; do sudo -n chmod o+rx \"$p\"; p=$(dirname \"$p\"); done; done" + + (f" && sudo -n chown -R {USER}:{USER} {mine}" if mine else "") + + f" && sudo -n chmod -R o+rX {q}", stdin=tar.stdout) + except Unavailable as e: + failed = e + finally: + tar.stdout.close() + code = tar.wait() + err.seek(0) + said = err.read().decode(errors="replace").strip()[-300:] + # tar's 1 is a file that changed as it was read: what the VM received still stands; a stream the + # VM stopped reading is the VM's failure, said by the VM + cut = "status 141" in said or "Broken pipe" in said + if (code > 1 and not (failed and cut)) or (code and failed and not cut): + raise Unavailable(f"could not pack what Boat VM {self.id} needs: {said}") + if failed: + raise failed + + def fetch(self, path: str) -> None: + """The folder back from the VM, as it is there: what the agent removed is removed here, a link + leaving the folder is not copied, nor a device or a pipe.""" + path = path.rstrip("/") + rsh = " ".join(shlex.quote(w) for w in self.ssh) + try: + r = subprocess.run(["rsync", "-rlt", "--delete", "--safe-links", "--no-devices", "--no-specials", + "--rsync-path=sudo -n rsync", "-e", rsh, f"{self.target}:{path}/", path + "/"], + capture_output=True, timeout=SSH_SECONDS) + except subprocess.TimeoutExpired: + raise Unavailable(f"the workspace did not come back from Boat VM {self.id} in {SSH_SECONDS} s") from None + if r.returncode: + raise Unavailable(f"the workspace did not come back from Boat VM {self.id}: " + + r.stderr.decode(errors="replace").strip()[-300:]) + + def argv(self, cmd: List[str], env: Dict[str, str], workdir: str, port: Optional[int]) -> List[str]: + """``cmd`` run in the VM as the agent's user, in ``workdir``, with ``env`` alone; its stdin and + stdout this process's (ACP, JSON lines). ``port``: the relay's, the same port in the VM. The + environment (the relay's word, a licence's value) never goes on a command line, here or in the + VM: it is sent first as a file only the agent's user reads, which its shell reads and removes.""" + lines = "".join(f"export {k}={shlex.quote(v)}\n" for k, v in env.items() if _NAME.match(k)) + with tempfile.TemporaryFile() as f: + f.write(lines.encode()) + f.seek(0) + made = self.sh(f"f=$(mktemp /tmp/.nullius-env.XXXXXXXX) && cat > \"$f\" && sudo -n chown {USER} \"$f\" && echo \"$f\"", + stdin=f).strip() + remote = (f"cd {shlex.quote(workdir)} && exec sudo -n -u {USER} env -i /bin/bash -c " + + shlex.quote('. "$0" && rm -f "$0" && exec "$@"') + f" {shlex.quote(made)} " + + " ".join(shlex.quote(w) for w in cmd)) + tunnel = ["-o", "ExitOnForwardFailure=yes", "-R", f"127.0.0.1:{port}:127.0.0.1:{port}"] if port else [] + return self.ssh + ["-T", *tunnel, self.target, remote] + + def delete(self) -> Optional[str]: + """Deletes the VM, a second time if Boat did not; why it is still there, or None.""" + if not self.id: + return None + for _ in range(2): + d = self.client.delete(self.id) + if gone(d): + break + else: + return said(d) + try: + os.remove(os.path.join(self.keys.dir, f"known-{self.id}")) + except OSError: + pass + return None + + +def remote_env(env: Dict[str, str], bin_dirs: List[str], local: Sequence[str] = ()) -> Dict[str, str]: + """A process's environment as the VM gives it: ``local``, the variables taken of this machine's + environment, dropped but for its locale; its ``PATH`` the install's programs, then the VM's.""" + out = {k: v for k, v in env.items() if k not in local or k in KEEP} + out["PATH"] = os.pathsep.join(bin_dirs + [VM_PATH]) + return out + + +MCP_REFUSAL = ("the benchmark's MCP servers run from this machine, which an agent in a Boat VM cannot start " + "(--backend local or container)") + + +def refusal(env_def: Optional[dict]) -> Optional[str]: + """Why an environment's agent cannot run in a Boat VM, or None: the public web it is granted + (``network``), a licence file or server of this machine it must reach.""" + if (env_def or {}).get("network"): + return "grants its agent the public web, which a Boat VM does not yet (--backend container)" + for s in (env_def or {}).get("needs_secrets") or []: + if s.get("to") == "agent" and s.get("kind") in ("file", "server"): + return f"gives its agent the licence {s['name']} of this machine, which a Boat VM cannot reach (--backend container)" + return None + + +class Backend: + """The boat backend for one run: its account, the label its VMs carry (this run's alone, though a + job's name may be used again), its SSH key, made with its first VM, and the VM each unit's thread + holds while its agent runs (:meth:`unit`, :meth:`place`). :meth:`close` deletes what is left.""" + + def __init__(self, job: str, env=os.environ, client: Optional[Client] = None, size: str = SIZE, + say: Callable[[str], None] = lambda s: None): + self.client = client or Client(key(env) or "") + self.env = env + self.size, self.say, self._cut = size, say, False + self.label = f"{job}.{os.urandom(4).hex()}" + self._keys: Optional[Keys] = None + self._roots: Dict[str, List[str]] = {} + self._lock = threading.Lock() + self._held = threading.local() + + def starts(self, n: int) -> Optional[str]: + return starts_problem(self.client, n) + + def most(self) -> Optional[int]: + """How many units may run at once: the VMs the account runs at once.""" + return slots(self.client) + + @contextmanager + def unit(self, trial: str, ws: str, home: str, stopped: Optional[threading.Event] = None, *, + inst: Optional[str] = None, here: Optional[str] = None, seconds: float = 0): + """The unit's VM while its agent or capability runs, given before it starts, each at its own + path: the install (``inst``) and the runtimes it links to outside itself, and the definition + (``here``). ``agent/boat.json`` says which VM, and how long it took to come up and be sent + all that. Its workspace and home (sent by :meth:`place`) are brought back + from it, then the VM deleted. A failure to bring them back after the agent ended is the + unit's ``Unavailable``; after the agent failed, its own failure stands, and after a Stop + nothing is brought back. ``seconds``: the longest the unit may take, its VM's auto-stop + with the sending and the bringing back (``DRIVER``); a trial's shorter one is said once.""" + with self._lock: + if self._keys is None: + self._keys = Keys() + if inst and inst not in self._roots: + self._roots[inst] = outside_roots(inst, self.env) if os.path.isdir(inst) else [] + read = [p for p in [inst, here, *self._roots.get(inst or "", [])] if p and os.path.isdir(p)] + t0 = time.time() + # once a trial's cut is known, asked at once: a refused create would count against the account's starts + vm = VM(self.client, self._keys, self.label, size=self.size, stopped=stopped, ttl=TTL if self._cut else seconds + DRIVER) + try: + if vm.ttl < seconds + DRIVER and not self._cut: + self._cut = True + self.say(f"Boat stops a free trial's VMs after {TTL // 3600} hours: a unit still running then ends there") + vm.send(read) + os.makedirs(os.path.join(trial, "agent"), exist_ok=True) + with open(os.path.join(trial, "agent", "boat.json"), "w") as f: + json.dump({"vm": vm.id, "size": self.size, "ttl_seconds": vm.ttl, "ready_seconds": vm.seconds, + "sent_seconds": round(time.time() - t0, 1)}, f) + self._held.vm = vm + try: + yield vm + except BaseException as e: + stop = isinstance(e, KeyboardInterrupt) or (stopped is not None and stopped.is_set()) + for d in () if stop else (ws, home): # what it left, for the evidence + try: + vm.fetch(d) + except Unavailable: + pass + raise + for d in (ws, home): + vm.fetch(d) + finally: + self._held.vm = None + why = vm.delete() + if why: + # still billing until its auto-stop: said, and kept with the unit's evidence + self.say(f"Boat did not delete VM {vm.id}: {why}. It stops by itself within {vm.ttl // 3600} hours; " + "delete it at boat.dev") + try: + with open(os.path.join(trial, "agent", "boat.json")) as f: + rec = json.load(f) + with open(os.path.join(trial, "agent", "boat.json"), "w") as f: + json.dump(dict(rec, delete_error=why), f) + except (OSError, ValueError): + pass + + def place(self, cmd: List[str], env: Dict[str, str], workdir: str, *, inst: Optional[str], owned: List[str], + relay_url: Optional[str], local: Sequence[str] = ()) -> tuple: + """``cmd`` and the environment to start it with here: ``ssh`` to this thread's VM, where it runs as + its locked user, the install's programs (``inst``) first on its ``PATH``. ``owned``, its home and + workspace as its setup left them, are sent first and become its user's.""" + vm = getattr(self._held, "vm", None) + if vm is None: + raise Unavailable("the unit has no Boat VM") + vm.send([], owned) + port = urllib.parse.urlsplit(relay_url).port if relay_url else None + env = remote_env(env, install.bin_dirs(inst) if inst else [], local) + return vm.argv(list(cmd), env, workdir, port), {k: os.environ[k] for k in ("PATH", "HOME", "LANG") if k in os.environ} + + def close(self) -> None: + sweep(self.label, self.client, self.say) + if self._keys is not None: + self._keys.close() diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index 81f8544d..9a803b58 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -6,8 +6,8 @@ PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P] PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION - PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container] - PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container] + PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container | --backend boat] + PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--units FILE] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--constraint KEY=VALUE ...] [--served-by [MODEL=]SLUG[,SLUG...] ...] [--unsandboxed | --backend container | --backend boat [--boat-size SIZE]] PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--units FILE] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N] PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID] @@ -19,6 +19,9 @@ ``--backend container`` (``docker``, its earlier name) runs each unit's agent or capability in a container whose only way out is the unit's relay, or a public web proxy when its environment declares ``network`` (``standard/container.py``), and needs no such yes. +``--backend boat`` runs each unit's agent or capability in a boat.dev VM of its +own, whose only way out is a tunnel to the unit's relay on this machine +(``standard/boat.py``); its API key is Settings → Sandbox's, or ``BOAT_API_KEY``. A benchmark's control (``nop``, ``oracle``: its ``controls``) runs no agent and calls no model - only a judged benchmark's judge - so it asks for neither. ``prove`` runs a benchmark's controls and checks its units and that nothing @@ -77,7 +80,7 @@ if __name__ == "__main__" and "--from-run" not in sys.argv[1:]: settings.load(paths.settings_file()) -from nullius.standard import conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard +from nullius.standard import boat, conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard ENV_FILE = paths.settings_file() @@ -101,8 +104,10 @@ def run_env(provider: str, env=os.environ) -> dict: """The environment a run started by Nullius reads its provider from: ``execution``'s launcher hands every run its one key as ``OPENAI_API_KEY`` and its endpoint as ``OPENAI_BASE_URL`` (empty for OpenAI's own), so they - are named here as the provider's own settings and nothing else is.""" - out = {k: v for k, v in env.items() if k in run.KEEP} + are named here as the provider's own settings and nothing else is - but + the Boat key, handed the same way to a run on Boat: on a host, the only + way it has one.""" + out = {k: v for k, v in env.items() if k in run.KEEP or k == boat.NAME} if env.get("OPENAI_API_KEY"): out[providers.key_setting(provider)] = env["OPENAI_API_KEY"] if provider == "custom" and env.get("OPENAI_BASE_URL"): @@ -527,7 +532,7 @@ def cmd_run(a): try: job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit, concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, - simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, + boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, constraints=given, **({"units_file": run.units_file(a.units)} if a.units is not None else {}), @@ -663,8 +668,11 @@ def main(argv=None): p.add_argument("--unsandboxed", action="store_true", help="allow an agent, or a setup check without Docker, to execute code on this machine as you") p.add_argument("--backend", default="local", type=definitions.names.backend, choices=list(run.BACKENDS), - help="local processes (the default), or container: each unit in a container, its relay beside it " - "(docker: its earlier name)") + help="local processes (the default); container: each unit in a container, its relay beside it " + "(docker: its earlier name); boat: each unit's agent in a boat.dev VM of its own") + if name == "run": + p.add_argument("--boat-size", default=boat.SIZE, choices=list(boat.SIZES), + help="on the boat backend, its VMs' machine type (default small: 2 vCPU, 4 GB)") p.add_argument("--from-run", action="store_true", help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read") p.add_argument("--option", action="append", metavar="NAME=VALUE", diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 6234e4e3..51a74390 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -525,7 +525,7 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str, provider: str, dataset: str, units: List[str], - concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", + concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "", boat_size: str = "", options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1, bench_options: Optional[dict] = None, simulator: str = "", prices: Optional[dict] = None, units_file: Optional[dict] = None, @@ -550,7 +550,8 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider, "needs": bench["needs"], "permissions_granted": granted, "unsandboxed": granted and backend == "local", "backend": backend, - **({"image": image} if image else {}), **({"judge": judge} if judge else {}), + **({"image": image} if image else {}), **({"boat_size": boat_size} if boat_size else {}), + **({"judge": judge} if judge else {}), **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other), **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}), **({"units_file": units_file} if units_file else {}), diff --git a/app/nullius/standard/kits/agent-workspace-v1/standard.json b/app/nullius/standard/kits/agent-workspace-v1/standard.json index f625e731..90a1e353 100644 --- a/app/nullius/standard/kits/agent-workspace-v1/standard.json +++ b/app/nullius/standard/kits/agent-workspace-v1/standard.json @@ -36,7 +36,8 @@ }, "score": { "kind": "pass", - "says": "the file holds the content; the test passes" + "says": "the file holds the content; the test passes", + "runs_code": true }, "limits": { "turn_seconds": 600 diff --git a/app/nullius/standard/names.py b/app/nullius/standard/names.py index 4ff6cf5e..d54fa174 100644 --- a/app/nullius/standard/names.py +++ b/app/nullius/standard/names.py @@ -1,7 +1,7 @@ """Canonical names, with read compatibility for earlier catalogues, jobs and recipes.""" from __future__ import annotations -BACKENDS = ("local", "container") +BACKENDS = ("local", "container", "boat") def backend(value): diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 5e5d759c..36aaf7d7 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -28,8 +28,11 @@ ``limit`` is its own. ``backend``: ``local`` processes, or ``container`` - each unit's agent or capability in a container, ``docker`` its earlier name - which a host must have Docker or Podman for (``docker`` in the result: the run checks it -before anything is sent). Each machine's constraints in Settings -(``nullius.constraints``) go on its command. +before anything is sent), or ``boat`` - each unit's agent or capability in a +boat.dev VM of its own (``boat.py``), started where the run runs, with this +machine's key. +Each machine's constraints in Settings (``nullius.constraints``) go on its +command. """ from __future__ import annotations @@ -40,7 +43,7 @@ from nullius import constraints as limits, hosts, providers from nullius.providers import PROVIDERS, STEP -from nullius.standard import hostreq, definitions, ledger, match, ports, prove, relay +from nullius.standard import boat, hostreq, definitions, ledger, match, ports, prove, relay CHOOSE_PROVIDER = "Choose a provider: OpenRouter, OpenAI, Anthropic or a custom provider." HARNESS = "standard" @@ -262,6 +265,8 @@ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]: def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, docker_unusable: Optional[Callable[[], Optional[str]]] = None, + boat_unusable: Optional[Callable[[], Optional[str]]] = None, + boat_size_problem: Optional[Callable[[str], Optional[str]]] = None, unpriced: Optional[Callable[[List[str]], List[str]]] = None) -> dict: """{errors, notes, review, jobs, confirm, script, parts, model} for one recipe; errors name the step @@ -274,7 +279,10 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, machine, why docker cannot run here (``container.unusable``) - a reason is the Where step's error. Not given, it is not asked: compiling stays a function of the definitions alone, and the routes that preview or start - a run pass it. + a run pass it. ``boat_unusable``: the same for the boat backend + (``boat.unusable``), asked with whether the run is on this machine: its + key, and there the programs that reach a VM (a host's are its prerequisites); + ``boat_size_problem``, why the account cannot start a VM of the size. ``max_usd``: the run's spending cap (``standard.ledger``), one amount for the whole run: its jobs that call a model - each dataset with each @@ -318,7 +326,7 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, # Several may be chosen: each is its own job of the one run, one after the other. backend = definitions.names.backend(recipe.get("backend") or "local") if backend not in BACKENDS: - add("where", "The backend is local processes or a container.") + add("where", "The backend is local processes, a container or Boat.") backend = "local" what = "an agent" if side == "agent" else f"a {_bench_page(b)['kind'] or 'capability'}" refs = chosen_of(recipe) @@ -498,6 +506,16 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, why = docker_unusable() if why: add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.") + elif backend == "boat" and boat_unusable is not None and (why := boat_unusable("local" in targets)): + add("where", f"Boat cannot run: {why}.") + elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES: + why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE)) + if why: + add("where", f"Boat cannot start a {recipe.get('boat_size') or boat.SIZE} VM on this account: {why}. Choose another size.") + boat_size = str(recipe.get("boat_size") or boat.SIZE) + if backend == "boat" and boat_size not in boat.SIZES: + add("where", f"Choose a Boat size: {', '.join(boat.SIZES)}.") + boat_size = boat.SIZE # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) @@ -600,7 +618,8 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--simulator-model", sim_model] if sim_model and not p["control"] else []), *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], - *(["--backend", "container"] if backend == "container" else []), + *(["--backend", backend] if backend != "local" else []), + *(["--boat-size", boat_size] if backend == "boat" and not control and not p.get("setup") else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch_of(d, p)] if of > 1 else []), *limits.cli_words(limits.of(targets[index - 1])), @@ -634,9 +653,11 @@ def script_for(d: dict, p: dict, command: str, index: int) -> str: out["script"] = "\n".join(pt["script"] for pt in out["shards"][0]["parts"]) if len(pairs) > 1 else out["shards"][0]["script"] out["parts"] = [pt for sh in out["shards"] for pt in sh["parts"]] out["jobs"] = [j for sh in out["shards"] for j in sh["jobs"]] - # docker where it runs: the container backend, or a control run in each unit's own image, whatever the backend + # docker where it runs: the container backend, a control run in each unit's own image, whatever the backend, + # or an agent's code checked on Boat (it comes back from the VM to the check container) own = any(definitions.own_image(definitions.environment_of(b, None, x)[1]) for x in dss) - out["docker"] = backend == "container" or (own and any(p["control"] for p in picks)) + checked = backend == "boat" and (b.get("score") or {}).get("runs_code") and any(not p["control"] and not p.get("setup") for p in picks) + out["docker"] = backend == "container" or (own and any(p["control"] for p in picks)) or bool(checked) out["model"] = model # Nothing in the run asks a model: no key is handed to it, and none need be set. out["keyless"] = only_controls and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"] @@ -699,7 +720,8 @@ def row(p: dict) -> list: + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []), *([[STEP["limits"], f"Spending cap per {noun}", f"${ledger.usd(per_unit)}"]] if per_unit else []), [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") - + ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]] + + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" + + (f" · {boat_size} VMs" if backend == "boat" else "")]] if agents_local: notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: " "the local backend is no sandbox.") @@ -707,9 +729,13 @@ def row(p: dict) -> list: notes.append(f"Each machine runs the units whose shard is its own, by their ids' hash: {of} jobs under one launch, " f"{name}, put together by cli merge.") docker_only = [n for n, e in (b.get("environments") or {}).items() if not e.get("local_ok")] - if docker_only and not only_controls and backend == "local": + if docker_only and not only_controls and backend in ("local", "boat"): notes.append(f"Units in {b['label']}'s environment{'s' if len(docker_only) > 1 else ''} {', '.join(docker_only)} " - "need the container backend: on local processes they are recorded as not run.") + f"need the container backend: on {'local processes' if backend == 'local' else 'Boat'} they are recorded as not run.") + web_only = [n for n, e in (b.get("environments") or {}).items() if e.get("network") and e.get("local_ok")] + if web_only and side == "agent" and not only_controls and backend == "boat": + notes.append(f"Units in {b['label']}'s environment{'s' if len(web_only) > 1 else ''} {', '.join(web_only)} " + "give the agent the public web, which a Boat VM does not yet: they are recorded as not run.") if any(p["control"] for p in picks) and any(definitions.own_image(e) for e in (b.get("environments") or {}).values()): notes.append(f"Each {(b.get('units') or {}).get('noun', 'unit')} of {b['label']} is its own image: the control " "runs in it, so docker is needed where it runs, whatever the backend.") @@ -759,7 +785,7 @@ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: errors.append("Choose a model or enter its ID.") backend = definitions.names.backend(backend) if backend not in BACKENDS: - errors.append("Choose local processes or a container for the checks.") + errors.append("Choose local processes, a container or Boat for the checks.") if kind == "agent" and backend == "local" and unsandboxed is not True: errors.append(f"{o['label']} runs on this machine as you while its kits run: the local backend is not a sandbox. Say yes to that to check it.") if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""): @@ -771,7 +797,7 @@ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: if errors: return {"errors": errors, "script": "", "jobs": [], "label": o["label"]} words = ["nullius", "standard", "conform", o["_ref"], "--model", model, "--provider", provider, - "--stamp", stamp, *(["--backend", "container"] if backend == "container" else []), + "--stamp", stamp, *(["--backend", backend] if backend != "local" else []), *(["--unsandboxed"] if kind == "agent" and backend == "local" else []), "--from-run"] command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"' script = ("#!/usr/bin/env bash\n" diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 20fbcc56..0b52ef2f 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -13,13 +13,16 @@ unit backs off on its own, as it always did. Its record sits in the trial's ``.work/relay``. -Two backends (``backend``). ``local``: processes on this machine, which are +Backends (``backend``). ``local``: processes on this machine, which are not a sandbox - what an agent runs, it runs as the user (``docs/standard.md``, "Safety"). ``docker``: the agent or the capability in a container of its own on an internal network with the unit's relay beside it (``container``); an own-image task may grant a public-address proxy with ``internet: true``, and an agent whose environment declares ``network`` gets a public web proxy. On docker the agent's or capability's packages are installed in the base image, on the libc and tools its container runs on. +A third, ``boat``: the agent's or capability's process alone in a Boat VM of +its own, its relay still here, reached through the VM's tunnel (``boat.Backend``); +a check or a setup on it runs on local processes. The driver, its environment installed here, stays on this machine either way: it owns the answers. So do a benchmark's own MCP servers (``mcp_servers``), in the driver's environment: on the local backend the @@ -71,7 +74,7 @@ from nullius import constraints as limits, paths, prices as price_list, progress, providers from nullius.redact import redact from nullius.harbor import feed as live_feed -from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards +from nullius.standard import acp, boat, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards # the totals a unit's end carries onto the live board, from its trial's agent_result FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd") @@ -392,13 +395,15 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None, max_usd_per_unit: Optional[float] = None, units_file: Optional[dict] = None, constraints: Optional[dict] = None, - served_by: Optional[Dict[str, List[str]]] = None): + served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for local agents, and for setup checks that run code without Docker. - ``backend``: ``local`` or ``container`` (``docker``, its earlier name), which needs - no such yes and is refused when docker is missing or not usable. + ``backend``: ``local``, ``container`` (``docker``, its earlier name), which needs + no such yes and is refused when docker is missing or not usable, or ``boat``; + ``boat_size``: its VMs' machine type (``boat.SIZES``), refused when the + account cannot start it. ``options``: the agent's or capability's options set for this run, by name; the rest keep their defaults, and the job's config records every value. @@ -447,6 +452,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas backend = definitions.names.backend(backend) if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") + if backend == "boat" and other.get("_kind") not in ("agent", "capability"): + backend = "local" # a check or a setup: nothing of an agent's runs, so nothing goes to a VM + if backend == "boat": + why = boat.unusable(env=env, which=which) + if why: + raise Refused(f"the Boat backend cannot run here: {why}") if backend == "local" and other.get("_kind") == "agent" and not unsandboxed: raise Refused(f"{other['label']} would run commands as you: the local backend is not a sandbox. " "Say so explicitly to go on (--unsandboxed), or run it in a container (--backend container).") @@ -473,6 +484,13 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas raise Refused(f"{bench['label']}'s check executes generated code, but Docker is unavailable: {why}. " "Make Docker usable, or explicitly allow execution as you (--unsandboxed).") self.container_check = not why + if backend == "boat" and (bench.get("score") or {}).get("runs_code"): + # the agent's code comes back from its VM to be checked here: in a throwaway container, never as you + why = container.unusable() + if why: + raise Refused(f"{bench['label']}'s check executes the agent's code on this machine, in a container, " + f"but Docker is unavailable: {why}. Make Docker usable here, or run it with --backend container.") + self.container_check = True self.bench, self.other, self.model, self.provider, self.backend = bench, other, model, provider, backend # A setup (definitions.setup): the upstream's own answerer, asked as one model (``asked``), the relay # sending another (``sent``); its units may be its own (a conversation where the paper batches questions). @@ -610,6 +628,15 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # each stage said as it goes, on the cli's stderr and in the job's progress.jsonl (nullius/progress.py) self.progress = progress.Reporter(self.job_dir) self._said = threading.Lock() + self.boat = boat.Backend(self.name, env, size=boat_size, say=say) if backend == "boat" else None + if self.boat is not None: + why = boat.size_problem(self.boat.client, boat_size) + if why: + raise Refused(f"Boat cannot start a {boat_size} VM: {why}") + most = self.boat.most() + if most is not None and self.concurrency > most: + say(f"Boat runs {most} VMs at once on this account: {most} units at a time, not {self.concurrency}") + self.concurrency = most # the job page's live board (nullius/runs/live.py): written once the units are known self.feed: Optional[live_feed.Writer] = None self._ended = 0 # units ended so far: the i of the next unit's line @@ -731,10 +758,15 @@ def start(self) -> str: mine = mine[: self.limit] if self.limit else mine units = [u for _, u in mine] self.n_units = len(units) + if getattr(self, "boat", None) is not None: + why = self.boat.starts(len(units) * self.repeat) + if why: + self.say(why) p.emit("units", "start", n=len(units), of=len(everything), noun=self.bench["units"]["nouns"]) job.write_config(self.job_dir, job=self.name, bench=self.bench, other=self.other, model=self.model, provider=self.provider, dataset=self.ds["id"], units=units, concurrency=self.concurrency, granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend, + boat_size=self.boat.size if getattr(self, "boat", None) is not None else "", image=self.setup_driver.image if self.setup_driver else self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, # a shard of a list splits the listed units: they are its whole @@ -813,6 +845,8 @@ def work_unit(iur): self.bench_relay.stop() if self.backend == "container": container.sweep(self.name) # whatever an interrupted unit left + if getattr(self, "boat", None) is not None: + self.boat.close() if self.loopback is not None: self.loopback.stop() budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]]) @@ -1045,6 +1079,8 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: own = definitions.own_image(env_def) if "internet" in setup and (not isinstance(setup["internet"], bool) or not own): raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image") + if self.other["_kind"] == "agent" and self.backend == "boat" and (mcp_machine.wanted(setup) or setup.get("mcp")): + raise _Ended("EnvironmentSetupError", "not run: " + boat.MCP_REFUSAL) if self.other["_kind"] == "agent" and self.backend == "local" and mcp_machine.wanted(setup): try: # the benchmark's own servers: the agent starts them from the driver's environment setup = dict(setup, mcp=mcp_machine.entries(self.bench, setup, install.driver_dir(self.bench), @@ -1068,23 +1104,24 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: box = self.own_box(tag, setup, rl, home) if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) - p0 = time.time() - self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" - else DRIVER_SECONDS if self.other["key"] == "oracle" else None) - try: - with self.agent_internet(setup, box, rl, trial): - if self.other["_kind"] == "control": - answer = self.control_answer(unit, ws, box, name) - elif self.upstream: - answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) - elif self.other["_kind"] == "agent": - answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) - else: - answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) - finally: - produce = (p0, time.time()) - if box is not None: # what the agent (or the control) left in the container, before the check - changes = self.box_changes(box) + with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts + p0 = time.time() + self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" + else DRIVER_SECONDS if self.other["key"] == "oracle" else None) + try: + with self.agent_internet(setup, box, rl, trial): + if self.other["_kind"] == "control": + answer = self.control_answer(unit, ws, box, name) + elif self.upstream: + answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) + elif self.other["_kind"] == "agent": + answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) + else: + answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + finally: + produce = (p0, time.time()) + if box is not None: # what the agent (or the control) left in the container, before the check + changes = self.box_changes(box) cut = self.cap_reached(rl, tag) if cut: # cut off at a spending cap: nothing of it is checked, and its judge is asked nothing raise _Ended(cut[0], cut[1], traj) @@ -1271,9 +1308,13 @@ def not_run(self, name: Optional[str], env_def: Optional[dict]) -> Optional[tupl refused the same way, for what would run in it (not a control).""" if name is None: return None - if self.backend == "local" and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"): + if self.backend in ("local", "boat") and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"): return ("EnvironmentSetupError", f"not run: the environment {name} needs the container backend " "(--backend container); it does not say local_ok") + if self.backend == "boat": + why = boat.refusal(env_def) + if why: + return ("EnvironmentSetupError", f"not run: the environment {name} {why}") missing = definitions.missing_secrets(env_def, self.licences) if missing: return ("EnvironmentSecretMissingError", "not run: needs " + ", ".join(missing)) @@ -1376,7 +1417,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] record = os.path.join(trial, ".work", "relay") side = tuple((self.other.get("model") or {}).get("side_models") or ()) key_header = self.key_header - if self.backend == "local": + if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None)).start() rl.unit_url = rl.url(tag) @@ -1506,6 +1547,11 @@ def place(self, rl, cmd: List[str], env: dict, workdir: str, home: str, interact if self.backend == "local": return limits.wrap(cmd, env, getattr(self, "scope_limits", {})) ws = os.path.join(os.path.dirname(home), "workspace") + if self.backend == "boat": + try: + return self.boat.place(cmd, env, workdir, inst=self.inst, owned=[ws, home], relay_url=rl.unit_url if rl is not None else None, local=KEEP) + except boat.Unavailable as e: + raise _Ended("EnvironmentSetupError", str(e)) # What the agent's container sees: its workspace and home, its own # install and definition, and the files its MCP servers need - never # this application's folder (its .env holds the keys) nor the @@ -1546,6 +1592,21 @@ def control_answer(self, unit, ws, box=None, name: str = "") -> str: return str(reply.get("answer") or "") # -- an own image: the unit's container ------------------------------------ + @contextmanager + def unit_vm(self, ws: str, home: str, trial: str, setup: dict): + """On the boat backend, the unit's VM while its agent or capability runs (``boat.Backend.unit``), + living as long as its every turn may take.""" + if getattr(self, "boat", None) is None: + yield + return + turns = self.max_turns if "agent.conversation/v1" in self.bench["needs"] else 1 + try: + with self.boat.unit(trial, ws, home, stopped=self._stopped, inst=self.inst, here=self.other.get("_here"), + seconds=self.turn_limit(setup) * turns): + yield + except boat.Unavailable as e: + raise _Ended("EnvironmentSetupError", str(e)) + @contextmanager def agent_internet(self, setup: dict, box, rl, trial: str): """The benchmark's task policy, never a run option: a public-address proxy diff --git a/app/static/brand/README.md b/app/static/brand/README.md index 38c01b5d..d32c31c7 100644 --- a/app/static/brand/README.md +++ b/app/static/brand/README.md @@ -18,3 +18,5 @@ third-party brand asset, not a Nullius mark or part of Nullius's font licensing. `docker.svg` is the Docker mark from [Simple Icons](https://github.com/simple-icons/simple-icons/blob/develop/icons/docker.svg), distributed under CC0, rendered in Docker blue. The Docker trademark belongs to Docker, Inc. It identifies Docker Hub under Data providers. `huggingface.svg` is Hugging Face’s official mark, copied from https://huggingface.co/front/assets/huggingface_logo-noborder.svg. It identifies the data provider; its trademark belongs to Hugging Face. + +`boat.svg` is Boat's official mark, copied unchanged from https://boat.dev/icon.svg, and `boat-dark.svg` its version for dark backgrounds, from https://boat.dev/icon-dark.svg. They identify the Boat sandbox provider in Settings and the builder's Where step; its trademark belongs to Boat. diff --git a/app/static/brand/boat-dark.svg b/app/static/brand/boat-dark.svg new file mode 100644 index 00000000..16c3dcd4 --- /dev/null +++ b/app/static/brand/boat-dark.svg @@ -0,0 +1,3 @@ + diff --git a/app/static/brand/boat.svg b/app/static/brand/boat.svg new file mode 100644 index 00000000..815ef042 --- /dev/null +++ b/app/static/brand/boat.svg @@ -0,0 +1,3 @@ + diff --git a/app/static/builder/command-builder-core.js b/app/static/builder/command-builder-core.js index 1d53e123..a58e76e4 100644 --- a/app/static/builder/command-builder-core.js +++ b/app/static/builder/command-builder-core.js @@ -172,7 +172,8 @@ const picked = Array.isArray(raw.others) ? raw.others.filter(k => typeof k === 'string') : typeof raw.other === 'string' && raw.other ? [raw.other] : null; if (picked) r.others = [...new Set(picked)]; if (Array.isArray(raw.where) && raw.where.length && raw.where.every(w => typeof w === 'string')) r.where = raw.where.length > 1 ? [...raw.where] : raw.where[0]; - if (['local', 'container', 'docker'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name + if (['local', 'container', 'docker', 'boat'].includes(raw.backend)) r.backend = raw.backend === 'docker' ? 'container' : raw.backend; // docker: its earlier name + if (['small', 'default', 'large', 'xlarge'].includes(raw.boat_size)) r.boat_size = raw.boat_size; if (raw.bench_options && typeof raw.bench_options === 'object' && !Array.isArray(raw.bench_options)) // the benchmark's own options, a word each r.bench_options = ownOptions(standardBench(r), Object.fromEntries(Object.entries(raw.bench_options).filter(([, v]) => typeof v === 'string'))); // Options per choice, {ref: {name: word}}; a single agent also accepts flat {name: word} options. diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index a64956ce..f3e37326 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -240,12 +240,17 @@ /* Compact settings rows use one shared control column, including paired fields. */ #nullius-builder .nb-two:has(> .ui-field-row) { grid-template-columns: minmax(0, 1fr); gap: 0; } -#nullius-builder .std-backends { grid-template-columns: repeat(2, minmax(0, 240px)); gap: 8px; } -#nullius-builder .std-backends .nb-tile { flex-direction: row; align-items: center; gap: 8px; padding: 10px 12px; } -#nullius-builder .std-backends .nb-tile-head { flex: 1; } +#nullius-builder .std-backends { grid-template-columns: repeat(3, minmax(0, 240px)); gap: 8px; } +@media (max-width: 860px) { #nullius-builder .std-backends { grid-template-columns: repeat(2, minmax(0, 240px)); } } +/* the name keeps the tile's width; why one cannot be used goes under it, so a row's tiles stay one height */ +#nullius-builder .std-backends .nb-tile { flex-direction: column; align-items: stretch; justify-content: center; gap: 0; padding: 10px 12px; } #nullius-builder .std-backends .nb-mark { width: 20px; height: 20px; } -#nullius-builder .std-backends .nb-tile-note { font-size: 11px; white-space: nowrap; } -@media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } } +#nullius-builder .std-backends .nb-tile-note { font-size: 11px; padding-left: 30px; } +#nullius-builder .std-boat-sizes { grid-template-columns: repeat(4, minmax(0, 160px)); grid-auto-rows: auto; } +#nullius-builder .std-boat-sizes .nb-tile { flex-direction: column; align-items: stretch; gap: 2px; } +#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; } +@media (max-width: 760px) { #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 180px)); } } +@media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 1fr)); } } /* Served by: a row after its model's row, as that row is: the label left, the picker right, the providers chosen under the picker in their order */ #nullius-builder .std-served-list { grid-column: 2; min-width: 0; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index a25326b6..1574913e 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -427,9 +427,11 @@ // Review's rule (nullius/standard/recipe.py): a setup's image installs what has an install; nothing else reaches its container const cannot = selected.some(o => o.setup) ? optionNeeds(rec).host.filter(n => (b.host || {})[n] && !b.host[n].install.length) : []; const containerWhy = dockerWhy || (cannot.length ? cannot.join(', ') + ' cannot be provided in a container: choose local processes.' : ''); + // boat: started where the run runs, with this machine's key (Settings → Sandbox); a host's programs are its prerequisites + const boatWhy = (list.includes('local') ? here.boat : here.boatKey) || ''; const agentRun = b && b.family === 'agent' && selected.some(o => !o.control && !o.setup); const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : '▣', title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? '$' : v === 'boat' ? '' : '▣', title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy return '
${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on local processes their units are recorded as not run.
` : '') + + (dockerOnly.length && backend !== 'container' && agentRun ? `${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.
` : '') + hostSetup(rec); } + const BOAT_SIZES = [['small', 'Small'], ['default', 'Default'], ['large', 'Large'], ['xlarge', 'XLarge']]; + function boatSizes(rec) { + const here = schema.standardHere || {}, why = here.boatSizes || {}, spec = here.boatSpecs || {}, size = rec.boat_size || 'small'; + const card = ([v, t]) => tile({attr: 'std-boat-size', key: v, on: size === v && !why[v], disabled: !!why[v], + note: why[v] ? 'Not on this account' : spec[v] || '', title: t}); + return 'Where agents run, apart from this machine.
Checking…