From 7e1a5d5d3c8175e33fb7086b9eb59562eaa929c2 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:06:51 +0100 Subject: [PATCH 01/36] feat(standard): add opt-in driver lifecycle hooks Pass participant kinds and agent setup limits to drivers that request them. Notify opted-in drivers once each agent unit ends, including failed and stopped units. Apply agent-only benchmark options in runs, recipes and the builder. Keep leak checks on the control setup contract and document the hooks. --- app/nullius/standard/definitions.py | 20 +- app/nullius/standard/jsonl.py | 7 +- app/nullius/standard/prove.py | 3 +- app/nullius/standard/recipe.py | 11 +- app/nullius/standard/run.py | 58 ++++- app/static/builder/command-builder.js | 3 +- app/tests/test_standard_driver_hooks.py | 328 ++++++++++++++++++++++++ docs/standard.md | 8 +- 8 files changed, 417 insertions(+), 21 deletions(-) create mode 100644 app/tests/test_standard_driver_hooks.py diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 125c2b2f..4b702922 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -484,8 +484,9 @@ def _bench_options(d: dict) -> List[str]: if ds is not None and (not isinstance(ds, list) or not ds or any(x not in ids for x in ds)): bad.append(f"{w}.datasets: some of the benchmark's dataset ids") on = o.get("setups") - if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours",) for x in on)): - bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)") + if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours", "agent") for x in on)): + bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability), " + "agent (an agent alone)") when = o.get("when") or {} if not _when_valid(name, when, opts): bad.append(f"{w}.when: another option -> the values of it this one applies under") @@ -570,14 +571,17 @@ def _host(d: dict) -> List[str]: return bad -def bench_options_of(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours") -> Tuple[dict, List[str]]: +def bench_options_of(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours", + kind: Optional[str] = None) -> Tuple[dict, List[str]]: """The benchmark's options for one run on ``dataset`` in ``setup`` (original, model, or yours: an agent or a capability in its port): each that applies to both, at the value ``given`` or its default; and what is wrong with the given ones. An option given for a dataset or a setup it does not apply to is ignored - a run of several carries one choice for all. An option applies to an agent's or a - capability's run unless its ``setups`` say otherwise.""" + capability's run unless its ``setups`` say otherwise; one whose ``setups`` name ``agent`` applies to a run + whose ``kind`` is an agent, never to a capability's or a control's.""" opts = {n: o for n, o in (bench.get("options") or {}).items() - if dataset in (o.get("datasets") or [dataset]) and setup in (o.get("setups") or ["yours"])} + if dataset in (o.get("datasets") or [dataset]) + and (setup in (o.get("setups") or ["yours"]) or (kind == "agent" and setup == "yours" and "agent" in (o.get("setups") or [])))} given = {n: v for n, v in (given or {}).items() if n in opts or n not in (bench.get("options") or {})} vals, bad = options_of(dict(options=opts, label=bench.get("label"), key=bench.get("key")), given) return vals, bad @@ -1278,6 +1282,10 @@ def problems(d: dict, kind: str) -> List[str]: bad += _install(dict(driver_install(d), install=drv["install"]), "driver.install") elif any("{install}" in w for w in drv.get("command") or [] if isinstance(w, str)): bad.append("driver.command: {install} names driver.install, which is not set") + if "participant" in drv and not isinstance(drv["participant"], bool): + bad.append("driver.participant: true or false - whether setup is told the participant's kind") + if "unit_end" in drv and not isinstance(drv["unit_end"], bool): + bad.append("driver.unit_end: true or false - whether the driver is told each unit's end") bad += _mcp_servers(d.get("mcp_servers"), isinstance(drv, dict) and "install" in drv) ds = d.get("datasets") if not isinstance(ds, list) or not ds: @@ -1313,7 +1321,7 @@ def problems(d: dict, kind: str) -> List[str]: if not isinstance(lim, dict): bad.append("limits: an object") else: - for f in ("turn_seconds", "turns"): + for f in ("turn_seconds", "turns", "setup_seconds"): v = lim.get(f) if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000): bad.append(f"limits.{f}: a whole number from 1") diff --git a/app/nullius/standard/jsonl.py b/app/nullius/standard/jsonl.py index 98bf61fd..e4c01fd9 100644 --- a/app/nullius/standard/jsonl.py +++ b/app/nullius/standard/jsonl.py @@ -46,8 +46,9 @@ def _read(self): self.lines.put(line) self.lines.put(None) - def request(self, op: str, timeout: float = 600, **kw) -> dict: - if self.stopped is not None and self.stopped.is_set(): + def request(self, op: str, timeout: float = 600, force: bool = False, **kw) -> dict: + """``force``: sent and waited for (up to ``timeout``) also once the run is stopped - a unit's clean-up.""" + if self.stopped is not None and self.stopped.is_set() and not force: raise CancelledError("the run was stopped") if self.p is None: self.start() @@ -57,7 +58,7 @@ def request(self, op: str, timeout: float = 600, **kw) -> dict: except (BrokenPipeError, OSError): raise Failed(f"{op}: the program had already ended ({self._tail()})") try: - line = wait_reply(self.lines, timeout, self.stopped) + line = wait_reply(self.lines, timeout, None if force else self.stopped) except queue.Empty: self.close() raise Failed(f"{op}: no reply in {int(timeout)} s") diff --git a/app/nullius/standard/prove.py b/app/nullius/standard/prove.py index 59ad318e..06316e87 100644 --- a/app/nullius/standard/prove.py +++ b/app/nullius/standard/prove.py @@ -148,7 +148,8 @@ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = a, b = os.path.join(driver.work, "leak", f"{i}", "a"), os.path.join(driver.work, "leak", f"{i}", "b") os.makedirs(a) try: - s = p.request("setup", unit=unit, dir=a, timeout=600) + sent, seconds = run.setup_request(driver.bench, "control") # the oracle's unit, as a run sends it + s = p.request("setup", unit=unit, dir=a, timeout=seconds, **sent) before = _files(a) shutil.copytree(a, b) o = p.request("oracle", unit=unit, dir=b, timeout=600) diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 9e397d1c..9da17363 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -371,13 +371,15 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, add("dataset", f"Like the leaderboard sets {(b.get('options') or {})[k].get('label') or k.replace('_', ' ')} to {v}: choose Any of your choice for {chosen_opts[k]}.") chosen_opts = {**(lb.get("bench_options") or {}), **chosen_opts} per_ds, needs = {}, {"embeddings": [], "reranker": [], "host": []} - scopes = list(dict.fromkeys(p["o"]["key"] if p.get("setup") else "yours" for p in picks)) or ["yours"] + # each participant's scope and kind: an option an agent's run alone takes (setups: agent) is an agent's + scopes = list(dict.fromkeys((p["o"]["key"], None) if p.get("setup") else ("yours", p["o"].get("_kind")) + for p in picks)) or [("yours", None)] for x in dss: # Original's retrievers have machine requirements too; collect the # options of every participant that the compiled commands will run. vals, bad = {}, [] - for scope in scopes: - chosen, problems = definitions.bench_options_of(b, x["id"], chosen_opts, scope) + for scope, kind in scopes: + chosen, problems = definitions.bench_options_of(b, x["id"], chosen_opts, scope, kind=kind) vals.update(chosen) bad.extend(problems) for why_not in dict.fromkeys(bad): @@ -517,7 +519,8 @@ def job_of(d: dict, p: dict, index: int) -> str: def options_for_pick(d: dict, p: dict) -> dict: """The benchmark's options for this dataset and this pick: a setup's own, or those of the port.""" - return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours")[0] + return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours", + kind=None if p.get("setup") else p["o"].get("_kind"))[0] def command_for(d: dict, p: dict, index: int) -> str: o, control, given = p["o"], p["control"], p["given"] diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 40251328..31d758f4 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -105,8 +105,9 @@ def driver_path() -> str: "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"}, } TURN_SECONDS = 900 -DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each +DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each; limits.setup_seconds for setup IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image) +END_SECONDS = 120 # a driver's answer to a unit's end (driver.unit_end) MAX_AGENT_SECONDS = 6 * 3600 # the longest a unit's own agent limit (setup's agent_seconds) may ask MAX_TURNS = 20 BACKENDS = definitions.names.BACKENDS @@ -117,6 +118,19 @@ class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" +def setup_request(bench: dict, kind: str, attempt: Optional[int] = None) -> tuple: + """What a unit's ``setup`` is sent besides the unit and its folder, and how long it may take, for a unit that + ``kind`` runs (agent, capability, control): ``attempt`` when the benchmark tells attempts apart + (``limits.attempts``), ``participant`` when its driver asks (``driver.participant``); the benchmark's + ``limits.setup_seconds`` for an agent's unit, + ``DRIVER_SECONDS`` for any other. A run and a proof's checks build the request here, the one way.""" + limits, drv = bench.get("limits") or {}, bench.get("driver") or {} + sent = {"attempt": attempt} if attempt and limits.get("attempts") else {} + if drv.get("participant"): + sent["participant"] = kind + return sent, float((limits.get("setup_seconds") if kind == "agent" else None) or DRIVER_SECONDS) + + def provider_endpoint(provider: str, env=os.environ) -> tuple: p = PROVIDERS.get(provider) if not p: @@ -483,7 +497,8 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # calls through a relay of the run's own (bench_relay), programs on this machine. # each option where it applies: a setup's own (a retriever, a top-k), or those of the benchmark's port self.bench_options, bad = definitions.bench_options_of(bench, self.ds["id"], bench_options, - other["key"] if self.upstream else "yours") + other["key"] if self.upstream else "yours", + kind=other.get("_kind")) if bad: raise Refused("; ".join(bad)) needs = definitions.option_needs(bench, self.bench_options) @@ -946,6 +961,29 @@ def attempt_kw(self) -> dict: a = getattr(self.local, "attempt", None) return {"attempt": a} if a and (self.bench.get("limits") or {}).get("attempts") else {} + def setup_kw(self) -> dict: + """What ``setup`` is told besides the unit (:func:`setup_request`).""" + return setup_request(self.bench, self.other["_kind"], getattr(self.local, "attempt", None))[0] + + def setup_seconds(self) -> float: + """How long a unit's ``setup`` may take (:func:`setup_request`).""" + return setup_request(self.bench, self.other["_kind"])[1] + + def unit_ended(self, unit: str, ws: str) -> None: + """A driver that asks for it (``driver.unit_end``) is told that a unit an agent ran has ended, scored or not - + also when the run was stopped - so that what its setup started for the agent ends now. The driver the unit + used is asked, never a new one; its answer is waited for ``END_SECONDS`` at most, and a failure is no + unit's.""" + if self.other.get("_kind") != "agent" or not (self.bench.get("driver") or {}).get("unit_end"): + return + d = getattr(self.local, "driver", None) + if d is None or d.p is None or d.p.poll() is not None: + return + try: + d.request("end", unit=unit, dir=ws, timeout=END_SECONDS, force=True) + except Exception: # noqa: BLE001 - the unit's result is written; its clean-up is not its result + pass + def trial_name(self, i: int, unit: str, r: int = 1) -> str: return f"{_slug(unit)}__u{i:04d}" + (f"r{r}" if self.repeat > 1 else "") @@ -954,6 +992,17 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout) def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: + """One unit (:meth:`_unit`); once its setup was asked, whatever happened after, the driver is told its end + (:meth:`unit_ended`).""" + self.local.set_up = None + try: + return self._unit(i, unit, r) + finally: + asked, self.local.set_up = self.local.set_up, None + if asked is not None: + self.unit_ended(*asked) + + def _unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()): return None # the endpoint refused the run: nothing more is begun if getattr(self, "ledger", None) is not None and self.ledger.closed(): @@ -972,7 +1021,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: os.makedirs(home, exist_ok=True) t0 = time.time() self.progress.emit("unit", "start", unit=unit) - self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS) + self.phase(name, unit, "env_setup", None if self.upstream else self.setup_seconds()) produce, score, exc, traj, logs = (None, None), None, None, None, {} p0, checking = None, (None, None) # when the agent began; the check's start and end setup: dict = {} @@ -981,7 +1030,8 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: try: try: # a setup's unit needs none: the upstream's answerer reads its data itself - setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=DRIVER_SECONDS, **self.attempt_kw()) + self.local.set_up = None if self.upstream else (unit, ws) + setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.setup_seconds(), **self.setup_kw()) except jsonl.Failed as e: raise _Ended("EnvironmentSetupError", str(e)) environment, env_def = self.unit_environment(setup) diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index f4eb338f..b1aa2b3f 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -454,7 +454,8 @@ const shared = on.includes('yours') && on.includes('original'); if (scope === 'shared') return shared; if (shared) return false; - return scope === 'setup' ? !on.includes('yours') && on.some(k => chosen.includes(k)) : on.includes('yours'); + // 'agent': an agent's run alone takes it, shown with the agents + return scope === 'setup' ? !on.includes('yours') && !on.includes('agent') && on.some(k => chosen.includes(k)) : on.includes('yours') || on.includes('agent'); }) .map(([n, o]) => { const cur = (rec.bench_options || {})[n] ?? o.default; diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py new file mode 100644 index 00000000..4398a312 --- /dev/null +++ b/app/tests/test_standard_driver_hooks.py @@ -0,0 +1,328 @@ +"""Opt-in driver setup and unit cleanup, with agent-only options and setup bounds.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import sys +import threading +import unittest + +from unittest import mock + +from nullius.standard import definitions, install, jsonl, prove, recipe, run + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard", "catalogue") +BENCH = { + "schema": "nullius.benchmark/v1", "key": "fixture", "label": "Fixture", "summary": "A task.", + "family": "agent", "needs": ["agent.task/v1"], + "driver": {"command": ["python3", "{here}/driver.py"]}, + "datasets": [{"id": "all", "units": 1}], + "units": {"noun": "task", "nouns": "tasks"}, + "score": {"kind": "pass", "says": "the answer is right"}, +} +# A driver that records what setup was told, in the unit's trial folder. +DRIVER = '''import json, os +from nullius.standard.jsonl import serve +def setup(unit, dir, **told): + with open(os.path.join(os.path.dirname(dir), "told.json"), "w") as f: + json.dump(told, f) + return {"prompt": "hi"} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, + "score": lambda unit, dir, transcript, answer: {"reward": 1.0, "passed": True, "detail": {}}}) +''' +# The same, recording each unit's end too. +ENDING_DRIVER = DRIVER.replace('serve({"units"', '''def end(unit, dir): + with open(os.path.join(os.path.dirname(dir), "ended.json"), "a") as f: + f.write(json.dumps({"unit": unit, "dir": dir}) + "\\n") + return {} +serve({"end": end, "units"''') +# Its setup failing: the unit ends before any agent or score. +FAILING_DRIVER = ENDING_DRIVER.replace(' return {"prompt": "hi"}', ' raise ValueError("no task")') +# The same with setup as most drivers write it: the unit and its folder, nothing else. +PLAIN_DRIVER = DRIVER.replace("def setup(unit, dir, **told):", "def setup(unit, dir):\n told = {}") + + +class SetupFields(unittest.TestCase): + def test_participant_and_setup_seconds_are_checked(self): + self.assertEqual(definitions.problems(dict(BENCH, driver=dict(BENCH["driver"], participant=True), + limits={"setup_seconds": 1800}), "benchmark"), []) + self.assertIn("driver.participant: true or false - whether setup is told the participant's kind", + definitions.problems(dict(BENCH, driver=dict(BENCH["driver"], participant="yes")), "benchmark")) + self.assertIn("limits.setup_seconds: a whole number from 1", + definitions.problems(dict(BENCH, limits={"setup_seconds": 0}), "benchmark")) + + def test_setup_seconds_rejects_noninteger_and_out_of_range_values(self): + for value in (True, -1, 1.5, "600", 100001): + with self.subTest(value=value): + self.assertIn("limits.setup_seconds: a whole number from 1", + definitions.problems(dict(BENCH, limits={"setup_seconds": value}), "benchmark")) + + def test_setup_preserves_attempts_and_sends_only_requested_fields(self): + for kind in ("agent", "capability", "control"): + with self.subTest(kind=kind): + self.assertEqual(run.setup_request(BENCH, kind, 2), ({}, 600.0)) + bench = dict(BENCH, driver=dict(BENCH["driver"], participant=True), + limits={"attempts": "A seed per attempt.", "setup_seconds": 1800}) + self.assertEqual(run.setup_request(bench, kind, 2), + ({"participant": kind, "attempt": 2}, 1800.0 if kind == "agent" else 600.0)) + bench["driver"]["participant"] = False + self.assertEqual(run.setup_request(bench, kind, 2)[0], {"attempt": 2}) + + def test_the_leak_check_sets_up_a_control_with_the_original_bound(self): + bench = dict(BENCH, driver=dict(BENCH["driver"], participant=True), limits={"setup_seconds": 1800}) + with tempfile.TemporaryDirectory() as tmp: + proc = mock.Mock() + proc.request.return_value = {} + driver = mock.Mock(work=tmp, bench=bench) + driver.start.return_value = proc + prove.leak_check(driver, ["a"], "agent") + op, kw = proc.request.call_args_list[0].args[0], proc.request.call_args_list[0].kwargs + self.assertEqual((op, kw["timeout"], kw["participant"]), ("setup", 600.0, "control")) + proc.close.assert_called_once() + + +class AgentOptions(unittest.TestCase): + def setUp(self): + self.option = {"label": "Steps", "summary": "The number of steps.", "values": ["10", "20"], + "default": "10", "setups": ["agent"]} + self.bench = dict(BENCH, options={"steps": self.option}) + + def test_agent_scope_is_validated(self): + self.assertEqual(definitions.problems(self.bench, "benchmark"), []) + for setups in ([], "agent", ["capability"]): + with self.subTest(setups=setups): + self.assertTrue(any("options.steps.setups" in p for p in definitions.problems( + dict(BENCH, options={"steps": dict(self.option, setups=setups)}), "benchmark"))) + + def test_only_an_agent_gets_and_validates_the_option(self): + for kind in ("agent", "capability", "control", "upstream", None): + for scope in ("yours", "original", "model"): + with self.subTest(kind=kind, scope=scope): + active = kind == "agent" and scope == "yours" + vals, bad = definitions.bench_options_of(self.bench, "all", {"steps": "20"}, scope, kind=kind) + self.assertEqual((vals, bad), ({"steps": "20"} if active else {}, [])) + self.assertEqual(bool(definitions.bench_options_of( + self.bench, "all", {"steps": "bad"}, scope, kind=kind)[1]), active) + + def test_a_recipe_carries_the_option_only_to_its_agent(self): + defs = {k: definitions.load(k, ROOT)[0] for k in ("benchmark", "agent", "capability")} + bench = definitions.get("benchmark", "fakebench", ROOT) + bench["options"] = self.bench["options"] + bench["controls"] = {"nop": {"source": "nullius", "from": "nothing"}} + defs["benchmark"] = [bench] + rec = {"version": 1, "harness": "standard", "name": "fixture", "benchmark": "fakebench", + "dataset": "all", "others": ["fake", "nop"], "provider": "openai", "model": "m", + "limit": 1, "concurrency": 1, "unsandboxed": True, "where": "local", "bench_options": {"steps": "20"}} + out = recipe.compile(rec, defs) + self.assertEqual(out["errors"], []) + self.assertIn("--bench-option steps=20", out["parts"][0]["script"]) + self.assertNotIn("--bench-option steps", out["parts"][1]["script"]) + rec["bench_options"]["steps"] = "bad" + self.assertTrue(recipe.compile(rec, defs)["errors"]) + rec["others"] = ["nop"] + self.assertEqual(recipe.compile(rec, defs)["errors"], []) + + @unittest.skipUnless(shutil.which("node"), "Node is required for builder options") + def test_the_builder_places_the_option_with_selected_agents(self): + source = (Path(__file__).resolve().parents[1] / "static/builder/command-builder.js").read_text() + function = source[source.index(" function benchOptions("):source.index("\n function ", source.index(" function benchOptions(") + 1)] + program = """const C = {chosenOf: r => r.others, dsIds: () => ['all']}; +const schema = {standardHere: {}}, esc = x => x, tile = x => x.title; +""" + function + """ +const b = {options: {steps: {label: 'Steps', summary: 'Steps', values: ['10'], default: '10', datasets: [], setups: ['agent']}}}; +const r = {others: ['fake', 'original']}; +process.stdout.write(JSON.stringify(['yours', 'setup', 'shared'].map(s => benchOptions(r, b, s)))); +""" + result = subprocess.run(["node", "-e", program], stdin=subprocess.DEVNULL, capture_output=True, text=True, check=True) + yours, setup, shared = json.loads(result.stdout) + self.assertIn("Steps", yours) + self.assertEqual((setup, shared), ("", "")) + + + +class ThroughARun(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + + self.env = {"MODEL_API_URL": "http://127.0.0.1:9/v1"} + for name, value in (("unit_relay", None), ("drive_agent", ("answer", None, {}))): + patch = mock.patch.object(run.Run, name, return_value=value) + patch.start() + self.addCleanup(patch.stop) + + + def tearDown(self): + install.HOME = self._home + shutil.rmtree(self.tmp, ignore_errors=True) + + + def go(self, code: str, other: dict, **bench) -> tuple: + here = os.path.join(self.tmp, "bench") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(code) + b = dict(BENCH, _here=here, _kind="benchmark", **bench) + job = run.run(b, other, model="m-1", provider="custom", jobs_root=self.tmp, concurrency=1, env=self.env, + say=lambda s: None, unsandboxed=True) + trial = next(os.path.join(job, d) for d in os.listdir(job) if "__u" in d) + with open(os.path.join(job, "config.json")) as f: + config = json.load(f) + with open(os.path.join(trial, "told.json")) as f: + told = json.load(f) + with open(os.path.join(trial, "result.json")) as f: + result = json.load(f) + return config, told, result + + + def fake_agent(self) -> dict: + return definitions.get("agent", "fake", ROOT) + + + def test_setup_is_told_the_participant_when_the_driver_asks(self): + _, told, _ = self.go(DRIVER, self.fake_agent(), driver=dict(BENCH["driver"], participant=True)) + self.assertEqual(told, {"participant": "agent"}) + control = definitions.control(dict(BENCH, controls={"nop": {"source": "nullius", "from": "nothing"}}), "nop") + _, told, _ = self.go(DRIVER, control, driver=dict(BENCH["driver"], participant=True), + controls={"nop": {"source": "nullius", "from": "nothing"}}) + self.assertEqual(told, {"participant": "control"}) + + + def test_a_driver_whose_setup_takes_nothing_more_runs_as_before(self): + config, told, result = self.go(PLAIN_DRIVER, self.fake_agent()) + self.assertEqual(told, {}) + self.assertIsNone(result.get("exception_info")) + + + def trial_of(self, code: str, other: dict = None, **bench) -> str: + here = os.path.join(self.tmp, "bench") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(code) + jobs = tempfile.mkdtemp(dir=self.tmp) + job = run.run(dict(BENCH, _here=here, _kind="benchmark", **bench), other or self.fake_agent(), model="m-1", + provider="custom", jobs_root=jobs, concurrency=1, env=self.env, say=lambda s: None, + unsandboxed=True) + return next(os.path.join(job, d) for d in os.listdir(job) if "__u" in d) + + + def test_the_null_controls_unit_gets_no_end(self): + asks = dict(BENCH["driver"], unit_end=True) + controls = {"nop": {"source": "nullius", "from": "nothing"}} + control = definitions.control(dict(BENCH, controls=controls), "nop") + trial = self.trial_of(ENDING_DRIVER, control, driver=asks, controls=controls) + self.assertFalse(os.path.exists(os.path.join(trial, "ended.json"))) + + + def test_the_end_still_goes_once_the_run_is_stopped_and_its_failure_is_no_units(self): + here = os.path.join(self.tmp, "bench") + os.makedirs(here, exist_ok=True) + record = os.path.join(here, "ended.txt") + answering = ("import sys, json\n" + "for line in sys.stdin:\n" + " open(sys.argv[1], 'a').write(json.loads(line)['op'] + '\\n')\n" + " print(sys.argv[2], flush=True)\n") + stopped = threading.Event() + for reply in ('{}', 'not json', '{"detail": ' + '[' * 3000 + '0' + ']' * 3000 + '}'): + with self.subTest(reply=reply[:12]): + r = run.Run.__new__(run.Run) + r.bench, r.other, r.local = dict(BENCH, driver=dict(BENCH["driver"], unit_end=True)), {"_kind": "agent"}, \ + threading.local() + r.local.driver = jsonl.Proc([sys.executable, "-c", answering, record, reply], here, dict(os.environ), + os.path.join(here, "driver.log"), stopped=stopped).start() + stopped.set() # the run is stopped once the unit has its driver + try: + r.unit_ended("a", here) # returns, whatever the driver answered + finally: + r.local.driver.close() + stopped.clear() + with open(record) as f: + self.assertEqual(f.read().split(), ["end", "end", "end"]) + + + def test_a_driver_that_asks_is_told_each_units_end(self): + asks = dict(BENCH["driver"], unit_end=True) + trial = self.trial_of(ENDING_DRIVER, driver=asks) + with open(os.path.join(trial, "ended.json")) as f: + self.assertEqual(json.load(f), {"unit": "a", "dir": os.path.join(trial, "workspace")}) + trial = self.trial_of(FAILING_DRIVER, driver=asks) + with open(os.path.join(trial, "result.json")) as f: + self.assertEqual(json.load(f)["exception_info"]["exception_type"], "EnvironmentSetupError") + self.assertTrue(os.path.isfile(os.path.join(trial, "ended.json"))) # told although setup failed + + + def test_a_driver_that_does_not_ask_is_sent_no_end(self): + self.assertIn("driver.unit_end: true or false - whether the driver is told each unit's end", + definitions.problems(dict(BENCH, driver=dict(BENCH["driver"], unit_end="yes")), "benchmark")) + trial = self.trial_of(ENDING_DRIVER) + self.assertFalse(os.path.exists(os.path.join(trial, "ended.json"))) + + def test_a_capability_is_named_but_gets_no_end(self): + capability = definitions.get("capability", "fakemem", ROOT) + with mock.patch.object(run.Run, "drive_feature", return_value=("answer", None)): + trial = self.trial_of(ENDING_DRIVER, capability, family="capability", needs=["cap.memory/v1"], + driver=dict(BENCH["driver"], participant=True, unit_end=True)) + with open(os.path.join(trial, "told.json")) as f: + self.assertEqual(json.load(f), {"participant": "capability"}) + self.assertFalse(os.path.exists(os.path.join(trial, "ended.json"))) + + def test_a_stopped_unit_gets_one_end_and_the_next_unit_never_starts(self): + def stop(r, *args): + r._stopped.set() + return "answer", None, {} + + with mock.patch.object(run.Run, "drive_agent", stop): + trial = self.trial_of(ENDING_DRIVER.replace('["a"]', '["a", "b"]'), + driver=dict(BENCH["driver"], unit_end=True)) + with open(os.path.join(trial, "ended.json")) as f: + self.assertEqual([json.loads(line) for line in f], [{"unit": "a", "dir": os.path.join(trial, "workspace")}]) + with open(os.path.join(trial, "result.json")) as f: + self.assertEqual(json.load(f)["exception_info"]["exception_type"], "CancelledError") + self.assertEqual([d for d in os.listdir(os.path.dirname(trial)) if "__u" in d], ["a__u0000"]) + + def test_an_end_never_starts_a_driver(self): + r = run.Run.__new__(run.Run) + r.bench, r.other, r.local = dict(BENCH, driver=dict(BENCH["driver"], unit_end=True)), {"_kind": "agent"}, threading.local() + for driver in (None, mock.Mock(p=None), mock.Mock()): + with self.subTest(driver=driver): + r.local.driver = driver + r.unit_ended("a", self.tmp) + if driver is not None: + driver.request.assert_not_called() + + def test_the_run_selects_agent_options_by_kind(self): + option = {"summary": "Steps.", "values": ["10"], "default": "10", "setups": ["agent"]} + bench = dict(BENCH, options={"steps": option}, controls={"nop": {"source": "nullius", "from": "nothing"}}) + for other in (self.fake_agent(), definitions.control(bench, "nop")): + with self.subTest(kind=other["_kind"]): + r = run.Run(bench, other, model="m", provider="custom", + env=self.env, unsandboxed=True, jobs_root=self.tmp) + self.assertEqual(r.bench_options, {"steps": "10"} if other["_kind"] == "agent" else {}) + + + def test_setup_gets_the_benchmarks_own_limit(self): + r = run.Run.__new__(run.Run) + r.bench, r.other, r.local = dict(BENCH, limits={"setup_seconds": 1800}), {"_kind": "agent"}, threading.local() + self.assertEqual(r.setup_seconds(), 1800.0) + r.other = {"_kind": "control"} + self.assertEqual(r.setup_seconds(), float(run.DRIVER_SECONDS)) + r.bench, r.other = dict(BENCH), {"_kind": "agent"} + self.assertEqual(r.setup_seconds(), float(run.DRIVER_SECONDS)) + slow = DRIVER.replace(' return {"prompt": "hi"}', ' import time; time.sleep(3)\n return {"prompt": "hi"}') + here = os.path.join(self.tmp, "bench") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(slow) + b = dict(BENCH, _here=here, _kind="benchmark", limits={"setup_seconds": 1}) + job = run.run(b, self.fake_agent(), model="m-1", provider="custom", jobs_root=self.tmp, concurrency=1, + env=self.env, say=lambda s: None, unsandboxed=True) + trial = next(os.path.join(job, d) for d in os.listdir(job) if "__u" in d) + with open(os.path.join(trial, "result.json")) as f: + failed = json.load(f)["exception_info"] + self.assertEqual(failed["exception_type"], "EnvironmentSetupError") + self.assertIn("setup: no reply in 1 s", failed["exception_message"]) diff --git a/docs/standard.md b/docs/standard.md index f6f0068b..514f90ed 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -406,12 +406,13 @@ use the source folders and repository checks. | Op | Sent | Replies | | --- | --- | --- | | `units` | `dataset`, `setup` | Every unit id | -| `setup` | `unit`, `dir` | `{prompt}`, any `mcp` servers, `environment`, an own image's `image`, `prepare`, `internet` and `verify`, and `agent_seconds`, the unit's own agent timeout (up to 6 hours, instead of `limits.turn_seconds`) | +| `setup` | `unit`, `dir`; `attempt` with `limits.attempts`; `participant` (`agent`, `capability` or `control`) with `driver.participant` | `{prompt}`, any `mcp` servers, `environment`, an own image's `image`, `prepare`, `internet` and `verify`, and `agent_seconds`, the unit's own agent timeout (up to 6 hours, instead of `limits.turn_seconds`) | | `turn` | The transcript so far; `simulator: {model, base_url, key}` when there is one; `judge` for an upstream that grades as the conversation ends; `attempt` with `limits.attempts` | The next message, `{done: true}`, or `restart` | | `produce` | A capability benchmark's recall, `base_url` and `key` | The reading, through the relay | | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | | `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | +| `end` | `{unit, dir}`, with `driver.unit_end`, once a unit an agent ran has ended, also after a stop | `{}` | | Field in `standard.json` | Meaning | | --- | --- | @@ -419,6 +420,9 @@ use the source folders and repository checks. | `simulator` | `{model, says}`, only with `agent.conversation/v1`: the user's model, on `-simulator`. The conversation opens with a `turn` on an empty transcript, its message after `setup`'s prompt and a blank line. A control never asks it. `cli run --simulator-model` asks another | | `leaderboard` | `{says, simulator?, judge?, bench_options?, repeat?}`: what the upstream's leaderboard fixes besides the ranked model, from its own submission rules - the user simulator's and judge's models, the benchmark's options it sets and the fewest attempts per unit. The builder then offers **Like the leaderboard** | | `limits.turns` | At most this many agent turns in a conversation (default 20) | +| `limits.setup_seconds` | How long `setup` may take, in seconds, for a unit an agent runs (a setup that starts a machine of its own for an agent); every other unit keeps 600 | +| `driver.participant` | `true`: `setup` is also sent `participant`, the kind of what runs the unit, for a driver that starts something only an agent uses. A driver whose `setup` takes `unit` and `dir` alone leaves it out | +| `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. A control's or a setup's unit gets none, nor does any other driver | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | @@ -455,7 +459,7 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | `about` | A line about each value | | `needs` | What a value needs: an `embeddings` model, a `reranker` model, `host` requirements, or named `files` | | `when` | Another option's values that make it apply | -| `setups` | `["original"]`, `["original", "model"]`: a setting of the paper's pipeline. Without it the option is the port's (`["yours"]`) | +| `setups` | `["original"]`, `["original", "model"]`: a setting of the paper's pipeline. Without it the option is the port's (`["yours"]`); `["agent"]`: an agent's run alone, never a capability's or a control's | - **Temperature** in GSM8K and HumanEval's Original setups accepts a non-negative decimal, such as `0.5`. **Provider default** omits the parameter; the provider/model decides. Some models restrict overrides. In the builder, leave the input empty for provider default; the CLI accepts `--bench-option temperature=0.5` or `temperature=default`. This does not override an agent's own sampling settings. - **Read** by the driver as `NULLIUS_OPTION_`; set with `cli run --bench-option NAME=VALUE`. From 34589a0784db9d521b67bb18d894d3beb79d2411 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:06:52 +0100 Subject: [PATCH 02/36] fix(standard): retain proofs when a candidate kit is refused Report benchmark-specific refusals per port and continue checking the remaining candidates. Save the proofs collected by the command while preserving existing records for refused kits. Keep explicit port refusals and command-wide failures fatal. Distinguish benchmark prerequisites and installs from agent installs, and validate provider credentials before checking kits. --- app/nullius/standard/conform.py | 19 ++++-- app/nullius/standard/run.py | 34 +++++++---- app/tests/test_standard_conform.py | 95 +++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/app/nullius/standard/conform.py b/app/nullius/standard/conform.py index ea378a28..fc15cdab 100644 --- a/app/nullius/standard/conform.py +++ b/app/nullius/standard/conform.py @@ -111,10 +111,13 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] jobs before they exist. ``backend``: where the kits run (``run.Run``). ``options``: the agent's options for the kits' runs; a record proven with any of them off its default says which. - ``repeat``: how many times each check runs (:func:`outcome`).""" + ``repeat``: how many times each check runs (:func:`outcome`). A benchmark-specific + refusal is reported and skipped when checking all candidates. An explicit + ``which`` or a refusal of the whole command still ends the check.""" import os env = env if env is not None else os.environ repeat = max(1, int(repeat or 1)) + run.provider_endpoint(provider, env) got = {} for p in which or candidates(other): kit = definitions.kit(p) @@ -122,10 +125,16 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] say(f"{p}: no kit yet") continue say(f"{p}: {kit['summary']}") - job_dir = run.run(kit, other, model=model, provider=provider, jobs_root=jobs_root, concurrency=2, - name=job_name(stamp or time.strftime(STAMP), kit["key"], definitions.job_word(other)), - say=lambda s: say(" " + s), env=env, proving=True, unsandboxed=unsandboxed, - backend=backend, options=options, repeat=repeat) + try: + job_dir = run.run(kit, other, model=model, provider=provider, jobs_root=jobs_root, concurrency=2, + name=job_name(stamp or time.strftime(STAMP), kit["key"], definitions.job_word(other)), + say=lambda s: say(" " + s), env=env, proving=True, unsandboxed=unsandboxed, + backend=backend, options=options, repeat=repeat) + except run.BenchmarkRefused as e: + if which: + raise + say(f"{p}: refused: {e}") + continue o = outcome(kit, _runs(parse.parse_job(job_dir)), repeat) got[p] = {"passed": o["passed"], "ok": o["ok"], "on": time.strftime("%Y-%m-%d"), "model": model, "provider": provider, "version": str((other.get("upstream") or {}).get("version") or ""), diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 31d758f4..a70b267f 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -118,6 +118,10 @@ class Refused(Exception): """The run cannot start: the pair does not fit, or a setting is missing.""" +class BenchmarkRefused(Refused): + """This benchmark cannot run here; another benchmark may still run.""" + + def setup_request(bench: dict, kind: str, attempt: Optional[int] = None) -> tuple: """What a unit's ``setup`` is sent besides the unit and its folder, and how long it may take, for a unit that ``kind`` runs (agent, capability, control): ``attempt`` when the benchmark tells attempts apart @@ -482,16 +486,16 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas # A control (definitions.control) calls no model: its run needs a key only for the benchmark's judge. ds = [d for d in bench["datasets"] if dataset in (None, d["id"])] if not ds: - raise Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") + raise BenchmarkRefused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}") self.ds = ds[0] try: _, env_def = definitions.environment_of(bench, None, self.ds) except ValueError as e: - raise Refused(str(e)) + raise BenchmarkRefused(str(e)) if backend != "container" and other.get("_kind") == "control" and definitions.own_image(env_def): why = container.unusable() if why: - raise Refused(f"each {(bench.get('units') or {}).get('noun', 'unit')} of {bench['label']} runs in its own image, " + raise BenchmarkRefused(f"each {(bench.get('units') or {}).get('noun', 'unit')} of {bench['label']} runs in its own image, " f"which needs docker: {why}") # The benchmark's own options on this dataset; what their values need: an embedding model the driver # calls through a relay of the run's own (bench_relay), programs on this machine. @@ -500,14 +504,14 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas other["key"] if self.upstream else "yours", kind=other.get("_kind")) if bad: - raise Refused("; ".join(bad)) + raise BenchmarkRefused("; ".join(bad)) needs = definitions.option_needs(bench, self.bench_options) self.files = needs["files"] self.file_env: Dict[str, str] = {} for m in needs["embeddings"]: why = providers.embedding_problem(m, provider) if why: - raise Refused(why) + raise BenchmarkRefused(why) self.embed_model = providers.served_as(needs["embeddings"][0], provider) if needs["embeddings"] else None self.rerank_model = providers.served_as(needs["reranker"][0], provider) if needs["reranker"] else None # What a call's cost is estimated from when the provider does not say it: the user's price list @@ -541,12 +545,12 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas imaged = hostreq.in_image(bench, on_host) cannot = [n for n in hostreq.requirements(bench, on_host) if n not in imaged] if cannot: - raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs " + raise BenchmarkRefused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs " f"{', '.join(cannot)}, which a container setup cannot provide: choose local processes") on_host = [] lacking = hostreq.problems(bench, on_host, which, self.constraints.get("gpu_ids")) if lacking: - raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " + raise BenchmarkRefused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine " + "; ".join(f"{n} ({why})" for n, why in lacking.items()) + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking))) else ". Run it on a machine that has it")) @@ -612,7 +616,7 @@ def driver(self) -> jsonl.Proc: try: cmd, extra = install.driver_argv(self.bench) # its driver.install: installed in ready(), found here except install.InstallFailed as e: - raise Refused(f"could not install the driver's environment: {e}") + raise BenchmarkRefused(f"could not install the driver's environment: {e}") env = base_env(os.path.join(self.job_dir, ".work", "driver-home"), {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "", "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1", @@ -863,7 +867,7 @@ def ready(self) -> None: from . import network_requirements why = network_requirements.check(self.bench, self.other) if why: - raise Refused("; ".join(why)) + raise BenchmarkRefused("; ".join(why)) p = self.progress self.scope_limits = self.scope_ready(limits.scoped(self.constraints), refuse=self.backend == "local") owns = any(definitions.own_image(e) for e in (self.bench.get("environments") or {}).values()) @@ -878,13 +882,14 @@ def ready(self) -> None: # every container the run starts takes the caps: its agents', relays' and installs' on the container # backend, its checks' and own images' on either if self.backend == "container" or self.container_check or owns: + refusal = Refused if self.backend == "container" else BenchmarkRefused try: # Docker can drop a limit with only a warning: a result would look limited when it was not not_applied = container.unenforced(self.image or limits.PROBE_IMAGE, self.container_limits) except container.Unavailable as e: - raise Refused(str(e)) + raise refusal(str(e)) if not_applied: - raise Refused("this machine's engine does not apply " + "; ".join( + raise refusal("this machine's engine does not apply " + "; ".join( f"{limits.BY_KEY[k].label} ({limits.explain(k, self.container_limits[k], why)})" for k, why in not_applied.items())) # on the container backend the agent's packages are installed in the base image, with the Node, uv and libc it runs on image = self.image if self.backend == "container" else None @@ -920,6 +925,11 @@ def ready(self) -> None: elif drv is not None: # the environment the upstream's own code needs, for the driver with p.stage("driver", cached=install.ready(drv), start=f"installing {self.bench['key']}'s driver environment"): install.driver_packages(self.bench) + except (install.InstallFailed, container.Unavailable) as e: + if self.backend == "container": + container.sweep(self.name) + raise BenchmarkRefused(f"could not install what the run needs: {e}") + try: self.inst = "" if self.other["_kind"] not in ("control", "upstream"): with p.stage("install", cached=install.ready(install.packages_dir(self.other, image=image)), agent=self.other.get("_ref") or self.other["key"]): @@ -935,7 +945,7 @@ def ready(self) -> None: with p.stage("runtime", start="fetching " + ", ".join(self.runtimes_needed()), name=",".join(self.runtimes_needed())): self.runtimes = container.runtime_mounts(self.runtimes_needed()) except container.Unavailable as e: - raise Refused(f"the agent's runtime for an own image: {e}") + raise BenchmarkRefused(f"the agent's runtime for an own image: {e}") # An option whose driver embeds or reranks: its calls go through a relay of the run's # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's. if (self.embed_model or self.rerank_model) and self.bench_relay is None: diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index cb61ee16..2e1eeb67 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -3,11 +3,15 @@ from __future__ import annotations import os +import io import shutil import tempfile import unittest +from contextlib import redirect_stdout +from types import SimpleNamespace +from unittest import mock -from nullius.standard import conform, definitions, install, match +from nullius.standard import cli, conform, container, definitions, install, match, run FIX = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "standard") @@ -71,6 +75,95 @@ def test_candidates_follow_the_family_and_the_form(self): self.assertEqual(conform.candidates(cmd), ["agent.task/v1", "agent.workspace/v1", "agent.search/v1"]) self.assertEqual(conform.candidates(definitions.get("capability", "fakemem", self.root)), ["cap.memory/v1"]) + def invoke(self, other, *args): + out = io.StringIO() + with mock.patch.object(cli, "_other", return_value=other), mock.patch.object(cli.settings, "load"), \ + mock.patch.object(cli, "_exit_on_term"), mock.patch.object(cli.progress, "ECHO", None), \ + mock.patch.dict(os.environ, {"MODEL_API_URL": "http://127.0.0.1:9/v1"}), redirect_stdout(out): + status = cli.main(["conform", "fake", "--model", "m", "--provider", "custom", "--unsandboxed", + "--jobs", os.path.join(self.tmp, "jobs"), *args]) + return status, out.getvalue() + + def test_all_ports_continue_after_a_kit_refusal_and_save_the_proofs(self): + other = definitions.get("agent", "fake", self.root) + old = other["proven"]["agent.workspace/v1"].copy() + ports = ["agent.task/v1", "agent.workspace/v1", "agent.conversation/v1"] + started = [] + real_run = run.run + + def attempt(kit, participant, **kwargs): + started.append(kit["needs"][0]) + if kit["needs"] == ["agent.workspace/v1"]: + kit = dict(kit, environments={"task": {"image": "own", "summary": "A task image."}}) + return real_run(kit, participant, **kwargs) + return os.path.join(self.tmp, kit["key"]) + + passed = SimpleNamespace(trials=[SimpleNamespace(task_name="kit/check", reward=1.0)]) + with mock.patch.object(conform, "candidates", return_value=ports), \ + mock.patch.object(run, "run", side_effect=attempt), \ + mock.patch.object(container, "unenforced", side_effect=container.Unavailable("no container engine")), \ + mock.patch.object(conform.parse, "parse_job", return_value=passed): + status, said = self.invoke(other) + self.assertEqual(status, 0, said) + self.assertEqual(started, ports) + self.assertIn("agent.workspace/v1: refused: no container engine", said) + saved = definitions.get("agent", "fake", self.root) + for port in (ports[0], ports[2]): + self.assertEqual(saved["proven"][port]["passed"], "1/1") + self.assertIn("job", saved["proven"][port]) + self.assertIn(port, saved["offers"]) + self.assertEqual(saved["proven"][ports[1]], old) + self.assertIn(ports[1], saved["offers"]) + + def test_an_explicit_port_refusal_keeps_exit_status_two(self): + other = definitions.get("agent", "fake", self.root) + with mock.patch.object(run, "run", side_effect=run.BenchmarkRefused("kit unavailable")) as attempt, \ + mock.patch.object(definitions, "save") as save: + status, said = self.invoke(other, "--port", "agent.workspace/v1") + self.assertEqual(status, 2) + self.assertIn("refused: kit unavailable", said) + self.assertEqual(attempt.call_count, 1) + save.assert_not_called() + + def test_a_command_refusal_still_ends_the_check_without_saving(self): + other = definitions.get("agent", "fake", self.root) + passed = SimpleNamespace(trials=[SimpleNamespace(task_name="kit/check", reward=1.0)]) + for reason in ("agent unavailable", "model unavailable", "key rejected"): + with self.subTest(reason=reason), \ + mock.patch.object(run, "run", side_effect=["passed-job", run.Refused(reason)]) as attempt, \ + mock.patch.object(conform.parse, "parse_job", return_value=passed), \ + mock.patch.object(definitions, "save") as save: + status, said = self.invoke(other) + self.assertEqual(status, 2) + self.assertIn("refused: " + reason, said) + self.assertEqual(attempt.call_count, 2) + save.assert_not_called() + + def test_missing_credentials_are_refused_before_any_kit(self): + other = definitions.get("agent", "fake", self.root) + with mock.patch.object(run, "run") as attempt, mock.patch.object(definitions, "save") as save: + with self.assertRaisesRegex(run.Refused, "OPENAI_API_KEY"): + conform.check(other, model="m", provider="openai", env={}, unsandboxed=True) + attempt.assert_not_called() + save.assert_not_called() + + def test_an_agent_install_refusal_is_not_a_kit_refusal(self): + other = definitions.get("agent", "fake", self.root) + with mock.patch.object(install, "packages", side_effect=install.InstallFailed("agent unavailable")), \ + mock.patch.object(definitions, "save") as save: + status, said = self.invoke(other) + self.assertEqual(status, 2) + self.assertIn("refused: could not install what the run needs: agent unavailable", said) + save.assert_not_called() + + def test_invalid_agent_options_still_end_the_command(self): + other = definitions.get("agent", "fake", self.root) + with mock.patch.object(definitions, "save") as save: + status, said = self.invoke(other, "--option", "missing=value") + self.assertEqual(status, 2) + self.assertIn("refused:", said) + save.assert_not_called() + if __name__ == "__main__": unittest.main() From 7cc74c1f04854b3e0f0e2267d0bba785f3ea4e76 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:13:42 +0100 Subject: [PATCH 03/36] feat(standard): serve the computer port to a desktop benchmark's own loop A desktop benchmark's runner serves one MCP server, computer, whose tools are the desktop's basic operations; the agent is a client alone. The benchmark's loop runs as it is and is handed a Bridge: each predict gives it the actions of the agent's next acting call, and the reading calls are served before it, one desktop operation at a time, in the order they came. An acting call is answered once its actions and their pause have run. - computer_port: the Desktop protocol (python, upload, read, capture, password), the Bridge (the step budget, the refusal of every call past it, the turn's end), the Server over a Unix socket (every connection of the session, one call log line per tool call: tool, times, failure), files sent once for a run by their sha256, and the runner's ready file, lifeline and one stop. - computer_pipe: the session's stdio pipe to the socket. A read of the input is counted under the lock the exit decision takes; the input's end is seen by its writer's hang-up while a full queue holds the reader; the pipe's end has one deadline from the input's end. - Tests on a fake desktop with a scripted MCP client, a stand-in loop that counts steps as the upstream's does, and the upstream's own run_single_example where a checkout of it is at hand. --- AGENTS.md | 2 +- app/nullius/standard/computer_pipe.py | 163 ++++ app/nullius/standard/computer_port.py | 710 ++++++++++++++++ .../desktop_env/__init__.py | 0 .../desktop_env/desktop_env.py | 99 +++ .../osworld1_fake_desktop/sitecustomize.py | 19 + app/tests/test_standard_computer_pipe.py | 352 ++++++++ app/tests/test_standard_computer_port.py | 780 ++++++++++++++++++ 8 files changed, 2124 insertions(+), 1 deletion(-) create mode 100644 app/nullius/standard/computer_pipe.py create mode 100644 app/nullius/standard/computer_port.py create mode 100644 app/tests/fixtures/osworld1_fake_desktop/desktop_env/__init__.py create mode 100644 app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py create mode 100644 app/tests/fixtures/osworld1_fake_desktop/sitecustomize.py create mode 100644 app/tests/test_standard_computer_pipe.py create mode 100644 app/tests/test_standard_computer_port.py diff --git a/AGENTS.md b/AGENTS.md index 8a336c0c..830c31b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,7 +61,7 @@ and its resources. The source layout stays under `app/`. `feed`: the live feed any Harbor writes, from its own trial hooks), `importers/` (`atif` ATIF, `otel` OpenTelemetry, and the `cli`), `judge/` (`questions`, `state`, `client`, `store`, `batch`, `reconcile`, `lanes`), `standard/` (the Nullius standard, `docs/standard.md`: `ports`, `names` (capability vocabulary and legacy aliases), `definitions` (with an agent's or capability's versions), `match`, - `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its + `jsonl`, `acp`, `cancellation` (interruptible protocol waits), `relay`, `ledger` (a run's spending cap), `pins` (image digests checked against registry tags), `install`, `run`, `container` (the container backend), `setup_container` (an Original setup's driver and its Linux dependencies in Docker), `forward` (a licence server's one door from it, or a benchmark's MCP server's), `web_services` (HTTP fixtures a driver's setup declares, each behind a unit-local Docker door), `mcp_machine` and `mcp_bridge` (a benchmark's MCP server run on the machine, in its driver's environment, and the agent's bridge to it), `computer_port` and `computer_pipe` (the computer port's MCP server, served by a desktop benchmark's runner over the benchmark's own loop, and the session's stdio pipe to it), `network_requirements` (an environment's declared public web access: the schema and the connectivity check), `hostreq` (what a benchmark's options need of the machine its driver runs on: the check and the install) and `egress` (an own image's check's proxy to public addresses), `job`, `conform`, `fidelity` (ACP against the agent's own CLI), `prove` (a benchmark's controls and proofs), `shard` and `merge` (one run over many machines), `recipe` (the builder's standard recipe), `howto` (the How to add steps) and the `cli`; its `payload` snapshots the running backend and effective catalogue for a remote host; conformance kits under `standard/kits/`, each a benchmark: a `driver` and, for the tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: diff --git a/app/nullius/standard/computer_pipe.py b/app/nullius/standard/computer_pipe.py new file mode 100644 index 00000000..9bc346b5 --- /dev/null +++ b/app/nullius/standard/computer_pipe.py @@ -0,0 +1,163 @@ +"""The stdio MCP server ``computer`` an agent's session starts: a pipe to the computer port's server. + + computer_pipe.py SOCKET + +The port's server (``computer_port.Server``) runs in the unit's runner and accepts the agent's session at SOCKET, a +Unix socket in a folder of the user's alone; it lives as long as the unit. An MCP client ends a stdio server it started +soon after closing its input: the server it starts is this pipe, and the port's server outlives it. Stdin is read apart +from the sends to the socket, and the socket is read apart from both, so that no direction waits on another: stdin to +the socket and the socket to stdout, unchanged, each write whole. At most QUEUE_CHUNKS reads of the input wait for the +socket: while they do, the input is not read, and its end is seen by its writer's hang-up instead. + +The pipe exits 0 only when both directions are complete: every byte of the input it read was sent (and, once the input +ended, the socket's writing side closed), and every byte it read from the socket was written. A read of the input is +counted as it is taken, under the lock that the decision to exit takes, so that no byte read is ever left out of it. + +- The end of its input (or its writer's hang-up) sets the pipe's one deadline, END_SECONDS later, and every wait of + the pipe's end counts to it. +- Once the input ended and all it read is sent, the socket's writing side is closed; when the socket's side then ends, + the pipe exits 0. +- When the socket's side ends first, the pipe reads no more input, and what it holds is sent by the deadline (the + input's, else END_SECONDS from then). What is left unsent is an exit 1, its count on stderr. +- At the deadline with the socket's side still open, the pipe takes no more of the socket, sends no more to it, and + exits 3: its end was cut with the server still connected. The bytes left either way are counted on stderr. + +Standard library only. +""" +import os +import queue +import select +import socket +import sys +import threading +import time + +END_SECONDS = 5 +CUT = 3 +QUEUE_CHUNKS = 64 # reads of the input, of up to 64 KiB each, waiting for the socket + + +def main(): + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.connect(sys.argv[1]) + except OSError as e: + sys.stderr.write(f"computer_pipe: the server's socket {sys.argv[1]}: {e}\n") + return 1 + input_ended, socket_ended, sent_all, cut, quiet = (threading.Event() for _ in range(5)) + chunks = queue.Queue(QUEUE_CHUNKS) + failed, lock = [], threading.Lock() + held = {"in": 0, "out": 0} # read and not yet sent; read from the socket and not yet written + ended_at = [] # when the input ended, as the reader saw it + hang_up = select.poll() + hang_up.register(0, select.POLLHUP) # the input's writer gone, even while its last bytes wait unread + + def count(way, n): + with lock: + held[way] += n + + def put(item): # a full queue holds the input back until the socket takes more + while not cut.is_set(): + try: + chunks.put(item, timeout=0.2) + return + except queue.Full: + pass + + def from_input(): + try: + while True: + readable = select.select([0], [], [], 0.2)[0] + with lock: # a read is counted before anyone can decide on what is held + if quiet.is_set() or cut.is_set(): + return + if not readable: + continue + data = os.read(0, 65536) + held["in"] += len(data) + if not data: + break + put(data) + except (OSError, ValueError) as e: + failed.append(e) + ended_at.append(time.monotonic()) + input_ended.set() + put(None) + + def to_server(): + try: + while not cut.is_set(): + try: + data = chunks.get(timeout=0.2) + except queue.Empty: + continue + if data is None: + conn.shutdown(socket.SHUT_WR) + sent_all.set() + return + conn.sendall(data) + count("in", -len(data)) + except OSError as e: + if not cut.is_set(): + failed.append(e) + + def from_server(): + try: + while not cut.is_set(): + data = conn.recv(65536) + if not data: + break + count("out", len(data)) + view = memoryview(data) + while view: # whole: what was read is written, also after the cut + n = os.write(1, view) + view = view[n:] + count("out", -n) + except OSError as e: + if not cut.is_set(): + failed.append(e) + socket_ended.set() + + def left(): + with lock: + return held["in"], held["out"] + + reader = threading.Thread(target=from_server, daemon=True) + for target in (from_input, to_server): + threading.Thread(target=target, daemon=True).start() + reader.start() + deadline = None # the pipe's one deadline, from the input's end + while not socket_ended.wait(0.2 if deadline is None else max(0.0, min(0.2, deadline - time.monotonic()))): + if deadline is None and (input_ended.is_set() or hang_up.poll(0)): + deadline = (ended_at[0] if ended_at else time.monotonic()) + END_SECONDS + if deadline is not None and time.monotonic() >= deadline: + cut.set() # no more is taken or sent + try: + conn.shutdown(socket.SHUT_RDWR) + except OSError: + pass + unsent, unwritten = left() + sys.stderr.write(f"computer_pipe: cut {END_SECONDS:g} s after the input's end, the server still connected: " + f"{unwritten} bytes from the server not written, {unsent} bytes of input not sent\n") + return CUT + with lock: # the socket's side ended: no more input is taken + quiet.set() + until = deadline if deadline is not None else time.monotonic() + END_SECONDS + while left()[0] and not failed and time.monotonic() < until: + time.sleep(0.05) + if input_ended.is_set(): + sent_all.wait(max(0.0, until - time.monotonic())) + if failed: + sys.stderr.write(f"computer_pipe: {type(failed[0]).__name__}: {failed[0]}\n") + return 1 + unsent = left()[0] + if unsent or (input_ended.is_set() and not sent_all.is_set()): + sys.stderr.write(f"computer_pipe: the server's side ended with {unsent} bytes of input not sent\n") + return 1 + return 0 + + +if __name__ == "__main__": + code = main() + sys.stdout.flush() + os._exit(code) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py new file mode 100644 index 00000000..7f2fcd70 --- /dev/null +++ b/app/nullius/standard/computer_port.py @@ -0,0 +1,710 @@ +"""The computer port, ``agent.computer/v1``, as a desktop benchmark's runner serves it (``docs/ports/agent-computer-v1.md``). + +A desktop benchmark's runner serves one MCP server, ``computer``, to the agent's session; its tools are the desktop's +basic operations, and the agent is a client alone. The benchmark's own loop runs as it is: it is handed a ``Bridge``, +which has the loop's ``reset`` and ``predict``. Standard library alone, so that a runner in the upstream's own Python +imports it with the package's folder on its path. + +- ``Desktop``: the five operations an entry or a kit gives over its desktop. +- ``Bridge``: the loop's agent. Each ``predict`` hands the loop the actions of the agent's next acting call (one + step), and serves the reading calls that come before it, in the thread of the loop: one desktop operation at a time, + in the order the calls arrived. An acting call is held until the loop has run its actions and their pause, then + answered. Once the steps are used up, every call but ``info`` is refused. The turn's end (``end_turn``) ends the + wait for the agent's next call; each later ``predict`` gives the loop no action. +- ``Server``: the MCP server over a Unix socket: JSON-RPC 2.0, one message a line; every connection of the unit's + session accepted, the task's state kept in the bridge, outside them; one line per tool call in the call log (the + tool, its start and end, whether it failed; no argument and no result). +- ``Held``: content sent once for a run, when the runner is given a folder for it. +- What a runner needs around them: the ready file (``ready``), the lifeline (``lifeline``, which also carries the + turn's end) and the one stop (``Stop``). + +The agent's session reaches the socket through ``computer_pipe.py``, a stdio server that joins its stdin and stdout to +it. +""" +from __future__ import annotations + +import base64 +import binascii +import collections +import contextlib +import hashlib +import json +import os +import re +import signal +import socket +import sys +import threading +import time +from typing import Optional, Protocol + +PROTOCOL = "2025-06-18" +LINE_BYTES = 24 * 1024 * 1024 # one message either way, its newline counted +CHUNK_BYTES = 4 * 1024 * 1024 # upload's data, a call +READ_BYTES = 12 * 1024 * 1024 # the largest file read returns +TURN_END = b"turn-end" # the lifeline's one message: the agent's turn has ended +APPEND = ("import os; s = open({part!r}, 'rb'); d = open({path!r}, 'ab'); d.write(s.read()); d.close(); s.close(); " + "os.remove({part!r}); print(os.path.getsize({path!r}))") + + +class Desktop(Protocol): + """What an entry or a kit gives over its desktop. Each call is one desktop operation; the bridge makes them one at + a time, from the thread of the benchmark's loop.""" + + def python(self, code: str) -> Optional[dict]: + """Python code run in the desktop as its user: the reply ``{status, output, error, returncode}``, or None when + the desktop gave none in time.""" + + def upload(self, path: str, data: bytes) -> None: + """The file at ``path`` in the desktop made to hold ``data``.""" + + def read(self, path: str) -> Optional[bytes]: + """The file at ``path`` in the desktop, or None when there is no such file.""" + + def capture(self) -> bytes: + """The screen, as PNG.""" + + def password(self) -> str: + """The desktop user's password.""" + + +POINT = {"x": {"type": "integer", "description": "Pixels from the screen's left edge"}, + "y": {"type": "integer", "description": "Pixels from the screen's top edge"}} +BUTTON = {"button": {"type": "string", "enum": ["left", "right", "middle"], "description": "Default left"}} +TOOLS = [ + ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " + "python waits for the desktop's reply, and the desktop user's password.", {}, []), + ("screenshot", "The screen now, as a PNG image at its own size.", {}, []), + ("read", "A file of the desktop: data is its content in base64, or null when there is no such file.", + {"path": {"type": "string"}}, ["path"]), + ("python", "Runs Python code in the desktop as its user and returns its reply: status, output, error, returncode. " + "For reading the desktop and for your own programs there; pointer and keyboard input goes through the " + "acting tools.", {"code": {"type": "string"}}, ["code"]), + ("upload", "Writes data to a file of the desktop and returns the file's size so far. data is base64, at most 4 MiB " + "a call; append false starts the file, true adds to it. sha256, optional, is the whole file's: when " + "the run already holds a file of that sha256, data may be left out.", + {"path": {"type": "string"}, "data": {"type": "string"}, "append": {"type": "boolean"}, + "sha256": {"type": "string"}}, ["path"]), + ("click", "One step: a click at the point.", + dict(POINT, **BUTTON, count={"type": "integer", "minimum": 1, "maximum": 3, "description": "Default 1"}), + ["x", "y"]), + ("move", "One step: the pointer moved to the point.", dict(POINT), ["x", "y"]), + ("drag", "One step: a press at (x, y), a move, and a release at (to_x, to_y).", + dict(POINT, to_x={"type": "integer"}, to_y={"type": "integer"}, **BUTTON), ["x", "y", "to_x", "to_y"]), + ("scroll", "One step: the wheel turned at the point, dy clicks down (negative: up) and dx clicks right " + "(negative: left).", dict(POINT, dy={"type": "integer"}, dx={"type": "integer"}), ["x", "y", "dy"]), + ("type", "One step: the text typed key by key.", {"text": {"type": "string"}}, ["text"]), + ("key", "One step: the keys pressed together, such as [\"ctrl\", \"s\"].", + {"keys": {"type": "array", "items": {"type": "string"}}}, ["keys"]), + ("wait", "One step: no action, the pause alone.", {}, []), + ("step", "One step of several actions, run in order: each is a line of Python run in the desktop with pyautogui " + "and time imported, or WAIT. An empty list is a step with no action. The screen comes back when " + "screenshot is true.", + {"actions": {"type": "array", "items": {"type": "string"}}, "response": {"type": "string"}, + "screenshot": {"type": "boolean"}}, ["actions"]), + ("done", "The last step: the task is complete.", {"message": {"type": "string"}}, []), + ("fail", "The last step: the task cannot be done.", {"reason": {"type": "string"}}, []), +] +NAMES = [t[0] for t in TOOLS] +SCREEN_TOOLS = ("click", "move", "drag", "scroll", "type", "key", "wait") # answered with the screen after the pause +ACTING = SCREEN_TOOLS + ("step", "done", "fail") +FINAL = {"done": "DONE", "fail": "FAIL"} + + +def listed() -> list: + return [{"name": n, "description": d, + "inputSchema": dict({"type": "object", "properties": props}, **({"required": req} if req else {}))} + for n, d, props, req in TOOLS] + + +class Refused(Exception): + """A call answered with ``isError`` and these words.""" + + +class Call: + """One tools/call: its tool and arguments, when it came, and how it is answered (once).""" + + def __init__(self, tool, args, reply): + self.tool, self.args, self._reply = tool, args, reply + self.start = time.time() + self.actions, self.response, self.screen = [], "", False + self.answered = False + self._lock = threading.Lock() + + def answer(self, value=None, image=None, error=None): + with self._lock: + if self.answered: + return + self.answered = True + self._reply(self, value, image, error) + + +def _number(args, name, low, high, default=None): + v = args.get(name, default) + if isinstance(v, bool) or not isinstance(v, (int, float)) or v != v: + raise Refused(f"{name} is a number") + v = int(round(v)) + if not low <= v <= high: + raise Refused(f"{name} is from {low} to {high}") + return v + + +def _text(args, name, default=None, empty=False): + v = args.get(name, default) + if not isinstance(v, str) or not (v or empty): + raise Refused(f"{name} is a text" + ("" if empty else ", not empty")) + return v + + +def checked(call, screen) -> None: + """The call's arguments checked, and for an acting call the actions it stands for (Python lines run with pyautogui + and time imported, or WAIT, DONE, FAIL), its response and whether its answer carries the screen. Refused when an + argument is wrong: the call is then answered at once and is no step.""" + a, (w, h), tool = call.args, screen, call.tool + if not isinstance(a, dict): + raise Refused("arguments is an object") + if tool in ("click", "move", "drag", "scroll"): + x, y = _number(a, "x", 0, w - 1), _number(a, "y", 0, h - 1) + button = a.get("button", "left") + if tool in ("click", "drag") and button not in ("left", "right", "middle"): + raise Refused("button is left, right or middle") + if tool == "click": + call.actions = [f"pyautogui.click({x}, {y}, clicks={_number(a, 'count', 1, 3, 1)}, button={button!r})"] + elif tool == "move": + call.actions = [f"pyautogui.moveTo({x}, {y})"] + elif tool == "drag": + to = (_number(a, "to_x", 0, w - 1), _number(a, "to_y", 0, h - 1)) + call.actions = [f"pyautogui.moveTo({x}, {y}); pyautogui.dragTo({to[0]}, {to[1]}, duration=0.5, " + f"button={button!r})"] + elif tool == "scroll": + dy, dx = _number(a, "dy", -100, 100), _number(a, "dx", -100, 100, 0) + if not (dy or dx): + raise Refused("dy or dx is not 0") + call.actions = ["; ".join([f"pyautogui.scroll({-dy}, x={x}, y={y})"] * bool(dy) + + [f"pyautogui.hscroll({dx}, x={x}, y={y})"] * bool(dx))] + elif tool == "type": + call.actions = [f"pyautogui.write({_text(a, 'text')!r})"] + elif tool == "key": + keys = a.get("keys") + if not (isinstance(keys, list) and 1 <= len(keys) <= 8 + and all(isinstance(k, str) and 0 < len(k) <= 32 for k in keys)): + raise Refused("keys is a list of 1 to 8 key names") + call.actions = [f"pyautogui.hotkey({', '.join(repr(k if len(k) == 1 else k.lower()) for k in keys)})"] + elif tool == "wait": + call.actions = ["WAIT"] + elif tool == "step": + actions = a.get("actions") + if not (isinstance(actions, list) and all(isinstance(x, str) and x.strip() for x in actions)): + raise Refused("actions is a list of texts, each a line of Python or WAIT") + if any(x.strip() in FINAL.values() for x in actions): + raise Refused("the task's end is the tool done or fail, not an action of step") + call.actions, call.response = list(actions), _text(a, "response", "", empty=True) + call.screen = a.get("screenshot") is True + elif tool in FINAL: + call.actions, call.response = [FINAL[tool]], _text(a, "message" if tool == "done" else "reason", "", empty=True) + elif tool in ("read", "upload"): + _text(a, "path") + elif tool == "python": + _text(a, "code") + if tool in SCREEN_TOOLS: + call.screen = True + if tool == "upload": + if "data" in a: + _text(a, "data", empty=True) + if a.get("sha256") is not None and not re.fullmatch(r"[0-9a-f]{64}", str(a.get("sha256"))): + raise Refused("sha256 is 64 lowercase hexadecimal digits") + if not isinstance(a.get("append", False), bool): + raise Refused("append is true or false") + + +class Held: + """Content sent once for a run: a file whose chunks, sent in this unit from its start, hash to the sha256 the agent + gave is kept in ``folder`` (the run's, the same for each of its units) under that name. A later upload of the same + sha256, in any unit of the run, takes it from there, checked again, and the agent sends no bytes; the unit's + desktop gets the file at that call, as it would have from the bytes. Nothing else passes from one unit to the + next.""" + + def __init__(self, folder): + self.folder, self.open = folder, {} + + def get(self, digest): + try: + with open(os.path.join(self.folder, digest), "rb") as f: + data = f.read() + except OSError: + return None + return data if hashlib.sha256(data).hexdigest() == digest else None + + def add(self, path, digest, data, append): + if not append: + self.open.pop(path, None) + if digest: + self.open[path] = (digest, hashlib.sha256(), bytearray()) + kept = self.open.get(path) + if kept is None: + return + kept[1].update(data) + kept[2].extend(data) + if kept[1].hexdigest() == kept[0]: + del self.open[path] + part = os.path.join(self.folder, f".{kept[0]}.{os.getpid()}.{threading.get_ident()}") + with open(part, "wb") as f: + f.write(kept[2]) + os.replace(part, os.path.join(self.folder, kept[0])) + + +class Bridge: + """The benchmark loop's agent, over a Desktop. + + ``steps`` is the loop's budget, ``pause`` the seconds it waits after each action, ``screen`` the screen's size in + pixels, ``python_seconds`` how long the desktop waits for a reply to python. ``on_first`` is called at the loop's + first ``predict``, before any call is served (a runner writes its ready file there). ``held`` is a Held, or None.""" + + def __init__(self, desktop, steps, pause, screen, python_seconds, on_first=None, held=None): + self.desktop, self.steps, self.pause = desktop, int(steps), pause + self.screen, self.python_seconds = (int(screen[0]), int(screen[1])), python_seconds + self.on_first, self.held = on_first, held + self.password = desktop.password() + self.used = 0 # acting calls handed to the loop: its steps + self.booked = 0 # acting calls taken in, handed or waiting + self.calls = collections.deque() # calls waiting for the loop, in the order they came + self.running = None # the acting call the loop is running + self.ended = None # why no call is taken any more + self.turn_ended = False + self.first = True + self.change = threading.Condition() + + # -- the loop's agent ------------------------------------------------------------------------------------------- + def reset(self, *_, **__): + """The loop's reset: nothing. The agent's calls wait for the first predict.""" + + def predict(self, instruction, obs): + """The agent's next acting call, as the loop's (response, actions); reading calls are served until it comes. + Once the turn has ended, no action.""" + if self.first: + self.first = False + if self.on_first: + self.on_first() + while True: + call = self._next() + if call is None: + return "", [] + if call.tool not in ACTING: + self._serve(call) + continue + if call.tool in FINAL: + call.answer({"steps_left": self.steps - self.used, "done": True}) + return call.response, list(call.actions) + return call.response, self._run(call) + + def _next(self): + refused = [] + with self.change: + while not self.calls and self.ended is None: + self.change.wait(0.5) + if not self.calls: + return None + call = self.calls.popleft() + if call.tool in FINAL: + self.used += 1 + refused = self._end(f"the task has ended ({call.tool})") + elif call.tool in ACTING: + self.used += 1 + self.running = call + for c in refused: + c.answer(error=self.ended) + return call + + def _run(self, call): + """The call's actions, one at a time as the loop asks; once the loop asks past the last, it has run them all + and their pause, and the call is answered.""" + ran = False + try: + yield from call.actions + ran = True + finally: + self._ran(call, ran) + + def _ran(self, call, ran): + with self.change: + self.running = None + refused = self._end("the steps are used up") if self.used >= self.steps else [] + value = {"steps_left": self.steps - self.used, "done": self.ended is not None} + turn_ended = self.turn_ended + if not ran: + call.answer(error="the step was not run to its end") + else: + try: + call.answer(value, self.desktop.capture() if call.screen and not turn_ended else None) + except Exception as e: # the screen could not be taken: the step itself was run + call.answer(error=f"the step was run; the screen could not be taken: {type(e).__name__}: {e}") + for c in refused: + c.answer(error=self.ended) + + def _serve(self, call): + try: + call.answer(**self._reading(call)) + except Refused as e: + call.answer(error=str(e)) + except Exception as e: # the desktop failed it: its words, and the agent may call again + call.answer(error=f"the desktop failed {call.tool}: {type(e).__name__}: {e}") + + def _reading(self, call): + a = call.args + if call.tool == "screenshot": + return {"image": self.desktop.capture()} + if call.tool == "read": + data = self.desktop.read(a["path"]) + if data is not None and len(data) > READ_BYTES: + raise Refused(f"{a['path']} is over {READ_BYTES} bytes") + return {"value": {"data": None if data is None else base64.b64encode(data).decode()}} + if call.tool == "python": + reply = self.desktop.python(a["code"]) + if reply is None: + raise Refused(f"the desktop gave no reply within {self.python_seconds:g} s") + return {"value": reply} + return {"value": self._upload(a)} + + def _upload(self, a): + path, append, digest = a["path"], a.get("append", False), a.get("sha256") + if "data" not in a: + data = self.held.get(digest) if self.held is not None and digest else None + if data is None: + raise Refused("upload needs data" + (": the run holds no file of that sha256" if digest else "")) + self.desktop.upload(path, data) + return {"size": len(data)} + try: + data = base64.b64decode(a["data"], validate=True) + except (binascii.Error, ValueError): + raise Refused("data is base64") + if len(data) > CHUNK_BYTES: + raise Refused(f"data is at most {CHUNK_BYTES} bytes a call") + if not append: + self.desktop.upload(path, data) + size = len(data) + else: # the chunk beside the file, then added to it in the desktop + part = path + ".part" + self.desktop.upload(part, data) + reply = self.desktop.python(APPEND.format(part=part, path=path)) + try: + size = int(str(reply.get("output")).strip()) + except (AttributeError, ValueError): + raise Refused(f"the desktop did not add the data to {path}: {json.dumps(reply)[:300]}") + if self.held is not None: + self.held.add(path, digest, data, append) + return {"size": size} + + # -- what the server and the runner call ------------------------------------------------------------------------ + def submit(self, call): + """A call that needs the desktop, for the loop's thread; refused at once once the task ended, and an acting + call once every step is taken.""" + with self.change: + refusal = self.ended + if refusal is None and call.tool in ACTING: + if self.booked >= self.steps: + refusal = "the steps are used up" + else: + self.booked += 1 + if refusal is None: + self.calls.append(call) + self.change.notify_all() + if refusal is not None: + call.answer(error=refusal) + + def info(self): + with self.change: + left = self.steps - self.used + return {"screen": list(self.screen), "steps": self.steps, "steps_left": left, "pause": self.pause, + "python_seconds": self.python_seconds, "password": self.password} + + def instructions(self): + w, h = self.screen + return (f"A desktop seen in screenshots and driven by pointer and keyboard actions. The screen is {w} by {h} " + f"pixels, counted from its top left corner. Each call of click, move, drag, scroll, type, key, wait, " + f"step, done or fail is one step, answered once its actions have run and a pause of {self.pause:g} s " + f"has passed; there are {self.steps} steps in all, and steps_left in each answer. Once they are used " + f"up, every call but info is refused. Pointer and keyboard input goes through those tools, not through " + f"python. A turn that ends with neither done nor fail ends the task as the desktop then is.") + + def end_turn(self): + """The agent's turn has ended: the wait for its next call ends, and no call is taken any more.""" + with self.change: + self.turn_ended = True + refused = self._end("the agent's turn has ended") + for c in refused: + c.answer(error=self.ended) + + def finish(self): + """The loop has returned, or failed: every call still waiting, and one the loop was running, is answered.""" + with self.change: + refused = self._end("the task has ended") + running, self.running = self.running, None + for c in refused + ([running] if running is not None else []): + c.answer(error=self.ended) + + def _end(self, why): + """No call taken any more (with the condition held); the calls waiting, to refuse once it is let go.""" + if self.ended is None: + self.ended = why + refused = list(self.calls) + self.calls.clear() + self.change.notify_all() + return refused + + +class Wire: + """One connection of the agent's session: JSON-RPC messages, one a line, each at most LINE_BYTES with its newline. + Once its input has ended, it is closed when every call it made has been answered.""" + + def __init__(self, conn): + self.conn, self.lock = conn, threading.Lock() + self.open_calls, self.input_ended, self.closed = 0, False, False + + def lines(self): + buf, scanned = bytearray(), 0 + while True: + try: + chunk = self.conn.recv(65536) + except OSError: + return + if not chunk: + return + buf.extend(chunk) + while True: + i = buf.find(b"\n", scanned) + if i < 0: + scanned = len(buf) + break + line = bytes(buf[:i]) + del buf[:i + 1] + scanned = 0 + yield line + if len(buf) >= LINE_BYTES: + self.send({"jsonrpc": "2.0", "id": None, + "error": {"code": -32600, "message": f"a message is at most {LINE_BYTES} bytes"}}) + return + + def send(self, message): + data = (json.dumps(message) + "\n").encode() + if len(data) > LINE_BYTES: + data = (json.dumps({"jsonrpc": "2.0", "id": message.get("id"), "error": { + "code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"}}) + "\n").encode() + with self.lock: + if self.closed: + return + try: + self.conn.sendall(data) + except OSError: + self._close() + + def opened(self): + with self.lock: + self.open_calls += 1 + + def answered(self): + with self.lock: + self.open_calls -= 1 + if self.input_ended and not self.open_calls: + self._close() + + def ended(self): + with self.lock: + self.input_ended = True + if not self.open_calls: + self._close() + + def close(self): + with self.lock: + self._close() + + def _close(self): + if not self.closed: + self.closed = True + with contextlib.suppress(OSError): + self.conn.shutdown(socket.SHUT_RDWR) + self.conn.close() + + +class Server: + """The MCP server ``computer`` at ``path``, a Unix socket (its folder the user's alone), over a Bridge. ``log``, + when given, gets one JSON line per tool call: ``{tool, start, end, isError}``.""" + + def __init__(self, bridge, path, log=None): + self.bridge, self.path, self.log = bridge, path, log + self.log_lock = threading.Lock() + self.wires = [] + self.listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.listener.bind(path) + self.listener.listen(16) + threading.Thread(target=self._accept, name="computer-accept", daemon=True).start() + + def close(self): + with contextlib.suppress(OSError): + self.listener.shutdown(socket.SHUT_RDWR) + self.listener.close() + for wire in list(self.wires): + wire.close() + with contextlib.suppress(OSError): + os.unlink(self.path) + + def _accept(self): + while True: + try: + conn = self.listener.accept()[0] + except OSError: + return + wire = Wire(conn) + self.wires.append(wire) + threading.Thread(target=self._serve, args=(wire,), name="computer-connection", daemon=True).start() + + def _serve(self, wire): + for line in wire.lines(): + try: + message = json.loads(line) + except ValueError: + wire.send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) + continue + reply = self.handle(message, wire) + if reply is not None: + wire.send(reply) + wire.ended() + + def handle(self, m, wire): + """One message in; its reply out, None for a notification or a tool call answered on the wire later.""" + if not isinstance(m, dict): + return {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "a message is a JSON object"}} + if "id" not in m: + return None + rid, method, params = m["id"], m.get("method"), m.get("params") + params = params if isinstance(params, dict) else {} + if method == "initialize": + result = {"protocolVersion": params.get("protocolVersion") or PROTOCOL, "capabilities": {"tools": {}}, + "serverInfo": {"name": "computer", "version": "1"}, "instructions": self.bridge.instructions()} + elif method == "tools/list": + result = {"tools": listed()} + elif method == "tools/call": + self.call(rid, params, wire) + return None + elif method == "ping": + result = {} + elif method in ("resources/list", "prompts/list"): + result = {method.split("/")[0]: []} + else: + return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": f"no method {method!r}"}} + return {"jsonrpc": "2.0", "id": rid, "result": result} + + def call(self, rid, params, wire): + name = params.get("name") + if name not in NAMES: + self.record({"tool": None, "start": time.time(), "end": time.time(), "isError": True}) + wire.send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"no tool {str(name)[:100]!r}"}}) + return + + def reply(call, value, image, error): + content = [] + if error is not None: + content.append({"type": "text", "text": str(error)}) + if value is not None: + content.append({"type": "text", "text": json.dumps(value)}) + if image is not None: + content.append({"type": "image", "data": base64.b64encode(image).decode(), "mimeType": "image/png"}) + self.record({"tool": call.tool, "start": call.start, "end": time.time(), "isError": error is not None}) + wire.send({"jsonrpc": "2.0", "id": rid, "result": {"content": content, "isError": error is not None}}) + wire.answered() + + wire.opened() + call = Call(name, params.get("arguments") or {}, reply) + if name == "info": + call.answer(self.bridge.info()) + return + try: + checked(call, self.bridge.screen) + except Refused as e: + call.answer(error=str(e)) + return + self.bridge.submit(call) + + def record(self, line): + if self.log: + with self.log_lock, open(self.log, "a", encoding="utf-8") as f: + f.write(json.dumps(line) + "\n") + + +# -- what a runner needs around them ---------------------------------------------------------------------------------- +def ready(path, **fields): + """The ready file, written whole, readable by this user alone: the driver reads it once, then removes it.""" + if not fields.get("ok"): # also in the log: the file is gone at the runner's exit + sys.stderr.write(f"runner: not ready: {fields.get('error')}\n") + part = path + ".part" + fd = os.open(part, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(fields, f) + os.replace(part, path) + + +def lifeline(fd, bridge=None): + """The read end of a pipe from the driver. A line TURN_END ends the agent's turn (``bridge.end_turn``); any other + line is ignored. Once the driver's end closes (the driver ended, was killed, or let the unit go), the runner stops + as on SIGTERM. Run it in a thread of its own.""" + buf = b"" + try: + while True: + chunk = os.read(fd, 4096) + if not chunk: + break + buf += chunk + while b"\n" in buf: + line, _, buf = buf.partition(b"\n") + if line.strip() == TURN_END and bridge is not None: + bridge.end_turn() + buf = buf[-4096:] + except OSError: + pass + os.kill(os.getpid(), signal.SIGTERM) + + +class Stop: + """The runner's one stop: SIGTERM, or the lifeline's end, taken once; a second does not cut the clean-up. While a + part that must not be cut runs (``deferred``), a stop waits for its end. ``seconds`` after the stop, a runner still + there calls ``last`` (its last resort) and exits 1.""" + + def __init__(self, seconds, last=None): + self.seconds, self.last = seconds, last + self.deferring, self.pending = 0, False + self.taken = threading.Event() + + def install(self): + """In the main thread, before anything the clean-up holds exists.""" + signal.signal(signal.SIGTERM, self.take) + threading.Thread(target=self._last_resort, name="last-resort", daemon=True).start() + + def take(self, *_): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + self.taken.set() + if self.deferring: + self.pending = True + return + sys.exit(1) + + @contextlib.contextmanager + def deferred(self, then_stop=True): + """A part no stop cuts. A stop that came meanwhile stops the runner at the part's end (``then_stop``), or is + dropped when the runner is ending anyway.""" + self.deferring += 1 + try: + yield + finally: + self.deferring -= 1 + if not self.deferring and self.pending: + self.pending = False + if then_stop: + sys.exit(1) + + def _last_resort(self): + self.taken.wait() + time.sleep(self.seconds) + if self.last is not None: + with contextlib.suppress(Exception): + self.last() + sys.stderr.write(f"runner: the clean-up did not end within {self.seconds:g} s of the stop\n") + os._exit(1) diff --git a/app/tests/fixtures/osworld1_fake_desktop/desktop_env/__init__.py b/app/tests/fixtures/osworld1_fake_desktop/desktop_env/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py b/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py new file mode 100644 index 00000000..0d38cb82 --- /dev/null +++ b/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py @@ -0,0 +1,99 @@ +"""A fake of the upstream's DesktopEnv for the computer port's tests: no VM, no docker, no network. + +It does what run_single_example and a Desktop over the upstream's controllers use of DesktopEnv: reset, _get_obs, +step, evaluate, close, the controller's execute_python_command, get_file and get_screenshot, the setup controller's +upload, the used mark and the guest's password. Guest code is not run: execute_python_command answers with the code's length and its first line, +and an uploaded file is kept in memory, readable with get_file. Every event goes to $FAKE_DESKTOP_LOG, one JSON line +each, with whether it ran inside env.step. evaluate gives 1 when the last action is the one the task expects (FAIL +for an infeasible task, DONE otherwise), 0 otherwise. +""" +import base64 +import json +import os + +PNG = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC") + + +class _Controller: + def __init__(self, env): + self.env = env + + def execute_python_command(self, command): + first = command.splitlines()[0] if command else "" + self.env.record("execute", first=first[:200]) + return {"status": "success", "output": f"{len(command)} {first}", "error": "", "returncode": 0} + + def get_file(self, path): + self.env.record("get_file", path=path) + return self.env.files.get(path) + + def get_screenshot(self): + return PNG + + def start_recording(self): + self.env.record("start_recording") + + def end_recording(self, dest): + self.env.record("end_recording") + with open(dest, "wb") as f: + f.write(b"\x00\x00\x00\x18ftypmp42") + + +class _SetupController: + def __init__(self, env): + self.env = env + + def _upload_file_setup(self, files): + for f in files: + with open(f["local_path"], "rb") as src: + self.env.files[f["path"]] = src.read() + self.env.record("upload", paths=[f["path"] for f in files]) + + +class DesktopEnv: + vm_ip = "127.0.0.1" + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.client_password = kwargs.get("client_password") or "password" + self.controller, self.setup_controller = _Controller(self), _SetupController(self) + self.files, self.actions = {}, [] + self.task, self.instruction, self.in_step = {}, None, False + self.is_environment_used = False + self.record("started", path_to_vm=kwargs.get("path_to_vm")) + + def record(self, event, **fields): + log = os.environ.get("FAKE_DESKTOP_LOG") + if log: + with open(log, "a", encoding="utf-8") as f: + f.write(json.dumps({"event": event, "in_step": self.in_step, **fields}) + "\n") + + def reset(self, task_config=None): + self.task = task_config or {} + self.instruction = self.task.get("instruction") + self.actions = [] + self.record("reset", task=self.task.get("id")) + + def _get_obs(self): + return {"screenshot": PNG} + + def step(self, action, pause=2): + self.actions.append(action) + done = action in ("DONE", "FAIL") + self.record("step", action=action, pause=pause) + if not done and action not in ("WAIT",): + self.in_step = True + try: + self.controller.execute_python_command(action) + finally: + self.in_step = False + return self._get_obs(), 0, done, {} + + def evaluate(self): + expected = "FAIL" if (self.task.get("evaluator") or {}).get("func") == "infeasible" else "DONE" + last = self.actions[-1] if self.actions else None + self.record("evaluate", last=last) + return 1.0 if last == expected else 0.0 + + def close(self): + self.record("close", used=self.is_environment_used) diff --git a/app/tests/fixtures/osworld1_fake_desktop/sitecustomize.py b/app/tests/fixtures/osworld1_fake_desktop/sitecustomize.py new file mode 100644 index 00000000..458a2f40 --- /dev/null +++ b/app/tests/fixtures/osworld1_fake_desktop/sitecustomize.py @@ -0,0 +1,19 @@ +"""The fake upstream's start-up hook, for the tests that run the upstream's loop: run_single_example's fixed waits (60 s +after the task's setup, 20 s before the evaluation) are recorded in $FAKE_DESKTOP_LOG instead of slept. Every other +sleep is slept.""" +import json +import os +import time + +_sleep = time.sleep + + +def _recorded(seconds): + if seconds in (20, 60) and os.environ.get("FAKE_DESKTOP_LOG"): + with open(os.environ["FAKE_DESKTOP_LOG"], "a", encoding="utf-8") as f: + f.write(json.dumps({"event": "wait", "seconds": seconds}) + "\n") + return + _sleep(seconds) + + +time.sleep = _recorded diff --git a/app/tests/test_standard_computer_pipe.py b/app/tests/test_standard_computer_pipe.py new file mode 100644 index 00000000..ec3a5c37 --- /dev/null +++ b/app/tests/test_standard_computer_pipe.py @@ -0,0 +1,352 @@ +"""computer_pipe.py, the stdio server ``computer`` an agent's session starts, against a socket of the test's or a +stand-in socket in the pipe's own process: both directions at once, what the server sent written out before the pipe +ends, the one deadline from the input's end, a bounded queue, and its exit status. Each run is bounded and its child +ended.""" +from __future__ import annotations + +import importlib.util +import os +import select +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +PIPE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "nullius", "standard", + "computer_pipe.py") + + +def pipe_module(): + spec = importlib.util.spec_from_file_location("pipe_module", PIPE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Pipe(unittest.TestCase): + def run_pipe(self, serve, data=b"request\n", seconds=30) -> tuple: + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + path = os.path.join(tmp, "s") + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(path) + listener.listen(1) + self.addCleanup(listener.close) + server = threading.Thread(target=lambda: serve(listener.accept()[0]), daemon=True) + server.start() + p = subprocess.Popen([sys.executable, PIPE, path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + t0 = time.monotonic() + try: + out, _ = p.communicate(data, timeout=seconds) + finally: + if p.poll() is None: + p.kill() + p.communicate() + return out, p.returncode, time.monotonic() - t0 + + def stand_in(self, socket_code: str, extra: str = "", stdin=subprocess.PIPE): + """The pipe in a process of the test's, its socket a stand-in (``socket_code``: a class Connected), and + ``extra`` run before main. sys.argv[2] is a file of the test's.""" + code = ("import importlib.util, os, sys, threading, time\n" + socket_code + + "spec = importlib.util.spec_from_file_location('pipe', sys.argv[1])\n" + "module = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(module)\n" + "module.socket.socket = lambda *a, **kw: Connected()\n" + extra + + "os._exit(module.main())\n") + p = subprocess.Popen([sys.executable, "-c", code, PIPE, self.tmp_file()], stdin=stdin, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + self.addCleanup(lambda: (p.poll() is None and p.kill(), p.wait())) + return p + + def tmp_file(self, name="sent"): + if not hasattr(self, "_tmp"): + self._tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self._tmp, True) + return os.path.join(self._tmp, name) + + def test_a_reply_after_the_inputs_end_is_written_out(self): + def serve(conn): + conn.makefile("rb").readline() + time.sleep(1) + conn.sendall(b"late reply\n") + conn.close() + self.assertEqual(self.run_pipe(serve)[:2], (b"late reply\n", 0)) + + def test_a_server_still_connected_gets_its_reply_written_and_a_cut_status(self): + held = [] + + def serve(conn): + held.append(conn) + conn.sendall(b"early reply\n") + out, code, seconds = self.run_pipe(serve) + self.assertEqual((out, code), (b"early reply\n", 3)) + self.assertLess(seconds, 15) + for conn in held: + conn.close() + + def test_a_server_that_writes_on_and_on_is_cut_at_the_deadline(self): + stop = threading.Event() + self.addCleanup(stop.set) + + def serve(conn): + try: + while not stop.is_set(): + conn.sendall(b"x" * 1024) + time.sleep(0.01) + except OSError: + pass + out, code, seconds = self.run_pipe(serve) + self.assertEqual(code, 3) + self.assertLess(seconds, 15) + self.assertTrue(out and set(out) == {ord("x")}) + + def test_a_server_that_always_has_more_to_send_is_cut_at_the_deadline(self): + # The socket is never empty: the server writes with no pause and the test reads the pipe's output slowly, so + # the pipe always finds more to read. The cut comes at the deadline all the same, and the run is bounded. + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + path = os.path.join(tmp, "s") + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(path) + listener.listen(1) + self.addCleanup(listener.close) + stop = threading.Event() + self.addCleanup(stop.set) + + def serve(): + conn = listener.accept()[0] + try: + while not stop.is_set(): + conn.sendall(b"x" * 65536) + except OSError: + pass + finally: + conn.close() + + threading.Thread(target=serve, daemon=True).start() + p = subprocess.Popen([sys.executable, PIPE, path], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + t0, got = time.monotonic(), 0 + try: + p.stdin.write(b"request\n") + p.stdin.close() + while time.monotonic() - t0 < 20: + if not select.select([p.stdout], [], [], max(0.0, 20 - (time.monotonic() - t0)))[0]: + break + chunk = os.read(p.stdout.fileno(), 65536) + if not chunk: + break + got += len(chunk) + time.sleep(0.02) + code = p.wait(timeout=5) if p.poll() is None and time.monotonic() - t0 < 20 else p.poll() + finally: + stop.set() + if p.poll() is None: + p.kill() + p.wait() + p.stdout.close() + self.assertEqual(code, 3) + self.assertLess(time.monotonic() - t0, 20) + self.assertGreater(got, 0) + + def test_the_cut_starts_at_the_inputs_end_while_a_send_waits(self): + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def sendall(self, data): time.sleep(30)\n" + " def recv(self, count):\n time.sleep(30)\n return b''\n" + " def shutdown(self, how): pass\n", + "module.END_SECONDS = 0.1\n") + try: + p.communicate(b"x" * 8192, timeout=5) + except subprocess.TimeoutExpired: + pass + self.assertEqual(p.wait(timeout=5), 3, "the input ended, and the pipe was not cut") + + def test_input_still_being_sent_when_the_servers_side_ends_is_sent_before_an_exit_0(self): + p = self.stand_in("started = threading.Event()\n" + "class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data):\n" + " started.set()\n time.sleep(0.5)\n" + " open(sys.argv[2], 'ab').write(data)\n" + " def recv(self, size):\n started.wait(2)\n return b''\n") + p.communicate(b"request\n", timeout=10) + self.assertEqual(p.returncode, 0) + with open(self.tmp_file(), "rb") as f: + self.assertEqual(f.read(), b"request\n") + + def test_input_left_unsent_when_the_servers_side_ends_is_an_exit_1(self): + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data): time.sleep(30)\n" + " def recv(self, size):\n time.sleep(0.3)\n return b''\n", + "module.END_SECONDS = 0.2\n") + p.stdin.write(b"request\n") + p.stdin.flush() # the input stays open: the server's side ends first + code = p.wait(timeout=10) + p.stdin.close() + self.assertEqual(code, 1) + self.assertIn(b"8 bytes of input not sent", p.stderr.read()) + + def test_input_waiting_for_a_stalled_socket_is_bounded_and_drained_once_it_takes_more(self): + # The socket takes the first chunk, then stalls until the test lets it go: the input is held back meanwhile, + # and all of it reaches the socket once the socket takes more. + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def sendall(self, data):\n" + " with open(sys.argv[2], 'ab') as f:\n f.write(data)\n" + " while not os.path.exists(sys.argv[2] + '.go'):\n time.sleep(0.05)\n" + " def shutdown(self, how):\n open(sys.argv[2] + '.shut', 'w').close()\n" + " def recv(self, size):\n" + " while not os.path.exists(sys.argv[2] + '.shut'):\n time.sleep(0.05)\n" + " return b''\n") + chunk, written = b"q" * 65536, [0] + p.stdin.write(chunk) + p.stdin.flush() + written[0] += len(chunk) + deadline = time.monotonic() + 10 + while self.sent() < len(chunk) and time.monotonic() < deadline: # the pipe runs and passes data on + time.sleep(0.05) + self.assertEqual(self.sent(), len(chunk), "nothing reached the socket") + stop = threading.Event() + + def feed(): + try: + while not stop.is_set() and written[0] < 64 * 1024 * 1024: + p.stdin.write(chunk) + p.stdin.flush() + written[0] += len(chunk) + except (OSError, ValueError): + pass + feeder = threading.Thread(target=feed, daemon=True) + feeder.start() + last = -1 + while time.monotonic() < deadline + 10 and written[0] != last: # until the input is held back + last = written[0] + time.sleep(0.5) + bound = 65536 * (3 + pipe_module().QUEUE_CHUNKS) + 4 * 1024 * 1024 # queued, one in each hand, the OS pipe + self.assertLess(written[0], bound, "the pipe read on with nothing taken by the socket") + self.assertEqual(self.sent(), len(chunk)) + stop.set() + open(self.tmp_file() + ".go", "w").close() # the socket takes more + feeder.join(10) + p.stdin.close() + self.assertEqual(p.wait(timeout=20), 0) + self.assertEqual(self.sent(), written[0]) + + def sent(self): + try: + return os.path.getsize(self.tmp_file()) + except OSError: + return 0 + + def test_a_cut_counts_what_it_left_unwritten(self): + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data): pass\n" + " def recv(self, size):\n" + " if not hasattr(self, 'sent'):\n" + " self.sent = True\n return b'x' * 1048576\n" + " time.sleep(30)\n return b''\n", + "module.END_SECONDS = 0.2\n", stdin=subprocess.DEVNULL) + code = p.wait(timeout=10) # nothing reads its output: most of what it read stays unwritten + err = p.stderr.read() + self.assertEqual(code, 3) + self.assertRegex(err.decode(), r"cut 0\.2 s after the input's end, the server still connected: [1-9][0-9]* bytes " + r"from the server not written") + + def test_both_directions_at_once_never_block_each_other(self): + big = 2 * 1024 * 1024 + + def serve(conn): + conn.sendall(b"r" * big) # before reading any of the request + got = 0 + while got < big: + chunk = conn.recv(65536) + if not chunk: + break + got += len(chunk) + conn.close() + out, code, _ = self.run_pipe(serve, data=b"q" * big) + self.assertEqual((len(out), code), (big, 0)) + + def test_the_inputs_end_is_seen_while_a_full_queue_holds_the_reader(self): + # The socket never takes more, and the queue is full: the reader waits to put its last read and cannot read + # on to the input's end. Its writer's hang-up is seen all the same, and the cut comes at the deadline. + read, write = os.pipe() + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data): time.sleep(60)\n" + " def recv(self, size):\n time.sleep(60)\n return b''\n", + "module.END_SECONDS = 0.5\nmodule.QUEUE_CHUNKS = 2\n", stdin=read) + os.close(read) + os.set_blocking(write, False) + quiet_since, chunk = time.monotonic(), b"q" * 65536 + while time.monotonic() - quiet_since < 1.0: # fill until the pipe takes nothing for a second + try: + os.write(write, chunk) + quiet_since = time.monotonic() + except BlockingIOError: + time.sleep(0.05) + os.close(write) # the input ends, with bytes still waiting + t0 = time.monotonic() + code = p.wait(timeout=10) + self.assertEqual(code, 3) + self.assertLess(time.monotonic() - t0, 3) + self.assertRegex(p.stderr.read().decode(), r"[1-9][0-9]* bytes of input not sent") + + def test_bytes_read_while_the_servers_side_ends_are_sent_or_counted(self): + # The input's read is held open across the socket's end: the read returns, and only then is it counted. + # Exit 0 must mean that what was read was sent. + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data):\n open(sys.argv[2], 'ab').write(data)\n" + " def recv(self, size):\n" + " while not os.path.exists(sys.argv[2] + '.read'):\n time.sleep(0.01)\n" + " return b''\n", + "real = os.read\n" + "def slow(fd, n):\n" + " data = real(fd, n)\n" + " if fd == 0 and data:\n" + " open(sys.argv[2] + '.read', 'w').close()\n time.sleep(0.5)\n" + " return data\n" + "module.os.read = slow\n") + p.stdin.write(b"request\n") + p.stdin.flush() + code = p.wait(timeout=10) + p.stdin.close() + err = p.stderr.read() + sent = open(self.tmp_file(), "rb").read() if os.path.exists(self.tmp_file()) else b"" + self.assertEqual((code, sent), (0, b"request\n"), f"what was read was not sent: {err!r}") + + def test_the_end_after_the_inputs_end_has_one_deadline(self): + # A write to stdout in progress (nothing reads the output) is not given a second period past the deadline. + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def shutdown(self, how): pass\n" + " def sendall(self, data): pass\n" + " def recv(self, size):\n" + " if not hasattr(self, 'sent'):\n" + " self.sent = True\n return b'x' * 1048576\n" + " time.sleep(30)\n return b''\n", + "module.END_SECONDS = 2.0\n" + "open(sys.argv[2], 'w').write(repr(time.monotonic()))\n", stdin=subprocess.DEVNULL) + code = p.wait(timeout=15) + ended = time.monotonic() + with open(self.tmp_file()) as f: + started = float(f.read()) + self.assertEqual(code, 3) + self.assertLess(ended - started, 3.0, "the pipe's end took a second period") + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py new file mode 100644 index 00000000..0c4da936 --- /dev/null +++ b/app/tests/test_standard_computer_port.py @@ -0,0 +1,780 @@ +"""The computer port's shared code (``nullius.standard.computer_port``): the MCP server ``computer`` and its bridge to a +benchmark's loop, on a fake desktop, with a scripted MCP client over the server's socket; the runner's ready file, +lifeline and one stop. A loop of a few lines stands in for an upstream's, counting steps as OSWorld's +run_single_example does. The class UpstreamLoop drives OSWorld's own run_single_example with the bridge, where a +checkout of the upstream is at hand. No VM, no docker, no network, no model.""" +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import json +import os +import select +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +from nullius.standard import computer_port as cp + +FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PNG = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC") + + +class FakeDesktop: + """A desktop of the test's, after the fake DesktopEnv of fixtures/osworld1_fake_desktop: its files in a folder of + the test's (relative paths only), python run there in a child process, one constant screen. Every operation is + recorded with its kind and start; two at once are counted as an overlap. ``fail`` names the operations that raise; + ``silent`` makes python give no reply.""" + + def __init__(self, root): + self.root, self.events, self.busy, self.overlaps = root, [], 0, 0 + self.fail, self.silent = set(), False + self.lock = threading.Lock() + + @contextlib.contextmanager + def op(self, kind, **fields): + with self.lock: + self.busy += 1 + self.overlaps += self.busy > 1 + self.events.append(dict(kind=kind, start=time.time(), **fields)) + try: + time.sleep(0.01) # long enough for two at once to meet + if kind in self.fail: + raise RuntimeError(f"{kind} failed in the desktop") + yield + finally: + with self.lock: + self.busy -= 1 + + def at(self, path): + assert not os.path.isabs(path) and ".." not in path.split("/"), path + return os.path.join(self.root, path) + + def python(self, code): + with self.op("python", code=code): + if self.silent: + return None + p = subprocess.run([sys.executable, "-I", "-c", code], cwd=self.root, capture_output=True, text=True, + timeout=30, stdin=subprocess.DEVNULL) + return {"status": "success" if p.returncode == 0 else "error", "output": p.stdout, "error": p.stderr, + "returncode": p.returncode} + + def upload(self, path, data): + with self.op("upload", path=path): + with open(self.at(path), "wb") as f: + f.write(data) + + def read(self, path): + with self.op("read", path=path): + try: + with open(self.at(path), "rb") as f: + return f.read() + except FileNotFoundError: + return None + + def capture(self): + with self.op("capture"): + return PNG + + def password(self): + return "password" + + def act(self, action): # the loop's own run of an action + with self.op("action", action=action): + pass + + def kinds(self): + return [e["kind"] for e in self.events] + + +def stand_in_loop(agent, desktop, steps, pause, ran, go): + """Stands in for an upstream's loop, counting as OSWorld's run_single_example does: after the agent's reset and the + environment's wait (``go``), one predict a step, its actions run one at a time with the pause after each, until + DONE or FAIL or the step budget; then the evaluation, of the desktop as it is.""" + agent.reset(None, vm_ip="127.0.0.1") + go.wait(30) + done, step = False, 0 + while not done and step < steps: + _, actions = agent.predict("the task", {"screenshot": PNG}) + for action in actions: + desktop.act(action) + time.sleep(pause) + ran.append(action) + done = action in ("DONE", "FAIL") + if done: + break + step += 1 + ran.append(("evaluated", step)) + + +class Client: + """A scripted MCP client: JSON-RPC lines over the server's socket, each answer waited for a bounded time.""" + + def __init__(self, path): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(path) + self.buf, self.got, self.last = b"", {}, 0 + + def send(self, method, params=None): + self.last += 1 + message = {"jsonrpc": "2.0", "id": self.last, "method": method} + if params is not None: + message["params"] = params + self.sock.sendall((json.dumps(message) + "\n").encode()) + return self.last + + def answer(self, rid, seconds=10.0): + deadline = time.monotonic() + seconds + while rid not in self.got: + while b"\n" in self.buf: + line, _, self.buf = self.buf.partition(b"\n") + message = json.loads(line) + self.got[message.get("id")] = message + if rid in self.got: + break + left = deadline - time.monotonic() + if left <= 0 or not select.select([self.sock], [], [], left)[0]: + raise AssertionError(f"no answer to {rid} within {seconds} s") + chunk = self.sock.recv(1 << 20) + if not chunk: + raise AssertionError(f"the server closed the connection before answering {rid}") + self.buf += chunk + return self.got.pop(rid) + + def ask(self, method, params=None, seconds=10.0): + return self.answer(self.send(method, params), seconds) + + def start(self, tool, **arguments): + return self.send("tools/call", {"name": tool, "arguments": arguments}) + + def result(self, rid, seconds=10.0): + return self.answer(rid, seconds)["result"] + + def call(self, tool, seconds=10.0, **arguments): + return self.result(self.start(tool, **arguments), seconds) + + def initialize(self): + return self.ask("initialize", {"protocolVersion": cp.PROTOCOL, "capabilities": {}, "clientInfo": {"name": "t"}}) + + def close(self): + self.sock.close() + + +def value(result): + return json.loads(next(c["text"] for c in result["content"] if c["type"] == "text")) + + +def image(result): + found = [base64.b64decode(c["data"]) for c in result["content"] if c["type"] == "image"] + return found[0] if found else None + + +def error(result): + assert result["isError"], result + return result["content"][0]["text"] + + +class Port(unittest.TestCase): + """A unit: a fake desktop, the bridge over it, the server at a socket of the test's and the stand-in loop in a + thread, as a runner holds them.""" + + def start(self, steps=3, pause=0.05, held=None, go=True): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + root = os.path.join(self.tmp, "desktop") + os.mkdir(root) + self.desktop = FakeDesktop(root) + self.first = threading.Event() + self.bridge = cp.Bridge(self.desktop, steps, pause, (1920, 1080), 30, on_first=self.first.set, held=held) + folder = os.path.join(self.tmp, "socket") + os.mkdir(folder, 0o700) + self.socket, self.log = os.path.join(folder, "s"), os.path.join(self.tmp, "calls.jsonl") + self.server = cp.Server(self.bridge, self.socket, self.log) + self.addCleanup(self.server.close) + self.ran, self.go = [], threading.Event() + if go: + self.go.set() + self.loop = threading.Thread(target=self._loop, args=(steps, pause), daemon=True) + self.loop.start() + self.addCleanup(self.bridge.finish) + return self.client() + + def _loop(self, steps, pause): + try: + stand_in_loop(self.bridge, self.desktop, steps, pause, self.ran, self.go) + finally: + self.bridge.finish() + + def client(self): + c = Client(self.socket) + self.addCleanup(c.close) + c.initialize() + return c + + def evaluated(self, seconds=10): + self.loop.join(seconds) + self.assertFalse(self.loop.is_alive(), "the loop did not end") + return self.ran[-1][1] + + def logged(self): + with open(self.log) as f: + return [json.loads(line) for line in f] + + +class Tools(Port): + def test_initialize_and_the_tool_list_answer_before_the_desktop_is_ready_and_state_the_limits(self): + c = self.start(steps=3, go=False) # the loop is in the environment's wait + init = c.initialize()["result"] + self.assertEqual(init["serverInfo"]["name"], "computer") + self.assertIn("1920 by 1080 pixels", init["instructions"]) + self.assertIn("there are 3 steps in all", init["instructions"]) + self.assertEqual([t["name"] for t in c.ask("tools/list")["result"]["tools"]], + ["info", "screenshot", "read", "python", "upload", "click", "move", "drag", "scroll", "type", + "key", "wait", "step", "done", "fail"]) + self.assertEqual(value(c.call("info")), {"screen": [1920, 1080], "steps": 3, "steps_left": 3, "pause": 0.05, + "python_seconds": 30, "password": "password"}) + shot = c.start("screenshot") + with self.assertRaises(AssertionError): # held until the loop's first predict + c.answer(shot, 0.5) + self.assertEqual(self.desktop.events, []) + self.go.set() + self.assertEqual(image(c.result(shot)), PNG) + self.assertTrue(self.first.is_set()) + + def test_the_reading_tools_take_no_step(self): + c = self.start(steps=3) + self.assertEqual(image(c.call("screenshot")), PNG) + self.assertEqual(value(c.call("python", code="print(6 * 7)"))["output"], "42\n") + self.assertEqual(value(c.call("read", path="missing.txt")), {"data": None}) + self.assertEqual(value(c.call("upload", path="a.txt", data=base64.b64encode(b"abc").decode())), {"size": 3}) + self.assertEqual(value(c.call("read", path="a.txt")), {"data": base64.b64encode(b"abc").decode()}) + self.assertEqual(value(c.call("info"))["steps_left"], 3) + self.assertEqual(self.ran, []) + + def test_each_pointer_and_keyboard_tool_is_one_action_answered_with_the_screen(self): + c = self.start(steps=8) + calls = [("click", {"x": 10, "y": 20}, "pyautogui.click(10, 20, clicks=1, button='left')"), + ("click", {"x": 5, "y": 6, "button": "right", "count": 2}, "pyautogui.click(5, 6, clicks=2, button='right')"), + ("move", {"x": 100, "y": 200}, "pyautogui.moveTo(100, 200)"), + ("drag", {"x": 1, "y": 2, "to_x": 30, "to_y": 40}, + "pyautogui.moveTo(1, 2); pyautogui.dragTo(30, 40, duration=0.5, button='left')"), + ("scroll", {"x": 50, "y": 60, "dy": 3, "dx": -2}, + "pyautogui.scroll(-3, x=50, y=60); pyautogui.hscroll(-2, x=50, y=60)"), + ("type", {"text": "it's 'fine'\n"}, "pyautogui.write(\"it's 'fine'\\n\")"), + ("key", {"keys": ["Ctrl", "S"]}, "pyautogui.hotkey('ctrl', 'S')"), + ("wait", {}, "WAIT")] + for i, (tool, args, _) in enumerate(calls, 1): + got = c.call(tool, **args) + self.assertFalse(got["isError"], got) + self.assertEqual(value(got), {"steps_left": 8 - i, "done": i == 8}, tool) + self.assertEqual(image(got), PNG, tool) + self.assertEqual(self.evaluated(), 8) + self.assertEqual(self.ran[:-1], [action for _, _, action in calls]) + + def test_step_runs_its_actions_in_order_as_one_step_and_an_empty_list_is_a_step(self): + c = self.start(steps=3) + got = c.call("step", actions=["pyautogui.press('a')", "WAIT", "pyautogui.press('b')"], response="pressing") + self.assertEqual((value(got), image(got)), ({"steps_left": 2, "done": False}, None)) + got = c.call("step", actions=[], screenshot=True) + self.assertEqual((value(got), image(got)), ({"steps_left": 1, "done": False}, PNG)) + self.assertEqual(self.ran, ["pyautogui.press('a')", "WAIT", "pyautogui.press('b')"]) + + def test_wrong_arguments_are_refused_at_once_and_take_no_step(self): + c = self.start(steps=3) + for tool, args, words in [("click", {"x": 1920, "y": 0}, "x is from 0 to 1919"), + ("click", {"x": 1, "y": 1, "button": "side"}, "button is left, right or middle"), + ("click", {"x": True, "y": 1}, "x is a number"), + ("key", {"keys": []}, "keys is a list"), + ("type", {"text": ""}, "text is a text, not empty"), + ("scroll", {"x": 1, "y": 1, "dy": 0}, "dy or dx is not 0"), + ("step", {"actions": ["DONE"]}, "the task's end is the tool done or fail"), + ("step", {"actions": "pyautogui.click()"}, "actions is a list"), + ("upload", {"path": "a", "data": "!!"}, "data is base64"), + ("upload", {"path": "a", "sha256": "ABC"}, "sha256 is 64 lowercase")]: + self.assertIn(words, error(c.call(tool, **args)), tool) + self.assertEqual(value(c.call("info"))["steps_left"], 3) + self.assertEqual(self.ran, []) + + +class Budget(Port): + def test_the_loop_counts_one_step_for_each_acting_call_and_nothing_else(self): + c = self.start(steps=3) + c.call("click", x=1, y=1) + c.call("screenshot") + c.call("step", actions=["pyautogui.press('a')", "pyautogui.press('b')"]) + c.call("python", code="pass") + last = c.call("key", keys=["enter"]) + self.assertEqual(value(last), {"steps_left": 0, "done": True}) + self.assertEqual(self.evaluated(), 3) + self.assertEqual(self.ran[:-1], ["pyautogui.click(1, 1, clicks=1, button='left')", "pyautogui.press('a')", + "pyautogui.press('b')", "pyautogui.hotkey('enter')"]) + + def test_every_call_but_info_is_refused_past_the_budget_and_the_desktop_is_not_touched(self): + c = self.start(steps=1) + c.call("click", x=1, y=1) + self.evaluated() + touched = len(self.desktop.events) + for tool, args in [("click", {"x": 1, "y": 1}), ("screenshot", {}), ("python", {"code": "pass"}), + ("read", {"path": "a"}), ("done", {})]: + self.assertEqual(error(c.call(tool, **args)), "the steps are used up", tool) + self.assertEqual(value(c.call("info"))["steps_left"], 0) + self.assertEqual(len(self.desktop.events), touched) + + def test_acting_calls_sent_together_past_the_budget_are_refused_at_once(self): + c = self.start(steps=2, pause=0.3) + ids = [c.start("click", x=i, y=i) for i in (1, 2, 3)] + third = c.result(ids[2], 0.25) # before the first has been answered + self.assertEqual(error(third), "the steps are used up") + self.assertEqual([value(c.result(i))["steps_left"] for i in ids[:2]], [1, 0]) + self.assertEqual(self.evaluated(), 2) + self.assertEqual(len(self.ran), 3) + + +class Held(Port): + def test_an_acting_call_is_answered_once_its_actions_and_their_pause_have_run(self): + c = self.start(steps=3, pause=0.4) + rid = c.start("click", x=3, y=4) + got = c.result(rid) + answered = time.time() + acted = next(e["start"] for e in self.desktop.events if e["kind"] == "action") + self.assertGreaterEqual(answered - acted, 0.4) + self.assertEqual(self.desktop.kinds(), ["action", "capture"]) + self.assertEqual(image(got), PNG) + + def test_calls_sent_during_a_step_wait_for_it_one_desktop_operation_at_a_time_in_their_order(self): + c = self.start(steps=3, pause=0.3) + ids = [c.start("click", x=1, y=1), c.start("screenshot"), c.start("python", code="print(1)"), + c.start("read", path="none"), c.start("move", x=2, y=2)] + order = [] + for rid in ids: + c.answer(rid) + order.append(rid) + self.assertEqual(self.desktop.kinds(), ["action", "capture", "capture", "python", "read", "action", "capture"]) + self.assertEqual(self.desktop.overlaps, 0) + + def test_the_answers_come_in_the_order_the_desktop_ran_the_calls(self): + c = self.start(steps=3, pause=0.3) + first, second = c.start("click", x=1, y=1), c.start("screenshot") + deadline = time.monotonic() + 10 + while len(c.got) < 2 and time.monotonic() < deadline: # read both answers as they come + with contextlib.suppress(AssertionError): + c.answer(-1, 0.1) + arrived = list(c.got) + self.assertEqual(arrived, [first, second]) + + +class TurnEnd(Port): + def test_a_turn_that_ends_with_no_done_ends_the_task_as_the_desktop_is(self): + c = self.start(steps=5) + c.call("click", x=1, y=1) + waiting = c.start("screenshot") + c.answer(waiting) + t0 = time.monotonic() + self.bridge.end_turn() # the loop is waiting for the agent's next call + self.assertEqual(self.evaluated(), 5) + self.assertLess(time.monotonic() - t0, 3) + self.assertEqual(self.ran, ["pyautogui.click(1, 1, clicks=1, button='left')", ("evaluated", 5)]) + self.assertEqual(error(c.call("click", x=1, y=1)), "the agent's turn has ended") + + def test_done_ends_the_task_and_a_later_turn_end_does_nothing(self): + c = self.start(steps=5) + c.call("click", x=1, y=1) + self.assertEqual(value(c.call("done", message="finished")), {"steps_left": 3, "done": True}) + self.assertEqual(self.evaluated(), 2) + self.assertEqual(self.ran[-2:], ["DONE", ("evaluated", 2)]) + self.bridge.end_turn() + self.assertEqual(self.ran[-2:], ["DONE", ("evaluated", 2)]) + self.assertIn("the task has ended (done)", error(c.call("screenshot"))) + + def test_fail_is_the_last_step(self): + c = self.start(steps=5) + self.assertEqual(value(c.call("fail", reason="no such menu")), {"steps_left": 4, "done": True}) + self.assertEqual(self.evaluated(), 1) + self.assertEqual(self.ran, ["FAIL", ("evaluated", 1)]) + self.assertIn("the task has ended (fail)", error(c.call("click", x=1, y=1))) + + def test_a_turn_end_on_the_lifeline_ends_the_turn_and_its_close_stops_the_runner(self): + self.start(steps=2) + read, write = os.pipe() + with mock.patch.object(cp.os, "kill") as kill: + reader = threading.Thread(target=cp.lifeline, args=(read, self.bridge), daemon=True) + reader.start() + os.write(write, b"something else\n" + cp.TURN_END + b"\n") + self.assertEqual(self.evaluated(), 2) # the turn's end reached the bridge + kill.assert_not_called() + os.close(write) + reader.join(5) + os.close(read) + self.assertFalse(reader.is_alive()) + kill.assert_called_once_with(os.getpid(), signal.SIGTERM) + + +class Errors(Port): + def test_a_failed_desktop_call_answers_with_its_words_and_the_next_succeeds(self): + c = self.start(steps=2) + self.desktop.fail = {"python", "capture"} + self.assertEqual(error(c.call("python", code="print(1)")), + "the desktop failed python: RuntimeError: python failed in the desktop") + self.assertIn("capture failed in the desktop", error(c.call("screenshot"))) + got = c.call("click", x=1, y=1) # the step ran; its screen could not be taken + self.assertIn("the step was run; the screen could not be taken", error(got)) + self.desktop.fail = set() + self.assertEqual(value(c.call("python", code="print(1)"))["output"], "1\n") + self.assertEqual(value(c.call("info"))["steps_left"], 1) + + def test_python_with_no_reply_in_time_is_an_error(self): + c = self.start() + self.desktop.silent = True + self.assertEqual(error(c.call("python", code="pass")), "the desktop gave no reply within 30 s") + + def test_an_unknown_tool_a_line_that_is_no_json_and_an_unknown_method_are_answered(self): + c = self.start() + self.assertEqual(c.ask("tools/call", {"name": "nope", "arguments": {}})["error"]["code"], -32602) + c.sock.sendall(b"not json\n") + self.assertEqual(c.answer(None)["error"]["code"], -32700) + self.assertEqual(c.ask("sampling/x")["error"]["code"], -32601) + self.assertEqual(c.ask("ping")["result"], {}) + c.sock.sendall(b'{"jsonrpc": "2.0", "method": "notifications/initialized"}\n') + self.assertEqual(image(c.call("screenshot")), PNG) + + def test_a_loop_that_fails_during_a_step_has_the_call_answered(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + desktop = FakeDesktop(self.tmp) + bridge = cp.Bridge(desktop, 3, 0, (100, 100), 30) + answers = [] + call = cp.Call("click", {"x": 1, "y": 1}, lambda c, v, i, e: answers.append(e)) + cp.checked(call, bridge.screen) + bridge.submit(call) + _, actions = bridge.predict("x", {}) + next(iter(actions)) # the loop takes the action, then fails in it + bridge.finish() + self.assertEqual(answers, ["the task has ended"]) + + +class Upload(Port): + def test_an_upload_in_chunks_is_the_file_with_its_size_so_far(self): + c = self.start() + parts = [b"first", b"-second", b"-third"] + sizes = [value(c.call("upload", path="f.bin", data=base64.b64encode(p).decode(), append=i > 0))["size"] + for i, p in enumerate(parts)] + self.assertEqual(sizes, [5, 12, 18]) + self.assertEqual(value(c.call("read", path="f.bin")), {"data": base64.b64encode(b"".join(parts)).decode()}) + self.assertEqual(sorted(os.listdir(self.desktop.root)), ["f.bin"]) + self.assertEqual(self.ran, []) + + def test_a_chunk_over_four_mib_is_refused(self): + c = self.start() + big = base64.b64encode(b"x" * (cp.CHUNK_BYTES + 1)).decode() + self.assertEqual(error(c.call("upload", path="f.bin", data=big)), f"data is at most {cp.CHUNK_BYTES} bytes a call") + self.assertNotIn("upload", self.desktop.kinds()) + + def test_a_file_sent_once_for_a_run_reaches_the_next_units_desktop_by_its_sha256(self): + run = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, run, True) + content = os.urandom(3000) + digest = hashlib.sha256(content).hexdigest() + c = self.start(held=cp.Held(run)) # the first unit sends the bytes + self.assertIn("the run holds no file of that sha256", error(c.call("upload", path="tool.bin", sha256=digest))) + c.call("upload", path="tool.bin", data=base64.b64encode(content[:1000]).decode(), sha256=digest) + c.call("upload", path="tool.bin", data=base64.b64encode(content[1000:]).decode(), append=True) + self.assertEqual(os.listdir(run), [digest]) + second = Port("run") # the next unit of the same run sends none + second.addCleanup = self.addCleanup + c2 = second.start(held=cp.Held(run)) + self.assertEqual(value(c2.call("upload", path="tool.bin", sha256=digest)), {"size": 3000}) + with open(os.path.join(second.desktop.root, "tool.bin"), "rb") as f: + self.assertEqual(f.read(), content) + self.assertEqual(second.desktop.kinds(), ["upload"]) + with open(os.path.join(run, digest), "r+b") as f: # a kept file that changed is not taken + f.write(b"\0") + self.assertIn("the run holds no file of that sha256", error(c2.call("upload", path="t2.bin", sha256=digest))) + + def test_chunks_that_do_not_hash_to_the_sha256_given_are_not_kept(self): + run = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, run, True) + c = self.start(held=cp.Held(run)) + c.call("upload", path="t.bin", data=base64.b64encode(b"abc").decode(), sha256="0" * 64) + self.assertEqual(os.listdir(run), []) + + def test_with_no_folder_for_the_run_an_upload_by_sha256_alone_is_refused(self): + c = self.start() + self.assertEqual(error(c.call("upload", path="t.bin", sha256="0" * 64)), + "upload needs data: the run holds no file of that sha256") + + +class Connections(Port): + def test_several_connections_share_the_task(self): + first = self.start(steps=3) + first.call("click", x=1, y=1) + first.close() + second = self.client() + self.assertEqual(value(second.call("info"))["steps_left"], 2) + third = self.client() + a, b = second.start("click", x=2, y=2), third.start("click", x=3, y=3) + self.assertEqual(sorted([value(second.result(a))["steps_left"], value(third.result(b))["steps_left"]]), [0, 1]) + self.assertEqual(self.evaluated(), 3) + + def test_a_connection_whose_input_ends_is_closed_once_its_calls_are_answered(self): + c = self.start(steps=3, pause=0.3) + rid = c.start("click", x=1, y=1) + c.sock.shutdown(socket.SHUT_WR) + self.assertEqual(value(c.result(rid))["steps_left"], 2) + c.sock.settimeout(5) + self.assertEqual(c.sock.recv(10), b"") + + +class CallLog(Port): + def test_every_tool_call_is_recorded_with_its_tool_times_and_failure_and_nothing_else(self): + c = self.start(steps=2, pause=0.3) + c.call("info") + c.call("screenshot") + c.call("python", code="print('a-secret-text')") + c.call("click", x=1, y=1) + c.call("click", x=99999, y=1) + c.ask("tools/call", {"name": "nope"}) + c.call("done") + c.call("screenshot") + lines = self.logged() + self.assertEqual([(x["tool"], x["isError"]) for x in lines], + [("info", False), ("screenshot", False), ("python", False), ("click", False), ("click", True), + (None, True), ("done", False), ("screenshot", True)]) + for x in lines: + self.assertEqual(set(x), {"tool", "start", "end", "isError"}) + self.assertLessEqual(x["start"], x["end"]) + self.assertGreaterEqual(lines[3]["end"] - lines[3]["start"], 0.3) # held while the loop ran it + with open(self.log) as f: + self.assertNotIn("a-secret-text", f.read()) + + +RUNNER_CHILD = """ +import os, signal, sys, threading, time +sys.path.insert(0, sys.argv[1]) +from nullius.standard import computer_port as cp +mode = sys.argv[2] +stop = cp.Stop(float(sys.argv[3]), last=lambda: print("last resort", flush=True)) +stop.install() +if mode == "lifeline": + threading.Thread(target=cp.lifeline, args=(int(sys.argv[4]),), daemon=True).start() + print("waiting", flush=True) + try: + time.sleep(30) + finally: + print("stopped", flush=True) +elif mode == "deferred": + try: + with stop.deferred(): + os.kill(os.getpid(), signal.SIGTERM) + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(0.3) + print("part ended", flush=True) + print("after the part", flush=True) + finally: + print("cleaned up", flush=True) +elif mode == "hung": + with stop.deferred(): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(30) +""" + + +class Runner(unittest.TestCase): + """The ready file, the lifeline and the one stop, each in a child process of the test's.""" + + def child(self, *args, pass_fds=()): + p = subprocess.Popen([sys.executable, "-c", RUNNER_CHILD, APP, *args], stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, pass_fds=pass_fds, start_new_session=True) + self.addCleanup(lambda: (p.poll() is None and os.killpg(p.pid, signal.SIGKILL), p.wait())) + return p + + def test_the_ready_file_is_the_users_alone_and_written_whole(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + old = os.umask(0o022) + try: + cp.ready(os.path.join(tmp, "ready.json"), ok=True, password="pw") + finally: + os.umask(old) + self.assertEqual(os.stat(os.path.join(tmp, "ready.json")).st_mode & 0o777, 0o600) + self.assertEqual(os.listdir(tmp), ["ready.json"]) + with open(os.path.join(tmp, "ready.json")) as f: + self.assertEqual(json.load(f), {"ok": True, "password": "pw"}) + + def test_the_lifelines_end_stops_the_runner_once(self): + read, write = os.pipe() + p = self.child("lifeline", "30", str(read), pass_fds=(read,)) + os.close(read) + ready = select.select([p.stdout], [], [], 10)[0] # a bounded wait: the child may stall + self.assertTrue(ready and os.read(p.stdout.fileno(), 8) == b"waiting\n", "the child did not start") + os.close(write) + out, _ = p.communicate(timeout=10) + self.assertEqual((p.returncode, out), (1, b"stopped\n")) + + def test_a_stop_during_a_deferred_part_lands_at_its_end_and_a_second_does_not_cut_it(self): + p = self.child("deferred", "30") + out, _ = p.communicate(timeout=10) + self.assertEqual((p.returncode, out), (1, b"part ended\ncleaned up\n")) + + def test_a_clean_up_that_hangs_takes_the_last_resort(self): + p = self.child("hung", "0.3") + t0 = time.monotonic() + out, err = p.communicate(timeout=10) + self.assertEqual((p.returncode, out), (1, b"last resort\n")) + self.assertLess(time.monotonic() - t0, 5) + self.assertIn(b"did not end within 0.3 s of the stop", err) + + +OSWORLD = os.environ.get("OSWORLD_DIR", "") +OSWORLD_PYTHON = os.environ.get("OSWORLD_PYTHON", "") +LOOP = ("lib_run_single.py", "lib_results_logger.py", "lib_run_errors.py", "mm_agents") +WHY = ("OSWORLD_DIR does not name a checkout of the upstream" if not all(os.path.exists(os.path.join(OSWORLD, n)) for n in LOOP) + else "OSWORLD_PYTHON does not name the upstream's Python" if not os.path.isfile(OSWORLD_PYTHON) + else "") +UPSTREAM_CHILD = """ +import argparse, os, sys, tempfile, threading +from nullius.standard import computer_port as cp +import lib_run_single +from desktop_env.desktop_env import DesktopEnv + +sock, ready, log, results, steps, lifeline = sys.argv[1:7] +env = DesktopEnv(path_to_vm="vm", action_space="pyautogui", provider_name="docker", screen_size=(1920, 1080), + headless=True, os_type="Ubuntu", require_a11y_tree=False, client_password="", enable_proxy=False) + + +class Desktop: # what an entry gives: one line each over the upstream's controllers + def python(self, code): + return env.controller.execute_python_command(code) + + def upload(self, path, data): + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(data) + try: + env.setup_controller._upload_file_setup([{"local_path": f.name, "path": path}]) + finally: + os.unlink(f.name) + + def read(self, path): + return env.controller.get_file(path) + + def capture(self): + return env.controller.get_screenshot() + + def password(self): + return env.client_password + + +bridge = cp.Bridge(Desktop(), int(steps), 0.1, (1920, 1080), 90, on_first=lambda: cp.ready(ready, ok=True)) +server = cp.Server(bridge, sock, log) +threading.Thread(target=cp.lifeline, args=(int(lifeline), bridge), daemon=True).start() +example = {"id": "task-1", "instruction": "Open the terminal.", "config": [], "evaluator": {"func": "check"}} +where = os.path.join(results, "os", "task-1") +os.makedirs(where) +try: + lib_run_single.run_single_example(bridge, env, example, int(steps), example["instruction"], + argparse.Namespace(sleep_after_execution=0.1, result_dir=results), where, []) +finally: + bridge.finish() + server.close() + env.close() +""" + + +@unittest.skipIf(WHY, WHY) +class UpstreamLoop(unittest.TestCase): + """OSWorld's own run_single_example, unmodified, driven to its end with the bridge as its agent, the fake + DesktopEnv of fixtures/osworld1_fake_desktop as its environment and a scripted client as the agent. It needs the + upstream's loop ($OSWORLD_DIR: lib_run_single.py, the two modules it imports and the agents' package one of its + lines imports) and a Python that holds its packages ($OSWORLD_PYTHON). The loop's fixed waits are recorded, not + slept (the fixture's sitecustomize.py).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + up = os.path.join(self.tmp, "upstream") + os.makedirs(up) + for name in LOOP: + os.symlink(os.path.join(OSWORLD, name), os.path.join(up, name)) + fake = os.path.join(FIXTURES, "osworld1_fake_desktop") + shutil.copytree(os.path.join(fake, "desktop_env"), os.path.join(up, "desktop_env")) + shutil.copyfile(os.path.join(fake, "sitecustomize.py"), os.path.join(up, "sitecustomize.py")) + folder = os.path.join(self.tmp, "socket") + os.mkdir(folder, 0o700) + self.socket, self.ready = os.path.join(folder, "s"), os.path.join(folder, "ready.json") + self.log, self.events = os.path.join(self.tmp, "calls.jsonl"), os.path.join(self.tmp, "desktop.jsonl") + self.results = os.path.join(self.tmp, "results") + self.up = up + + def run_unit(self, steps): + read, self.lifeline = os.pipe() + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": self.tmp, "LANG": "C.UTF-8", + "TMPDIR": tempfile.gettempdir(), "PYTHONPATH": os.pathsep.join([self.up, APP]), + "PYTHONDONTWRITEBYTECODE": "1", "FAKE_DESKTOP_LOG": self.events} + p = subprocess.Popen([OSWORLD_PYTHON, "-c", UPSTREAM_CHILD, self.socket, self.ready, self.log, self.results, + str(steps), str(read)], cwd=self.tmp, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, pass_fds=(read,), start_new_session=True) + os.close(read) + self.addCleanup(lambda: (p.poll() is None and os.killpg(p.pid, signal.SIGKILL), p.wait(), + p.stdout.close())) + self.addCleanup(os.close, self.lifeline) + deadline = time.monotonic() + 60 + while not os.path.exists(self.ready): + if p.poll() is not None: + self.fail("the unit ended before it was ready: " + p.stdout.read().decode()[-3000:]) + self.assertLess(time.monotonic(), deadline, "not ready within 60 s") + time.sleep(0.1) + c = Client(self.socket) + self.addCleanup(c.close) + c.initialize() + return p, c + + def ended(self, p): + out, _ = p.communicate(timeout=60) + self.assertEqual(p.returncode, 0, out.decode()[-3000:]) + task = os.path.join(self.results, "os", "task-1") + with open(os.path.join(task, "result.txt")) as f: + result = f.read() + with open(os.path.join(task, "traj.jsonl")) as f: + traj = [json.loads(line) for line in f] + with open(self.events) as f: + events = [json.loads(line) for line in f] + return result, traj, events + + def test_the_upstreams_loop_runs_the_agents_steps_to_done(self): + p, c = self.run_unit(steps=5) + self.assertEqual(image(c.call("screenshot")), PNG) + self.assertEqual(value(c.call("click", x=10, y=20))["steps_left"], 4) + self.assertEqual(value(c.call("step", actions=["pyautogui.press('enter')"], response="enter"))["steps_left"], 3) + self.assertEqual(value(c.call("done")), {"steps_left": 2, "done": True}) + result, traj, events = self.ended(p) + self.assertEqual(result, "1.0\n") + self.assertEqual([(t["step_num"], t["action"], t["response"]) for t in traj], + [(1, "pyautogui.click(10, 20, clicks=1, button='left')", ""), + (2, "pyautogui.press('enter')", "enter"), (3, "DONE", "")]) + self.assertEqual([e["action"] for e in events if e["event"] == "step"], + ["pyautogui.click(10, 20, clicks=1, button='left')", "pyautogui.press('enter')", "DONE"]) + self.assertEqual([e["seconds"] for e in events if e["event"] == "wait"], [60, 20]) + with open(self.log) as f: + self.assertEqual([json.loads(line)["tool"] for line in f], ["screenshot", "click", "step", "done"]) + + def test_a_turn_that_ends_with_no_done_has_the_desktop_evaluated_as_it_is(self): + p, c = self.run_unit(steps=4) + c.call("click", x=10, y=20) + os.write(self.lifeline, cp.TURN_END + b"\n") + result, traj, events = self.ended(p) + self.assertEqual(result, "0.0\n") # the fake gives 1 only to a task ended by DONE + self.assertEqual([t["action"] for t in traj], ["pyautogui.click(10, 20, clicks=1, button='left')"]) + self.assertEqual([e["last"] for e in events if e["event"] == "evaluate"], + ["pyautogui.click(10, 20, clicks=1, button='left')"]) + + +if __name__ == "__main__": + unittest.main() From 54027f1d77f9391edde4412e5062e0f66235d69e Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:13:51 +0100 Subject: [PATCH 04/36] docs(standard): the computer port's tools and rules A page for agent.computer/v1: its fifteen tools, what a step is, the step budget, the turn's end, uploads by content, the call log, and what a desktop benchmark's runner gives the shared code. The standard's row of the port links to it, and the docs folder map names ports/. --- docs/README.md | 1 + docs/ports/agent-computer-v1.md | 73 +++++++++++++++++++++++++++++++++ docs/standard.md | 2 +- 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/ports/agent-computer-v1.md diff --git a/docs/README.md b/docs/README.md index 74665cfd..fcd002e3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,6 +38,7 @@ Nullius runs AI evaluations on your machines and opens every score into the evid | Path | Holds | | --- | --- | | `settings/`, `cli/`, `benchmarks/`, `agents/`, `capabilities/` | The sections above | +| `ports/` | A port's tools and rules, a page each, linked from the standard's table of ports | | `releases/` | Reviewed notes for each release | | `examples/new-benchmark/` | A worked Harbor entry to copy | | `assets/` | The screenshots these pages use | diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md new file mode 100644 index 00000000..6403a14c --- /dev/null +++ b/docs/ports/agent-computer-v1.md @@ -0,0 +1,73 @@ +# The computer port: `agent.computer/v1` + +A desktop the agent sees in screenshots and drives with pointer and keyboard actions. A desktop benchmark's runner serves one MCP server, `computer`, to the agent's session; its tools are the desktop's basic operations. The agent is a client alone. An agent with a driver of its own installs it with `upload` and `python` and sends its actions with `step`. Any other agent uses the pointer and keyboard tools. Every agent gets the same tool list. + +Coordinates are pixels of the screen, from its top left corner. A result's text is JSON. + +## Reading + +None of these is a step. + +| Tool | Arguments | Result | +| --- | --- | --- | +| `info` | none | `{screen: [width, height], steps, steps_left, pause, python_seconds, password}` | +| `screenshot` | none | An image: the screen as PNG, at its own size | +| `read` | `path`, in the desktop | `{data}`: base64, or null when there is no such file. A file over 12 MiB is an error | +| `python` | `code` | The desktop's reply, `{status, output, error, returncode}`; an error when it gave none within `python_seconds` | +| `upload` | `path`, in the desktop; `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's | `{size}`: the file's size so far | + +`python` runs as the desktop's user, in the desktop's Python. It is for reading the desktop and for the agent's own programs there. + +## Acting + +One call is one step. + +| Tool | Arguments | What the benchmark runs | +| --- | --- | --- | +| `click` | `x`, `y`; `button`: `left`, `right` or `middle`, default `left`; `count`: 1 to 3, default 1 | A click at the point | +| `move` | `x`, `y` | The pointer moved to the point | +| `drag` | `x`, `y`, `to_x`, `to_y`; `button`, default `left` | A press at the first point, a move, a release at the second | +| `scroll` | `x`, `y`; `dy`, and `dx` with default 0, in wheel clicks: `dy` down, `dx` right, negative the other way | The wheel turned at the point | +| `type` | `text` | The text typed key by key | +| `key` | `keys`: a list pressed together, such as `["ctrl", "s"]` | The chord | +| `wait` | none | No action, the pause alone | +| `step` | `actions`: a list of texts, each Python run in the desktop with `pyautogui` and `time` imported, or `WAIT`; `response`, the agent's words, optional; `screenshot`, default false | The actions in order, as one step. An empty list is a step with no action | +| `done` | `message`, optional | The task is complete | +| `fail` | `reason`, optional | The task cannot be done | + +An acting call answers once its actions have run and the benchmark's pause has passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. A pointer or keyboard tool and `wait` add the screen after the pause as an image; `step` adds it when `screenshot` is true. `done` and `fail` answer as the benchmark takes them. + +## Rules + +| Rule | Detail | +| --- | --- | +| A step | One call of an acting tool; the benchmark's own loop counts these calls. Wrong arguments are refused at once, and the call is no step | +| The limits | In `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result | +| The step budget | The benchmark's loop stops at its budget. After the last step, every tool but `info` answers with an error, and the benchmark evaluates the desktop as it is. An acting call sent while every step is taken is refused at once | +| Model calls | A benchmark may bound the agent's model calls at its step budget | +| The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call, and each step left passes with no action | +| Input | Pointer and keyboard input goes through the acting tools. Input sent through `python` is outside the port | +| One at a time | The server runs one desktop operation at a time, in the order the calls arrive. A call sent during a step waits for it | +| Before the desktop is ready | `initialize`, `tools/list` and `info` answer at once; any other call waits. The session starts once the desktop is ready | +| Errors | A call the desktop failed answers with `isError` and the desktop's words. The agent may call again | +| Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first. Nothing else passes from one unit to the next | +| Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A connection whose input has ended is closed once its calls are answered | +| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records. No argument and no result is recorded | +| More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | + +## For a benchmark's runner + +The shared code is `app/nullius/standard/computer_port.py`, with the standard library alone, and `computer_pipe.py` beside it. + +| Piece | What the runner gives or does | +| --- | --- | +| `Desktop` | Five methods over its desktop: `python(code)`, `upload(path, data)`, `read(path)`, `capture()`, `password()` | +| `Bridge` | Handed to the benchmark's own loop as its agent: `reset` does nothing, and each `predict` returns the actions of the agent's next acting call. Given the loop's step budget and pause, the screen's size and `python_seconds`; `finish` once the loop has returned | +| `Server` | The MCP server at a Unix socket in a folder of the user's alone, with the path of the call log | +| `Held` | The run's folder for files sent once, when the runner keeps one | +| The session's server | Setup's `mcp` entry runs `computer_pipe.py SOCKET`, which joins the session's stdio to the socket | +| `ready`, `lifeline`, `Stop` | The ready file, written at the loop's first `predict`; the lifeline from the driver, whose line `turn-end` is the turn's end and whose close is a stop; the one stop, with its last resort | + +## The kit + +None yet. Until it exists, no agent offers this port. diff --git a/docs/standard.md b/docs/standard.md index f6f0068b..6232f65a 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -31,7 +31,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark** | `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said | | `agent.search/v1` | Search an HTTP collection with the agent's own tools, retrieve a source and answer with evidence | Find a source, choose the current one, follow a reference (3 checks) | | `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark | -| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions | Defined; no kit yet | +| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions, through [the tools of an MCP server the benchmark's runner serves](ports/agent-computer-v1.md) | Defined; no kit yet | | `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential | - **The last message** follows ACP `messageId` boundaries and other events (a tool call, a notice). Without IDs, session metadata parts messages but never erases the final answer. From 570cb40dfbafa3cf34bb8bde92d6fca254ea2d22 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:20:36 +0100 Subject: [PATCH 05/36] feat(standard): add the computer kit's desktop, fixture and first verdicts The computer port's conformance kit needs a small real desktop on which an agent shows that it sees the screen, points and types, decided by a program. This adds the desktop and the fixture the agent acts on; the kit's driver, runner and standard.json come with the port's server. - desktop.Dockerfile: an environment of the kit on the base image, tagged nullius/env-agent-computer-v1-desktop: as container.env_image tags any environment. Xvfb at 1920 x 1080 on :0, Openbox, a session bus for uid 1000, Python 3 with Tk, pyautogui and Pillow pinned by version and hash (desktop-requirements.txt). Two users: user (uid 1000, no sudo) and fixture. - session.py: the container's command; the display, its cookie, the bus and the window manager, then a ready file. - desktop.py: start a desktop (no network, no new privileges), the five operations a benchmark's side gives the port's server (python, upload, read, capture, password), one action run as user with its start and end read on the desktop's clock by a root process, the fixture's start and log, and removal. - fixture.py: a Tk program that draws codes and labels as bitmaps, with Tk's send removed, and logs each press, release, scroll and key with its time and what it hit, in a folder only fixture reads. - checks.py: the values each check draws, and the verdicts of see, click and type from the fixture's log, the actions' times and the last message; click and type fail on any input outside an action's time. - pyproject.toml: the kits' Dockerfiles and requirement files go in the package. Tests: the verdicts on hand-written logs, the action's times taken from the desktop's reply, the image's declaration; the docker tier starts real desktops and judges a click and a word sent by actions right, and the same click sent through python wrong. --- .../standard/kits/agent-computer-v1/checks.py | 113 ++++++ .../desktop-requirements.txt | 27 ++ .../kits/agent-computer-v1/desktop.Dockerfile | 20 ++ .../kits/agent-computer-v1/desktop.py | 257 ++++++++++++++ .../kits/agent-computer-v1/fixture.py | 206 +++++++++++ .../kits/agent-computer-v1/session.py | 107 ++++++ app/tests/test_computer_kit.py | 333 ++++++++++++++++++ pyproject.toml | 2 +- 8 files changed, 1064 insertions(+), 1 deletion(-) create mode 100644 app/nullius/standard/kits/agent-computer-v1/checks.py create mode 100644 app/nullius/standard/kits/agent-computer-v1/desktop-requirements.txt create mode 100644 app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile create mode 100644 app/nullius/standard/kits/agent-computer-v1/desktop.py create mode 100644 app/nullius/standard/kits/agent-computer-v1/fixture.py create mode 100644 app/nullius/standard/kits/agent-computer-v1/session.py create mode 100644 app/tests/test_computer_kit.py diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py new file mode 100644 index 00000000..ce2b5c07 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -0,0 +1,113 @@ +"""The computer kit's first checks: what each one draws, and its verdict. + +A verdict is a function of three records and nothing else: the fixture's log +(``fixture.py``: one dict an event, ``kind`` and ``t`` on the desktop's +clock), the actions' times (``desktop.Desktop.act``: one dict an action, +``start`` and ``end`` on the same clock) and the agent's last message. + +- ``see``: the last message holds the code the window drew; +- ``click``: the log holds a left press on the named button and on none of + the five others; +- ``type``: the field held exactly the word at the last Enter. + +``click`` and ``type`` pass only when every pointer and key event lies +inside an action's time: input sent any other way, through the desktop's +Python for one, fails them. An event counts as inside from the action's start +to ``SLACK`` seconds after its end, the time the fixture may take to take an +event in once the action has sent it. The kit's pause after each action is +longer, so nothing sent after an action has answered falls in it. +""" +import re +import secrets + +SLACK = 0.5 +CODE_LENGTH = 12 +ALPHABET = "ACDEFGHJKMNPRTWXY34679" # no pair a glance confuses: 0/O, 1/I/L, 5/S, 2/Z, 8/B, U/V +LABELS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "IVORY", "JUMBO", + "KNOT", "LEMON", "MAPLE", "NORTH", "OCEAN", "PIANO", "QUILT", "RIVER", "STONE", "TIGER", + "UMBRA", "VELVET", "WHEAT", "YEAST") +POINTER = ("press", "release", "scroll") +KEYS = ("key", "keyup") +CHECKS = ("see", "click", "type") + + +def word(rng=None, length=8): + """A pronounceable word that is no word: consonant, vowel, and again.""" + rng = rng or secrets.SystemRandom() + return "".join(rng.choice("bdfgklmnprstvz" if i % 2 == 0 else "aeiou") for i in range(length)) + + +def draw(check, rng=None): + """What the fixture draws for ``check`` (its stdin) and what the verdict + expects: ``(values, expected)``. Nothing the verdict expects is in the + values but what the screen shows.""" + rng = rng or secrets.SystemRandom() + if check == "see": + code = "".join(rng.choice(ALPHABET) for _ in range(CODE_LENGTH)) + return {"screen": "see", "code": code}, {"code": code} + if check == "click": + labels = rng.sample(LABELS, 6) + return {"screen": "click", "labels": labels}, {"target": rng.choice(labels), "labels": labels} + if check == "type": + return {"screen": "type"}, {"word": word(rng)} + raise ValueError(f"no such check: {check!r}") + + +def inside(t, actions, slack=SLACK): + """Whether the time ``t`` lies in one action's time.""" + return any(a["start"] <= t <= a["end"] + slack for a in actions) + + +def outside(events, actions, slack=SLACK): + """The pointer and key events that lie in no action's time.""" + return [e for e in events if e.get("kind") in POINTER + KEYS and not inside(e.get("t", -1.0), actions, slack)] + + +def _flat(text): + return re.sub(r"[^A-Za-z0-9]", "", str(text or "")).upper() + + +def _verdict(why): + return {"passed": not why, "why": why} + + +def _input_rule(events, actions): + stray = outside(events, actions) + if stray: + first = stray[0] + return f"{len(stray)} input event(s) outside every action, the first a {first['kind']} at {first.get('t')}" + return "" + + +def see(expected, events, actions, message): + code = expected["code"] + return _verdict("" if code in _flat(message) else "the last message does not hold the code") + + +def click(expected, events, actions, message): + target = expected["target"] + presses = [e for e in events if e.get("kind") == "press" and e.get("button") == 1] + hits = {e.get("hit") for e in presses} + if target not in hits: + return _verdict(f"no left press on the button {target}") + others = sorted(h for h in hits if h != target and h in expected.get("labels", ())) + if others: + return _verdict(f"a press on another button as well: {', '.join(others)}") + return _verdict(_input_rule(events, actions)) + + +def type_word(expected, events, actions, message): + entered = [e for e in events if e.get("kind") == "enter"] + if not entered: + return _verdict("Enter was never pressed in the field") + if entered[-1].get("value") != expected["word"]: + return _verdict(f"the field held {entered[-1].get('value')!r} at the last Enter") + return _verdict(_input_rule(events, actions)) + + +VERDICTS = {"see": see, "click": click, "type": type_word} + + +def verdict(check, expected, events, actions, message): + """``{passed, why}`` for one check; ``why`` is empty when it passed.""" + return VERDICTS[check](expected, events, actions, message) diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop-requirements.txt b/app/nullius/standard/kits/agent-computer-v1/desktop-requirements.txt new file mode 100644 index 00000000..22fd76ee --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/desktop-requirements.txt @@ -0,0 +1,27 @@ +# The desktop's Python packages: pyautogui and Pillow, and what pyautogui needs on Linux, +# each pinned by version and by the sha256 of its file on PyPI. +pyautogui==0.9.54 \ + --hash=sha256:dd1d29e8fd118941cb193f74df57e5c6ff8e9253b99c7b04f39cfc69f3ae04b2 +pillow==12.0.0 \ + --hash=sha256:b87843e225e74576437fd5b6a4c2205d422754f84a06942cfaf1dc32243e45a8 \ + --hash=sha256:82240051c6ca513c616f7f9da06e871f61bfd7805f566275841af15015b8f98d \ + --hash=sha256:55f818bd74fe2f11d4d7cbc65880a843c4075e0ac7226bc1a23261dbea531953 \ + --hash=sha256:2fa5f0b6716fc88f11380b88b31fe591a06c6315e955c096c35715788b339e3f +python3-xlib==0.15 \ + --hash=sha256:dc4245f3ae4aa5949c1d112ee4723901ade37a96721ba9645f2bfa56e5b383f8 +pymsgbox==2.0.1 \ + --hash=sha256:5de8ec19bca2ca7e6c09d39c817c83f17c75cee80275235f43a9931db699f73b \ + --hash=sha256:98d055c49a511dcc10fa08c3043e7102d468f5e4b3a83c6d3c61df722c7d798d +pytweening==1.2.0 \ + --hash=sha256:243318b7736698066c5f362ec5c2b6434ecf4297c3c8e7caa8abfe6af4cac71b +pyscreeze==1.0.1 \ + --hash=sha256:cf1662710f1b46aa5ff229ee23f367da9e20af4a78e6e365bee973cad0ead4be +pygetwindow==0.0.9 \ + --hash=sha256:17894355e7d2b305cd832d717708384017c1698a90ce24f6f7fbf0242dd0a688 +pyrect==0.2.0 \ + --hash=sha256:f65155f6df9b929b67caffbd57c0947c5ae5449d3b580d178074bffb47a09b78 +mouseinfo==0.1.3 \ + --hash=sha256:2c62fb8885062b8e520a3cce0a297c657adcc08c60952eb05bc8256ef6f7f6e7 +pyperclip==1.11.0 \ + --hash=sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273 \ + --hash=sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6 diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile b/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile new file mode 100644 index 00000000..6520f834 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile @@ -0,0 +1,20 @@ +# The computer kit's desktop, a layer on Nullius's base image (Nullius writes the FROM line): +# a virtual X display at 1920 x 1080 on :0, the Openbox window manager, a session bus for uid 1000, +# Python 3 with Tk, and pyautogui and Pillow pinned by version and hash. +# Two users: `user` (uid 1000), as whom the agent's Python and its actions run, with no sudo; +# `fixture`, as whom the kit's fixture runs, its log in a folder only it reads. +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + xvfb openbox dbus python3-tk python3-setuptools python3-wheel fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* +COPY desktop-requirements.txt /opt/desktop/requirements.txt +RUN uv pip install --system --break-system-packages --no-cache --require-hashes --no-build-isolation \ + -r /opt/desktop/requirements.txt +# Ubuntu's image has a user at uid 1000 already: it becomes `user` +RUN (userdel -r ubuntu 2>/dev/null || true) \ + && groupadd -g 1000 user && useradd -u 1000 -g 1000 -m -d /home/user -s /bin/bash user \ + && groupadd -g 1001 fixture && useradd -u 1001 -g 1001 -M -d /var/lib/fixture -s /usr/sbin/nologin fixture \ + && install -d -o fixture -g fixture -m 0700 /var/lib/fixture \ + && install -d -m 0755 /run/desktop +COPY session.py fixture.py /opt/desktop/ +CMD ["python3", "-I", "/opt/desktop/session.py"] diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.py b/app/nullius/standard/kits/agent-computer-v1/desktop.py new file mode 100644 index 00000000..e75df7bb --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.py @@ -0,0 +1,257 @@ +"""The computer kit's desktop: one container of the kit's image, a small X +desktop the agent sees and acts on, and the five operations a benchmark's +side gives the port's server. + +**The image** is an environment of the kit, as a benchmark's environments +are (``container.env_image``): ``desktop.Dockerfile`` on Nullius's base +image, its build context the three files :data:`KIT` lists, tagged +``nullius/env-agent-computer-v1-desktop:`` by the sha256 of that text +and those files, and built the first time it is asked for (:func:`image`). + +**A desktop** (:meth:`Desktop.start`) is that image run with the engine's +init, no network and no new privileges; its command (``session.py``) starts +the X display, the session bus and the window manager, and the desktop is +ready once the window manager holds the display. Everything after is +``docker exec``: + +- :meth:`Desktop.python` runs code as ``user`` in the desktop's Python and + answers ``{status, output, error, returncode}``, or None when it did not + end within its bound (it is killed then); +- :meth:`Desktop.upload` and :meth:`Desktop.read` write and read a file as + ``user``; :meth:`Desktop.capture` is the screen as PNG bytes; + :meth:`Desktop.password` is ``""``: ``user`` has no password and no sudo; +- :meth:`Desktop.act` runs one action, Python text with ``pyautogui`` and + ``time`` imported, as ``user``, and keeps its start and end, read on the + desktop's clock by the root process that runs it: the agent's code never + writes them. + +The kit's own reads go apart: :meth:`Desktop.start_fixture` and +:meth:`Desktop.events` run as ``fixture``, whose folder ``user`` cannot read. +Standard library only, and the engine's program. +""" +from __future__ import annotations + +import json +import os +import secrets +import subprocess +from typing import Dict, List, Optional + +from nullius.standard import container + +HERE = os.path.dirname(os.path.abspath(__file__)) +ENVIRONMENT = "desktop" +KIT = {"key": "agent-computer-v1", "_here": HERE, + "environments": {ENVIRONMENT: {"dockerfile": "desktop.Dockerfile", + "files": ["desktop-requirements.txt", "fixture.py", "session.py"]}}} +SCREEN = (1920, 1080) +USER = "1000:1000" +FIXTURE = "1001:1001" +ROOT = "0:0" +HOME = "/home/user" +FIXTURE_LOG = "/var/lib/fixture/events.jsonl" +PYTHON_SECONDS = 90 +START_SECONDS = 60 +OUTPUT_LIMIT = 4 * 1024 * 1024 # of each of output and error, as the reply gives them +ACTION_PREFIX = "import pyautogui\nimport time\npyautogui.FAILSAFE = False\n" + +# Run as root in the desktop: the request on stdin, the code as `user`, the times around it. +_RUN = r""" +import json, os, signal, subprocess, sys, tempfile, time +req = json.loads(sys.stdin.buffer.read()) +env = {"HOME": "/home/user", "USER": "user", "LOGNAME": "user", "SHELL": "/bin/bash", "LANG": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "DISPLAY": ":0", + "XDG_RUNTIME_DIR": "/run/user/1000", "DBUS_SESSION_BUS_ADDRESS": "unix:path=/run/user/1000/bus"} +out, err = tempfile.TemporaryFile(), tempfile.TemporaryFile() +start = time.time() +p = subprocess.Popen(["python3", "-c", req["code"]], user=1000, group=1000, extra_groups=[], env=env, + cwd="/home/user", stdin=subprocess.DEVNULL, stdout=out, stderr=err, start_new_session=True) +try: + rc, late = p.wait(timeout=req["seconds"]), False +except subprocess.TimeoutExpired: + os.killpg(p.pid, signal.SIGKILL) + rc, late = p.wait(), True +end = time.time() +def text(f): + f.seek(0) + return f.read(req["limit"]).decode("utf-8", "replace") +print(json.dumps({"start": start, "end": end, "timeout": late, "returncode": rc, "output": text(out), "error": text(err)})) +""" + +_UPLOAD = r""" +import os, sys +path = os.path.join(os.path.expanduser("~"), sys.argv[1]) +if os.path.dirname(path): + os.makedirs(os.path.dirname(path), exist_ok=True) +with open(path, "ab" if sys.argv[2] == "append" else "wb") as f: + f.write(sys.stdin.buffer.read()) +print(os.path.getsize(path)) +""" + +_READ = r""" +import os, sys +try: + f = open(os.path.join(os.path.expanduser("~"), sys.argv[1]), "rb") +except OSError: + sys.exit(3) +with f: + sys.stdout.buffer.write(f.read()) +""" + +_CAPTURE = r""" +import io, sys +from PIL import ImageGrab +b = io.BytesIO() +ImageGrab.grab(xdisplay=":0").save(b, "PNG") +sys.stdout.buffer.write(b.getvalue()) +""" + +_READY = "i=0; while [ ! -e /run/desktop/ready ]; do i=$((i+1)); [ $i -gt {n} ] && exit 1; sleep 0.1; done" + + +class DesktopError(Exception): + """The desktop failed a request: it is gone, or answered nothing a request answers.""" + + +def image(log_dir: Optional[str] = None) -> str: + """The desktop's image, built the first time it is asked for, on the base image.""" + return container.env_image(KIT, ENVIRONMENT, log_dir) + + +def tag() -> str: + """The desktop image's tag on this machine, built or not.""" + return container.env_tag(KIT, ENVIRONMENT, container.base_tag()) + + +def _engine(args: List[str], *, input: Optional[bytes] = None, timeout: float) -> subprocess.CompletedProcess: + try: + return subprocess.run([container.DOCKER, *args], input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, env=container.client_env()) + except FileNotFoundError: + raise DesktopError(f"{container.DOCKER} is not installed on this machine") + + +def _said(r: subprocess.CompletedProcess) -> str: + return (r.stderr or r.stdout or b"").decode("utf-8", "replace").strip()[-300:] + + +class Desktop: + """One running desktop, by its container's name.""" + + def __init__(self, name: str): + self.name = name + self.actions: List[Dict[str, float]] = [] + + @classmethod + def start(cls, image: str, name: Optional[str] = None, *, job: Optional[str] = None, + seconds: float = START_SECONDS) -> "Desktop": + """Start a desktop of ``image`` and wait until it is ready. ``job`` + labels the container as a unit's are, so a sweep of the job removes it.""" + d = cls(name or f"nullius-desktop-{secrets.token_hex(4)}") + label = ["--label", f"{container.LABEL}={job}"] if job else [] + r = _engine(["run", "-d", "--init", "--name", d.name, "--network", "none", "--security-opt", "no-new-privileges", + "--shm-size", "256m", *label, *container.capped(), image], timeout=120) + if r.returncode != 0: + raise DesktopError(f"starting the desktop {d.name}: {_said(r)}") + try: + r = _engine(["exec", d.name, "sh", "-c", _READY.format(n=int(seconds * 10))], timeout=seconds + 30) + except subprocess.TimeoutExpired: + r = None + if r is None or r.returncode != 0: + logs = _engine(["logs", "--tail", "20", d.name], timeout=30) + d.remove() + raise DesktopError(f"the desktop {d.name} was not ready in {seconds:.0f} s: {_said(logs)}") + return d + + def _exec(self, user: str, argv: List[str], *, input: Optional[bytes] = None, timeout: float, + env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess: + flags = ["-i"] if input is not None else [] + for k, v in (env or {}).items(): + flags += ["-e", f"{k}={v}"] + return _engine(["exec", *flags, "-u", user, self.name, *argv], input=input, timeout=timeout) + + def _run(self, code: str, seconds: float) -> Optional[dict]: + """``code`` run as ``user`` by the desktop's root: the times and the reply, or None past ``seconds``.""" + req = json.dumps({"code": code, "seconds": seconds, "limit": OUTPUT_LIMIT}).encode() + try: + r = self._exec(ROOT, ["python3", "-I", "-c", _RUN], input=req, timeout=seconds + 30) + except subprocess.TimeoutExpired: + return None + try: + said = json.loads(r.stdout.decode("utf-8", "replace").strip().splitlines()[-1]) + except (ValueError, IndexError): + raise DesktopError(f"the desktop {self.name} did not run the code: {_said(r)}") + return said + + def python(self, code: str, seconds: float = PYTHON_SECONDS) -> Optional[dict]: + """Run ``code`` as ``user`` in the desktop's Python: ``{status, output, + error, returncode}``, or None when it did not end within ``seconds``.""" + said = self._run(code, seconds) + if said is None or said["timeout"]: + return None + return {"status": "success", "output": said["output"], "error": said["error"], "returncode": said["returncode"]} + + def act(self, code: str, seconds: float = PYTHON_SECONDS) -> dict: + """Run one action, ``code`` with ``pyautogui`` and ``time`` imported: + ``{start, end, reply}``, the times on the desktop's clock, ``reply`` as + :meth:`python` gives it. The times are kept in :attr:`actions`, also + for an action that ran past its bound.""" + said = self._run(ACTION_PREFIX + code, seconds) + if said is None: # the desktop did not answer: it gave no time to keep + return {"start": None, "end": None, "reply": None} + self.actions.append({"start": said["start"], "end": said["end"]}) + reply = None if said["timeout"] else {"status": "success", "output": said["output"], "error": said["error"], + "returncode": said["returncode"]} + return {"start": said["start"], "end": said["end"], "reply": reply} + + def upload(self, path: str, data: bytes, *, append: bool = False) -> int: + """Write ``data`` to ``path`` as ``user`` (a relative path is under + its home), its folders made; the file's size after.""" + r = self._exec(USER, ["python3", "-I", "-c", _UPLOAD, path, "append" if append else "write"], + input=data, timeout=600, env={"HOME": HOME}) + if r.returncode != 0: + raise DesktopError(f"upload to {path}: {_said(r)}") + return int(r.stdout.decode().strip()) + + def read(self, path: str) -> Optional[bytes]: + """The bytes of ``path`` as ``user`` reads them, or None when it cannot.""" + r = self._exec(USER, ["python3", "-I", "-c", _READ, path], timeout=120, env={"HOME": HOME}) + if r.returncode == 3: + return None + if r.returncode != 0: + raise DesktopError(f"read of {path}: {_said(r)}") + return r.stdout + + def capture(self) -> bytes: + """The screen, as PNG bytes.""" + r = self._exec(ROOT, ["python3", "-I", "-c", _CAPTURE], timeout=60) + if r.returncode != 0 or not r.stdout.startswith(b"\x89PNG\r\n\x1a\n"): + raise DesktopError(f"the capture: {_said(r)}") + return r.stdout + + def password(self) -> str: + return "" + + def start_fixture(self, values: dict, seconds: float = 60) -> None: + """Start the fixture as ``fixture``, ``values`` on its stdin; returns once its window is up.""" + r = self._exec(FIXTURE, ["python3", "-I", "/opt/desktop/fixture.py"], input=json.dumps(values).encode(), + timeout=seconds, env={"DISPLAY": ":0", "HOME": "/var/lib/fixture"}) + if r.returncode != 0: + raise DesktopError(f"the fixture did not start: {_said(r)}") + + def events(self) -> List[dict]: + """The fixture's log, an event a dict; lines that are not JSON are left out.""" + r = self._exec(FIXTURE, ["cat", FIXTURE_LOG], timeout=60) + if r.returncode != 0: + return [] + out = [] + for line in r.stdout.decode("utf-8", "replace").splitlines(): + try: + out.append(json.loads(line)) + except ValueError: + pass + return out + + def remove(self) -> None: + """Remove the desktop's container, running or not.""" + _engine(["rm", "-f", self.name], timeout=120) diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py new file mode 100644 index 00000000..72a8ff3e --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -0,0 +1,206 @@ +"""The computer kit's fixture: a Tk program the agent acts on, run in the +desktop as ``fixture``. + +It reads what to draw from the kit on its stdin, as one JSON object: + +- ``{"screen": "see", "code": "..."}``: the code, drawn large; +- ``{"screen": "click", "labels": [six words]}``: six buttons, each labelled; +- ``{"screen": "type"}``: a text field. + +Codes and labels are drawn as bitmaps from the glyphs below: no widget holds +their text, and Tk's ``send`` is removed, so no other program can read them +but from the screen. The window fills the screen, so every press lands on it. + +Each press, release, scroll and key goes to ``events.jsonl`` in the fixture's +home, a folder only ``fixture`` reads, one JSON object a line: ``kind``, the +time ``t`` on the desktop's clock, the point or the key, and ``hit``, what it +landed on (a button's label, ``field`` or ``background``). The first line is +``ready``, with where each target is on the screen; Enter in the field writes +``enter`` with the field's text. + +The program leaves the caller once its window is up: it prints ``ready`` and +the window lives on, on its own, until the desktop ends. +""" +import json +import os +import sys +import time + +HOME = "/var/lib/fixture" +LOG = os.path.join(HOME, "events.jsonl") +ERRORS = os.path.join(HOME, "fixture.err") +TITLE = "Check" +BACKGROUND = "#eceff1" +INK = "#111111" + +# 5 x 7 glyphs, a row a string +_GLYPHS = """ +A .###. #...# #...# ##### #...# #...# #...# +B ####. #...# #...# ####. #...# #...# ####. +C .###. #...# #.... #.... #.... #...# .###. +D ###.. #..#. #...# #...# #...# #..#. ###.. +E ##### #.... #.... ####. #.... #.... ##### +F ##### #.... #.... ####. #.... #.... #.... +G .###. #...# #.... #.### #...# #...# .#### +H #...# #...# #...# ##### #...# #...# #...# +I .###. ..#.. ..#.. ..#.. ..#.. ..#.. .###. +J ..### ...#. ...#. ...#. ...#. #..#. .##.. +K #...# #..#. #.#.. ##... #.#.. #..#. #...# +L #.... #.... #.... #.... #.... #.... ##### +M #...# ##.## #.#.# #.#.# #...# #...# #...# +N #...# #...# ##..# #.#.# #..## #...# #...# +O .###. #...# #...# #...# #...# #...# .###. +P ####. #...# #...# ####. #.... #.... #.... +Q .###. #...# #...# #...# #.#.# #..#. .##.# +R ####. #...# #...# ####. #.#.. #..#. #...# +S .#### #.... #.... .###. ....# ....# ####. +T ##### ..#.. ..#.. ..#.. ..#.. ..#.. ..#.. +U #...# #...# #...# #...# #...# #...# .###. +V #...# #...# #...# #...# #...# .#.#. ..#.. +W #...# #...# #...# #.#.# #.#.# #.#.# .#.#. +X #...# #...# .#.#. ..#.. .#.#. #...# #...# +Y #...# #...# .#.#. ..#.. ..#.. ..#.. ..#.. +Z ##### ....# ...#. ..#.. .#... #.... ##### +0 .###. #...# #..## #.#.# ##..# #...# .###. +1 ..#.. .##.. ..#.. ..#.. ..#.. ..#.. .###. +2 .###. #...# ....# ...#. ..#.. .#... ##### +3 ##### ...#. ..#.. ...#. ....# #...# .###. +4 ...#. ..##. .#.#. #..#. ##### ...#. ...#. +5 ##### #.... ####. ....# ....# #...# .###. +6 ..##. .#... #.... ####. #...# #...# .###. +7 ##### ....# ...#. ..#.. .#... .#... .#... +8 .###. #...# #...# .###. #...# #...# .###. +9 .###. #...# #...# .#### ....# ...#. .##.. +""" +GLYPHS = {line.split()[0]: tuple(line.split()[1:]) for line in _GLYPHS.strip().splitlines()} + + +def pixels(text): + """The lit cells of ``text`` in glyph units, a blank column between characters: (columns, rows, cells).""" + cells = [] + for i, ch in enumerate(text): + for y, row in enumerate(GLYPHS[ch]): + cells += [(i * 6 + x, y) for x, c in enumerate(row) if c == "#"] + return max(len(text) * 6 - 1, 1), 7, cells + + +def bitmap(tk, master, text, scale, ink=INK, paper=BACKGROUND): + """``text`` as a Tk image, ``scale`` screen pixels a glyph cell.""" + cols, rows, cells = pixels(text) + img = tk.PhotoImage(master=master, width=cols * scale, height=rows * scale) + img.put(paper, to=(0, 0, cols * scale, rows * scale)) + for x, y in cells: + img.put(ink, to=(x * scale, y * scale, (x + 1) * scale, (y + 1) * scale)) + return img + + +class Fixture: + def __init__(self, values, log): + import tkinter as tk + self.tk, self.values, self.log_file = tk, values, log + self.root = tk.Tk() + self.root.tk.call("rename", "send", "") # no other X client may run Tcl in this one + self.root.title(TITLE) + self.root.configure(background=BACKGROUND) + self.root.attributes("-fullscreen", True) + self.images, self.names, self.field = [], {}, None + getattr(self, "_" + values["screen"])() + for seq in ("", "", "", "", ""): + self.root.bind_all(seq, self.on_event, add="+") + + def _see(self): + img = bitmap(self.tk, self.root, self.values["code"], 9) + self.images.append(img) + self.tk.Label(self.root, image=img, background=BACKGROUND, borderwidth=0).place(relx=0.5, rely=0.5, anchor="center") + + def _click(self): + frame = self.tk.Frame(self.root, background=BACKGROUND) + frame.place(relx=0.5, rely=0.5, anchor="center") + labels = self.values["labels"] + for i, label in enumerate(labels): + img = bitmap(self.tk, frame, label, 5, paper="#ffffff") + self.images.append(img) + b = self.tk.Button(frame, image=img, background="#ffffff", activebackground="#ffffff", + relief="raised", borderwidth=4, padx=36, pady=24, highlightthickness=0) + b.grid(row=i // 3, column=i % 3, padx=60, pady=60) + self.names[str(b)] = label + + def _type(self): + self.field = self.tk.Entry(self.root, font=("DejaVu Sans", 28), width=24, relief="sunken", borderwidth=3) + self.field.place(relx=0.5, rely=0.5, anchor="center") + self.names[str(self.field)] = "field" + for key in ("", ""): + self.field.bind(key, lambda e: self.write("enter", value=self.field.get()), add="+") + self.field.focus_set() + + def write(self, kind, **fields): + self.log_file.write(json.dumps({"kind": kind, "t": round(time.time(), 6), **fields}) + "\n") + self.log_file.flush() + + def on_event(self, e): + hit = self.names.get(str(e.widget), "background") + point = {"x": e.x_root, "y": e.y_root, "hit": hit} + t = getattr(e.type, "name", str(e.type)) + if t == "ButtonPress": + if e.num in (4, 5, 6, 7): + self.write("scroll", button=e.num, **point) + else: + self.write("press", button=e.num, **point) + elif t == "ButtonRelease": + if e.num not in (4, 5, 6, 7): + self.write("release", button=e.num, **point) + elif t == "MouseWheel": + self.write("scroll", delta=e.delta, **point) + elif t == "KeyPress": + self.write("key", keysym=e.keysym, char=e.char, hit=hit) + elif t == "KeyRelease": + self.write("keyup", keysym=e.keysym, hit=hit) + + def box(self, w): + return [w.winfo_rootx(), w.winfo_rooty(), w.winfo_rootx() + w.winfo_width(), w.winfo_rooty() + w.winfo_height()] + + def up(self, seconds=10.0): + """Wait until the window fills the screen, then write ``ready`` with where each target is.""" + end = time.monotonic() + seconds + self.root.update() + while time.monotonic() < end and (self.root.winfo_width() < self.root.winfo_screenwidth() + or not self.root.winfo_viewable()): + time.sleep(0.05) + self.root.update() + self.root.focus_force() + if self.field is not None: + self.field.focus_set() + self.root.update() + boxes = {label: self.box(self.root.nametowidget(path)) for path, label in self.names.items()} + self.write("ready", screen=self.values["screen"], size=[self.root.winfo_width(), self.root.winfo_height()], + boxes=boxes) + + +def main(): + values = json.loads(sys.stdin.read()) + if values.get("screen") not in ("see", "click", "type"): + sys.exit(f"fixture: no such screen: {values.get('screen')!r}") + r, w = os.pipe() + if os.fork(): # the caller's side: wait for the window, then return + os.close(w) + with os.fdopen(r) as f: + said = f.read() + print(said.strip() or "fixture: the window did not come up") + sys.exit(0 if said.startswith("ready") else 1) + os.close(r) + os.setsid() + null = os.open(os.devnull, os.O_RDWR) + err = os.open(ERRORS, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + os.dup2(null, 0) + os.dup2(err, 1) + os.dup2(err, 2) + with open(LOG, "a", buffering=1) as log: + fixture = Fixture(values, log) + fixture.up() + os.write(w, b"ready\n") + os.close(w) + fixture.root.mainloop() + + +if __name__ == "__main__": + main() diff --git a/app/nullius/standard/kits/agent-computer-v1/session.py b/app/nullius/standard/kits/agent-computer-v1/session.py new file mode 100644 index 00000000..84e4baf1 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/session.py @@ -0,0 +1,107 @@ +"""The desktop's session, the container's command, run as root under the +engine's init: the X display, then the session bus and the window manager as +``user``. Each of root, ``user`` and ``fixture`` gets the display's cookie in +its ``~/.Xauthority``, under this host's name and for any host, as +``python3-xlib`` (pyautogui's) and libX11 look it up; access control stays +off, so a client without it connects too. Once the window manager has +claimed the display it writes ``/run/desktop/ready``; it lives as long as the +display does, and its end ends the container. Standard library only, but for +the Xlib check of the window manager.""" +import os +import signal +import socket +import struct +import subprocess +import sys +import time + +DISPLAY = ":0" +SCREEN = "1920x1080x24" +UID = 1000 +HOME = "/home/user" +RUNTIME = f"/run/user/{UID}" +BUS = f"unix:path={RUNTIME}/bus" +READY = "/run/desktop/ready" +X_SOCKET = "/tmp/.X11-unix/X0" +HOMES = (("/root", 0), (HOME, UID), ("/var/lib/fixture", 1001)) + + +def user_env(): + return {"HOME": HOME, "USER": "user", "LOGNAME": "user", "SHELL": "/bin/bash", "LANG": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "DISPLAY": DISPLAY, "XDG_RUNTIME_DIR": RUNTIME, "DBUS_SESSION_BUS_ADDRESS": BUS} + + +def as_user(argv): + return subprocess.Popen(argv, user=UID, group=UID, extra_groups=[], env=user_env(), cwd=HOME, + stdin=subprocess.DEVNULL, start_new_session=True) + + +def xauthority(cookie: bytes, host: str) -> bytes: + """The display's cookie as an Xauthority file: for this host by name (FamilyLocal), and for any (FamilyWild).""" + def field(b): + return struct.pack(">H", len(b)) + b + return b"".join(struct.pack(">H", family) + field(addr) + field(b"0") + field(b"MIT-MAGIC-COOKIE-1") + field(cookie) + for family, addr in ((256, host.encode()), (0xFFFF, b""))) + + +def wait_for(what, ok, seconds=30.0): + end = time.monotonic() + seconds + while not ok(): + if time.monotonic() > end: + sys.exit(f"desktop: {what} did not come up in {seconds:.0f} s") + time.sleep(0.05) + + +def window_manager_up(): + """Whether a window manager has said so on the root window (EWMH).""" + try: + from Xlib import display + d = display.Display(DISPLAY) + try: + prop = d.screen().root.get_full_property(d.intern_atom("_NET_SUPPORTING_WM_CHECK"), 0) + return bool(prop and prop.value) + finally: + d.close() + except Exception: + return False + + +def main(): + os.makedirs("/tmp/.X11-unix", exist_ok=True) + os.chmod("/tmp/.X11-unix", 0o1777) + data = xauthority(os.urandom(16), socket.gethostname()) + for home, uid in HOMES: + path = os.path.join(home, ".Xauthority") + with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "wb") as f: + f.write(data) + os.chown(path, uid, uid) + # access control off: every local account draws and is drawn on; the only way in is the socket + xvfb = subprocess.Popen(["Xvfb", DISPLAY, "-screen", "0", SCREEN, "-auth", "/root/.Xauthority", "-nolisten", "tcp", "-ac"], + stdin=subprocess.DEVNULL) + wait_for("the X display", lambda: os.path.exists(X_SOCKET) or xvfb.poll() is not None) + if xvfb.poll() is not None: + sys.exit("desktop: Xvfb ended") + os.makedirs(RUNTIME, exist_ok=True) + os.chown(RUNTIME, UID, UID) + os.chmod(RUNTIME, 0o700) + bus = as_user(["dbus-daemon", "--session", f"--address={BUS}", "--nofork", "--nopidfile"]) + wait_for("the session bus", lambda: os.path.exists(f"{RUNTIME}/bus")) + wm = as_user(["openbox"]) + wait_for("the window manager", window_manager_up) + with open(READY, "w") as f: + f.write(f"{time.time():.6f}\n") + + def stop(*_, code=0): + for p in (wm, bus, xvfb): + if p.poll() is None: + p.terminate() + sys.exit(code) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + xvfb.wait() + stop(code=1) # the display ended on its own + + +if __name__ == "__main__": + main() diff --git a/app/tests/test_computer_kit.py b/app/tests/test_computer_kit.py new file mode 100644 index 00000000..b2730a14 --- /dev/null +++ b/app/tests/test_computer_kit.py @@ -0,0 +1,333 @@ +"""The computer kit's desktop and fixture: the verdicts of ``see``, ``click`` +and ``type`` on hand-written logs, the action's times taken from the desktop +and never from this machine, and the image declared as an environment of the +kit. The docker tier starts real desktops: the screen drawn, a click and a +word sent by actions and judged right, the same click sent through the +desktop's Python judged wrong.""" +from __future__ import annotations + +import fnmatch +import importlib.util +import json +import os +import random +import re +import secrets +import struct +import subprocess +import time +import unittest +from unittest import mock + +from nullius.standard import container, definitions +from tests import docker_tier + +KIT = os.path.join(definitions.KITS, "agent-computer-v1") +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def _module(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(KIT, file)) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +checks = _module("computer_kit_checks", "checks.py") +desktop = _module("computer_kit_desktop", "desktop.py") +fixture = _module("computer_kit_fixture", "fixture.py") + +ACTIONS = [{"start": 100.0, "end": 101.0}, {"start": 110.0, "end": 112.0}] + + +def ev(kind, t, **kw): + return {"kind": kind, "t": t, **kw} + + +def press(t, hit, button=1): + return [ev("press", t, button=button, x=10, y=10, hit=hit), ev("release", t + 0.05, button=button, x=10, y=10, hit=hit)] + + +def keys(t, text): + out = [] + for i, ch in enumerate(text): + out += [ev("key", t + i * 0.01, keysym=ch, char=ch, hit="field"), ev("keyup", t + i * 0.01 + 0.005, keysym=ch, hit="field")] + return out + + +CLICK = {"target": "OCEAN", "labels": ["APPLE", "BRICK", "OCEAN", "RIVER", "STONE", "TIGER"]} + + +class Verdicts(unittest.TestCase): + def test_see_passes_on_the_code_in_any_case_and_spacing_and_fails_without_it(self): + exp = {"code": "ACDE34679HJK"} + self.assertTrue(checks.verdict("see", exp, [], [], "The code is acde-3467 9hjk.")["passed"]) + self.assertFalse(checks.verdict("see", exp, [], [], "The code is ACDE34679HJ.")["passed"]) + self.assertFalse(checks.verdict("see", exp, [], [], "")["passed"]) + self.assertFalse(checks.verdict("see", exp, [], [], None)["passed"]) + + def test_click_passes_on_a_left_press_on_the_named_button_inside_an_action(self): + events = [ev("ready", 99.0)] + press(100.5, "OCEAN") + self.assertEqual(checks.verdict("click", CLICK, events, ACTIONS, "done"), {"passed": True, "why": ""}) + + def test_click_fails_without_the_press_or_with_another_button_or_another_mouse_button(self): + self.assertIn("no left press", checks.verdict("click", CLICK, press(100.5, "RIVER"), ACTIONS, "")["why"]) + self.assertIn("no left press", checks.verdict("click", CLICK, press(100.5, "OCEAN", button=3), ACTIONS, "")["why"]) + both = press(100.2, "RIVER") + press(110.5, "OCEAN") + self.assertIn("another button", checks.verdict("click", CLICK, both, ACTIONS, "")["why"]) + self.assertFalse(checks.verdict("click", CLICK, [], ACTIONS, "")["passed"]) + # a press on the background is no other button + self.assertTrue(checks.verdict("click", CLICK, press(100.2, "background") + press(110.5, "OCEAN"), ACTIONS, "")["passed"]) + + def test_click_fails_when_any_input_lies_outside_every_action(self): + by_python = press(105.0, "OCEAN") # between two actions + v = checks.verdict("click", CLICK, by_python, ACTIONS, "") + self.assertFalse(v["passed"]) + self.assertIn("outside every action", v["why"]) + before = press(99.0, "OCEAN") # before the first action started + self.assertFalse(checks.verdict("click", CLICK, before, ACTIONS, "")["passed"]) + stray_key = press(100.5, "OCEAN") + [ev("key", 120.0, keysym="a", char="a", hit="background")] + self.assertFalse(checks.verdict("click", CLICK, stray_key, ACTIONS, "")["passed"]) + self.assertFalse(checks.verdict("click", CLICK, press(100.5, "OCEAN"), [], "")["passed"]) + + def test_an_event_taken_in_just_after_its_action_ended_counts_inside_and_one_later_does_not(self): + late = [ev("press", 101.0 + checks.SLACK * 0.6, button=1, hit="OCEAN")] + self.assertTrue(checks.verdict("click", CLICK, late, ACTIONS, "")["passed"]) + later = [ev("press", 101.0 + checks.SLACK * 1.4, button=1, hit="OCEAN")] + self.assertFalse(checks.verdict("click", CLICK, later, ACTIONS, "")["passed"]) + self.assertLess(checks.SLACK, 1.0) # under the kit's pause after an action + + def test_type_passes_on_exactly_the_word_at_the_last_enter_inside_actions(self): + exp = {"word": "kotabiru"} + events = keys(100.1, "kotabiru") + [ev("enter", 100.5, value="kotabiru")] + self.assertEqual(checks.verdict("type", exp, events, ACTIONS, ""), {"passed": True, "why": ""}) + fixed = keys(100.1, "kotabir") + [ev("enter", 100.3, value="kotabir")] + keys(110.1, "u") + [ev("enter", 110.2, value="kotabiru")] + self.assertTrue(checks.verdict("type", exp, fixed, ACTIONS, "")["passed"]) + + def test_type_fails_on_another_text_a_later_change_no_enter_or_keys_outside_actions(self): + exp = {"word": "kotabiru"} + self.assertIn("held 'kotabirux'", checks.verdict("type", exp, [ev("enter", 100.5, value="kotabirux")], ACTIONS, "")["why"]) + self.assertFalse(checks.verdict("type", exp, [ev("enter", 100.5, value="Kotabiru")], ACTIONS, "")["passed"]) + spoiled = [ev("enter", 100.5, value="kotabiru"), ev("enter", 110.5, value="kotabiru2")] + self.assertFalse(checks.verdict("type", exp, spoiled, ACTIONS, "")["passed"]) + self.assertIn("never pressed", checks.verdict("type", exp, keys(100.1, "kotabiru"), ACTIONS, "")["why"]) + by_python = keys(105.0, "kotabiru") + [ev("enter", 105.2, value="kotabiru")] + self.assertIn("outside every action", checks.verdict("type", exp, by_python, ACTIONS, "")["why"]) + + def test_input_rule_counts_pointer_and_key_events_only(self): + events = [ev("ready", 1.0), ev("enter", 105.0, value="x"), ev("press", 100.5, button=1, hit="OCEAN")] + self.assertEqual(checks.outside(events, ACTIONS), []) + self.assertEqual([e["kind"] for e in checks.outside([ev("scroll", 105.0, button=4)], ACTIONS)], ["scroll"]) + + +class Draw(unittest.TestCase): + def test_each_check_draws_from_its_own_values_and_expects_what_the_screen_shows(self): + rng = random.Random(7) + values, exp = checks.draw("see", rng) + self.assertEqual(len(exp["code"]), checks.CODE_LENGTH) + self.assertTrue(set(exp["code"]) <= set(checks.ALPHABET)) + self.assertEqual(values, {"screen": "see", "code": exp["code"]}) + values, exp = checks.draw("click", rng) + self.assertEqual(len(set(values["labels"])), 6) + self.assertIn(exp["target"], values["labels"]) + values, exp = checks.draw("type", rng) + self.assertEqual(values, {"screen": "type"}) # the word is the prompt's, never the screen's + self.assertRegex(exp["word"], r"^[a-z]{8}$") + with self.assertRaises(ValueError): + checks.draw("drag") + + def test_every_character_drawn_has_a_glyph_of_five_by_seven(self): + for ch in checks.ALPHABET + "".join(checks.LABELS): + self.assertIn(ch, fixture.GLYPHS) + for ch, rows in fixture.GLYPHS.items(): + self.assertEqual((len(rows), {len(r) for r in rows}), (7, {5}), ch) + cols, rows, cells = fixture.pixels("AC") + self.assertEqual((cols, rows), (11, 7)) + self.assertIn((0, 3), cells) # A's bar + self.assertNotIn((5, 3), cells) # the blank column + + +class Image(unittest.TestCase): + def test_the_desktop_is_an_environment_of_the_kit_tagged_by_its_text_and_files(self): + base = "nullius/base:0123456789ab" + tag = container.env_tag(desktop.KIT, desktop.ENVIRONMENT, base) + self.assertRegex(tag, r"^nullius/env-agent-computer-v1-desktop:[0-9a-f]{12}$") + text, files = container.env_context(desktop.KIT, desktop.ENVIRONMENT, base) + self.assertTrue(text.startswith(f"FROM {base}\n")) + body = text.split("\n", 1)[1] + self.assertNotRegex(body, r"(?im)^\s*(FROM|ADD)\b") + for line in re.findall(r"(?im)^COPY\s+(.+)$", body): + for src in line.split()[:-1]: + self.assertIn(src, files) + self.assertRegex(desktop.tag(), r"^nullius/env-agent-computer-v1-desktop:[0-9a-f]{12}$") + + def test_every_python_package_is_pinned_by_version_and_hash(self): + with open(os.path.join(KIT, "desktop-requirements.txt")) as f: + reqs = re.sub(r"\\\n", " ", f.read()) + lines = [ln for ln in reqs.splitlines() if ln.strip() and not ln.startswith("#")] + names = set() + for ln in lines: + self.assertRegex(ln, r"^[A-Za-z0-9_.-]+==[0-9][^ ]*\s+(--hash=sha256:[0-9a-f]{64}\s*)+$", ln) + names.add(ln.split("==")[0].lower()) + self.assertTrue({"pyautogui", "pillow"} <= names) + + def test_the_package_carries_every_file_the_image_is_built_from(self): + with open(os.path.join(ROOT, "pyproject.toml")) as f: + line = next(ln for ln in f if ln.startswith('"nullius.standard"')) + patterns = re.findall(r'"(kits/[^"]+)"', line) + for rel in [desktop.KIT["environments"]["desktop"]["dockerfile"], *desktop.KIT["environments"]["desktop"]["files"]]: + self.assertTrue(any(fnmatch.fnmatch(f"kits/agent-computer-v1/{rel}", p) for p in patterns), rel) + + +def _engine_answering(said): + """A stand-in for the engine: every exec answers ``said`` as the desktop's root process prints it.""" + sent = [] + + def run(args, *, input=None, timeout): + sent.append((args, input)) + out = b"" if said is None else (json.dumps(said) + "\n").encode() + return subprocess.CompletedProcess(args, 0, out, b"") + return run, sent + + +class Actions(unittest.TestCase): + def test_an_action_keeps_the_times_the_desktop_read_never_this_machines(self): + said = {"start": 5000.25, "end": 5001.5, "timeout": False, "returncode": 0, "output": "ok\n", "error": ""} + run, sent = _engine_answering(said) + d = desktop.Desktop("d") + with mock.patch.object(desktop, "_engine", run), mock.patch("time.time", return_value=1.0): + got = d.act("pyautogui.click(10, 20)") + self.assertEqual(d.actions, [{"start": 5000.25, "end": 5001.5}]) + self.assertEqual(got["reply"], {"status": "success", "output": "ok\n", "error": "", "returncode": 0}) + args, payload = sent[0] + self.assertEqual(args[:4], ["exec", "-i", "-u", desktop.ROOT]) + code = json.loads(payload)["code"] + self.assertTrue(code.startswith(desktop.ACTION_PREFIX)) + self.assertTrue(code.endswith("pyautogui.click(10, 20)")) + self.assertIn("import pyautogui", code) + self.assertIn("import time", code) + + def test_python_is_no_action_and_an_action_past_its_bound_keeps_its_times(self): + said = {"start": 7.0, "end": 9.0, "timeout": True, "returncode": -9, "output": "", "error": ""} + run, _ = _engine_answering(said) + d = desktop.Desktop("d") + with mock.patch.object(desktop, "_engine", run): + self.assertIsNone(d.python("import time; time.sleep(99)", seconds=2)) + self.assertEqual(d.actions, []) + self.assertIsNone(d.act("time.sleep(99)", seconds=2)["reply"]) + self.assertEqual(d.actions, [{"start": 7.0, "end": 9.0}]) + + def test_a_desktop_that_gives_no_answer_is_an_error_and_keeps_no_time(self): + run, _ = _engine_answering(None) + d = desktop.Desktop("d") + with mock.patch.object(desktop, "_engine", run): + with self.assertRaises(desktop.DesktopError): + d.act("pyautogui.press('a')") + self.assertEqual(d.actions, []) + + def late(args, *, input=None, timeout): + raise subprocess.TimeoutExpired(args, timeout) + with mock.patch.object(desktop, "_engine", late): + self.assertIsNone(d.python("print(1)")) + self.assertIsNone(d.act("pyautogui.press('a')")["reply"]) + self.assertEqual(d.actions, []) + + +def _png_size(data): + return struct.unpack(">II", data[16:24]) + + +# What the desktop's Python reads back from its own screen: the cells of ink, a glyph cell 9 pixels square, +# counted from the ink's top left corner. +_READ_INK = r""" +import json +from PIL import Image, ImageChops, ImageGrab +im = ImageGrab.grab(xdisplay=":0").convert("RGB") +mask = ImageChops.difference(im, Image.new("RGB", im.size, (17, 17, 17))).convert("L").point(lambda v: 255 if v == 0 else 0) +x0, y0, x1, y1 = mask.getbbox() +px = mask.load() +print(json.dumps([[c, r] for r in range((y1 - y0) // 9) for c in range((x1 - x0) // 9) if px[x0 + c * 9 + 4, y0 + r * 9 + 4]])) +""" + + +def _from_corner(cells): + c0, r0 = min(c for c, _ in cells), min(r for _, r in cells) + return sorted((c - c0, r - r0) for c, r in cells) + + +PAUSE = 1.0 +_PREFIX = os.environ.get("NULLIUS_TEST_CONTAINER_PREFIX", "nullius-test") + + +@unittest.skipIf(docker_tier.why_not(desktop.tag()), docker_tier.why_not(desktop.tag()) or "") +class DockerDesktop(unittest.TestCase): + """Real desktops of the kit's image, one at a time.""" + + def setUp(self): + self.d = desktop.Desktop.start(desktop.tag(), f"{_PREFIX}-desktop-{secrets.token_hex(4)}") + self.addCleanup(self.d.remove) + + def test_the_five_operations(self): + png = self.d.capture() + self.assertEqual(_png_size(png), desktop.SCREEN) + said = self.d.python("import os, pyautogui; print(os.getuid(), os.environ['HOME'], pyautogui.size())") + self.assertEqual(said["status"], "success") + self.assertEqual(said["returncode"], 0, said["error"]) + self.assertEqual(said["output"].split()[:2], ["1000", "/home/user"]) + self.assertIn("1920", said["output"]) + self.assertEqual(self.d.python("import sys; sys.exit(4)")["returncode"], 4) + self.assertIsNone(self.d.python("import time; time.sleep(30)", seconds=2)) + self.assertEqual(self.d.upload("dir/file.bin", b"\x00\x01" * 1000), 2000) + self.assertEqual(self.d.upload("dir/file.bin", b"end", append=True), 2003) + self.assertEqual(self.d.read("/home/user/dir/file.bin"), b"\x00\x01" * 1000 + b"end") + self.assertIsNone(self.d.read("no/such/file")) + self.assertEqual(self.d.password(), "") + sudo = self.d.python("import shutil; print(shutil.which('sudo'))") + self.assertEqual(sudo["output"].strip(), "None") + + def test_the_fixture_is_read_from_the_screen_alone(self): + values, exp = checks.draw("see") + self.d.start_fixture(values) + self.assertIsNone(self.d.read(desktop.FIXTURE_LOG)) # user cannot read the log + said = self.d.python(f"open({desktop.FIXTURE_LOG!r})") + self.assertIn("PermissionError", said["error"]) + seen = self.d.python(_READ_INK) + self.assertEqual(seen["returncode"], 0, seen["error"]) + _, _, cells = fixture.pixels(exp["code"]) + self.assertEqual(_from_corner(json.loads(seen["output"])), _from_corner(cells)) + self.assertTrue(checks.verdict("see", exp, self.d.events(), self.d.actions, exp["code"])["passed"]) + + def test_a_click_by_an_action_passes_and_the_same_click_by_python_fails(self): + values, exp = checks.draw("click") + self.d.start_fixture(values) + ready = self.d.events()[0] + self.assertEqual(ready["kind"], "ready") + x0, y0, x1, y1 = ready["boxes"][exp["target"]] + x, y = (x0 + x1) // 2, (y0 + y1) // 2 + self.assertEqual(self.d.act(f"pyautogui.click({x}, {y})")["reply"]["returncode"], 0) + self.assertEqual(checks.verdict("click", exp, self.d.events(), self.d.actions, ""), {"passed": True, "why": ""}) + time.sleep(PAUSE) # the kit's pause after an action + self.d.python(f"import pyautogui; pyautogui.click({x}, {y})") + v = checks.verdict("click", exp, self.d.events(), self.d.actions, "") + self.assertFalse(v["passed"]) + self.assertIn("outside every action", v["why"]) + + def test_a_word_typed_by_an_action_passes(self): + values, exp = checks.draw("type") + self.d.start_fixture(values) + x0, y0, x1, y1 = self.d.events()[0]["boxes"]["field"] + self.d.act(f"pyautogui.click({(x0 + x1) // 2}, {(y0 + y1) // 2})") + self.d.act(f"pyautogui.write({exp['word']!r}, interval=0.02)\npyautogui.press('enter')") + events = self.d.events() + self.assertEqual(checks.verdict("type", exp, events, self.d.actions, ""), {"passed": True, "why": ""}) + self.assertEqual([e["value"] for e in events if e["kind"] == "enter"], [exp["word"]]) + + def test_removal_leaves_no_container(self): + self.d.remove() + left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"name=^{self.d.name}$"], + capture_output=True, text=True).stdout.split() + self.assertEqual(left, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 1b2a9d66..19440a7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,6 @@ dependencies = {file = ["app/requirements.txt"]} [tool.setuptools.package-data] "nullius.catalogue" = ["agents/*/*.json", "agents/*/*.py", "agents/*/*.toml", "agents/*/*.yaml", "benchmarks/*/env/*", "benchmarks/*/*.json", "benchmarks/*/*.py", "capabilities/*/*.json", "capabilities/*/*.py"] -"nullius.standard" = ["kits/*/*.json", "kits/*/*.py"] +"nullius.standard" = ["kits/*/*.json", "kits/*/*.py", "kits/*/*.Dockerfile", "kits/*/*.txt"] "nullius.judge" = ["floors.json"] _nullius_app = ["templates/**/*.html", "static/**/*.css", "static/**/*.js", "static/**/*.svg", "static/**/*.png", "static/**/*.json", "static/**/*.woff2", "static/**/*.ttf", "static/**/*.txt", "static/**/*.md", "requirements.txt", ".env.example"] From c99c850a5e1281fc5320ade127a19ae30eccf53f Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:39:45 +0100 Subject: [PATCH 06/36] fix(standard): send a unit's end only once its setup was sent A run stopped after a unit passed its stop check, and before its setup request was written, still sent that unit's end to the driver. A unit now counts as set up once its setup line is written. A request cut short by a stop leaves its late reply in the driver's queue, where a forced end would read it as its own. Such a driver is no longer sent an end. Document that a driver closed after a request got no reply, or one left with an unanswered request, is not sent an end. Test the forced wait after a stop, and replace a test whose mock passed for the wrong reason. --- app/nullius/standard/jsonl.py | 11 ++- app/nullius/standard/run.py | 18 +++-- app/tests/test_standard_driver_hooks.py | 102 ++++++++++++++++++++++-- docs/standard.md | 4 +- 4 files changed, 117 insertions(+), 18 deletions(-) diff --git a/app/nullius/standard/jsonl.py b/app/nullius/standard/jsonl.py index e4c01fd9..677e76af 100644 --- a/app/nullius/standard/jsonl.py +++ b/app/nullius/standard/jsonl.py @@ -30,6 +30,7 @@ def __init__(self, cmd: List[str], cwd: str, env: dict, log: str, stopped=None): self.stopped = stopped self.p: Optional[subprocess.Popen] = None self.lines: "queue.Queue[Optional[str]]" = queue.Queue() + self.unanswered = False # a request the stop cut short: its late reply may still come def start(self) -> "Proc": if self.stopped is not None and self.stopped.is_set(): @@ -46,8 +47,9 @@ def _read(self): self.lines.put(line) self.lines.put(None) - def request(self, op: str, timeout: float = 600, force: bool = False, **kw) -> dict: - """``force``: sent and waited for (up to ``timeout``) also once the run is stopped - a unit's clean-up.""" + def request(self, op: str, timeout: float = 600, force: bool = False, on_sent=None, **kw) -> dict: + """``force``: sent and waited for (up to ``timeout``) also once the run is stopped - a unit's clean-up. + ``on_sent``: called once the request is written, before its reply is waited for.""" if self.stopped is not None and self.stopped.is_set() and not force: raise CancelledError("the run was stopped") if self.p is None: @@ -57,11 +59,16 @@ def request(self, op: str, timeout: float = 600, force: bool = False, **kw) -> d self.p.stdin.flush() except (BrokenPipeError, OSError): raise Failed(f"{op}: the program had already ended ({self._tail()})") + if on_sent is not None: + on_sent() try: line = wait_reply(self.lines, timeout, None if force else self.stopped) except queue.Empty: self.close() raise Failed(f"{op}: no reply in {int(timeout)} s") + except CancelledError: + self.unanswered = True + raise if line is None: raise Failed(f"{op}: the program ended without replying ({self._tail()})") try: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index a70b267f..2d2e72d5 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -982,12 +982,13 @@ def setup_seconds(self) -> float: def unit_ended(self, unit: str, ws: str) -> None: """A driver that asks for it (``driver.unit_end``) is told that a unit an agent ran has ended, scored or not - also when the run was stopped - so that what its setup started for the agent ends now. The driver the unit - used is asked, never a new one; its answer is waited for ``END_SECONDS`` at most, and a failure is no - unit's.""" + used is asked, never a new one, and only while it runs with no request the stop left unanswered (that + request's late reply would be read as this one's); its answer is waited for ``END_SECONDS`` at most, and a + failure is no unit's.""" if self.other.get("_kind") != "agent" or not (self.bench.get("driver") or {}).get("unit_end"): return d = getattr(self.local, "driver", None) - if d is None or d.p is None or d.p.poll() is not None: + if d is None or d.p is None or d.p.poll() is not None or d.unanswered: return try: d.request("end", unit=unit, dir=ws, timeout=END_SECONDS, force=True) @@ -1002,7 +1003,7 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout) def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: - """One unit (:meth:`_unit`); once its setup was asked, whatever happened after, the driver is told its end + """One unit (:meth:`_unit`); once its setup was sent, whatever happened after, the driver is told its end (:meth:`unit_ended`).""" self.local.set_up = None try: @@ -1039,9 +1040,12 @@ def _unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: rl, environment, box, changes = None, None, None, None try: try: - # a setup's unit needs none: the upstream's answerer reads its data itself - self.local.set_up = None if self.upstream else (unit, ws) - setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.setup_seconds(), **self.setup_kw()) + # a setup's unit needs none: the upstream's answerer reads its data itself. The unit is set up + # once its request is written: a stop before that leaves the driver nothing to end + def sent(): + self.local.set_up = (unit, ws) + setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=self.setup_seconds(), + on_sent=sent, **self.setup_kw()) except jsonl.Failed as e: raise _Ended("EnvironmentSetupError", str(e)) environment, env_def = self.unit_environment(setup) diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py index 4398a312..dbc949ed 100644 --- a/app/tests/test_standard_driver_hooks.py +++ b/app/tests/test_standard_driver_hooks.py @@ -3,12 +3,14 @@ import json import os +from concurrent.futures import CancelledError from pathlib import Path import shutil import subprocess import tempfile import sys import threading +import time import unittest from unittest import mock @@ -285,15 +287,101 @@ def stop(r, *args): self.assertEqual(json.load(f)["exception_info"]["exception_type"], "CancelledError") self.assertEqual([d for d in os.listdir(os.path.dirname(trial)) if "__u" in d], ["a__u0000"]) - def test_an_end_never_starts_a_driver(self): + def ending(self) -> "run.Run": r = run.Run.__new__(run.Run) r.bench, r.other, r.local = dict(BENCH, driver=dict(BENCH["driver"], unit_end=True)), {"_kind": "agent"}, threading.local() - for driver in (None, mock.Mock(p=None), mock.Mock()): - with self.subTest(driver=driver): - r.local.driver = driver - r.unit_ended("a", self.tmp) - if driver is not None: - driver.request.assert_not_called() + return r + + def driver_proc(self, code: str, *args, stopped=None) -> jsonl.Proc: + here = tempfile.mkdtemp(dir=self.tmp) + return jsonl.Proc([sys.executable, "-c", code, *args], here, dict(os.environ), os.path.join(here, "driver.log"), + stopped=stopped) + + def test_an_end_never_starts_a_driver(self): + r = self.ending() + r.local.driver = None + r.unit_ended("a", self.tmp) # no driver: nothing is started + never = self.driver_proc("import sys; sys.stdin.read()") + ended = self.driver_proc("pass").start() + ended.p.wait(timeout=10) + running = self.driver_proc("import sys\nfor line in sys.stdin: print('{}', flush=True)").start() + try: + for driver, asked in ((never, False), (ended, False), (running, True)): + with self.subTest(started=driver.p is not None, asked=asked): + r.local.driver = driver + with mock.patch.object(driver, "request", wraps=driver.request) as request: + r.unit_ended("a", self.tmp) + self.assertEqual(request.called, asked) + self.assertIsNone(never.p) # a driver never started stays so + finally: + for driver in (ended, running): + driver.close() + + def test_an_end_is_waited_for_once_the_run_is_stopped(self): + here = tempfile.mkdtemp(dir=self.tmp) + mark = os.path.join(here, "ended") + answering = ("import sys, time\n" + "for line in sys.stdin:\n" + " time.sleep(1.0)\n" + " open(sys.argv[1], 'w').write('ended')\n" + " print('{}', flush=True)\n") + stopped = threading.Event() + r = self.ending() + r.local.driver = self.driver_proc(answering, mark, stopped=stopped).start() + stopped.set() + try: + r.unit_ended("a", here) + self.assertTrue(os.path.exists(mark), "returned before the driver answered its end") + finally: + r.local.driver.close() + + def test_a_driver_with_a_request_the_stop_left_unanswered_gets_no_end(self): + here = tempfile.mkdtemp(dir=self.tmp) + ops = os.path.join(here, "ops") + late = ("import sys, json, time\n" + "for line in sys.stdin:\n" + " op = json.loads(line)['op']\n" + " open(sys.argv[1], 'a').write(op + '\\n')\n" + " time.sleep(1.0)\n" + " print(json.dumps({'to': op}), flush=True)\n") + stopped = threading.Event() + r = self.ending() + d = r.local.driver = self.driver_proc(late, ops, stopped=stopped).start() + threading.Timer(0.3, stopped.set).start() + try: + with self.assertRaises(CancelledError): + d.request("setup", unit="a", dir=here, timeout=60) # its reply comes after the stop + r.unit_ended("a", here) + time.sleep(1.5) # the setup's late reply, then anything after it + finally: + d.close() + with open(ops) as f: + self.assertEqual(f.read().split(), ["setup"]) + + def test_a_stop_before_setup_is_sent_sends_no_end(self): + real_phase = run.Run.phase + + def phase(r, name, unit, ph, timeout=None): + if unit == "b" and ph == "env_setup": + r._stopped.set() # b passed the stop check; its setup is not sent yet + return real_phase(r, name, unit, ph, timeout) + + noting = ENDING_DRIVER.replace("def setup(unit, dir, **told):\n", + "def setup(unit, dir, **told):\n" + " open(os.path.join(os.path.dirname(dir), 'setup-sent'), 'w').write(unit)\n") + with mock.patch.object(run.Run, "phase", phase): + job = os.path.dirname(self.trial_of(noting.replace('["a"]', '["a", "b"]'), driver=dict(BENCH["driver"], unit_end=True))) + a, b = os.path.join(job, "a__u0000"), os.path.join(job, "b__u0001") + self.assertTrue(os.path.isfile(os.path.join(a, "ended.json"))) # a: set up and ended + self.assertFalse(os.path.exists(os.path.join(b, "setup-sent"))) + self.assertFalse(os.path.exists(os.path.join(b, "ended.json"))) + + def test_a_setup_past_its_time_gets_no_end_from_its_closed_driver(self): + slow = ENDING_DRIVER.replace(' return {"prompt": "hi"}', ' import time; time.sleep(3)\n return {"prompt": "hi"}') + trial = self.trial_of(slow, driver=dict(BENCH["driver"], unit_end=True), limits={"setup_seconds": 1}) + with open(os.path.join(trial, "result.json")) as f: + self.assertEqual(json.load(f)["exception_info"]["exception_type"], "EnvironmentSetupError") + self.assertFalse(os.path.exists(os.path.join(trial, "ended.json"))) def test_the_run_selects_agent_options_by_kind(self): option = {"summary": "Steps.", "values": ["10"], "default": "10", "setups": ["agent"]} diff --git a/docs/standard.md b/docs/standard.md index 514f90ed..b940ca18 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -412,7 +412,7 @@ use the source folders and repository checks. | `answer` | `unit, dir, setup, model, base_url, key` | `{answer}`: the upstream's own answerer, for a [setup](#setups-as-published-your-model-your-agent-or-capability) | | `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | -| `end` | `{unit, dir}`, with `driver.unit_end`, once a unit an agent ran has ended, also after a stop | `{}` | +| `end` | `{unit, dir}`, with `driver.unit_end`, once a unit an agent ran has ended, also after a stop, when its `setup` was sent and the driver still runs with no request left unanswered | `{}` | | Field in `standard.json` | Meaning | | --- | --- | @@ -422,7 +422,7 @@ use the source folders and repository checks. | `limits.turns` | At most this many agent turns in a conversation (default 20) | | `limits.setup_seconds` | How long `setup` may take, in seconds, for a unit an agent runs (a setup that starts a machine of its own for an agent); every other unit keeps 600 | | `driver.participant` | `true`: `setup` is also sent `participant`, the kind of what runs the unit, for a driver that starts something only an agent uses. A driver whose `setup` takes `unit` and `dir` alone leaves it out | -| `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. A control's or a setup's unit gets none, nor does any other driver | +| `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. Only a unit whose `setup` was sent gets it, and only while that driver still runs: a driver closed after a request got no reply in its time (its process group ended with it) is not asked, nor is one whose request a stop left unanswered. A control's or a setup's unit gets none, nor does any other driver | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | From 27eaac2b9e20c0544cd53faa53d60bd72b6caffe Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:40:35 +0100 Subject: [PATCH 07/36] fix(standard): keep a unit's setup time limit a whole number The setup time limit was turned into a float, so the live feed recorded the env_setup phase's timeout as 600.0 for every run, where it had recorded 600. The limit is now kept as given. Test the env_setup phase's timeout in the live feed for an agent and a control, with and without limits.setup_seconds. --- app/nullius/standard/run.py | 4 ++-- app/tests/test_standard_driver_hooks.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 2d2e72d5..06736ad4 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -132,7 +132,7 @@ def setup_request(bench: dict, kind: str, attempt: Optional[int] = None) -> tupl sent = {"attempt": attempt} if attempt and limits.get("attempts") else {} if drv.get("participant"): sent["participant"] = kind - return sent, float((limits.get("setup_seconds") if kind == "agent" else None) or DRIVER_SECONDS) + return sent, (limits.get("setup_seconds") if kind == "agent" else None) or DRIVER_SECONDS def provider_endpoint(provider: str, env=os.environ) -> tuple: @@ -975,7 +975,7 @@ def setup_kw(self) -> dict: """What ``setup`` is told besides the unit (:func:`setup_request`).""" return setup_request(self.bench, self.other["_kind"], getattr(self.local, "attempt", None))[0] - def setup_seconds(self) -> float: + def setup_seconds(self) -> int: """How long a unit's ``setup`` may take (:func:`setup_request`).""" return setup_request(self.bench, self.other["_kind"])[1] diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py index dbc949ed..0eda0950 100644 --- a/app/tests/test_standard_driver_hooks.py +++ b/app/tests/test_standard_driver_hooks.py @@ -393,6 +393,24 @@ def test_the_run_selects_agent_options_by_kind(self): self.assertEqual(r.bench_options, {"steps": "10"} if other["_kind"] == "agent" else {}) + def env_setup_timeouts(self, other: dict, **bench) -> list: + """The ``timeout_sec`` of each unit's ``env_setup`` phase in the job's live feed, as written.""" + trial = self.trial_of(DRIVER, other, **bench) + with open(os.path.join(os.path.dirname(trial), "events.jsonl")) as f: + return [e["timeout_sec"] for e in map(json.loads, f) if e.get("phase") == "env_setup"] + + def test_the_setup_phase_takes_the_setup_limit(self): + controls = {"nop": {"source": "nullius", "from": "nothing"}} + control = definitions.control(dict(BENCH, controls=controls), "nop") + for other, limits, expected in ((self.fake_agent(), None, 600), (control, None, 600), + (self.fake_agent(), {"setup_seconds": 1800}, 1800), + (control, {"setup_seconds": 1800}, 600)): + with self.subTest(kind=other["_kind"], limits=limits): + bench = dict(controls=controls, **({"limits": limits} if limits else {})) + got = self.env_setup_timeouts(other, **bench) + self.assertEqual(got, [expected]) + self.assertIs(type(got[0]), int) # recorded as a whole number, as given + def test_setup_gets_the_benchmarks_own_limit(self): r = run.Run.__new__(run.Run) r.bench, r.other, r.local = dict(BENCH, limits={"setup_seconds": 1800}), {"_kind": "agent"}, threading.local() From acc16c76219dace7641da1533cc361d73a85e8fc Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:41:30 +0100 Subject: [PATCH 08/36] fix(standard): accept an agent-only option alone and on agent benchmarks An option whose setups named agent beside another scope passed the check, though the builder never showed it in a setup's tab. One on a capability benchmark passed too, where no agent ever runs it. The check now accepts agent only on its own, and only on a benchmark that agents run. --- app/nullius/standard/definitions.py | 8 +++++--- app/tests/test_standard_driver_hooks.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 4b702922..fc9127c5 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -484,9 +484,11 @@ def _bench_options(d: dict) -> List[str]: if ds is not None and (not isinstance(ds, list) or not ds or any(x not in ids for x in ds)): bad.append(f"{w}.datasets: some of the benchmark's dataset ids") on = o.get("setups") - if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours", "agent") for x in on)): - bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability), " - "agent (an agent alone)") + # agent: an agent's run alone, on a benchmark agents run, and never beside another scope + if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours", "agent") for x in on) + or "agent" in on and (on != ["agent"] or d.get("family") != "agent")): + bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability); " + "or agent alone (an agent's run), on a benchmark agents run") when = o.get("when") or {} if not _when_valid(name, when, opts): bad.append(f"{w}.when: another option -> the values of it this one applies under") diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py index 0eda0950..6e288e5e 100644 --- a/app/tests/test_standard_driver_hooks.py +++ b/app/tests/test_standard_driver_hooks.py @@ -100,6 +100,16 @@ def test_agent_scope_is_validated(self): self.assertTrue(any("options.steps.setups" in p for p in definitions.problems( dict(BENCH, options={"steps": dict(self.option, setups=setups)}), "benchmark"))) + def test_agent_stands_alone_on_a_benchmark_agents_run(self): + for setups in (["agent", "original"], ["agent", "yours"], ["model", "agent"]): + with self.subTest(setups=setups): + self.assertTrue(any("options.steps.setups" in p for p in definitions.problems( + dict(self.bench, options={"steps": dict(self.option, setups=setups)}), "benchmark"))) + capability = dict(self.bench, family="capability", needs=["cap.memory/v1"]) + self.assertTrue(any("options.steps.setups" in p for p in definitions.problems(capability, "benchmark"))) + self.assertFalse(any("options.steps" in p for p in definitions.problems( + dict(capability, options={"steps": dict(self.option, setups=["yours"])}), "benchmark"))) + def test_only_an_agent_gets_and_validates_the_option(self): for kind in ("agent", "capability", "control", "upstream", None): for scope in ("yours", "original", "model"): From 281c6781409b4f91752ba323cc6c641c7eb3b63a Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:42:35 +0100 Subject: [PATCH 09/36] fix(standard): exit conform with 1 when a kit was refused Conform without a port named went on past a kit refused for what its benchmark needs of the machine, and then exited 0 as if every port had been proven. When every kit was refused it still saved the definition and exited 0. A refused kit is now returned as refused beside the proofs gained, which are written, and the command exits 1. When every kit is refused, nothing is written and the check is refused, with exit status 2. The key is checked before the first kit only when some port has a kit: a command that names only ports without one says so and needs no key. Document the refusals of conform and its exit statuses. --- app/nullius/standard/conform.py | 27 +++++++++++++++------- app/tests/test_standard_conform.py | 36 +++++++++++++++++++++++++++++- docs/cli/conform.md | 2 ++ docs/cli/exit-codes.md | 1 + 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/app/nullius/standard/conform.py b/app/nullius/standard/conform.py index fc15cdab..42006282 100644 --- a/app/nullius/standard/conform.py +++ b/app/nullius/standard/conform.py @@ -111,16 +111,24 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] jobs before they exist. ``backend``: where the kits run (``run.Run``). ``options``: the agent's options for the kits' runs; a record proven with any of them off its default says which. - ``repeat``: how many times each check runs (:func:`outcome`). A benchmark-specific - refusal is reported and skipped when checking all candidates. An explicit - ``which`` or a refusal of the whole command still ends the check.""" + ``repeat``: how many times each check runs (:func:`outcome`). + + A kit refused for a reason of its own (``run.BenchmarkRefused``: what its + benchmark needs of this machine) is said and passed over when checking all + candidates: the others' proofs are written, its record stays as it was, and + it is returned as ``{ok: False, refused: why}``. When every kit is refused, + nothing is written and the check is refused. An explicit ``which``, or a + refusal of the whole command (the agent, its options, the key), still ends + the check with nothing written.""" import os env = env if env is not None else os.environ repeat = max(1, int(repeat or 1)) - run.provider_endpoint(provider, env) - got = {} - for p in which or candidates(other): - kit = definitions.kit(p) + kits = [(p, definitions.kit(p)) for p in which or candidates(other)] + if any(kit for _, kit in kits): + run.provider_endpoint(provider, env) # the key is the whole command's: missing, no kit is begun + got: Dict[str, dict] = {} + refused: Dict[str, str] = {} + for p, kit in kits: if kit is None: say(f"{p}: no kit yet") continue @@ -134,6 +142,7 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] if which: raise say(f"{p}: refused: {e}") + refused[p] = str(e) continue o = outcome(kit, _runs(parse.parse_job(job_dir)), repeat) got[p] = {"passed": o["passed"], "ok": o["ok"], "on": time.strftime("%Y-%m-%d"), "model": model, "provider": provider, @@ -145,6 +154,8 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] got[p]["options"] = changed for line in said(p, got[p]): say(" " + line) + if refused and not got: + raise run.Refused("every kit was refused here: " + ", ".join(refused)) if write: def change(cur): # on the definition as it is now: another proof or an edit since the start stays offers = [p for p in cur.get("offers") or [] if not (p in got and not got[p]["ok"])] @@ -155,4 +166,4 @@ def change(cur): # on the definition as it is now: another proof cur["offers"] = [p for p in ports.PORTS if p in offers] cur["proven"] = proven definitions.save(other, change) - return got + return {**got, **{p: {"ok": False, "refused": why} for p, why in refused.items()}} diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index 2e1eeb67..748c38a5 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -104,7 +104,7 @@ def attempt(kit, participant, **kwargs): mock.patch.object(container, "unenforced", side_effect=container.Unavailable("no container engine")), \ mock.patch.object(conform.parse, "parse_job", return_value=passed): status, said = self.invoke(other) - self.assertEqual(status, 0, said) + self.assertEqual(status, 1, said) # a kit was refused: not every port is proven self.assertEqual(started, ports) self.assertIn("agent.workspace/v1: refused: no container engine", said) saved = definitions.get("agent", "fake", self.root) @@ -115,6 +115,40 @@ def attempt(kit, participant, **kwargs): self.assertEqual(saved["proven"][ports[1]], old) self.assertIn(ports[1], saved["offers"]) + def test_a_refused_kit_is_returned_as_refused_beside_the_proofs(self): + other = definitions.get("agent", "fake", self.root) + passed = SimpleNamespace(trials=[SimpleNamespace(task_name="kit/check", reward=1.0)]) + with mock.patch.object(conform, "candidates", return_value=["agent.task/v1", "agent.workspace/v1"]), \ + mock.patch.object(run, "run", side_effect=["passed-job", run.BenchmarkRefused("no engine")]), \ + mock.patch.object(conform.parse, "parse_job", return_value=passed): + got = conform.check(other, model="m", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1"}, unsandboxed=True) + self.assertTrue(got["agent.task/v1"]["ok"]) + self.assertEqual(got["agent.workspace/v1"], {"ok": False, "refused": "no engine"}) + + def test_every_kit_refused_saves_nothing_and_exits_two(self): + other = definitions.get("agent", "fake", self.root) + ports = ["agent.task/v1", "agent.workspace/v1"] + with mock.patch.object(conform, "candidates", return_value=ports), \ + mock.patch.object(run, "run", side_effect=run.BenchmarkRefused("no engine")) as attempt, \ + mock.patch.object(definitions, "save") as save: + status, said = self.invoke(other) + self.assertEqual(status, 2, said) + self.assertEqual(attempt.call_count, 2) # each kit was tried, each refusal said + for port in ports: + self.assertIn(f"{port}: refused: no engine", said) + save.assert_not_called() + + def test_ports_without_a_kit_need_no_key(self): + other = definitions.get("agent", "fake", self.root) + lines = [] + with mock.patch.object(definitions, "kit", return_value=None), mock.patch.object(run, "run") as attempt: + got = conform.check(other, model="m", provider="openai", which=["agent.workspace/v1"], env={}, + jobs_root=os.path.join(self.tmp, "jobs"), say=lines.append, unsandboxed=True) + self.assertEqual(got, {}) + self.assertEqual(lines, ["agent.workspace/v1: no kit yet"]) + attempt.assert_not_called() + def test_an_explicit_port_refusal_keeps_exit_status_two(self): other = definitions.get("agent", "fake", self.root) with mock.patch.object(run, "run", side_effect=run.BenchmarkRefused("kit unavailable")) as attempt, \ diff --git a/docs/cli/conform.md b/docs/cli/conform.md index e3b1cced..71c0496f 100644 --- a/docs/cli/conform.md +++ b/docs/cli/conform.md @@ -24,6 +24,8 @@ nullius standard conform [@version] --model M [--provider P] [--port PORT | Skills | A benchmark's `needs_skills` fits only an agent whose proof shows them | | Replaced | A pass on the current kit replaces an older proof; a fail takes the port back; a `fidelity` record stays | +Without `--port`, a kit refused for what its benchmark needs of this machine (a container engine, a program, a connection, its own installs) is said as `PORT: refused: why` and passed over: the other kits still run, their proofs are written, the refused port's record and offer stay as they were, and the command exits 1. When every kit is refused, nothing is written and it exits 2. A refusal of the command itself (the agent, its options, the key) and a refused port named with `--port` end it with 2 and nothing written. + !!! note "A new version runs nothing" Until `conform KEY@VERSION` passes; bumping a pin in a file without `versions` asks again. The app's *Prove its ports* is the same. diff --git a/docs/cli/exit-codes.md b/docs/cli/exit-codes.md index 70f1ff75..d180d574 100644 --- a/docs/cli/exit-codes.md +++ b/docs/cli/exit-codes.md @@ -17,6 +17,7 @@ a diagnostic and no planned files written. `--apply` performs the copy; the default is a preview. A refusal is one line on stderr, nothing partial written; the standard's commands exit non-zero on a failed check or a differing fidelity field. +One exception: `conform` without `--port` passes over a kit refused for what its benchmark needs of this machine, writes the proofs of the other kits and exits 1; when every kit is refused, it writes nothing and exits 2. ## Output From d2baa074c67b096af4756ce81af170bb5719583e Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:51:45 +0100 Subject: [PATCH 10/36] fix(standard): harden the computer port's server and bridge against any client - Each connection's answers are written by a thread of its own, at most OUT_BYTES waiting: a client that stops reading holds up neither the loop nor another connection, and a close never waits for a write. A connection accepted as the server closes is closed, and a closed one leaves the server's list. - One table gives what tools/list says and what a call may hold. A call is checked in full, an upload's base64 decoded, before it is queued; the JSON-RPC envelope is checked before any tool, a frame's length with its newline, and a number past any integer is refused and recorded. An answer over the bound is recorded as failed. - The turn's end stops the actions of a step not yet run and refuses that call; a done or fail taken before it still reaches the loop. - Uploads: an append goes through a name of the server's own, removed on every path; a file taken by its sha256 starts the file, and append is refused with it; the chunks being kept are the unit's own, dropped when the file starts again and at the unit's end, up to KEEP_BYTES; a kept file is written through an exclusive temporary file and read only when it is no link. - The last resort gives its own callback LAST_SECONDS before it exits. - The budget test holds the first action behind an event in place of a 250 ms wait. --- app/nullius/standard/computer_port.py | 597 ++++++++++++++--------- app/tests/test_standard_computer_port.py | 354 +++++++++++++- 2 files changed, 726 insertions(+), 225 deletions(-) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index 7f2fcd70..df1f81c2 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -6,15 +6,20 @@ imports it with the package's folder on its path. - ``Desktop``: the five operations an entry or a kit gives over its desktop. +- The tools: one table (``tools``) gives both what ``tools/list`` says and what ``checked`` accepts. A call is checked + in full, its upload's data decoded, before it is queued; a call refused there takes no step and never reaches the + desktop. - ``Bridge``: the loop's agent. Each ``predict`` hands the loop the actions of the agent's next acting call (one step), and serves the reading calls that come before it, in the thread of the loop: one desktop operation at a time, in the order the calls arrived. An acting call is held until the loop has run its actions and their pause, then answered. Once the steps are used up, every call but ``info`` is refused. The turn's end (``end_turn``) ends the - wait for the agent's next call; each later ``predict`` gives the loop no action. + wait for the agent's next call and the actions of a step not yet run; a ``done`` or ``fail`` already taken still + reaches the loop, and each later ``predict`` gives the loop no action. - ``Server``: the MCP server over a Unix socket: JSON-RPC 2.0, one message a line; every connection of the unit's - session accepted, the task's state kept in the bridge, outside them; one line per tool call in the call log (the - tool, its start and end, whether it failed; no argument and no result). -- ``Held``: content sent once for a run, when the runner is given a folder for it. + session accepted, the task's state kept in the bridge, outside them; each connection's answers written by a thread + of its own, at most OUT_BYTES waiting, so that a client that stops reading holds up nothing else; one line per tool + call in the call log (the tool, its start and end, whether it failed; no argument and no result). +- ``Held``: whole files kept by their sha256 for a run, when the runner is given a folder for them. - What a runner needs around them: the ready file (``ready``), the lifeline (``lifeline``, which also carries the turn's end) and the one stop (``Stop``). @@ -27,24 +32,31 @@ import binascii import collections import contextlib +import functools import hashlib import json import os +import posixpath import re +import secrets import signal import socket import sys +import tempfile import threading import time from typing import Optional, Protocol PROTOCOL = "2025-06-18" LINE_BYTES = 24 * 1024 * 1024 # one message either way, its newline counted +OUT_BYTES = 2 * LINE_BYTES # answers waiting for a connection's client to read them CHUNK_BYTES = 4 * 1024 * 1024 # upload's data, a call READ_BYTES = 12 * 1024 * 1024 # the largest file read returns +KEEP_BYTES = 256 * 1024 * 1024 # the largest file kept by its sha256 +LAST_SECONDS = 0.5 # what the last resort gives its own callback TURN_END = b"turn-end" # the lifeline's one message: the agent's turn has ended -APPEND = ("import os; s = open({part!r}, 'rb'); d = open({path!r}, 'ab'); d.write(s.read()); d.close(); s.close(); " - "os.remove({part!r}); print(os.path.getsize({path!r}))") +APPEND = ("import os\ntry:\n s = open({part!r}, 'rb'); d = open({path!r}, 'ab'); d.write(s.read()); d.close(); " + "s.close()\nfinally:\n os.remove({part!r})\nprint(os.path.getsize({path!r}))") class Desktop(Protocol): @@ -68,66 +80,143 @@ def password(self) -> str: """The desktop user's password.""" -POINT = {"x": {"type": "integer", "description": "Pixels from the screen's left edge"}, - "y": {"type": "integer", "description": "Pixels from the screen's top edge"}} -BUTTON = {"button": {"type": "string", "enum": ["left", "right", "middle"], "description": "Default left"}} -TOOLS = [ - ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " - "python waits for the desktop's reply, and the desktop user's password.", {}, []), - ("screenshot", "The screen now, as a PNG image at its own size.", {}, []), - ("read", "A file of the desktop: data is its content in base64, or null when there is no such file.", - {"path": {"type": "string"}}, ["path"]), - ("python", "Runs Python code in the desktop as its user and returns its reply: status, output, error, returncode. " - "For reading the desktop and for your own programs there; pointer and keyboard input goes through the " - "acting tools.", {"code": {"type": "string"}}, ["code"]), - ("upload", "Writes data to a file of the desktop and returns the file's size so far. data is base64, at most 4 MiB " - "a call; append false starts the file, true adds to it. sha256, optional, is the whole file's: when " - "the run already holds a file of that sha256, data may be left out.", - {"path": {"type": "string"}, "data": {"type": "string"}, "append": {"type": "boolean"}, - "sha256": {"type": "string"}}, ["path"]), - ("click", "One step: a click at the point.", - dict(POINT, **BUTTON, count={"type": "integer", "minimum": 1, "maximum": 3, "description": "Default 1"}), - ["x", "y"]), - ("move", "One step: the pointer moved to the point.", dict(POINT), ["x", "y"]), - ("drag", "One step: a press at (x, y), a move, and a release at (to_x, to_y).", - dict(POINT, to_x={"type": "integer"}, to_y={"type": "integer"}, **BUTTON), ["x", "y", "to_x", "to_y"]), - ("scroll", "One step: the wheel turned at the point, dy clicks down (negative: up) and dx clicks right " - "(negative: left).", dict(POINT, dy={"type": "integer"}, dx={"type": "integer"}), ["x", "y", "dy"]), - ("type", "One step: the text typed key by key.", {"text": {"type": "string"}}, ["text"]), - ("key", "One step: the keys pressed together, such as [\"ctrl\", \"s\"].", - {"keys": {"type": "array", "items": {"type": "string"}}}, ["keys"]), - ("wait", "One step: no action, the pause alone.", {}, []), - ("step", "One step of several actions, run in order: each is a line of Python run in the desktop with pyautogui " - "and time imported, or WAIT. An empty list is a step with no action. The screen comes back when " - "screenshot is true.", - {"actions": {"type": "array", "items": {"type": "string"}}, "response": {"type": "string"}, - "screenshot": {"type": "boolean"}}, ["actions"]), - ("done", "The last step: the task is complete.", {"message": {"type": "string"}}, []), - ("fail", "The last step: the task cannot be done.", {"reason": {"type": "string"}}, []), -] -NAMES = [t[0] for t in TOOLS] +# -- the tools: one table for what tools/list says and what a call may hold ------------------------------------------ +NAMES = ["info", "screenshot", "read", "python", "upload", "click", "move", "drag", "scroll", "type", "key", "wait", + "step", "done", "fail"] SCREEN_TOOLS = ("click", "move", "drag", "scroll", "type", "key", "wait") # answered with the screen after the pause ACTING = SCREEN_TOOLS + ("step", "done", "fail") FINAL = {"done": "DONE", "fail": "FAIL"} - - -def listed() -> list: - return [{"name": n, "description": d, - "inputSchema": dict({"type": "object", "properties": props}, **({"required": req} if req else {}))} - for n, d, props, req in TOOLS] +B64_CHARS = 4 * -(-CHUNK_BYTES // 3) +REFUSALS = {("scroll", "anyOf"): "dy or dx is not 0", ("upload", "anyOf"): "upload needs data, or a sha256", + ("upload", "if"): "append needs data: a file taken by its sha256 starts the file"} +PATTERN_WORDS = {"\\S": "is not blank", "^[0-9a-f]{64}$": "is 64 lowercase hexadecimal digits"} + + +@functools.lru_cache(maxsize=8) +def tools(screen): + """The tools for a screen of ``screen`` (width, height) pixels: [(name, description, input schema)].""" + w, h = screen + x = {"type": "integer", "minimum": 0, "maximum": w - 1, "description": "Pixels from the screen's left edge"} + y = {"type": "integer", "minimum": 0, "maximum": h - 1, "description": "Pixels from the screen's top edge"} + button = {"type": "string", "enum": ["left", "right", "middle"], "description": "Default left"} + clicks = {"type": "integer", "minimum": -100, "maximum": 100} + text = {"type": "string", "minLength": 1} + obj = lambda props, required=(), **more: dict({"type": "object", "properties": props}, + **({"required": list(required)} if required else {}), **more) + return [ + ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " + "python waits for the desktop's reply, and the desktop user's password.", obj({})), + ("screenshot", "The screen now, as a PNG image at its own size.", obj({})), + ("read", "A file of the desktop: data is its content in base64, or null when there is no such file.", + obj({"path": text}, ["path"])), + ("python", "Runs Python code in the desktop as its user and returns its reply: status, output, error, " + "returncode. For reading the desktop and for your own programs there; pointer and keyboard input " + "goes through the acting tools.", obj({"code": text}, ["code"])), + ("upload", "Writes data to a file of the desktop and returns the file's size so far. data is base64, at most " + "4 MiB a call; append false starts the file, true adds to it. sha256, optional, is the whole " + "file's: when the run already holds a file of that sha256, it may be given with no data, and it " + "starts the file.", + obj({"path": text, "data": {"type": "string", "contentEncoding": "base64", "maxLength": B64_CHARS}, + "append": {"type": "boolean"}, "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}}, ["path"], + anyOf=[{"required": ["data"]}, {"required": ["sha256"]}], + **{"if": {"not": {"required": ["data"]}}, "then": {"properties": {"append": {"const": False}}}})), + ("click", "One step: a click at the point.", + obj({"x": x, "y": y, "button": button, + "count": {"type": "integer", "minimum": 1, "maximum": 3, "description": "Default 1"}}, ["x", "y"])), + ("move", "One step: the pointer moved to the point.", obj({"x": x, "y": y}, ["x", "y"])), + ("drag", "One step: a press at (x, y), a move, and a release at (to_x, to_y).", + obj({"x": x, "y": y, "to_x": x, "to_y": y, "button": button}, ["x", "y", "to_x", "to_y"])), + ("scroll", "One step: the wheel turned at the point, dy clicks down (negative: up) and dx clicks right " + "(negative: left); at least one is not 0.", + obj({"x": x, "y": y, "dy": clicks, "dx": clicks}, ["x", "y"], + anyOf=[{"required": ["dy"], "properties": {"dy": {"not": {"const": 0}}}}, + {"required": ["dx"], "properties": {"dx": {"not": {"const": 0}}}}])), + ("type", "One step: the text typed key by key.", obj({"text": text}, ["text"])), + ("key", "One step: the keys pressed together, such as [\"ctrl\", \"s\"].", + obj({"keys": {"type": "array", "minItems": 1, "maxItems": 8, + "items": {"type": "string", "minLength": 1, "maxLength": 32}}}, ["keys"])), + ("wait", "One step: no action, the pause alone.", obj({})), + ("step", "One step of several actions, run in order: each is a line of Python run in the desktop with " + "pyautogui and time imported, or WAIT. An empty list is a step with no action. The screen comes back " + "when screenshot is true.", + obj({"actions": {"type": "array", "items": { + "type": "string", "pattern": "\\S", + "not": {"pattern": "^\\s*(DONE|FAIL)\\s*$", + "description": "the task's end is the tool done or fail, not an action of step"}}}, + "response": {"type": "string"}, "screenshot": {"type": "boolean"}}, ["actions"])), + ("done", "The last step: the task is complete.", obj({"message": {"type": "string"}})), + ("fail", "The last step: the task cannot be done.", obj({"reason": {"type": "string"}})), + ] + + +def listed(screen) -> list: + return [{"name": n, "description": d, "inputSchema": s} for n, d, s in tools(tuple(screen))] class Refused(Exception): """A call answered with ``isError`` and these words.""" +def _passes(value, schema, tool, name): + try: + _valid(value, schema, tool, name) + return True + except Refused: + return False + + +def _valid(value, schema, tool, name): + """``value`` checked against ``schema``, in the part of JSON Schema the tool table uses; the value with integers + as int. Refused, naming the argument, when it does not hold.""" + kind = schema.get("type") + if kind == "integer": + if isinstance(value, bool) or not (isinstance(value, int) or (isinstance(value, float) and value.is_integer())): + raise Refused(f"{name} is an integer") + value = int(value) + if not schema.get("minimum", value) <= value <= schema.get("maximum", value): + raise Refused(f"{name} is from {schema['minimum']} to {schema['maximum']}") + elif kind is not None and not isinstance(value, {"string": str, "boolean": bool, "array": list, "object": dict}[kind]): + raise Refused(f"{name} is " + {"string": "a text", "boolean": "true or false", "array": "a list", + "object": "an object"}[kind]) + if isinstance(value, str): + if len(value) < schema.get("minLength", 0): + raise Refused(f"{name} is a text, not empty") + if len(value) > schema.get("maxLength", len(value)): + raise Refused(f"{name} is at most {schema['maxLength']} characters") + if "pattern" in schema and not re.search(schema["pattern"], value): + raise Refused(f"{name} " + PATTERN_WORDS.get(schema["pattern"], "is not of its form")) + if isinstance(value, list): + if not schema.get("minItems", 0) <= len(value) <= schema.get("maxItems", len(value)): + raise Refused(f"{name} holds {schema.get('minItems', 0)} to {schema['maxItems']} items") + value = [_valid(v, schema.get("items", {}), tool, f"{name}[{i}]") for i, v in enumerate(value)] + if isinstance(value, dict): + for key in schema.get("required", []): + if key not in value: + raise Refused(f"{key} is missing") + props = schema.get("properties", {}) + value = {k: _valid(v, props[k], tool, k) if k in props else v for k, v in value.items()} + if "anyOf" in schema and not any(_passes(value, s, tool, name) for s in schema["anyOf"]): + raise Refused(REFUSALS.get((tool, "anyOf"), f"{name} is not of its form")) + if "if" in schema and _passes(value, schema["if"], tool, name) and not _passes(value, schema["then"], tool, name): + raise Refused(REFUSALS.get((tool, "if"), f"{name} is not of its form")) + if "enum" in schema and value not in schema["enum"]: + raise Refused(f"{name} is one of {', '.join(map(str, schema['enum']))}") + if "const" in schema and (value != schema["const"] or type(value) is not type(schema["const"])): + raise Refused(f"{name} is {json.dumps(schema['const'])}") + if "not" in schema and _passes(value, schema["not"], tool, name): + raise Refused(f"{name}: {schema['not'].get('description', 'not of its form')}") + return value + + class Call: - """One tools/call: its tool and arguments, when it came, and how it is answered (once).""" + """One tools/call: its tool and arguments, when it came, and how it is answered (once). ``checked`` adds what + the call stands for: its arguments as checked, its actions, its response, whether its answer has the screen, and + an upload's data, decoded.""" def __init__(self, tool, args, reply): self.tool, self.args, self._reply = tool, args, reply self.start = time.time() - self.actions, self.response, self.screen = [], "", False + self.clean, self.actions, self.response, self.screen, self.data = {}, [], "", False, None self.answered = False self._lock = threading.Lock() @@ -139,118 +228,76 @@ def answer(self, value=None, image=None, error=None): self._reply(self, value, image, error) -def _number(args, name, low, high, default=None): - v = args.get(name, default) - if isinstance(v, bool) or not isinstance(v, (int, float)) or v != v: - raise Refused(f"{name} is a number") - v = int(round(v)) - if not low <= v <= high: - raise Refused(f"{name} is from {low} to {high}") - return v - - -def _text(args, name, default=None, empty=False): - v = args.get(name, default) - if not isinstance(v, str) or not (v or empty): - raise Refused(f"{name} is a text" + ("" if empty else ", not empty")) - return v - - def checked(call, screen) -> None: - """The call's arguments checked, and for an acting call the actions it stands for (Python lines run with pyautogui - and time imported, or WAIT, DONE, FAIL), its response and whether its answer carries the screen. Refused when an - argument is wrong: the call is then answered at once and is no step.""" - a, (w, h), tool = call.args, screen, call.tool - if not isinstance(a, dict): - raise Refused("arguments is an object") - if tool in ("click", "move", "drag", "scroll"): - x, y = _number(a, "x", 0, w - 1), _number(a, "y", 0, h - 1) - button = a.get("button", "left") - if tool in ("click", "drag") and button not in ("left", "right", "middle"): - raise Refused("button is left, right or middle") + """The call checked against its tool's schema, in full and before it is queued; for an acting call, the actions + it stands for (Python lines run with pyautogui and time imported, or WAIT, DONE, FAIL). Refused when it does not + hold: the call is then answered at once, takes no step and never reaches the desktop.""" + schema = next(s for n, _, s in tools(tuple(screen)) if n == call.tool) + a = call.clean = _valid(call.args, schema, call.tool, "arguments") + tool, button = call.tool, repr(a.get("button", "left")) if tool == "click": - call.actions = [f"pyautogui.click({x}, {y}, clicks={_number(a, 'count', 1, 3, 1)}, button={button!r})"] + call.actions = [f"pyautogui.click({a['x']}, {a['y']}, clicks={a.get('count', 1)}, button={button})"] elif tool == "move": - call.actions = [f"pyautogui.moveTo({x}, {y})"] + call.actions = [f"pyautogui.moveTo({a['x']}, {a['y']})"] elif tool == "drag": - to = (_number(a, "to_x", 0, w - 1), _number(a, "to_y", 0, h - 1)) - call.actions = [f"pyautogui.moveTo({x}, {y}); pyautogui.dragTo({to[0]}, {to[1]}, duration=0.5, " - f"button={button!r})"] + call.actions = [f"pyautogui.moveTo({a['x']}, {a['y']}); pyautogui.dragTo({a['to_x']}, {a['to_y']}, " + f"duration=0.5, button={button})"] elif tool == "scroll": - dy, dx = _number(a, "dy", -100, 100), _number(a, "dx", -100, 100, 0) - if not (dy or dx): - raise Refused("dy or dx is not 0") + dy, dx, x, y = a.get("dy", 0), a.get("dx", 0), a["x"], a["y"] call.actions = ["; ".join([f"pyautogui.scroll({-dy}, x={x}, y={y})"] * bool(dy) + [f"pyautogui.hscroll({dx}, x={x}, y={y})"] * bool(dx))] elif tool == "type": - call.actions = [f"pyautogui.write({_text(a, 'text')!r})"] + call.actions = [f"pyautogui.write({a['text']!r})"] elif tool == "key": - keys = a.get("keys") - if not (isinstance(keys, list) and 1 <= len(keys) <= 8 - and all(isinstance(k, str) and 0 < len(k) <= 32 for k in keys)): - raise Refused("keys is a list of 1 to 8 key names") - call.actions = [f"pyautogui.hotkey({', '.join(repr(k if len(k) == 1 else k.lower()) for k in keys)})"] + call.actions = [f"pyautogui.hotkey({', '.join(repr(k if len(k) == 1 else k.lower()) for k in a['keys'])})"] elif tool == "wait": call.actions = ["WAIT"] elif tool == "step": - actions = a.get("actions") - if not (isinstance(actions, list) and all(isinstance(x, str) and x.strip() for x in actions)): - raise Refused("actions is a list of texts, each a line of Python or WAIT") - if any(x.strip() in FINAL.values() for x in actions): - raise Refused("the task's end is the tool done or fail, not an action of step") - call.actions, call.response = list(actions), _text(a, "response", "", empty=True) - call.screen = a.get("screenshot") is True + call.actions, call.response, call.screen = list(a["actions"]), a.get("response", ""), a.get("screenshot", False) elif tool in FINAL: - call.actions, call.response = [FINAL[tool]], _text(a, "message" if tool == "done" else "reason", "", empty=True) - elif tool in ("read", "upload"): - _text(a, "path") - elif tool == "python": - _text(a, "code") + call.actions, call.response = [FINAL[tool]], a.get("message" if tool == "done" else "reason", "") + elif tool == "upload" and "data" in a: + try: + call.data = base64.b64decode(a["data"], validate=True) + except (binascii.Error, ValueError): + raise Refused("data is base64") + if len(call.data) > CHUNK_BYTES: + raise Refused(f"data is at most {CHUNK_BYTES} bytes a call") if tool in SCREEN_TOOLS: call.screen = True - if tool == "upload": - if "data" in a: - _text(a, "data", empty=True) - if a.get("sha256") is not None and not re.fullmatch(r"[0-9a-f]{64}", str(a.get("sha256"))): - raise Refused("sha256 is 64 lowercase hexadecimal digits") - if not isinstance(a.get("append", False), bool): - raise Refused("append is true or false") class Held: - """Content sent once for a run: a file whose chunks, sent in this unit from its start, hash to the sha256 the agent - gave is kept in ``folder`` (the run's, the same for each of its units) under that name. A later upload of the same - sha256, in any unit of the run, takes it from there, checked again, and the agent sends no bytes; the unit's - desktop gets the file at that call, as it would have from the bytes. Nothing else passes from one unit to the - next.""" + """Whole files kept for a run by their sha256, in ``folder`` (the run's, the same for each of its units): a file + is kept once the chunks a unit sent for it from its start hash to the sha256 given with the first (the bridge + holds those chunks, for its unit alone). A later upload of that sha256 with no data, in any unit of the run, takes + the file from here, checked again, and the agent sends no bytes. Nothing else passes from one unit to the next.""" def __init__(self, folder): - self.folder, self.open = folder, {} + self.folder = folder def get(self, digest): try: - with open(os.path.join(self.folder, digest), "rb") as f: - data = f.read() + fd = os.open(os.path.join(self.folder, digest), os.O_RDONLY | os.O_NOFOLLOW) + with os.fdopen(fd, "rb") as f: + data = f.read(KEEP_BYTES + 1) except OSError: return None return data if hashlib.sha256(data).hexdigest() == digest else None - def add(self, path, digest, data, append): - if not append: - self.open.pop(path, None) - if digest: - self.open[path] = (digest, hashlib.sha256(), bytearray()) - kept = self.open.get(path) - if kept is None: + def keep(self, digest, data): + """``data`` kept under its sha256 ``digest``, through a temporary file of its own in the folder (created + anew, the user's alone), removed on every path.""" + if hashlib.sha256(data).hexdigest() != digest: return - kept[1].update(data) - kept[2].extend(data) - if kept[1].hexdigest() == kept[0]: - del self.open[path] - part = os.path.join(self.folder, f".{kept[0]}.{os.getpid()}.{threading.get_ident()}") - with open(part, "wb") as f: - f.write(kept[2]) - os.replace(part, os.path.join(self.folder, kept[0])) + fd, part = tempfile.mkstemp(prefix=".keep-", dir=self.folder) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(part, os.path.join(self.folder, digest)) + finally: + with contextlib.suppress(OSError): + os.unlink(part) class Bridge: @@ -272,6 +319,7 @@ def __init__(self, desktop, steps, pause, screen, python_seconds, on_first=None, self.ended = None # why no call is taken any more self.turn_ended = False self.first = True + self.partial = {} # this unit's uploads being kept: path -> [sha256, hash, bytes] self.change = threading.Condition() # -- the loop's agent ------------------------------------------------------------------------------------------- @@ -280,7 +328,7 @@ def reset(self, *_, **__): def predict(self, instruction, obs): """The agent's next acting call, as the loop's (response, actions); reading calls are served until it comes. - Once the turn has ended, no action.""" + Once the turn has ended, no action, but for a done or fail taken before it.""" if self.first: self.first = False if self.on_first: @@ -292,7 +340,7 @@ def predict(self, instruction, obs): if call.tool not in ACTING: self._serve(call) continue - if call.tool in FINAL: + if call.tool in FINAL: # the loop asks nothing after them: answered as it takes them call.answer({"steps_left": self.steps - self.used, "done": True}) return call.response, list(call.actions) return call.response, self._run(call) @@ -316,12 +364,16 @@ def _next(self): return call def _run(self, call): - """The call's actions, one at a time as the loop asks; once the loop asks past the last, it has run them all - and their pause, and the call is answered.""" + """The call's actions, one at a time as the loop asks, none once the turn has ended; once the loop asks past + the last, it has run them all and their pause, and the call is answered.""" ran = False try: - yield from call.actions - ran = True + for action in call.actions: + if self.turn_ended: + break + yield action + else: + ran = True finally: self._ran(call, ran) @@ -332,7 +384,7 @@ def _ran(self, call, ran): value = {"steps_left": self.steps - self.used, "done": self.ended is not None} turn_ended = self.turn_ended if not ran: - call.answer(error="the step was not run to its end") + call.answer(error="the agent's turn has ended" if turn_ended else "the step was not run to its end") else: try: call.answer(value, self.desktop.capture() if call.screen and not turn_ended else None) @@ -350,7 +402,7 @@ def _serve(self, call): call.answer(error=f"the desktop failed {call.tool}: {type(e).__name__}: {e}") def _reading(self, call): - a = call.args + a = call.clean if call.tool == "screenshot": return {"image": self.desktop.capture()} if call.tool == "read": @@ -363,37 +415,55 @@ def _reading(self, call): if reply is None: raise Refused(f"the desktop gave no reply within {self.python_seconds:g} s") return {"value": reply} - return {"value": self._upload(a)} + return {"value": self._upload(call)} - def _upload(self, a): + def _upload(self, call): + a, data = call.clean, call.data path, append, digest = a["path"], a.get("append", False), a.get("sha256") - if "data" not in a: - data = self.held.get(digest) if self.held is not None and digest else None + if not append: + self.partial.pop(path, None) # every start of the file drops what was being kept for it + if data is None: # the run's file of that sha256: it starts the file + data = self.held.get(digest) if self.held is not None else None if data is None: - raise Refused("upload needs data" + (": the run holds no file of that sha256" if digest else "")) + raise Refused("the run holds no file of that sha256: send its data") self.desktop.upload(path, data) return {"size": len(data)} - try: - data = base64.b64decode(a["data"], validate=True) - except (binascii.Error, ValueError): - raise Refused("data is base64") - if len(data) > CHUNK_BYTES: - raise Refused(f"data is at most {CHUNK_BYTES} bytes a call") if not append: self.desktop.upload(path, data) size = len(data) - else: # the chunk beside the file, then added to it in the desktop - part = path + ".part" - self.desktop.upload(part, data) - reply = self.desktop.python(APPEND.format(part=part, path=path)) + if digest and self.held is not None: + self.partial[path] = [digest, hashlib.sha256(), bytearray()] + else: try: - size = int(str(reply.get("output")).strip()) - except (AttributeError, ValueError): - raise Refused(f"the desktop did not add the data to {path}: {json.dumps(reply)[:300]}") - if self.held is not None: - self.held.add(path, digest, data, append) + size = self._append(path, data) + except BaseException: + self.partial.pop(path, None) + raise + kept = self.partial.get(path) + if kept is not None: + kept[1].update(data) + kept[2].extend(data) + if len(kept[2]) > KEEP_BYTES: + del self.partial[path] + elif kept[1].hexdigest() == kept[0]: + del self.partial[path] + self.held.keep(kept[0], bytes(kept[2])) return {"size": size} + def _append(self, path, data): + """``data`` added to the file at ``path``: written beside it under a name of the server's own, then added to + it in the desktop; that name is removed on every path.""" + part = posixpath.join(posixpath.dirname(path), f".upload-{secrets.token_hex(8)}") + self.desktop.upload(part, data) + reply = self.desktop.python(APPEND.format(part=part, path=path)) + try: + return int(str(reply.get("output")).strip()) + except (AttributeError, ValueError): + if reply is None: # the desktop gave no reply: the part may still be there + with contextlib.suppress(Exception): + self.desktop.python(f"import os; os.remove({part!r})") + raise Refused(f"the desktop did not add the data to {path}: {json.dumps(reply)[:300]}") + # -- what the server and the runner call ------------------------------------------------------------------------ def submit(self, call): """A call that needs the desktop, for the loop's thread; refused at once once the task ended, and an acting @@ -427,38 +497,55 @@ def instructions(self): f"python. A turn that ends with neither done nor fail ends the task as the desktop then is.") def end_turn(self): - """The agent's turn has ended: the wait for its next call ends, and no call is taken any more.""" + """The agent's turn has ended: the wait for its next call ends, the actions of a step not yet run are not + run, and no call is taken any more. A done or fail already taken stays, for the loop.""" with self.change: self.turn_ended = True - refused = self._end("the agent's turn has ended") + refused = self._end("the agent's turn has ended", keep_final=True) for c in refused: c.answer(error=self.ended) def finish(self): - """The loop has returned, or failed: every call still waiting, and one the loop was running, is answered.""" + """The loop has returned, or failed: every call still waiting, and one the loop was running, is answered, and + what this unit was keeping of its uploads is dropped.""" with self.change: refused = self._end("the task has ended") running, self.running = self.running, None + self.partial.clear() for c in refused + ([running] if running is not None else []): c.answer(error=self.ended) - def _end(self, why): - """No call taken any more (with the condition held); the calls waiting, to refuse once it is let go.""" + def _end(self, why, keep_final=False): + """No call taken any more (with the condition held); the calls waiting, to refuse once it is let go, but the + first done or fail when ``keep_final``.""" if self.ended is None: self.ended = why - refused = list(self.calls) + final = next((c for c in self.calls if c.tool in FINAL), None) if keep_final else None + refused = [c for c in self.calls if c is not final] self.calls.clear() + if final is not None: + self.calls.append(final) self.change.notify_all() return refused +def frame(message) -> bytes: + return (json.dumps(message) + "\n").encode() + + class Wire: """One connection of the agent's session: JSON-RPC messages, one a line, each at most LINE_BYTES with its newline. - Once its input has ended, it is closed when every call it made has been answered.""" - - def __init__(self, conn): - self.conn, self.lock = conn, threading.Lock() - self.open_calls, self.input_ended, self.closed = 0, False, False + Its answers are written by a thread of its own; at most OUT_BYTES of them wait for the client to read, past which + the connection is closed. ``close`` never waits for a write. Once its input has ended, it is closed when every + call it made has been answered and written. ``on_closed`` is called once, when it is closed.""" + + def __init__(self, conn, on_closed=None): + self.conn, self.on_closed = conn, on_closed + self.lock = threading.Lock() # the wire's state; never held across a call on the socket + self.ready = threading.Condition(self.lock) + self.out, self.out_bytes = collections.deque(), 0 + self.open_calls, self.input_ended, self.closing, self.closed = 0, False, False, False + threading.Thread(target=self._write, name="computer-writer", daemon=True).start() def lines(self): buf, scanned = bytearray(), 0 @@ -475,6 +562,8 @@ def lines(self): if i < 0: scanned = len(buf) break + if i + 1 > LINE_BYTES: + break line = bytes(buf[:i]) del buf[:i + 1] scanned = 0 @@ -485,17 +574,37 @@ def lines(self): return def send(self, message): - data = (json.dumps(message) + "\n").encode() - if len(data) > LINE_BYTES: - data = (json.dumps({"jsonrpc": "2.0", "id": message.get("id"), "error": { - "code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"}}) + "\n").encode() + """``message`` (an object, or a framed line) queued for the writer; False when the wire is closed, or closed + now because its client has left OUT_BYTES unread.""" + data = message if isinstance(message, bytes) else frame(message) with self.lock: if self.closed: - return + return False + overflow = self.out_bytes + len(data) > OUT_BYTES + if not overflow: + self.out.append(data) + self.out_bytes += len(data) + self.ready.notify_all() + if overflow: + self.close() + return False + return True + + def _write(self): + while True: + with self.lock: + while not self.out and not self.closing and not self.closed: + self.ready.wait() + if self.closed or not self.out: # closed, or closing with every answer written + break + data = self.out.popleft() try: self.conn.sendall(data) except OSError: - self._close() + break + with self.lock: + self.out_bytes -= len(data) + self.close() def opened(self): with self.lock: @@ -505,24 +614,33 @@ def answered(self): with self.lock: self.open_calls -= 1 if self.input_ended and not self.open_calls: - self._close() + self.closing = True + self.ready.notify_all() def ended(self): with self.lock: self.input_ended = True if not self.open_calls: - self._close() + self.closing = True + self.ready.notify_all() def close(self): with self.lock: - self._close() - - def _close(self): - if not self.closed: + if self.closed: + return self.closed = True - with contextlib.suppress(OSError): - self.conn.shutdown(socket.SHUT_RDWR) + self.out.clear() + self.ready.notify_all() + with contextlib.suppress(OSError): + self.conn.shutdown(socket.SHUT_RDWR) # also ends a write in progress on a real socket + with contextlib.suppress(OSError): self.conn.close() + if self.on_closed is not None: + self.on_closed(self) + + +def _good_id(rid): + return isinstance(rid, str) or (isinstance(rid, int) and not isinstance(rid, bool)) class Server: @@ -530,19 +648,26 @@ class Server: when given, gets one JSON line per tool call: ``{tool, start, end, isError}``.""" def __init__(self, bridge, path, log=None): - self.bridge, self.path, self.log = bridge, path, log - self.log_lock = threading.Lock() - self.wires = [] + self._state(bridge, path, log) self.listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.listener.bind(path) self.listener.listen(16) threading.Thread(target=self._accept, name="computer-accept", daemon=True).start() + def _state(self, bridge, path, log): + self.bridge, self.path, self.log = bridge, path, log + self.log_lock, self.lock = threading.Lock(), threading.Lock() + self.closing, self.wires = False, set() + def close(self): + with self.lock: + self.closing = True + wires = list(self.wires) with contextlib.suppress(OSError): self.listener.shutdown(socket.SHUT_RDWR) - self.listener.close() - for wire in list(self.wires): + with contextlib.suppress(OSError): + self.listener.close() + for wire in wires: wire.close() with contextlib.suppress(OSError): os.unlink(self.path) @@ -553,35 +678,58 @@ def _accept(self): conn = self.listener.accept()[0] except OSError: return - wire = Wire(conn) - self.wires.append(wire) + with self.lock: + wire = None if self.closing else Wire(conn, on_closed=self._forget) + if wire is not None: + self.wires.add(wire) + if wire is None: # came as the server closed + with contextlib.suppress(OSError): + conn.shutdown(socket.SHUT_RDWR) + with contextlib.suppress(OSError): + conn.close() + return threading.Thread(target=self._serve, args=(wire,), name="computer-connection", daemon=True).start() + def _forget(self, wire): + with self.lock: + self.wires.discard(wire) + def _serve(self, wire): - for line in wire.lines(): - try: - message = json.loads(line) - except ValueError: - wire.send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) - continue - reply = self.handle(message, wire) - if reply is not None: - wire.send(reply) + try: + for line in wire.lines(): + try: + message = json.loads(line) + except (ValueError, RecursionError): + wire.send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) + continue + reply = self.handle(message, wire) + if reply is not None: + wire.send(reply) + except Exception as e: # a failure of this handler: its connection is closed + sys.stderr.write(f"computer: a connection's handler failed: {type(e).__name__}: {e}\n") + wire.close() + return wire.ended() def handle(self, m, wire): """One message in; its reply out, None for a notification or a tool call answered on the wire later.""" if not isinstance(m, dict): return {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "a message is a JSON object"}} + rid, method, params = m.get("id"), m.get("method"), m.get("params", {}) + if m.get("jsonrpc") != "2.0" or not isinstance(method, str) or not isinstance(params, dict) or ( + "id" in m and not _good_id(rid)): + if "id" not in m: + return None + return {"jsonrpc": "2.0", "id": rid if _good_id(rid) else None, + "error": {"code": -32600, "message": "not a JSON-RPC 2.0 request: jsonrpc \"2.0\", a method, an " + "id that is a text or an integer, params an object"}} if "id" not in m: return None - rid, method, params = m["id"], m.get("method"), m.get("params") - params = params if isinstance(params, dict) else {} if method == "initialize": result = {"protocolVersion": params.get("protocolVersion") or PROTOCOL, "capabilities": {"tools": {}}, "serverInfo": {"name": "computer", "version": "1"}, "instructions": self.bridge.instructions()} elif method == "tools/list": - result = {"tools": listed()} + result = {"tools": listed(self.bridge.screen)} elif method == "tools/call": self.call(rid, params, wire) return None @@ -595,7 +743,7 @@ def handle(self, m, wire): def call(self, rid, params, wire): name = params.get("name") - if name not in NAMES: + if not isinstance(name, str) or name not in NAMES: self.record({"tool": None, "start": time.time(), "end": time.time(), "isError": True}) wire.send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"no tool {str(name)[:100]!r}"}}) return @@ -608,20 +756,28 @@ def reply(call, value, image, error): content.append({"type": "text", "text": json.dumps(value)}) if image is not None: content.append({"type": "image", "data": base64.b64encode(image).decode(), "mimeType": "image/png"}) - self.record({"tool": call.tool, "start": call.start, "end": time.time(), "isError": error is not None}) - wire.send({"jsonrpc": "2.0", "id": rid, "result": {"content": content, "isError": error is not None}}) + data = frame({"jsonrpc": "2.0", "id": rid, "result": {"content": content, "isError": error is not None}}) + failed = error is not None + if len(data) > LINE_BYTES: # the answer's size is known before its outcome is recorded + data, failed = frame({"jsonrpc": "2.0", "id": rid, "error": { + "code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"}}), True + self.record({"tool": call.tool, "start": call.start, "end": time.time(), "isError": failed}) + wire.send(data) wire.answered() wire.opened() - call = Call(name, params.get("arguments") or {}, reply) - if name == "info": - call.answer(self.bridge.info()) - return + call = Call(name, params.get("arguments", {}), reply) try: checked(call, self.bridge.screen) + if name == "info": + call.answer(self.bridge.info()) + return except Refused as e: call.answer(error=str(e)) return + except Exception as e: # checked failed: the call is refused, and recorded + call.answer(error=f"the arguments could not be checked: {type(e).__name__}") + return self.bridge.submit(call) def record(self, line): @@ -666,7 +822,7 @@ def lifeline(fd, bridge=None): class Stop: """The runner's one stop: SIGTERM, or the lifeline's end, taken once; a second does not cut the clean-up. While a part that must not be cut runs (``deferred``), a stop waits for its end. ``seconds`` after the stop, a runner still - there calls ``last`` (its last resort) and exits 1.""" + there calls ``last`` (its last resort), gives it LAST_SECONDS at most, and exits 1.""" def __init__(self, seconds, last=None): self.seconds, self.last = seconds, last @@ -704,7 +860,12 @@ def _last_resort(self): self.taken.wait() time.sleep(self.seconds) if self.last is not None: - with contextlib.suppress(Exception): - self.last() + last = threading.Thread(target=self._quietly, name="last-resort-callback", daemon=True) + last.start() + last.join(LAST_SECONDS) sys.stderr.write(f"runner: the clean-up did not end within {self.seconds:g} s of the stop\n") os._exit(1) + + def _quietly(self): + with contextlib.suppress(Exception): + self.last() diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index 0c4da936..649aceb6 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -38,6 +38,8 @@ class FakeDesktop: def __init__(self, root): self.root, self.events, self.busy, self.overlaps = root, [], 0, 0 self.fail, self.silent = set(), False + self.gate = threading.Event() # the loop's actions wait while it is clear + self.gate.set() self.lock = threading.Lock() @contextlib.contextmanager @@ -89,6 +91,7 @@ def password(self): return "password" def act(self, action): # the loop's own run of an action + self.gate.wait(30) with self.op("action", action=action): pass @@ -291,9 +294,9 @@ def test_step_runs_its_actions_in_order_as_one_step_and_an_empty_list_is_a_step( def test_wrong_arguments_are_refused_at_once_and_take_no_step(self): c = self.start(steps=3) for tool, args, words in [("click", {"x": 1920, "y": 0}, "x is from 0 to 1919"), - ("click", {"x": 1, "y": 1, "button": "side"}, "button is left, right or middle"), - ("click", {"x": True, "y": 1}, "x is a number"), - ("key", {"keys": []}, "keys is a list"), + ("click", {"x": 1, "y": 1, "button": "side"}, "button is one of left, right, middle"), + ("click", {"x": True, "y": 1}, "x is an integer"), + ("key", {"keys": []}, "keys holds 1 to 8 items"), ("type", {"text": ""}, "text is a text, not empty"), ("scroll", {"x": 1, "y": 1, "dy": 0}, "dy or dx is not 0"), ("step", {"actions": ["DONE"]}, "the task's end is the tool done or fail"), @@ -330,10 +333,13 @@ def test_every_call_but_info_is_refused_past_the_budget_and_the_desktop_is_not_t self.assertEqual(len(self.desktop.events), touched) def test_acting_calls_sent_together_past_the_budget_are_refused_at_once(self): - c = self.start(steps=2, pause=0.3) + c = self.start(steps=2) + self.desktop.gate.clear() # the first click is held in the loop ids = [c.start("click", x=i, y=i) for i in (1, 2, 3)] - third = c.result(ids[2], 0.25) # before the first has been answered - self.assertEqual(error(third), "the steps are used up") + self.assertEqual(error(c.result(ids[2])), "the steps are used up") + self.assertEqual(c.got, {}, "a click was answered while the first was held") + self.assertNotIn("action", self.desktop.kinds()) + self.desktop.gate.set() self.assertEqual([value(c.result(i))["steps_left"] for i in ids[:2]], [1, 0]) self.assertEqual(self.evaluated(), 2) self.assertEqual(len(self.ran), 3) @@ -509,7 +515,7 @@ def test_chunks_that_do_not_hash_to_the_sha256_given_are_not_kept(self): def test_with_no_folder_for_the_run_an_upload_by_sha256_alone_is_refused(self): c = self.start() self.assertEqual(error(c.call("upload", path="t.bin", sha256="0" * 64)), - "upload needs data: the run holds no file of that sha256") + "the run holds no file of that sha256: send its data") class Connections(Port): @@ -584,6 +590,11 @@ def test_every_tool_call_is_recorded_with_its_tool_times_and_failure_and_nothing with stop.deferred(): os.kill(os.getpid(), signal.SIGTERM) time.sleep(30) +elif mode == "stalled-last": + stop.last = lambda: time.sleep(30) + with stop.deferred(): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(30) """ @@ -632,6 +643,335 @@ def test_a_clean_up_that_hangs_takes_the_last_resort(self): self.assertLess(time.monotonic() - t0, 5) self.assertIn(b"did not end within 0.3 s of the stop", err) + def test_a_last_resort_whose_own_callback_hangs_still_exits(self): + p = self.child("stalled-last", "0.05") + t0 = time.monotonic() + try: + p.communicate(timeout=3) + except subprocess.TimeoutExpired: + self.fail("the last resort waited for its callback") + self.assertEqual(p.returncode, 1) + self.assertLess(time.monotonic() - t0, 3) + + +class FakeConnection: + """A socket of the test's for a Wire: ``chunks`` to read, then the end; what is sent, parsed.""" + + def __init__(self, chunks=()): + self.chunks, self.sent, self.closed = iter(chunks), [], False + + def recv(self, n): + return next(self.chunks, b"") + + def sendall(self, data): + self.sent.append(json.loads(data)) + + def shutdown(self, how): + pass + + def close(self): + self.closed = True + + +def until(condition, seconds=5.0): + deadline = time.monotonic() + seconds + while not condition(): + if time.monotonic() > deadline: + return False + time.sleep(0.01) + return True + + +class Bare(unittest.TestCase): + """A bridge and a server with no socket and no loop: the calls are made and served by the test.""" + + def bridge(self, steps=5, held=None): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + self.desktop = FakeDesktop(tmp) + b = cp.Bridge(self.desktop, steps, 0.01, (100, 100), 30, held=held) + self.addCleanup(b.finish) + return b + + def server(self, b): + s = cp.Server.__new__(cp.Server) # no socket bound + s._state(b, os.path.join(tempfile.gettempdir(), "no-such-socket"), None) + s.records = [] + s.record = s.records.append + return s + + def call(self, b, tool, **args): + answers = [] + call = cp.Call(tool, args, lambda c, v, i, e: answers.append((v, i, e))) + call.answers = answers + cp.checked(call, b.screen) + b.submit(call) + return call + + def upload(self, b, **args): + call = cp.Call("upload", args, None) + cp.checked(call, b.screen) + return b._upload(call) + + +class Wires(Bare): + def test_a_client_that_stops_reading_holds_up_neither_the_loop_nor_the_close(self): + entered, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + + class Stalled(FakeConnection): + def sendall(self, data): + entered.set() + release.wait(10) + raise OSError("let go") + + b = self.bridge() + wire = cp.Wire(Stalled()) + self.server(b).call(1, {"name": "screenshot"}, wire) + served = threading.Thread(target=b._serve, args=(b.calls.popleft(),), daemon=True) + served.start() + served.join(2) + self.assertFalse(served.is_alive(), "the loop's thread waited for the client to read") + self.assertTrue(entered.wait(5)) + closer = threading.Thread(target=wire.close, daemon=True) + closer.start() + closer.join(2) + self.assertFalse(closer.is_alive(), "the close waited for a write in progress") + + def test_answers_left_unread_past_the_bound_close_the_connection(self): + release = threading.Event() + self.addCleanup(release.set) + + class Stalled(FakeConnection): + def sendall(self, data): + release.wait(10) + + wire = cp.Wire(Stalled()) + with mock.patch.object(cp, "OUT_BYTES", 1000): + self.assertTrue(wire.send({"x": "a" * 400})) # taken by the writer, which stalls + self.assertTrue(until(lambda: not wire.out)) + self.assertTrue(wire.send({"x": "b" * 400})) + self.assertFalse(wire.send({"x": "c" * 400})) + self.assertTrue(wire.closed and wire.conn.closed) + + def test_a_connection_accepted_as_the_server_closes_is_closed(self): + accepted, release = threading.Event(), threading.Event() + self.addCleanup(release.set) + conn = FakeConnection() + + class Listener: + closed = False + + def accept(self): + if self.closed: + raise OSError("closed") + accepted.set() + release.wait(10) # a connection taken in, not yet registered + return conn, None + + def shutdown(self, how): + self.closed = True + + def close(self): + self.closed = True + + s = self.server(self.bridge()) + s.listener = Listener() + acceptor = threading.Thread(target=s._accept, daemon=True) + acceptor.start() + self.assertTrue(accepted.wait(5)) + s.close() + release.set() + acceptor.join(5) + self.assertTrue(conn.closed, "a connection accepted as the server closed outlived it") + self.assertEqual(s.wires, set()) + + def test_a_complete_message_over_the_bound_is_refused(self): + conn = FakeConnection([b" " * 127, b"{}\n"]) + with mock.patch.object(cp, "LINE_BYTES", 128): + self.assertEqual(list(cp.Wire(conn).lines()), []) + self.assertTrue(until(lambda: conn.sent)) + self.assertEqual(conn.sent[0]["error"]["code"], -32600) + + def test_json_too_deep_to_read_is_answered_and_the_connection_closed(self): + s = self.server(self.bridge()) + conn = FakeConnection([b"[" * 100000 + b"]" * 100000 + b"\n"]) + s._serve(cp.Wire(conn)) + self.assertTrue(until(lambda: conn.closed), "the connection stayed open") + self.assertEqual(conn.sent[0]["error"]["code"], -32700) + + +class Envelope(Bare): + VALID = {"info": {}, "screenshot": {}, "read": {"path": "a"}, "python": {"code": "pass"}, + "upload": {"path": "a", "data": "YQ=="}, "click": {"x": 1, "y": 2}, "move": {"x": 1, "y": 2}, + "drag": {"x": 1, "y": 2, "to_x": 3, "to_y": 4}, "scroll": {"x": 1, "y": 2, "dy": 1}, + "type": {"text": "a"}, "key": {"keys": ["a"]}, "wait": {}, "step": {"actions": ["WAIT"]}, "done": {}, + "fail": {}} + + def test_a_number_past_any_integer_is_refused_and_recorded(self): + b = self.bridge() + s = self.server(b) + s.handle(json.loads('{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "click", ' + '"arguments": {"x": 1e999, "y": 1}}}'), cp.Wire(FakeConnection())) + self.assertEqual([(r["tool"], r["isError"]) for r in s.records], [("click", True)]) + self.assertEqual(b.booked, 0) + + def test_a_request_that_is_not_json_rpc_2_takes_no_step(self): + b = self.bridge() + s = self.server(b) + for m in [{"jsonrpc": "1.0", "id": 1, "method": "tools/call", "params": {"name": "wait"}}, + {"jsonrpc": "2.0", "id": [], "method": "tools/call", "params": {"name": "wait"}}, + {"jsonrpc": "2.0", "id": True, "method": "tools/call", "params": {"name": "wait"}}, + {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": [{"name": "wait"}]}, + {"id": 1, "method": "tools/call", "params": {"name": "wait"}}]: + self.assertEqual(s.handle(m, cp.Wire(FakeConnection()))["error"]["code"], -32600, m) + self.assertEqual((b.booked, len(b.calls), s.records), (0, 0, [])) + + def test_calls_against_their_schema_are_refused_before_they_are_queued(self): + for tool, args in [("click", {"x": 1.5, "y": 1}), ("step", {"actions": [], "screenshot": "yes"}), + ("wait", []), ("upload", {"path": "a", "data": "!!"}), ("info", [])]: + with self.subTest(tool=tool, args=args): + b = self.bridge() + s = self.server(b) + s.call(1, {"name": tool, "arguments": args}, cp.Wire(FakeConnection())) + self.assertEqual((b.booked, len(b.calls)), (0, 0)) + self.assertEqual(s.records[0]["isError"], True) + + def test_each_listed_schema_is_what_the_calls_are_checked_against(self): + listed = {t["name"]: t["inputSchema"] for t in cp.listed((100, 100))} + self.assertEqual(sorted(listed), sorted(self.VALID)) + wrong = {"integer": "1", "string": 1, "boolean": "true", "array": "a"} + for tool, schema in listed.items(): + cp.checked(cp.Call(tool, dict(self.VALID[tool]), None), (100, 100)) # the valid call holds + for name in schema.get("required", []): + with self.subTest(tool=tool, missing=name), self.assertRaises(cp.Refused): + cp.checked(cp.Call(tool, {k: v for k, v in self.VALID[tool].items() if k != name}, None), (100, 100)) + for name, prop in schema["properties"].items(): + with self.subTest(tool=tool, wrong=name), self.assertRaises(cp.Refused): + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: wrong[prop["type"]]}), None), (100, 100)) + if "maximum" in prop: + with self.subTest(tool=tool, over=name), self.assertRaises(cp.Refused): + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"] + 1}), None), (100, 100)) + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"]}), None), (100, 100)) + + def test_an_answer_over_the_bound_is_recorded_as_failed(self): + b = self.bridge() + s, conn = self.server(b), FakeConnection() + s.call(1, {"name": "python", "arguments": {"code": "print('x' * 2000)"}}, cp.Wire(conn)) + with mock.patch.object(cp, "LINE_BYTES", 512): + b._serve(b.calls.popleft()) + self.assertTrue(until(lambda: conn.sent)) + self.assertIn("error", conn.sent[0]) + self.assertTrue(s.records[0]["isError"]) + + +class TurnEndAndSteps(Bare): + def test_the_turn_end_stops_the_actions_of_a_step_not_yet_run(self): + b = self.bridge() + call = self.call(b, "step", actions=["first()", "second()"]) + _, actions = b.predict("", {}) + self.assertEqual(next(actions), "first()") + b.end_turn() + self.assertEqual(list(actions), []) + self.assertEqual(call.answers, [(None, None, "the agent's turn has ended")]) + + def test_a_done_or_fail_taken_before_the_turn_end_reaches_the_loop(self): + for tool, action in (("done", "DONE"), ("fail", "FAIL")): + with self.subTest(tool=tool): + b = self.bridge() + click, final = self.call(b, "click", x=1, y=1), self.call(b, tool) + b.end_turn() + self.assertEqual(click.answers, [(None, None, "the agent's turn has ended")]) + _, actions = b.predict("", {}) + self.assertEqual(list(actions), [action]) + self.assertEqual(final.answers[0][0]["done"], True) + self.assertEqual(b.predict("", {}), ("", [])) + + +class Uploads(Bare): + def test_an_append_leaves_a_file_named_like_its_scratch_alone(self): + b = self.bridge() + b64 = lambda d: base64.b64encode(d).decode() + self.upload(b, path="file", data=b64(b"A")) + self.upload(b, path="file.part", data=b64(b"unrelated")) + self.assertEqual(self.upload(b, path="file", data=b64(b"B"), append=True), {"size": 2}) + with open(os.path.join(self.desktop.root, "file.part"), "rb") as f: + self.assertEqual(f.read(), b"unrelated") + self.assertEqual(sorted(os.listdir(self.desktop.root)), ["file", "file.part"]) + + def held(self): + run = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, run, True) + return cp.Held(run) + + def test_a_file_taken_by_its_sha256_starts_the_file_and_append_is_refused(self): + held = self.held() + digest = hashlib.sha256(b"B").hexdigest() + held.keep(digest, b"B") + b = self.bridge(held=held) + self.upload(b, path="f", data=base64.b64encode(b"A").decode()) + with self.assertRaisesRegex(cp.Refused, "append needs data"): + self.upload(b, path="f", sha256=digest, append=True) + self.assertEqual(self.upload(b, path="f", sha256=digest), {"size": 1}) + with open(os.path.join(self.desktop.root, "f"), "rb") as f: + self.assertEqual(f.read(), b"B") + + def test_a_file_taken_by_its_sha256_drops_what_was_being_kept_for_its_path(self): + held = self.held() + hash_b, hash_ac = hashlib.sha256(b"B").hexdigest(), hashlib.sha256(b"AC").hexdigest() + held.keep(hash_b, b"B") + b = self.bridge(held=held) + self.upload(b, path="f", data=base64.b64encode(b"A").decode(), sha256=hash_ac) + self.upload(b, path="f", sha256=hash_b) + self.upload(b, path="f", data=base64.b64encode(b"C").decode(), append=True) + with open(os.path.join(self.desktop.root, "f"), "rb") as f: + self.assertEqual(f.read(), b"BC") + self.assertIsNone(held.get(hash_ac)) + + def test_what_a_unit_was_keeping_does_not_pass_to_the_next(self): + held = self.held() + digest = hashlib.sha256(b"AB").hexdigest() + first = self.bridge(held=held) + self.upload(first, path="f", data=base64.b64encode(b"A").decode(), sha256=digest) + first.finish() + second = self.bridge(held=held) + self.upload(second, path="f", data=base64.b64encode(b"B").decode(), append=True) + self.assertIsNone(held.get(digest)) + self.assertEqual(os.listdir(held.folder), []) + + def test_a_file_over_the_bound_is_not_kept(self): + held = self.held() + digest = hashlib.sha256(b"ABC").hexdigest() + b = self.bridge(held=held) + with mock.patch.object(cp, "KEEP_BYTES", 2): + self.upload(b, path="f", data=base64.b64encode(b"AB").decode(), sha256=digest) + self.upload(b, path="f", data=base64.b64encode(b"C").decode(), append=True) + self.assertEqual(os.listdir(held.folder), []) + + def test_a_stop_while_a_file_is_kept_leaves_no_temporary_file(self): + held = self.held() + digest = hashlib.sha256(b"content").hexdigest() + with mock.patch.object(cp.os, "replace", side_effect=SystemExit(1)), self.assertRaises(SystemExit): + held.keep(digest, b"content") + self.assertEqual(os.listdir(held.folder), []) + + def test_a_link_in_place_of_a_kept_file_is_not_followed(self): + held = self.held() + elsewhere = os.path.join(self.held().folder, "content") + with open(elsewhere, "wb") as f: + f.write(b"content") + digest = hashlib.sha256(b"content").hexdigest() + os.symlink(elsewhere, os.path.join(held.folder, digest)) + self.assertIsNone(held.get(digest)) + + +class ConnectionsList(Port): + def test_a_closed_connection_leaves_the_servers_list(self): + c = self.start() + self.assertTrue(until(lambda: len(self.server.wires) == 1)) + c.close() + self.assertTrue(until(lambda: not self.server.wires), "the closed connection stayed in the list") + OSWORLD = os.environ.get("OSWORLD_DIR", "") OSWORLD_PYTHON = os.environ.get("OSWORLD_PYTHON", "") From 6eabd5d68fb4aca7b09c41e22b0736cc286a20a7 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:51:45 +0100 Subject: [PATCH 11/36] fix(standard): exit 0 from the computer pipe only after its input's end Once the server's side ends, the pipe reads its input on to its end and sends it, by its one deadline; it exits 0 only when it saw that end and sent everything, and 1 otherwise. The queue test waits for the first send and measures bytes, assuming no read's size, and runs again with reads of 4 KiB. --- app/nullius/standard/computer_pipe.py | 20 ++++----- app/tests/test_standard_computer_pipe.py | 53 +++++++++++++++++++----- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/app/nullius/standard/computer_pipe.py b/app/nullius/standard/computer_pipe.py index 9bc346b5..5fc498d6 100644 --- a/app/nullius/standard/computer_pipe.py +++ b/app/nullius/standard/computer_pipe.py @@ -17,8 +17,9 @@ the pipe's end counts to it. - Once the input ended and all it read is sent, the socket's writing side is closed; when the socket's side then ends, the pipe exits 0. -- When the socket's side ends first, the pipe reads no more input, and what it holds is sent by the deadline (the - input's, else END_SECONDS from then). What is left unsent is an exit 1, its count on stderr. +- When the socket's side ends first, the pipe reads on to its input's end and sends what it reads, by the deadline + (the input's, else END_SECONDS from then). It exits 0 only when it saw its input's end and sent all of it; else it + exits 1, the bytes left unsent on stderr. - At the deadline with the socket's side still open, the pipe takes no more of the socket, sends no more to it, and exits 3: its end was cut with the server still connected. The bytes left either way are counted on stderr. @@ -140,19 +141,18 @@ def left(): sys.stderr.write(f"computer_pipe: cut {END_SECONDS:g} s after the input's end, the server still connected: " f"{unwritten} bytes from the server not written, {unsent} bytes of input not sent\n") return CUT - with lock: # the socket's side ended: no more input is taken - quiet.set() until = deadline if deadline is not None else time.monotonic() + END_SECONDS - while left()[0] and not failed and time.monotonic() < until: - time.sleep(0.05) - if input_ended.is_set(): - sent_all.wait(max(0.0, until - time.monotonic())) + while not failed and time.monotonic() < until and not (input_ended.is_set() and sent_all.is_set()): + time.sleep(0.05) # the socket's side ended: the input is read on to its end, and sent + with lock: # no more input is taken + quiet.set() if failed: sys.stderr.write(f"computer_pipe: {type(failed[0]).__name__}: {failed[0]}\n") return 1 unsent = left()[0] - if unsent or (input_ended.is_set() and not sent_all.is_set()): - sys.stderr.write(f"computer_pipe: the server's side ended with {unsent} bytes of input not sent\n") + if unsent or not input_ended.is_set() or not sent_all.is_set(): + sys.stderr.write(f"computer_pipe: the server's side ended with {unsent} bytes of input not sent" + + ("" if input_ended.is_set() else ", its input not read to its end") + "\n") return 1 return 0 diff --git a/app/tests/test_standard_computer_pipe.py b/app/tests/test_standard_computer_pipe.py index ec3a5c37..5d6a8cfb 100644 --- a/app/tests/test_standard_computer_pipe.py +++ b/app/tests/test_standard_computer_pipe.py @@ -192,11 +192,11 @@ def test_input_left_unsent_when_the_servers_side_ends_is_an_exit_1(self): code = p.wait(timeout=10) p.stdin.close() self.assertEqual(code, 1) - self.assertIn(b"8 bytes of input not sent", p.stderr.read()) + self.assertIn(b"8 bytes of input not sent, its input not read to its end", p.stderr.read()) - def test_input_waiting_for_a_stalled_socket_is_bounded_and_drained_once_it_takes_more(self): - # The socket takes the first chunk, then stalls until the test lets it go: the input is held back meanwhile, - # and all of it reaches the socket once the socket takes more. + def queue_scenario(self, extra=""): + """The socket takes its first send, then stalls until the test lets it go: the input is held back meanwhile, + and all of it reaches the socket once the socket takes more. No read's size is assumed.""" p = self.stand_in("class Connected:\n" " def connect(self, path): pass\n" " def sendall(self, data):\n" @@ -205,15 +205,16 @@ def test_input_waiting_for_a_stalled_socket_is_bounded_and_drained_once_it_takes " def shutdown(self, how):\n open(sys.argv[2] + '.shut', 'w').close()\n" " def recv(self, size):\n" " while not os.path.exists(sys.argv[2] + '.shut'):\n time.sleep(0.05)\n" - " return b''\n") + " return b''\n", extra) chunk, written = b"q" * 65536, [0] p.stdin.write(chunk) p.stdin.flush() written[0] += len(chunk) deadline = time.monotonic() + 10 - while self.sent() < len(chunk) and time.monotonic() < deadline: # the pipe runs and passes data on + while not self.sent() and time.monotonic() < deadline: # the pipe runs and passes data on time.sleep(0.05) - self.assertEqual(self.sent(), len(chunk), "nothing reached the socket") + first = self.sent() + self.assertGreater(first, 0, "nothing reached the socket") stop = threading.Event() def feed(): @@ -232,7 +233,7 @@ def feed(): time.sleep(0.5) bound = 65536 * (3 + pipe_module().QUEUE_CHUNKS) + 4 * 1024 * 1024 # queued, one in each hand, the OS pipe self.assertLess(written[0], bound, "the pipe read on with nothing taken by the socket") - self.assertEqual(self.sent(), len(chunk)) + self.assertEqual(self.sent(), first, "the stalled socket took more") stop.set() open(self.tmp_file() + ".go", "w").close() # the socket takes more feeder.join(10) @@ -240,6 +241,37 @@ def feed(): self.assertEqual(p.wait(timeout=20), 0) self.assertEqual(self.sent(), written[0]) + def test_input_waiting_for_a_stalled_socket_is_bounded_and_drained_once_it_takes_more(self): + self.queue_scenario() + + def test_the_bounded_queue_holds_with_reads_of_any_size(self): + self.queue_scenario("real_read = module.os.read\nmodule.os.read = lambda fd, n: real_read(fd, min(n, 4096))\n") + + def test_input_the_pipe_never_read_is_sent_or_its_exit_is_not_0(self): + # The server's side ends at once; the input, a file, is read only after a delay. The pipe reads on to the + # input's end and sends it, or does not exit 0. + request = b"request that the half-closed server can still receive\n" + with open(self.tmp_file("input"), "wb") as f: + f.write(request) + with open(self.tmp_file("input"), "rb") as src: + p = self.stand_in("class Connected:\n" + " def connect(self, path): pass\n" + " def sendall(self, data):\n open(sys.argv[2], 'ab').write(data)\n" + " def recv(self, size): return b''\n" + " def shutdown(self, how): pass\n", + "real_select = module.select.select\n" + "def delayed(*args):\n time.sleep(0.2)\n return real_select(*args)\n" + "module.select.select = delayed\n", stdin=src) + code = p.wait(timeout=10) + self.assertTrue(code != 0 or self.sent_bytes() == request, f"exit {code}, sent {self.sent_bytes()!r}") + + def sent_bytes(self): + try: + with open(self.tmp_file(), "rb") as f: + return f.read() + except OSError: + return b"" + def sent(self): try: return os.path.getsize(self.tmp_file()) @@ -321,11 +353,10 @@ def test_bytes_read_while_the_servers_side_ends_are_sent_or_counted(self): " return data\n" "module.os.read = slow\n") p.stdin.write(b"request\n") - p.stdin.flush() + p.stdin.close() # the input ends once the request is read code = p.wait(timeout=10) - p.stdin.close() err = p.stderr.read() - sent = open(self.tmp_file(), "rb").read() if os.path.exists(self.tmp_file()) else b"" + sent = self.sent_bytes() self.assertEqual((code, sent), (0, b"request\n"), f"what was read was not sent: {err!r}") def test_the_end_after_the_inputs_end_has_one_deadline(self): From a406a60bd24640504e7dda127942ca13fb3d5fe8 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:51:45 +0100 Subject: [PATCH 12/36] docs(standard): state the computer port's checked calls, turn's end and bounds The page says that a call is checked against its listed schema before it waits, what the turn's end does to a step not yet run and to a done or fail taken before it, that a session may connect again, the bounds on a connection's waiting answers and on a kept file, what an error is, and that done and fail answer before the loop runs them. --- docs/ports/agent-computer-v1.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 6403a14c..780d77b5 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -14,9 +14,9 @@ None of these is a step. | `screenshot` | none | An image: the screen as PNG, at its own size | | `read` | `path`, in the desktop | `{data}`: base64, or null when there is no such file. A file over 12 MiB is an error | | `python` | `code` | The desktop's reply, `{status, output, error, returncode}`; an error when it gave none within `python_seconds` | -| `upload` | `path`, in the desktop; `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's | `{size}`: the file's size so far | +| `upload` | `path`, in the desktop; `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's. With `sha256` and no `data`, the call takes the run's file of that `sha256` and starts the file; `append` is then refused | `{size}`: the file's size so far | -`python` runs as the desktop's user, in the desktop's Python. It is for reading the desktop and for the agent's own programs there. +`python` runs as the desktop's user, in the desktop's Python. It is for reading the desktop and for the agent's own programs there. A reply whose code failed, with its `returncode`, is the tool's result, as a shell's would be. ## Acting @@ -27,7 +27,7 @@ One call is one step. | `click` | `x`, `y`; `button`: `left`, `right` or `middle`, default `left`; `count`: 1 to 3, default 1 | A click at the point | | `move` | `x`, `y` | The pointer moved to the point | | `drag` | `x`, `y`, `to_x`, `to_y`; `button`, default `left` | A press at the first point, a move, a release at the second | -| `scroll` | `x`, `y`; `dy`, and `dx` with default 0, in wheel clicks: `dy` down, `dx` right, negative the other way | The wheel turned at the point | +| `scroll` | `x`, `y`; `dy` and `dx`, each from -100 to 100 and default 0, in wheel clicks: `dy` down, `dx` right, negative the other way; at least one is not 0 | The wheel turned at the point | | `type` | `text` | The text typed key by key | | `key` | `keys`: a list pressed together, such as `["ctrl", "s"]` | The chord | | `wait` | none | No action, the pause alone | @@ -35,24 +35,25 @@ One call is one step. | `done` | `message`, optional | The task is complete | | `fail` | `reason`, optional | The task cannot be done | -An acting call answers once its actions have run and the benchmark's pause has passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. A pointer or keyboard tool and `wait` add the screen after the pause as an image; `step` adds it when `screenshot` is true. `done` and `fail` answer as the benchmark takes them. +An acting call answers once its actions have run and the benchmark's pause has passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. A pointer or keyboard tool and `wait` add the screen after the pause as an image; `step` adds it when `screenshot` is true. `done` and `fail` answer as the benchmark takes them, before it runs them: the benchmark's loop asks for nothing after them. ## Rules | Rule | Detail | | --- | --- | -| A step | One call of an acting tool; the benchmark's own loop counts these calls. Wrong arguments are refused at once, and the call is no step | +| A step | One call of an acting tool; the benchmark's own loop counts these calls | +| Calls | Each call is checked against its tool's schema in `tools/list`, an upload's base64 decoded, before it waits for the desktop. A call that does not hold is refused at once: it takes no step and never reaches the desktop. A message is JSON-RPC 2.0, one a line, at most 24 MiB with its newline; any other is refused | | The limits | In `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result | | The step budget | The benchmark's loop stops at its budget. After the last step, every tool but `info` answers with an error, and the benchmark evaluates the desktop as it is. An acting call sent while every step is taken is refused at once | | Model calls | A benchmark may bound the agent's model calls at its step budget | -| The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call, and each step left passes with no action | +| The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call. The actions of a step not yet run are not run, and that call is refused. A `done` or `fail` taken before the message still reaches the benchmark. Each step left then passes with no action: the benchmark's loop counts those steps up to its budget, its own records list no action for them, and the call log holds the agent's own steps | | Input | Pointer and keyboard input goes through the acting tools. Input sent through `python` is outside the port | | One at a time | The server runs one desktop operation at a time, in the order the calls arrive. A call sent during a step waits for it | | Before the desktop is ready | `initialize`, `tools/list` and `info` answer at once; any other call waits. The session starts once the desktop is ready | -| Errors | A call the desktop failed answers with `isError` and the desktop's words. The agent may call again | -| Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first. Nothing else passes from one unit to the next | -| Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A connection whose input has ended is closed once its calls are answered | -| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records. No argument and no result is recorded | +| Errors | An error (`isError`) is a call refused, or a call the desktop could not run or did not answer in time; it has the desktop's words. The agent may call again | +| Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first, up to 256 MiB. Those chunks are the unit's own: they are dropped when the file starts again and at the unit's end. Nothing else passes from one unit to the next | +| Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again: a connection that closes during a step does not end the task. The turn's end does, which Nullius sends when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | +| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded | | More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | ## For a benchmark's runner @@ -64,9 +65,9 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `Desktop` | Five methods over its desktop: `python(code)`, `upload(path, data)`, `read(path)`, `capture()`, `password()` | | `Bridge` | Handed to the benchmark's own loop as its agent: `reset` does nothing, and each `predict` returns the actions of the agent's next acting call. Given the loop's step budget and pause, the screen's size and `python_seconds`; `finish` once the loop has returned | | `Server` | The MCP server at a Unix socket in a folder of the user's alone, with the path of the call log | -| `Held` | The run's folder for files sent once, when the runner keeps one | +| `Held` | The run's folder for files sent once, when the runner keeps one. A file is kept through a temporary file of its own, and read only when it is no link | | The session's server | Setup's `mcp` entry runs `computer_pipe.py SOCKET`, which joins the session's stdio to the socket | -| `ready`, `lifeline`, `Stop` | The ready file, written at the loop's first `predict`; the lifeline from the driver, whose line `turn-end` is the turn's end and whose close is a stop; the one stop, with its last resort | +| `ready`, `lifeline`, `Stop` | The ready file, written at the loop's first `predict`; the lifeline from the driver, whose line `turn-end` is the turn's end and whose close is a stop; the one stop, whose last resort gives its own callback half a second before it exits | ## The kit From 674fb528eefd88acf69969c8d79fc59e13f2eda9 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:53:46 +0100 Subject: [PATCH 13/36] docs(standard): plainer wording for the computer port's turn's end and connections --- docs/ports/agent-computer-v1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 780d77b5..78cac14b 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -46,13 +46,13 @@ An acting call answers once its actions have run and the benchmark's pause has p | The limits | In `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result | | The step budget | The benchmark's loop stops at its budget. After the last step, every tool but `info` answers with an error, and the benchmark evaluates the desktop as it is. An acting call sent while every step is taken is refused at once | | Model calls | A benchmark may bound the agent's model calls at its step budget | -| The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call. The actions of a step not yet run are not run, and that call is refused. A `done` or `fail` taken before the message still reaches the benchmark. Each step left then passes with no action: the benchmark's loop counts those steps up to its budget, its own records list no action for them, and the call log holds the agent's own steps | +| The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call. The actions of a step not yet run are not run, and that call is refused. A `done` or `fail` taken before the message still reaches the benchmark. Each step left then passes with no action. The benchmark's loop counts those steps up to its budget, and its own records list no action for them. The call log holds the agent's own steps | | Input | Pointer and keyboard input goes through the acting tools. Input sent through `python` is outside the port | | One at a time | The server runs one desktop operation at a time, in the order the calls arrive. A call sent during a step waits for it | | Before the desktop is ready | `initialize`, `tools/list` and `info` answer at once; any other call waits. The session starts once the desktop is ready | | Errors | An error (`isError`) is a call refused, or a call the desktop could not run or did not answer in time; it has the desktop's words. The agent may call again | | Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first, up to 256 MiB. Those chunks are the unit's own: they are dropped when the file starts again and at the unit's end. Nothing else passes from one unit to the next | -| Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again: a connection that closes during a step does not end the task. The turn's end does, which Nullius sends when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | +| Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again; the task goes on when a connection closes during a step. The turn's end ends the task: Nullius sends it when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | | The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded | | More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | From f031bd23b4cb997741ffbd2937159dbc3515b08d Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:54:20 +0100 Subject: [PATCH 14/36] test(standard): hold the late connection open until the server closes it --- app/tests/test_standard_computer_port.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index 649aceb6..db4f3efc 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -755,9 +755,19 @@ def sendall(self, data): self.assertTrue(wire.closed and wire.conn.closed) def test_a_connection_accepted_as_the_server_closes_is_closed(self): - accepted, release = threading.Event(), threading.Event() + accepted, release, shut = threading.Event(), threading.Event(), threading.Event() self.addCleanup(release.set) - conn = FakeConnection() + self.addCleanup(shut.set) + + class Pending(FakeConnection): # a client that sends nothing and stays + def recv(self, n): + shut.wait(10) + return b"" + + def shutdown(self, how): + shut.set() + + conn = Pending() class Listener: closed = False From eeb23dcd37a83cd4e2634eabb45a04ed8d7a016c Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:55:36 +0100 Subject: [PATCH 15/36] fix(standard): run the computer kit's X server unprivileged, with its cookie and no key repeat The desktop's X server ran as root with access control off, and a key held from one action to the next repeated between them, which the verdicts count as input outside every action. - The server runs as `display`, an account of its own, with access control on: root, user and fixture connect with the cookie the session writes. - The core key repeat is turned off, and the server's repeat delay is the longest XKB allows, which also covers the input-test device. - The session ends, and the container with it, when the display, the session bus or the window manager ends, not the display alone. - The image adds the accessibility bus (at-spi2-core), which the session bus starts on demand for programs that look for it. --- .../kits/agent-computer-v1/desktop.Dockerfile | 9 ++- .../kits/agent-computer-v1/session.py | 81 +++++++++++++------ 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile b/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile index 6520f834..224eec07 100644 --- a/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.Dockerfile @@ -1,11 +1,12 @@ # The computer kit's desktop, a layer on Nullius's base image (Nullius writes the FROM line): # a virtual X display at 1920 x 1080 on :0, the Openbox window manager, a session bus for uid 1000, -# Python 3 with Tk, and pyautogui and Pillow pinned by version and hash. +# the accessibility bus (AT-SPI) for programs that look for it, Python 3 with Tk, and pyautogui and Pillow +# pinned by version and hash. # Two users: `user` (uid 1000), as whom the agent's Python and its actions run, with no sudo; -# `fixture`, as whom the kit's fixture runs, its log in a folder only it reads. +# `fixture`, as whom the kit's fixture runs, its log in a folder only it reads. The X server runs as `display`. RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - xvfb openbox dbus python3-tk python3-setuptools python3-wheel fonts-dejavu-core \ + xvfb openbox dbus at-spi2-core python3-tk python3-setuptools python3-wheel fonts-dejavu-core \ && rm -rf /var/lib/apt/lists/* COPY desktop-requirements.txt /opt/desktop/requirements.txt RUN uv pip install --system --break-system-packages --no-cache --require-hashes --no-build-isolation \ @@ -15,6 +16,8 @@ RUN (userdel -r ubuntu 2>/dev/null || true) \ && groupadd -g 1000 user && useradd -u 1000 -g 1000 -m -d /home/user -s /bin/bash user \ && groupadd -g 1001 fixture && useradd -u 1001 -g 1001 -M -d /var/lib/fixture -s /usr/sbin/nologin fixture \ && install -d -o fixture -g fixture -m 0700 /var/lib/fixture \ + && groupadd -g 1002 display && useradd -u 1002 -g 1002 -M -d /var/lib/display -s /usr/sbin/nologin display \ + && install -d -o display -g display -m 0700 /var/lib/display \ && install -d -m 0755 /run/desktop COPY session.py fixture.py /opt/desktop/ CMD ["python3", "-I", "/opt/desktop/session.py"] diff --git a/app/nullius/standard/kits/agent-computer-v1/session.py b/app/nullius/standard/kits/agent-computer-v1/session.py index 84e4baf1..22073308 100644 --- a/app/nullius/standard/kits/agent-computer-v1/session.py +++ b/app/nullius/standard/kits/agent-computer-v1/session.py @@ -1,12 +1,16 @@ """The desktop's session, the container's command, run as root under the -engine's init: the X display, then the session bus and the window manager as -``user``. Each of root, ``user`` and ``fixture`` gets the display's cookie in -its ``~/.Xauthority``, under this host's name and for any host, as -``python3-xlib`` (pyautogui's) and libX11 look it up; access control stays -off, so a client without it connects too. Once the window manager has -claimed the display it writes ``/run/desktop/ready``; it lives as long as the -display does, and its end ends the container. Standard library only, but for -the Xlib check of the window manager.""" +engine's init. + +The X display runs as ``display``, an account of its own, with access +control on: a client connects with the display's cookie, which this session +writes into the ``~/.Xauthority`` of root, ``user`` and ``fixture``, under +this host's name and for any host, as ``python3-xlib`` (pyautogui's) and +libX11 look it up. The server's key repeat is off: a key held between two +actions sends nothing in between. Then the session bus and the window +manager start as ``user``. Once the window manager has claimed the display +the session writes ``/run/desktop/ready``; it ends, and the container with +it, when the display, the bus or the window manager ends. Standard library +only, but for Xlib.""" import os import signal import socket @@ -19,6 +23,8 @@ SCREEN = "1920x1080x24" UID = 1000 HOME = "/home/user" +SERVER_UID = 1002 # `display`, as whom the X server runs +SERVER_AUTH = "/var/lib/display/Xauthority" RUNTIME = f"/run/user/{UID}" BUS = f"unix:path={RUNTIME}/bus" READY = "/run/desktop/ready" @@ -45,6 +51,23 @@ def field(b): for family, addr in ((256, host.encode()), (0xFFFF, b""))) +def write_owned(path, data, uid): + with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "wb") as f: + f.write(data) + os.chown(path, uid, uid) + + +# XKB keeps the repeat delay in 16 bits: the longest it can wait before a held key repeats, 65.5 s +REPEAT_DELAY_MS = 65535 + + +def server_argv(): + """The X server's command: the screen, its cookie file, no TCP, access control left on, and every + keyboard's repeat delayed as long as XKB allows (the core switch below misses the input-test device).""" + return ["Xvfb", DISPLAY, "-screen", "0", SCREEN, "-auth", SERVER_AUTH, "-nolisten", "tcp", + "-ardelay", str(REPEAT_DELAY_MS), "-arinterval", str(REPEAT_DELAY_MS)] + + def wait_for(what, ok, seconds=30.0): end = time.monotonic() + seconds while not ok(): @@ -53,44 +76,55 @@ def wait_for(what, ok, seconds=30.0): time.sleep(0.05) -def window_manager_up(): - """Whether a window manager has said so on the root window (EWMH).""" +def _with_display(fn): try: from Xlib import display d = display.Display(DISPLAY) try: - prop = d.screen().root.get_full_property(d.intern_atom("_NET_SUPPORTING_WM_CHECK"), 0) - return bool(prop and prop.value) + return fn(d) finally: d.close() except Exception: return False +def repeat_off(d): + """The server's key repeat turned off, for every keyboard: a held key sends one press and one release.""" + from Xlib import X + d.change_keyboard_control(auto_repeat_mode=X.AutoRepeatModeOff) + d.sync() + return d.get_keyboard_control().global_auto_repeat == X.AutoRepeatModeOff + + +def window_manager_up(d): + """Whether a window manager has said so on the root window (EWMH).""" + prop = d.screen().root.get_full_property(d.intern_atom("_NET_SUPPORTING_WM_CHECK"), 0) + return bool(prop and prop.value) + + def main(): os.makedirs("/tmp/.X11-unix", exist_ok=True) os.chmod("/tmp/.X11-unix", 0o1777) data = xauthority(os.urandom(16), socket.gethostname()) + write_owned(SERVER_AUTH, data, SERVER_UID) for home, uid in HOMES: - path = os.path.join(home, ".Xauthority") - with open(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "wb") as f: - f.write(data) - os.chown(path, uid, uid) - # access control off: every local account draws and is drawn on; the only way in is the socket - xvfb = subprocess.Popen(["Xvfb", DISPLAY, "-screen", "0", SCREEN, "-auth", "/root/.Xauthority", "-nolisten", "tcp", "-ac"], - stdin=subprocess.DEVNULL) + write_owned(os.path.join(home, ".Xauthority"), data, uid) + xvfb = subprocess.Popen(server_argv(), user=SERVER_UID, group=SERVER_UID, extra_groups=[], cwd="/var/lib/display", + env={"PATH": user_env()["PATH"], "HOME": "/var/lib/display"}, + stdin=subprocess.DEVNULL, start_new_session=True) wait_for("the X display", lambda: os.path.exists(X_SOCKET) or xvfb.poll() is not None) if xvfb.poll() is not None: sys.exit("desktop: Xvfb ended") + wait_for("the key repeat's switch", lambda: _with_display(repeat_off)) os.makedirs(RUNTIME, exist_ok=True) os.chown(RUNTIME, UID, UID) os.chmod(RUNTIME, 0o700) bus = as_user(["dbus-daemon", "--session", f"--address={BUS}", "--nofork", "--nopidfile"]) wait_for("the session bus", lambda: os.path.exists(f"{RUNTIME}/bus")) wm = as_user(["openbox"]) - wait_for("the window manager", window_manager_up) + wait_for("the window manager", lambda: _with_display(window_manager_up)) with open(READY, "w") as f: - f.write(f"{time.time():.6f}\n") + f.write(f"{time.monotonic():.6f}\n") def stop(*_, code=0): for p in (wm, bus, xvfb): @@ -99,8 +133,9 @@ def stop(*_, code=0): sys.exit(code) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) - xvfb.wait() - stop(code=1) # the display ended on its own + while all(p.poll() is None for p in (xvfb, bus, wm)): + time.sleep(0.5) + stop(code=1) # one of the three ended on its own if __name__ == "__main__": From e288af29fb892b9874dc92c3c0b3d0e19c861363 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:55:36 +0100 Subject: [PATCH 16/36] fix(standard): earn the computer kit's passes and make its desktop fail loudly A pass must come from input sent inside actions, and a correct agent must not be failed by the desktop's own timing. - An action answers only once the verdicts' slack has passed on the desktop, so a request sent after it cannot fall inside its time. - The action's times and the fixture's events are read on one monotonic clock: a step of the wall clock no longer fails a correct action. - click fails on a press of any mouse button on another button; Enter in the field is input whose time is checked, as keys are. - The fixture says ready only when its window is mapped at the screen's size and holds the focus within its deadline. - read gives a regular file alone, at most 12 MiB, and refuses the rest as the agent's error. - A start that fails or is interrupted removes the container it made, its log read or not; remove checks the engine's answer and that the container is gone; a fixture log that cannot be read, or holds a line that is no event, is an error. - The docker tier's desktop tests run wherever an engine answers and build the image first; the removal test's probe is bounded and must succeed. --- .../standard/kits/agent-computer-v1/checks.py | 26 +- .../kits/agent-computer-v1/desktop.py | 136 +++++--- .../kits/agent-computer-v1/fixture.py | 33 +- app/tests/test_computer_kit.py | 308 +++++++++++++++++- 4 files changed, 430 insertions(+), 73 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index ce2b5c07..8ba3ff67 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -6,16 +6,17 @@ ``start`` and ``end`` on the same clock) and the agent's last message. - ``see``: the last message holds the code the window drew; -- ``click``: the log holds a left press on the named button and on none of - the five others; +- ``click``: the log holds a left press on the named button and no press, + of any mouse button, on the five others; - ``type``: the field held exactly the word at the last Enter. -``click`` and ``type`` pass only when every pointer and key event lies -inside an action's time: input sent any other way, through the desktop's -Python for one, fails them. An event counts as inside from the action's start -to ``SLACK`` seconds after its end, the time the fixture may take to take an -event in once the action has sent it. The kit's pause after each action is -longer, so nothing sent after an action has answered falls in it. +``click`` and ``type`` pass only when every pointer and key event, Enter in +the field included, lies inside an action's time: input sent any other way, +through the desktop's Python for one, fails them. An event counts as inside +from the action's start to ``SLACK`` seconds after its end, the time the +fixture may need to receive an event the action sent. An action answers only +once that time has passed on the desktop (``desktop.SETTLE``): nothing sent +after it has answered falls in it. """ import re import secrets @@ -27,7 +28,7 @@ "KNOT", "LEMON", "MAPLE", "NORTH", "OCEAN", "PIANO", "QUILT", "RIVER", "STONE", "TIGER", "UMBRA", "VELVET", "WHEAT", "YEAST") POINTER = ("press", "release", "scroll") -KEYS = ("key", "keyup") +KEYS = ("key", "keyup", "enter") CHECKS = ("see", "click", "type") @@ -86,11 +87,10 @@ def see(expected, events, actions, message): def click(expected, events, actions, message): target = expected["target"] - presses = [e for e in events if e.get("kind") == "press" and e.get("button") == 1] - hits = {e.get("hit") for e in presses} - if target not in hits: + presses = [e for e in events if e.get("kind") == "press"] + if not any(e.get("hit") == target and e.get("button") == 1 for e in presses): return _verdict(f"no left press on the button {target}") - others = sorted(h for h in hits if h != target and h in expected.get("labels", ())) + others = sorted({e.get("hit") for e in presses} & set(expected.get("labels", ())) - {target}) if others: return _verdict(f"a press on another button as well: {', '.join(others)}") return _verdict(_input_rule(events, actions)) diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.py b/app/nullius/standard/kits/agent-computer-v1/desktop.py index e75df7bb..d2ee4dbd 100644 --- a/app/nullius/standard/kits/agent-computer-v1/desktop.py +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.py @@ -18,16 +18,20 @@ answers ``{status, output, error, returncode}``, or None when it did not end within its bound (it is killed then); - :meth:`Desktop.upload` and :meth:`Desktop.read` write and read a file as - ``user``; :meth:`Desktop.capture` is the screen as PNG bytes; + ``user``, a read bounded at 12 MiB and of a regular file alone; + :meth:`Desktop.capture` is the screen as PNG bytes; :meth:`Desktop.password` is ``""``: ``user`` has no password and no sudo; - :meth:`Desktop.act` runs one action, Python text with ``pyautogui`` and ``time`` imported, as ``user``, and keeps its start and end, read on the - desktop's clock by the root process that runs it: the agent's code never - writes them. + desktop's monotonic clock (the fixture's) by the root process that runs it: + the agent's code never writes them. It answers only :data:`SETTLE` seconds + after the action's end, by that clock, so no later request falls in the + time a verdict grants the action. The kit's own reads go apart: :meth:`Desktop.start_fixture` and :meth:`Desktop.events` run as ``fixture``, whose folder ``user`` cannot read. -Standard library only, and the engine's program. +A start that fails removes what it made; :meth:`Desktop.remove` checks that +the container is gone. Standard library only, and the engine's program. """ from __future__ import annotations @@ -53,6 +57,8 @@ PYTHON_SECONDS = 90 START_SECONDS = 60 OUTPUT_LIMIT = 4 * 1024 * 1024 # of each of output and error, as the reply gives them +READ_LIMIT = 12 * 1024 * 1024 # the largest file `read` gives +SETTLE = 0.5 # an action answers this long after its end: at least the verdicts' slack ACTION_PREFIX = "import pyautogui\nimport time\npyautogui.FAILSAFE = False\n" # Run as root in the desktop: the request on stdin, the code as `user`, the times around it. @@ -63,7 +69,7 @@ "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "DISPLAY": ":0", "XDG_RUNTIME_DIR": "/run/user/1000", "DBUS_SESSION_BUS_ADDRESS": "unix:path=/run/user/1000/bus"} out, err = tempfile.TemporaryFile(), tempfile.TemporaryFile() -start = time.time() +start = time.monotonic() p = subprocess.Popen(["python3", "-c", req["code"]], user=1000, group=1000, extra_groups=[], env=env, cwd="/home/user", stdin=subprocess.DEVNULL, stdout=out, stderr=err, start_new_session=True) try: @@ -71,7 +77,10 @@ except subprocess.TimeoutExpired: os.killpg(p.pid, signal.SIGKILL) rc, late = p.wait(), True -end = time.time() +end = time.monotonic() +rest = end + req.get("settle", 0) - time.monotonic() +if rest > 0: + time.sleep(rest) def text(f): f.seek(0) return f.read(req["limit"]).decode("utf-8", "replace") @@ -89,13 +98,19 @@ def text(f): """ _READ = r""" -import os, sys +import os, stat, sys +limit = int(sys.argv[2]) try: - f = open(os.path.join(os.path.expanduser("~"), sys.argv[1]), "rb") + fd = os.open(os.path.join(os.path.expanduser("~"), sys.argv[1]), os.O_RDONLY | os.O_NONBLOCK) except OSError: sys.exit(3) -with f: - sys.stdout.buffer.write(f.read()) +if not stat.S_ISREG(os.fstat(fd).st_mode): + sys.exit(4) +with os.fdopen(fd, "rb") as f: + data = f.read(limit + 1) +if len(data) > limit: + sys.exit(5) +sys.stdout.buffer.write(data) """ _CAPTURE = r""" @@ -113,6 +128,11 @@ class DesktopError(Exception): """The desktop failed a request: it is gone, or answered nothing a request answers.""" +class Refused(Exception): + """A request the desktop does not serve: a read of a file that is not a regular one, or of one past + :data:`READ_LIMIT`. The agent's error, never the desktop's.""" + + def image(log_dir: Optional[str] = None) -> str: """The desktop's image, built the first time it is asked for, on the base image.""" return container.env_image(KIT, ENVIRONMENT, log_dir) @@ -135,6 +155,11 @@ def _said(r: subprocess.CompletedProcess) -> str: return (r.stderr or r.stdout or b"").decode("utf-8", "replace").strip()[-300:] +def _absent(r: subprocess.CompletedProcess) -> bool: + """Whether the engine said the container does not exist.""" + return "no such container" in (r.stderr or b"").decode("utf-8", "replace").lower() + + class Desktop: """One running desktop, by its container's name.""" @@ -149,18 +174,30 @@ def start(cls, image: str, name: Optional[str] = None, *, job: Optional[str] = N labels the container as a unit's are, so a sweep of the job removes it.""" d = cls(name or f"nullius-desktop-{secrets.token_hex(4)}") label = ["--label", f"{container.LABEL}={job}"] if job else [] - r = _engine(["run", "-d", "--init", "--name", d.name, "--network", "none", "--security-opt", "no-new-privileges", - "--shm-size", "256m", *label, *container.capped(), image], timeout=120) - if r.returncode != 0: - raise DesktopError(f"starting the desktop {d.name}: {_said(r)}") try: - r = _engine(["exec", d.name, "sh", "-c", _READY.format(n=int(seconds * 10))], timeout=seconds + 30) - except subprocess.TimeoutExpired: - r = None - if r is None or r.returncode != 0: - logs = _engine(["logs", "--tail", "20", d.name], timeout=30) - d.remove() - raise DesktopError(f"the desktop {d.name} was not ready in {seconds:.0f} s: {_said(logs)}") + try: + r = _engine(["run", "-d", "--init", "--name", d.name, "--network", "none", "--security-opt", + "no-new-privileges", "--shm-size", "256m", *label, *container.capped(), image], timeout=120) + except subprocess.TimeoutExpired: + raise DesktopError(f"starting the desktop {d.name}: {container.DOCKER} run did not answer in 120 s") + if r.returncode != 0: + raise DesktopError(f"starting the desktop {d.name}: {_said(r)}") + try: + r = _engine(["exec", d.name, "sh", "-c", _READY.format(n=int(seconds * 10))], timeout=seconds + 30) + except subprocess.TimeoutExpired: + r = None + if r is None or r.returncode != 0: + try: + said = _said(_engine(["logs", "--tail", "20", d.name], timeout=30)) + except (subprocess.TimeoutExpired, DesktopError): + said = "its log could not be read" + raise DesktopError(f"the desktop {d.name} was not ready in {seconds:.0f} s: {said}") + except BaseException: # an error or an interrupt: what this start made goes + try: + d.remove() + except (DesktopError, subprocess.TimeoutExpired): + pass + raise return d def _exec(self, user: str, argv: List[str], *, input: Optional[bytes] = None, timeout: float, @@ -170,11 +207,12 @@ def _exec(self, user: str, argv: List[str], *, input: Optional[bytes] = None, ti flags += ["-e", f"{k}={v}"] return _engine(["exec", *flags, "-u", user, self.name, *argv], input=input, timeout=timeout) - def _run(self, code: str, seconds: float) -> Optional[dict]: - """``code`` run as ``user`` by the desktop's root: the times and the reply, or None past ``seconds``.""" - req = json.dumps({"code": code, "seconds": seconds, "limit": OUTPUT_LIMIT}).encode() + def _run(self, code: str, seconds: float, settle: float = 0.0) -> Optional[dict]: + """``code`` run as ``user`` by the desktop's root: the times and the reply, or None past ``seconds``. + The reply comes ``settle`` seconds after the code's end, on the desktop's clock.""" + req = json.dumps({"code": code, "seconds": seconds, "limit": OUTPUT_LIMIT, "settle": settle}).encode() try: - r = self._exec(ROOT, ["python3", "-I", "-c", _RUN], input=req, timeout=seconds + 30) + r = self._exec(ROOT, ["python3", "-I", "-c", _RUN], input=req, timeout=seconds + settle + 30) except subprocess.TimeoutExpired: return None try: @@ -193,10 +231,11 @@ def python(self, code: str, seconds: float = PYTHON_SECONDS) -> Optional[dict]: def act(self, code: str, seconds: float = PYTHON_SECONDS) -> dict: """Run one action, ``code`` with ``pyautogui`` and ``time`` imported: - ``{start, end, reply}``, the times on the desktop's clock, ``reply`` as - :meth:`python` gives it. The times are kept in :attr:`actions`, also - for an action that ran past its bound.""" - said = self._run(ACTION_PREFIX + code, seconds) + ``{start, end, reply}``, the times on the desktop's monotonic clock, + ``reply`` as :meth:`python` gives it. It answers :data:`SETTLE` seconds + after ``end``. The times are kept in :attr:`actions`, also for an + action that ran past its bound.""" + said = self._run(ACTION_PREFIX + code, seconds, SETTLE) if said is None: # the desktop did not answer: it gave no time to keep return {"start": None, "end": None, "reply": None} self.actions.append({"start": said["start"], "end": said["end"]}) @@ -214,10 +253,15 @@ def upload(self, path: str, data: bytes, *, append: bool = False) -> int: return int(r.stdout.decode().strip()) def read(self, path: str) -> Optional[bytes]: - """The bytes of ``path`` as ``user`` reads them, or None when it cannot.""" - r = self._exec(USER, ["python3", "-I", "-c", _READ, path], timeout=120, env={"HOME": HOME}) + """The bytes of ``path`` as ``user`` reads them, or None when it cannot open it. A path that is + not a regular file, or a file past :data:`READ_LIMIT`, is :class:`Refused`.""" + r = self._exec(USER, ["python3", "-I", "-c", _READ, path, str(READ_LIMIT)], timeout=120, env={"HOME": HOME}) if r.returncode == 3: return None + if r.returncode == 4: + raise Refused(f"{path} is not a regular file") + if r.returncode == 5: + raise Refused(f"{path} is over {READ_LIMIT // (1024 * 1024)} MiB") if r.returncode != 0: raise DesktopError(f"read of {path}: {_said(r)}") return r.stdout @@ -240,18 +284,34 @@ def start_fixture(self, values: dict, seconds: float = 60) -> None: raise DesktopError(f"the fixture did not start: {_said(r)}") def events(self) -> List[dict]: - """The fixture's log, an event a dict; lines that are not JSON are left out.""" + """The fixture's log, an event a dict. A log that cannot be read, or a line that is not an + event, is a :class:`DesktopError`: a verdict is never given on part of the log.""" r = self._exec(FIXTURE, ["cat", FIXTURE_LOG], timeout=60) if r.returncode != 0: - return [] + raise DesktopError(f"the fixture's log could not be read: {_said(r)}") out = [] - for line in r.stdout.decode("utf-8", "replace").splitlines(): + for i, line in enumerate(r.stdout.decode("utf-8", "replace").splitlines(), 1): try: - out.append(json.loads(line)) + e = json.loads(line) except ValueError: - pass + e = None + if not isinstance(e, dict) or not isinstance(e.get("kind"), str) or not isinstance(e.get("t"), (int, float)): + raise DesktopError(f"line {i} of the fixture's log is not an event") + out.append(e) return out def remove(self) -> None: - """Remove the desktop's container, running or not.""" - _engine(["rm", "-f", self.name], timeout=120) + """Remove the desktop's container, running or not, and check that it is gone. A container that + was gone already is no error; one that stays is a :class:`DesktopError`.""" + try: + r = _engine(["rm", "-f", self.name], timeout=120) + except subprocess.TimeoutExpired: + raise DesktopError(f"removing the desktop {self.name}: {container.DOCKER} rm did not answer in 120 s") + if r.returncode != 0 and not _absent(r): + raise DesktopError(f"removing the desktop {self.name}: {_said(r)}") + try: + r = _engine(["container", "inspect", "--format", "{{.Id}}", self.name], timeout=60) + except subprocess.TimeoutExpired: + raise DesktopError(f"the desktop {self.name}'s removal could not be checked") + if r.returncode == 0 or not _absent(r): + raise DesktopError(f"the desktop {self.name} is still there after its removal: {_said(r)}") diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py index 72a8ff3e..864e5c12 100644 --- a/app/nullius/standard/kits/agent-computer-v1/fixture.py +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -13,9 +13,11 @@ Each press, release, scroll and key goes to ``events.jsonl`` in the fixture's home, a folder only ``fixture`` reads, one JSON object a line: ``kind``, the -time ``t`` on the desktop's clock, the point or the key, and ``hit``, what it +time ``t`` on the desktop's monotonic clock (the one an action's times are read +on), the point or the key, and ``hit``, what it landed on (a button's label, ``field`` or ``background``). The first line is -``ready``, with where each target is on the screen; Enter in the field writes +``ready``, with where each target is on the screen, written once the window is +mapped at the screen's size and holds the focus; Enter in the field writes ``enter`` with the field's text. The program leaves the caller once its window is up: it prints ``ready`` and @@ -134,7 +136,7 @@ def _type(self): self.field.focus_set() def write(self, kind, **fields): - self.log_file.write(json.dumps({"kind": kind, "t": round(time.time(), 6), **fields}) + "\n") + self.log_file.write(json.dumps({"kind": kind, "t": round(time.monotonic(), 6), **fields}) + "\n") self.log_file.flush() def on_event(self, e): @@ -159,23 +161,32 @@ def on_event(self, e): def box(self, w): return [w.winfo_rootx(), w.winfo_rooty(), w.winfo_rootx() + w.winfo_width(), w.winfo_rooty() + w.winfo_height()] + def is_up(self): + """Mapped, at the screen's size, and with the keyboard's focus where the screen wants it.""" + r = self.root + if not (r.winfo_viewable() and r.winfo_width() == r.winfo_screenwidth() + and r.winfo_height() == r.winfo_screenheight()): + return False + focus = r.focus_get() + return focus is not None and (self.field is None or focus == self.field) + def up(self, seconds=10.0): - """Wait until the window fills the screen, then write ``ready`` with where each target is.""" + """Wait until the window is up (:meth:`is_up`), then write ``ready`` with where each target is. + Past ``seconds`` it raises, and writes nothing.""" end = time.monotonic() + seconds self.root.update() - while time.monotonic() < end and (self.root.winfo_width() < self.root.winfo_screenwidth() - or not self.root.winfo_viewable()): + while not self.is_up(): + if time.monotonic() >= end: + raise RuntimeError("the fixture's window was not up in time") + self.root.focus_force() + if self.field is not None: + self.field.focus_set() time.sleep(0.05) self.root.update() - self.root.focus_force() - if self.field is not None: - self.field.focus_set() - self.root.update() boxes = {label: self.box(self.root.nametowidget(path)) for path, label in self.names.items()} self.write("ready", screen=self.values["screen"], size=[self.root.winfo_width(), self.root.winfo_height()], boxes=boxes) - def main(): values = json.loads(sys.stdin.read()) if values.get("screen") not in ("see", "click", "type"): diff --git a/app/tests/test_computer_kit.py b/app/tests/test_computer_kit.py index b2730a14..e52f7e3f 100644 --- a/app/tests/test_computer_kit.py +++ b/app/tests/test_computer_kit.py @@ -6,8 +6,10 @@ desktop's Python judged wrong.""" from __future__ import annotations +import contextlib import fnmatch import importlib.util +import io import json import os import random @@ -15,6 +17,8 @@ import secrets import struct import subprocess +import sys +import tempfile import time import unittest from unittest import mock @@ -36,6 +40,7 @@ def _module(name, file): checks = _module("computer_kit_checks", "checks.py") desktop = _module("computer_kit_desktop", "desktop.py") fixture = _module("computer_kit_fixture", "fixture.py") +session = _module("computer_kit_session", "session.py") ACTIONS = [{"start": 100.0, "end": 101.0}, {"start": 110.0, "end": 112.0}] @@ -114,10 +119,11 @@ def test_type_fails_on_another_text_a_later_change_no_enter_or_keys_outside_acti by_python = keys(105.0, "kotabiru") + [ev("enter", 105.2, value="kotabiru")] self.assertIn("outside every action", checks.verdict("type", exp, by_python, ACTIONS, "")["why"]) - def test_input_rule_counts_pointer_and_key_events_only(self): - events = [ev("ready", 1.0), ev("enter", 105.0, value="x"), ev("press", 100.5, button=1, hit="OCEAN")] + def test_input_rule_counts_pointer_and_key_events_and_enter_only(self): + events = [ev("ready", 1.0), ev("enter", 100.6, value="x"), ev("press", 100.5, button=1, hit="OCEAN")] self.assertEqual(checks.outside(events, ACTIONS), []) - self.assertEqual([e["kind"] for e in checks.outside([ev("scroll", 105.0, button=4)], ACTIONS)], ["scroll"]) + stray = [ev("scroll", 105.0, button=4), ev("enter", 105.1, value="x"), ev("ready", 105.2)] + self.assertEqual([e["kind"] for e in checks.outside(stray, ACTIONS)], ["scroll", "enter"]) class Draw(unittest.TestCase): @@ -233,6 +239,227 @@ def late(args, *, input=None, timeout): self.assertEqual(d.actions, []) +class EarnedPasses(unittest.TestCase): + """A pass comes from input sent inside actions and seen by the fixture, nothing else.""" + + def test_a_press_of_any_mouse_button_on_another_button_fails_click(self): + exp = {"target": "OCEAN", "labels": ["OCEAN", "APPLE"]} + for button in (1, 2, 3): + events = [ev("press", 100.1, button=button, hit="APPLE"), ev("press", 100.2, button=1, hit="OCEAN")] + self.assertIn("another button", checks.click(exp, events, [{"start": 100, "end": 101}], "")["why"]) + + def test_enter_is_input_whose_time_is_checked(self): + exp = {"word": "kotabiru"} + self.assertFalse(checks.type_word(exp, [ev("enter", 100, value="kotabiru")], [], "")["passed"]) + late = [ev("key", 100.1, char="k", hit="field"), ev("enter", 105, value="kotabiru")] + v = checks.type_word(exp, late, [{"start": 100, "end": 101}], "") + self.assertIn("outside every action", v["why"]) + + def test_a_press_and_its_release_in_two_actions_pass(self): + events = [ev("press", 100.2, hit="OCEAN", button=1), ev("release", 110.2, hit="OCEAN", button=1)] + actions = [{"start": 100, "end": 101}, {"start": 110, "end": 111}] + self.assertTrue(checks.click({"target": "OCEAN", "labels": ["OCEAN", "APPLE"]}, events, actions, "")["passed"]) + + +def _run_in_desktop(request, *, monotonic=None, wall=None, wait=None): + """The desktop's root process (``desktop._RUN``) run here with its process start faked: its printed reply, + and what it slept.""" + p = mock.Mock() + p.wait.side_effect = wait or (lambda **kw: 0) + slept = [] + out = io.StringIO() + patches = [mock.patch.object(sys, "stdin", mock.Mock(buffer=io.BytesIO(json.dumps(request).encode()))), + mock.patch("subprocess.Popen", return_value=p), + mock.patch("tempfile.TemporaryFile", side_effect=lambda: io.BytesIO()), + mock.patch("time.sleep", side_effect=slept.append)] + if monotonic is not None: + patches.append(mock.patch("time.monotonic", side_effect=monotonic)) + if wall is not None: + patches.append(mock.patch("time.time", side_effect=wall)) + with contextlib.ExitStack() as stack: + for patch in patches: + stack.enter_context(patch) + with contextlib.redirect_stdout(out): + exec(desktop._RUN, {}) + return json.loads(out.getvalue()), slept + + +class ActionTimes(unittest.TestCase): + def test_an_action_answers_only_once_the_verdicts_slack_has_passed_on_the_desktop(self): + self.assertGreaterEqual(desktop.SETTLE, checks.SLACK) + run, sent = _engine_answering({"start": 1.0, "end": 2.0, "timeout": False, "returncode": 0, "output": "", "error": ""}) + with mock.patch.object(desktop, "_engine", run): + desktop.Desktop("d").act("pass") + desktop.Desktop("d").python("pass") + self.assertEqual([json.loads(payload)["settle"] for _, payload in sent], [desktop.SETTLE, 0.0]) + said, slept = _run_in_desktop({"code": "pass", "seconds": 5, "limit": 100, "settle": 0.5}, + monotonic=[10.0, 11.0, 11.1]) + self.assertEqual((said["start"], said["end"]), (10.0, 11.0)) + self.assertEqual([round(x, 6) for x in slept], [0.4]) # the reply waits until 11.5 + + def test_a_wall_clock_step_does_not_fail_a_correct_action(self): + f = fixture.Fixture.__new__(fixture.Fixture) + f.log_file = io.StringIO() + + def the_click(**kw): + f.write("press", button=1, hit="OCEAN") + return 0 + said, _ = _run_in_desktop({"code": "pass", "seconds": 1, "limit": 4096}, wall=[100, 100.1, 95, 95, 95], + wait=the_click) + events = [json.loads(line) for line in f.log_file.getvalue().splitlines()] + self.assertTrue(checks.click({"target": "OCEAN", "labels": ["OCEAN"]}, events, [said], "")["passed"]) + + +class _FakeX: + AutoRepeatModeOff = 0 + AutoRepeatModeOn = 1 + + +class Session(unittest.TestCase): + def test_the_x_server_runs_with_access_control_on_and_as_its_own_account(self): + argv = session.server_argv() + self.assertNotIn("-ac", argv) + self.assertEqual(argv[argv.index("-auth") + 1], session.SERVER_AUTH) + self.assertEqual(argv[argv.index("-ardelay") + 1], "65535") # the longest XKB allows + self.assertNotIn(session.SERVER_UID, (0, session.UID, 1001)) + with open(os.path.join(KIT, "desktop.Dockerfile")) as f: + self.assertIn(f"useradd -u {session.SERVER_UID} ", f.read()) + + def test_the_servers_key_repeat_is_turned_off(self): + d = mock.Mock() + d.get_keyboard_control.return_value = mock.Mock(global_auto_repeat=_FakeX.AutoRepeatModeOff) + with mock.patch.dict(sys.modules, {"Xlib": mock.Mock(X=_FakeX), "Xlib.X": _FakeX}): + self.assertTrue(session.repeat_off(d)) + d.change_keyboard_control.assert_called_once_with(auto_repeat_mode=_FakeX.AutoRepeatModeOff) + + def test_the_cookie_file_names_this_host_and_any_host(self): + data = session.xauthority(b"c" * 16, "desktop") + self.assertTrue(data.startswith(struct.pack(">H", 256) + struct.pack(">H", 7) + b"desktop")) + self.assertIn(struct.pack(">H", 0xFFFF) + struct.pack(">H", 0), data) + self.assertEqual(data.count(b"MIT-MAGIC-COOKIE-1"), 2) + + +def _response(args, rc=0, stdout=b"", stderr=b""): + return subprocess.CompletedProcess(args, rc, stdout, stderr) + + +class Lifecycle(unittest.TestCase): + def test_a_start_that_times_out_removes_what_it_made(self): + calls = [] + + def engine(args, **kw): + calls.append(args) + if args[0] == "run": + raise subprocess.TimeoutExpired(args, kw["timeout"]) + return _response(args, rc=1, stderr=b"Error: No such container: review") + with mock.patch.object(desktop, "_engine", engine): + with self.assertRaises(desktop.DesktopError): + desktop.Desktop.start("image", "review") + self.assertIn(["rm", "-f", "review"], calls) + + def test_a_desktop_never_ready_is_removed_even_when_its_log_cannot_be_read(self): + calls = [] + + def engine(args, **kw): + calls.append(args) + if args[0] == "logs": + raise subprocess.TimeoutExpired(args, kw["timeout"]) + if args[0] == "container": + return _response(args, rc=1, stderr=b"Error: No such container: review") + return _response(args, rc=1 if args[0] == "exec" else 0) + with mock.patch.object(desktop, "_engine", engine): + with self.assertRaises(desktop.DesktopError) as e: + desktop.Desktop.start("image", "review") + self.assertIn("not ready", str(e.exception)) + self.assertIn(["rm", "-f", "review"], calls) + + def test_remove_fails_when_the_engine_does_or_the_container_stays(self): + with mock.patch.object(desktop, "_engine", return_value=_response([], rc=1, stderr=b"busy")): + with self.assertRaises(desktop.DesktopError): + desktop.Desktop("review").remove() + with mock.patch.object(desktop, "_engine", return_value=_response([], rc=0, stdout=b"abc")): + with self.assertRaises(desktop.DesktopError): # removed, yet inspect still finds it + desktop.Desktop("review").remove() + gone = _response([], rc=1, stderr=b"Error: No such container: review") + + def busy_then_gone(args, **kw): # the engine's own failure is said + return _response(args, rc=1, stderr=b"busy") if args[0] == "rm" else gone + with mock.patch.object(desktop, "_engine", busy_then_gone): + with self.assertRaises(desktop.DesktopError): + desktop.Desktop("review").remove() + with mock.patch.object(desktop, "_engine", return_value=gone): + desktop.Desktop("review").remove() # gone already: no error + + def test_the_fixture_says_ready_only_when_its_window_is_up_in_time(self): + def fake(viewable, size, focus): + f = fixture.Fixture.__new__(fixture.Fixture) + f.root = mock.Mock() + f.root.winfo_viewable.return_value = viewable + f.root.winfo_width.return_value, f.root.winfo_height.return_value = size + f.root.winfo_screenwidth.return_value, f.root.winfo_screenheight.return_value = desktop.SCREEN + f.root.focus_get.return_value = focus + f.field, f.names, f.values = None, {}, {"screen": "see"} + f.log_file = io.StringIO() + return f + for viewable, size, focus in ((False, (1, 1), mock.Mock()), (True, (1920, 1), mock.Mock()), + (True, desktop.SCREEN, None)): + f = fake(viewable, size, focus) + with self.assertRaises(RuntimeError): + f.up(seconds=0) + self.assertNotIn('"ready"', f.log_file.getvalue()) + f = fake(True, desktop.SCREEN, mock.Mock()) + f.up(seconds=0) + self.assertIn('"ready"', f.log_file.getvalue()) + + def test_a_log_that_cannot_be_read_or_holds_a_line_that_is_no_event_is_an_error(self): + for answer in (_response([], rc=1, stderr=b"No such file"), + _response([], stdout=b'{"kind": "ready", "t": 1.0}\n{"kind": "press"\n'), + _response([], stdout=b'{"kind": "press"}\n')): + with mock.patch.object(desktop, "_engine", return_value=answer): + with self.assertRaises(desktop.DesktopError): + desktop.Desktop("d").events() + with mock.patch.object(desktop, "_engine", return_value=_response([], stdout=b'{"kind": "ready", "t": 1.5}\n')): + self.assertEqual(desktop.Desktop("d").events(), [{"kind": "ready", "t": 1.5}]) + + +class Reads(unittest.TestCase): + """``desktop._READ`` run here, as the desktop runs it: a regular file alone, and at most the bound.""" + + def read(self, path, limit): + return subprocess.run([sys.executable, "-I", "-c", desktop._READ, path, str(limit)], stdin=subprocess.DEVNULL, + capture_output=True, timeout=20) + + def test_a_read_is_bounded_and_of_a_regular_file_alone(self): + with tempfile.TemporaryDirectory() as tmp: + small = os.path.join(tmp, "small") + with open(small, "wb") as f: + f.write(b"abcd") + r = self.read(small, 4) + self.assertEqual((r.returncode, r.stdout), (0, b"abcd")) + self.assertEqual(self.read(small, 3).returncode, 5) + self.assertEqual(self.read(os.path.join(tmp, "none"), 4).returncode, 3) + self.assertEqual(self.read(tmp, 4).returncode, 4) + fifo = os.path.join(tmp, "fifo") + os.mkfifo(fifo) + self.assertEqual(self.read(fifo, 4).returncode, 4) # never waits for a writer + self.assertEqual(self.read("/dev/zero", 4).returncode, 4) + self.assertEqual(desktop.READ_LIMIT, 12 * 1024 * 1024) + + def test_a_refused_read_is_the_agents_error_and_the_bound_goes_to_the_desktop(self): + sent = [] + for rc, outcome in ((4, desktop.Refused), (5, desktop.Refused), (3, None)): + def engine(args, **kw): + sent.append(args) + return _response(args, rc=rc) + with mock.patch.object(desktop, "_engine", engine): + if outcome is None: + self.assertIsNone(desktop.Desktop("d").read("x")) + else: + with self.assertRaises(outcome): + desktop.Desktop("d").read("x") + self.assertEqual(sent[0][-2:], ["x", str(desktop.READ_LIMIT)]) + + def _png_size(data): return struct.unpack(">II", data[16:24]) @@ -255,16 +482,37 @@ def _from_corner(cells): return sorted((c - c0, r - r0) for c, r in cells) -PAUSE = 1.0 _PREFIX = os.environ.get("NULLIUS_TEST_CONTAINER_PREFIX", "nullius-test") -@unittest.skipIf(docker_tier.why_not(desktop.tag()), docker_tier.why_not(desktop.tag()) or "") +def why_no_desktop() -> str: + """Why the desktop's tests cannot run: the tier was not asked for, or no engine answers. A missing + image is no reason: the tests build it.""" + if not os.environ.get(docker_tier.VAR): + return "the docker tier runs with parallel.py --docker" + return container.unusable() or "" + + +class TierGate(unittest.TestCase): + def test_the_desktop_tests_run_wherever_an_engine_answers_and_build_the_image(self): + with mock.patch.dict(os.environ, {docker_tier.VAR: "1"}), mock.patch.object(container, "unusable", return_value=None), \ + mock.patch.object(container, "present", return_value=False): + self.assertEqual(why_no_desktop(), "") + with mock.patch.object(desktop, "image", side_effect=container.Unavailable("no build")): + with self.assertRaises(container.Unavailable): + DockerDesktop.setUpClass() + + +@unittest.skipIf(why_no_desktop(), why_no_desktop() or "") class DockerDesktop(unittest.TestCase): - """Real desktops of the kit's image, one at a time.""" + """Real desktops of the kit's image, one at a time; the image is built first when it is missing.""" + + @classmethod + def setUpClass(cls): + cls.image = desktop.image() def setUp(self): - self.d = desktop.Desktop.start(desktop.tag(), f"{_PREFIX}-desktop-{secrets.token_hex(4)}") + self.d = desktop.Desktop.start(self.image, f"{_PREFIX}-desktop-{secrets.token_hex(4)}") self.addCleanup(self.d.remove) def test_the_five_operations(self): @@ -306,7 +554,6 @@ def test_a_click_by_an_action_passes_and_the_same_click_by_python_fails(self): x, y = (x0 + x1) // 2, (y0 + y1) // 2 self.assertEqual(self.d.act(f"pyautogui.click({x}, {y})")["reply"]["returncode"], 0) self.assertEqual(checks.verdict("click", exp, self.d.events(), self.d.actions, ""), {"passed": True, "why": ""}) - time.sleep(PAUSE) # the kit's pause after an action self.d.python(f"import pyautogui; pyautogui.click({x}, {y})") v = checks.verdict("click", exp, self.d.events(), self.d.actions, "") self.assertFalse(v["passed"]) @@ -324,9 +571,48 @@ def test_a_word_typed_by_an_action_passes(self): def test_removal_leaves_no_container(self): self.d.remove() - left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"name=^{self.d.name}$"], - capture_output=True, text=True).stdout.split() - self.assertEqual(left, []) + probe = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"name=^{self.d.name}$"], + capture_output=True, text=True, timeout=60) + self.assertEqual(probe.returncode, 0, probe.stderr) + self.assertEqual(probe.stdout.split(), []) + + def test_the_x_server_is_unprivileged_and_refuses_a_client_without_the_cookie(self): + said = self.d.python("import os\nfor p in os.listdir('/proc'):\n if p.isdigit():\n" + " try:\n s = open(f'/proc/{p}/status').read()\n except OSError:\n" + " continue\n if 'Name:\\tXvfb' in s:\n" + " print([l.split()[1] for l in s.splitlines() if l.startswith('Uid:')][0])") + self.assertEqual(said["output"].split(), [str(session.SERVER_UID)]) + refused = self.d.python("import os\nopen('/home/user/no-cookie', 'w').close()\n" + "os.environ['XAUTHORITY'] = '/home/user/no-cookie'\n" + "from Xlib import display\ndisplay.Display(':0')") + self.assertIn("Display", refused["error"]) # the server's refusal, not a missing file + self.assertNotEqual(refused["returncode"], 0) + self.assertEqual(self.d.python("from Xlib import display\ndisplay.Display(':0').close()")["returncode"], 0) + + def test_a_key_held_from_one_action_to_the_next_sends_no_repeat(self): + values, exp = checks.draw("type") + self.d.start_fixture(values) + self.d.act("pyautogui.keyDown('a')\ntime.sleep(1.5)") + self.d.act("time.sleep(1.0)") + self.d.act("pyautogui.keyUp('a')\npyautogui.press('enter')") + events = self.d.events() + self.assertEqual([e["kind"] for e in events if e["kind"] in ("key", "keyup") and e.get("keysym") == "a"], + ["key", "keyup"]) + self.assertEqual(checks.outside(events, self.d.actions), []) + + def test_the_desktop_ends_when_its_window_manager_does(self): + self.d.python("import os, signal\nfor p in os.listdir('/proc'):\n if p.isdigit():\n try:\n" + " name = open(f'/proc/{p}/comm').read().strip()\n except OSError:\n continue\n" + " if name == 'openbox':\n os.kill(int(p), signal.SIGKILL)") + state = "" + for _ in range(40): + r = subprocess.run([container.DOCKER, "container", "inspect", "--format", "{{.State.Running}}", self.d.name], + capture_output=True, text=True, timeout=60) + state = r.stdout.strip() + if state == "false": + break + time.sleep(0.25) + self.assertEqual(state, "false") if __name__ == "__main__": From 17e81947da43fe0bf93e234595392f0d77eca9b0 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:57:59 +0100 Subject: [PATCH 17/36] fix(standard): keep a refused port's proof when conform writes the others When conform passed over a refused kit and wrote the other kits' proofs, it dropped every record of a known port that was not in the offers. A port proven earlier whose offer had been left out lost its proof although its kit never ran. The record of a port whose kit was refused in the same command now stays as it was. --- app/nullius/standard/conform.py | 6 ++++-- app/tests/test_standard_conform.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/conform.py b/app/nullius/standard/conform.py index 42006282..80bbcdcd 100644 --- a/app/nullius/standard/conform.py +++ b/app/nullius/standard/conform.py @@ -160,8 +160,10 @@ def check(other: dict, *, model: str, provider: str, which: Optional[List[str]] def change(cur): # on the definition as it is now: another proof or an edit since the start stays offers = [p for p in cur.get("offers") or [] if not (p in got and not got[p]["ok"])] offers += [p for p, r in got.items() if r["ok"] and p not in offers] - # a port's record goes with its offer; another proof (fidelity) is not this run's to take back - proven = {p: r for p, r in (cur.get("proven") or {}).items() if p in offers or not ports.known(p)} + # a port's record goes with its offer; another proof (fidelity), or the record of a port whose kit + # was refused here, is not this run's to take back + proven = {p: r for p, r in (cur.get("proven") or {}).items() + if p in offers or p in refused or not ports.known(p)} proven.update({p: {k: v for k, v in r.items() if k != "ok"} for p, r in got.items() if r["ok"]}) cur["offers"] = [p for p in ports.PORTS if p in offers] cur["proven"] = proven diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index 748c38a5..0b74fb5b 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -4,6 +4,7 @@ import os import io +import json import shutil import tempfile import unittest @@ -126,6 +127,26 @@ def test_a_refused_kit_is_returned_as_refused_beside_the_proofs(self): self.assertTrue(got["agent.task/v1"]["ok"]) self.assertEqual(got["agent.workspace/v1"], {"ok": False, "refused": "no engine"}) + def test_a_refused_ports_record_stays_also_without_its_offer(self): + path = os.path.join(self.root, "agents", "fake", "agent.json") + with open(path) as f: + raw = json.load(f) + raw["offers"].remove("agent.workspace/v1") # a record kept without its offer + kept = dict(raw["proven"]["agent.workspace/v1"]) + with open(path, "w") as f: + json.dump(raw, f) + other = definitions.get("agent", "fake", self.root) + self.assertEqual(definitions.problems(other, "agent"), []) + passed = SimpleNamespace(trials=[SimpleNamespace(task_name="kit/check", reward=1.0)]) + with mock.patch.object(conform, "candidates", return_value=["agent.task/v1", "agent.workspace/v1"]), \ + mock.patch.object(run, "run", side_effect=["passed-job", run.BenchmarkRefused("no engine")]), \ + mock.patch.object(conform.parse, "parse_job", return_value=passed): + conform.check(other, model="m", provider="custom", jobs_root=os.path.join(self.tmp, "jobs"), + say=lambda s: None, env={"MODEL_API_URL": "http://127.0.0.1:9/v1"}, unsandboxed=True) + saved = definitions.get("agent", "fake", self.root) + self.assertEqual(saved["proven"].get("agent.workspace/v1"), kept) + self.assertIn("job", saved["proven"]["agent.task/v1"]) + def test_every_kit_refused_saves_nothing_and_exits_two(self): other = definitions.get("agent", "fake", self.root) ports = ["agent.task/v1", "agent.workspace/v1"] From 60d58814c4c8769c17e681f6e21fa39ea898fb33 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:59:00 +0100 Subject: [PATCH 18/36] docs(standard): drop the promise that a closed driver's process group ends The driver.unit_end row said that a driver closed after a request got no reply has its process group ended with it. A child that redirects its streams can outlive that. The row keeps saying that such a driver is not sent an end. --- docs/standard.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/standard.md b/docs/standard.md index b940ca18..2b2a1e4a 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -422,7 +422,7 @@ use the source folders and repository checks. | `limits.turns` | At most this many agent turns in a conversation (default 20) | | `limits.setup_seconds` | How long `setup` may take, in seconds, for a unit an agent runs (a setup that starts a machine of its own for an agent); every other unit keeps 600 | | `driver.participant` | `true`: `setup` is also sent `participant`, the kind of what runs the unit, for a driver that starts something only an agent uses. A driver whose `setup` takes `unit` and `dir` alone leaves it out | -| `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. Only a unit whose `setup` was sent gets it, and only while that driver still runs: a driver closed after a request got no reply in its time (its process group ended with it) is not asked, nor is one whose request a stop left unanswered. A control's or a setup's unit gets none, nor does any other driver | +| `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. Only a unit whose `setup` was sent gets it, and only while that driver still runs: a driver closed after a request got no reply in its time is not asked, nor is one whose request a stop left unanswered. A control's or a setup's unit gets none, nor does any other driver | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id | From a7a628cb6f8c4a8cb1f98bd4d52727c4fa339307 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 12:59:00 +0100 Subject: [PATCH 19/36] test(standard): stop the run from the setup's own send in the unanswered-request test The test stopped the run from a timer started before setup was sent; on a slow machine the stop could come before the line was written and fail the test for the wrong reason. The stop now comes from the request's on_sent call, and the test waits for the driver to read all it was sent instead of sleeping. --- app/tests/test_standard_driver_hooks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py index 6e288e5e..49bc626c 100644 --- a/app/tests/test_standard_driver_hooks.py +++ b/app/tests/test_standard_driver_hooks.py @@ -10,7 +10,6 @@ import tempfile import sys import threading -import time import unittest from unittest import mock @@ -357,12 +356,12 @@ def test_a_driver_with_a_request_the_stop_left_unanswered_gets_no_end(self): stopped = threading.Event() r = self.ending() d = r.local.driver = self.driver_proc(late, ops, stopped=stopped).start() - threading.Timer(0.3, stopped.set).start() try: - with self.assertRaises(CancelledError): - d.request("setup", unit="a", dir=here, timeout=60) # its reply comes after the stop + with self.assertRaises(CancelledError): # stopped once setup is written: its reply comes late + d.request("setup", unit="a", dir=here, timeout=60, on_sent=stopped.set) r.unit_ended("a", here) - time.sleep(1.5) # the setup's late reply, then anything after it + d.p.stdin.close() # the driver reads all it was sent, then ends + d.p.wait(timeout=30) finally: d.close() with open(ops) as f: From dfa439b05bcc6b90d72fe6f480fd732021c8a40f Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:15:16 +0100 Subject: [PATCH 20/36] feat(catalogue): run OSWorld1 with an agent through the computer port For a unit an agent runs (driver.participant), setup starts runner.py behind the start gate. The runner builds OSWorld's DesktopEnv as the null control does, serves the computer port's MCP server over a Desktop of five methods on the upstream's controllers, and calls the upstream's own run_single_example, unmodified, with the port's bridge as the loop's agent. Setup answers once the loop asks for the agent's first step, with the prompt and the session's server computer: the port's pipe, whose path the driver takes from the package it imports, joined to the runner's socket. Score sends the turn's end on the runner's lifeline, waits for the runner within its bound and reads the upstream's result.txt; end (driver.unit_end) reaps the runner. - The bounds follow from the unit's limits (limits.setup_seconds, score's and end's); the runner's clean-up, now the bridge, the server and the desktop's bounded close, fits inside the driver's reap. A reap the driver's end cuts is handed to a reaper of its own. - The options Steps (100) and Pause (3.0 s) apply to an agent's run alone; driver.command names runner.py for a host's copy of the entry. - A unit with no agent runs as before. --- AGENTS.md | 2 +- .../benchmarks/osworld1/benchmark.json | 5 +- .../catalogue/benchmarks/osworld1/driver.py | 513 ++++++++++++++-- .../catalogue/benchmarks/osworld1/runner.py | 177 ++++++ .../benchmarks/osworld1/standard.json | 34 +- .../osworld1/upstream-settings.json | 14 + app/tests/test_osworld1_agent_side.py | 323 ++++++++++ app/tests/test_osworld1_entry_parts.py | 572 ++++++++++++++++++ docs/benchmarks/supported/osworld1.md | 9 +- 9 files changed, 1605 insertions(+), 44 deletions(-) create mode 100644 app/nullius/catalogue/benchmarks/osworld1/runner.py create mode 100644 app/tests/test_osworld1_agent_side.py create mode 100644 app/tests/test_osworld1_entry_parts.py diff --git a/AGENTS.md b/AGENTS.md index 830c31b4..b5057a3e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,7 +67,7 @@ and its resources. The source layout stays under `app/`. tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue: a benchmark's `standard.json` and `driver` in its folder (τ³-bench's also the MCP server of its domains' tools, `mcp_server`: the upstream's own, run on the machine; OSWorld1's also its null control's - agent, `nop`), `catalogue/agents//` + agent, `nop`, and the `runner` of a unit an agent runs: the upstream's loop with the computer port's bridge), `catalogue/agents//` (the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities//` (the keyword-memory `capability`); `workspace` (and `workspace_cli`, its shell command), `cli` and `__main__` (the unified `nullius` and `python -m nullius` commands), diff --git a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json index 7deaac73..dd1226d0 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json @@ -9,6 +9,7 @@ ], "origin": "official", "good_to_know": [ + "An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. Steps (100) and Pause (3.0 seconds) set that loop.", "Original runs one of OSWorld's own agents, chosen by the Agent option (M3 by default), with a model it is published with.", "Providers: OpenAI for OpenAI CUA; OpenAI-compatible for Muse Spark (Meta's API) and for EvoCUA, AutoGLM-OS and Mano (your model server); OpenRouter for the others.", "Only M3, PromptAgent and Qwen-CUA have completed tasks through Nullius. The other agents were checked up to their first model call.", @@ -25,7 +26,9 @@ "Each VM's ports are open on every network interface; one runs any command in the VM. Bind Docker's ports to 127.0.0.1, or keep the machine off untrusted networks." ], "limitations": [ - "No agent from the Agents page can run it yet.", + "An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop only, not on real VMs.", + "Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it.", + "A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs.", "Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach.", "The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested.", "A task still running after 2 hours is stopped and gets no score." diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index 59e1ba5e..6e672e79 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -1,15 +1,25 @@ """OSWorld's driver for the Nullius standard (JSON lines on stdin and stdout). -An agent benchmark whose agent side waits for agent.computer/v1's kit: today -it runs as its setups and its null control. Everything that acts and scores is -OSWorld's own code at the tested commit ($NULLIUS_UPSTREAM), run as published; -this driver names the task, starts the upstream's script and reads what it -wrote. Each unit's script runs in the unit's folder, so its logs, its task +An agent benchmark: it runs as its setups, its null control and an agent on +agent.computer/v1. Everything that acts and scores is OSWorld's own code at +the tested commit ($NULLIUS_UPSTREAM), run as published; this driver names the +task, starts the upstream's script (or, for an agent, runner.py beside it) and +reads what it wrote. Each unit's script runs in the unit's folder, so its logs, its task cache and its results stay with the trial. units - every task of the dataset (one of the upstream's evaluation_examples/test_*.json), as domain__id, in the file's order -setup - the task's instruction, for the null control +setup - the task's instruction, for the null control. For an agent + (participant "agent", which the definition's driver.participant + asks Nullius to send): runner.py started in the trial's + .work/osworld/, behind the start gate. The runner holds the + upstream's DesktopEnv and calls its run_single_example, unchanged, + with the computer port's bridge as the loop's agent. Once the loop + asks for the agent's first step, setup answers the prompt (the + instruction, a blank line and the desktop user's password line), + the MCP server computer (the port's pipe, joined to the runner's + socket) and the agent's time. The options max_steps and pause are + the loop's answer - a setup's (original, model): the script of the run's agent option (agents.json: the upstream's scripts/python/run_*.py) as published, on the one task (a test_all_meta file naming it), with the arguments @@ -25,9 +35,17 @@ score - for a setup, the upstream's result.txt: what the agent's runner (lib_run_single's, or its script's own) wrote from DesktopEnv.evaluate() on the VM's final state. + For an agent: the turn's end on the runner's lifeline; the runner's + loop passes its steps left with no action and evaluates; score + waits for the runner to end and reads result.txt from the runner's + folder, never the workspace, and only from a runner this driver + started that ended by itself with status 0, through folders opened + without following a link. It copies result.txt and the call log to + the trial's verifier/osworld/. For the null control, nop.py: the upstream's run_single_example with an agent that answers DONE at its first step, its result.txt read the same way +end - an agent's unit has ended: its runner, if it still has one, goes The VM image is the upstream's own: its docker provider's DockerVMManager downloads it into the upstream's docker_vm_data once, under a lock, for every @@ -39,23 +57,49 @@ agent's own included - mounts its image from that folder. A unit stopped by Nullius (its time ran out) has its script's processes killed and those containers removed by a reaper of its own session, which outlives this driver. + +An agent's runner holds the read end of a pipe from this driver (its +lifeline): a line turn-end ends the agent's turn, and the pipe's close stops +the runner, which closes its desktop and ends - at its unit's score, at the +unit's end (end, which the definition's driver.unit_end asks Nullius to send), +at the next unit's setup, and when this driver ends, by itself or killed. + +An agent's unit has one budget, the unit's own limits as the core keeps them: +setup's (the definition's limits.setup_seconds), the agent's (AGENT_SECONDS, +which setup answers), score's and end's (the core's CORE_DRIVER_SECONDS and +CORE_END_SECONDS). Each bound here follows from it. A runner let go has +REAP_SECONDS to end by itself, more than its own clean-up takes (runner.py's +STOP_SECONDS); then its group is killed and the unit's VMs removed, the docker +commands within REMOVE_SECONDS. Every wait in setup ends at the setup's +deadline, early enough for that reap to end before the core's: the start gate +a unit waits for, its runner's VM start, its task's setup. A unit holds the +start gate until its VM start has ended, with no bound of its own. Score waits +SCORE_SECONDS for the runner, the reap fitting after it within the core's. +When this driver's input ends, each runner left, and one being reaped, is let +go and handed to a reaper in a session of its own, which outlives the driver +and does the same reap. """ import fcntl import glob +import importlib.util import json import os import queue +import shutil import signal +import stat import subprocess import sys +import tempfile import threading import time HERE = os.path.dirname(os.path.abspath(__file__)) UP = os.environ["NULLIUS_UPSTREAM"] -# driver.py AGENTS_JSON NOP_PY: the definition's command names both, so a host's runner snapshot carries them -_ARGS = sys.argv[1:3] if sys.argv[1:2] != ["--reap"] else [] -AGENTS_FILE, NOP = (_ARGS + [os.path.join(HERE, "agents.json"), os.path.join(HERE, "nop.py")][len(_ARGS):])[:2] +# driver.py AGENTS_JSON NOP_PY RUNNER_PY: the definition's command names them, so a host's runner snapshot carries them +_ARGS = sys.argv[1:4] if sys.argv[1:2] not in (["--reap"], ["--reap-runner"]) else [] +AGENTS_FILE, NOP, RUNNER_FILE = (_ARGS + [os.path.join(HERE, "agents.json"), os.path.join(HERE, "nop.py"), + os.path.join(HERE, "runner.py")][len(_ARGS):])[:3] with open(AGENTS_FILE, encoding="utf-8") as _f: AGENTS = json.load(_f) VM =os.path.join(UP, "docker_vm_data", "Ubuntu.qcow2") # where the upstream's DockerVMManager puts it @@ -65,6 +109,37 @@ START_LOCK = "/tmp/nullius-osworld1-vm-start.lck" STARTED = "Started container with ports" # the provider's line once the container runs START_SECONDS = 120 +# an agent's unit (agent.computer/v1): the desktop user's password line, as OSWorld's own agents are told it; the +# agent's time, the setups'; the unit's budget (the module's text) and the bounds that follow from it +PASSWORD_LINE = "If you need a password for sudo, the password of the computer is '{password}'." +AGENT_SECONDS = 7200 +CORE_DRIVER_SECONDS = 600 # the core's wait on score, and on setup when no limit is given +CORE_END_SECONDS = 120 # the core's wait on end +ANSWER_SECONDS = 10 # this driver's own answer once a bound is reached +REMOVE_SECONDS = 30 # the docker commands that remove a unit's VMs, together +RUNNERS = {} # the runner folder of each unit this driver set up -> its process + + +def _load(path): + spec = importlib.util.spec_from_file_location("osworld1_runner", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _setup_seconds(): + """The definition's limits.setup_seconds (standard.json beside this file), the core's wait on an agent's setup.""" + try: + with open(os.path.join(HERE, "standard.json"), encoding="utf-8") as f: + return float(json.load(f)["limits"]["setup_seconds"]) + except (OSError, ValueError, KeyError, TypeError): + return float(CORE_DRIVER_SECONDS) + + +RUNNER = _load(RUNNER_FILE) +REAP_SECONDS = RUNNER.STOP_SECONDS + 5 # a runner let go ends by itself first, its clean-up done +SETUP_SECONDS = _setup_seconds() +SCORE_SECONDS = CORE_DRIVER_SECONDS - REAP_SECONDS - REMOVE_SECONDS - ANSWER_SECONDS def _task(unit): @@ -79,8 +154,13 @@ def units(dataset=None, setup=None): return {"units": [f"{d}__{e}" for d, ids in meta.items() for e in ids]} -def setup(unit, dir, **_): - return {"prompt": _task(unit)[2]["instruction"]} +def setup(unit, dir, participant=None, **_): + until = time.time() + SETUP_SECONDS - REAP_SECONDS - REMOVE_SECONDS - ANSWER_SECONDS # an agent's setup's own + instruction = _task(unit)[2]["instruction"] + if participant != "agent": + return {"prompt": instruction} + _reap_all() # a unit before this one that never reached score + return _start_runner(unit, dir, instruction, until) # -- the upstream's processes: started apart, stopped by the upstream's own cleanup -------------------- @@ -115,20 +195,32 @@ def reap(pgid, dir): _remove_vms(dir) -def _remove_vms(dir): +def _remove_vms(dir, seconds=None): """Remove the VM containers that mount the unit's image link: a stopped unit's, or those a script left - behind when it ended without closing its environment.""" + behind when it ended without closing its environment. ``seconds``: the docker commands' bound, together (an + agent's unit); past it the rest is left.""" + until = None if seconds is None else time.time() + seconds + + def docker(*argv, **kw): + left = None if until is None else until - time.time() + if left is not None and left <= 0: + raise subprocess.TimeoutExpired(argv, 0) + return subprocess.run(["docker", *argv], timeout=left, **kw) + mine = os.path.join(dir, "vm") + os.sep - out = subprocess.run(["docker", "ps", "-a", "-q", "--filter", "ancestor=happysixd/osworld-docker"], - capture_output=True, text=True).stdout.split() - ids = [] - for cid in out: - sources = subprocess.run(["docker", "inspect", "-f", "{{range .Mounts}}{{.Source}}\n{{end}}", cid], - capture_output=True, text=True).stdout.split() - if any(s.startswith(mine) for s in sources): - ids.append(cid) - if ids: - subprocess.run(["docker", "rm", "-f", *ids], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + out = docker("ps", "-a", "-q", "--filter", "ancestor=happysixd/osworld-docker", + capture_output=True, text=True).stdout.split() + ids = [] + for cid in out: + sources = docker("inspect", "-f", "{{range .Mounts}}{{.Source}}\n{{end}}", cid, + capture_output=True, text=True).stdout.split() + if any(s.startswith(mine) for s in sources): + ids.append(cid) + if ids: + docker("rm", "-f", *ids, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.TimeoutExpired: + sys.stderr.write(f"driver: the unit's VMs were not all removed within {seconds} s ({dir})\n") def _started(p, log, what): @@ -287,6 +379,334 @@ def answer(unit, dir, setup, model, base_url, key, **_): return {"answer": last if isinstance(last, str) else json.dumps(last), "model": model} +# -- an agent's unit: the runner, its readiness, its end ---------------------------------------------------------- +# A runner's life is tied to this driver's: it holds the read end of a pipe whose write end this driver keeps (its +# lifeline), and closes its desktop and ends once that pipe closes - this driver ended, was killed, or let the unit go. +# Each unit's runner stays in RUNNERS until it is gone. Its exit is seen without reaping it (_exited), so that its +# process group is signalled only while its leader still holds the group's number. + + +def _port(): + """The computer port's shared code, from the nullius package on this driver's import path.""" + from nullius.standard import computer_port + return computer_port + + +def _runner_dir(dir): + """The runner's folder: the trial's .work/osworld/, beside the agent's workspace and outside it.""" + return os.path.join(os.path.dirname(os.path.abspath(dir)), ".work", "osworld") + + +def _held_dir(dir): + """The run's folder of files sent once: the job's .work/osworld-held/, shared by the run's units; it goes with + the job.""" + return os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(dir))), ".work", "osworld-held") + + +def _option(name, default, kind): + return kind(os.environ.get("NULLIUS_OPTION_" + name.upper(), default)) + + +def _runner_env(): + """The runner's environment: this driver's, its import path the upstream's folder and the nullius package's.""" + package = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(_port().__file__)))) + return dict(os.environ, PYTHONPATH=os.pathsep.join([UP, package]), PYTHONIOENCODING="utf-8") + + +LOST = "lost" # _exited: the runner was reaped elsewhere; its number is not ours + + +def _exited(p): + """None while the runner runs; once it has ended, (whether it exited by itself, its status), without reaping it; + LOST once it was reaped elsewhere (waitid finds no such child): its number may be another process's now.""" + if p.returncode is not None: + return (p.returncode >= 0, p.returncode) + try: + info = os.waitid(os.P_PID, p.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) + except ChildProcessError: + return LOST + if info is None: + return None + return (info.si_code == os.CLD_EXITED, info.si_status) + + +def _start_time(pid): + """The process's start time (field 22 of /proc/PID/stat), which tells it from a later process with its number.""" + try: + with open(f"/proc/{pid}/stat", encoding="utf-8") as f: + return int(f.read().rsplit(")", 1)[1].split()[19]) + except (OSError, IndexError, ValueError): + return None + + +def _start_runner(unit, dir, instruction, until): + domain, eid, example = _task(unit) + where = _runner_dir(dir) + os.makedirs(where, exist_ok=True) + vm = _workspace(where) + task = os.path.join(where, f"{domain}__{eid}.json") + with open(task, "w", encoding="utf-8") as f: + json.dump(example, f) + held = _held_dir(dir) + os.makedirs(held, mode=0o700, exist_ok=True) + log = os.path.join(where, "runner.log") + sock_dir = life_r = life_w = lock = p = None + try: # until RUNNERS holds it, what was made here is undone here + sock_dir = tempfile.mkdtemp(prefix="osworld1-") # 0700, a short path (a socket's limit): the socket, ready.json + ready = os.path.join(sock_dir, "ready.json") + life_r, life_w = os.pipe() + argv = [sys.executable, RUNNER_FILE, "--example", task, "--result-dir", os.path.join(where, RESULTS), + "--socket", os.path.join(sock_dir, "s"), "--ready", ready, "--lifeline", str(life_r), + "--steps", str(_option("max_steps", "100", int)), "--pause", str(_option("pause", "3.0", float)), + "--vm", vm, "--calls", os.path.join(where, "computer-calls.jsonl"), "--held", held] + lock = open(START_LOCK, "a") + while True: # VM starts one after another, as the setups' do + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + left = until - time.time() + if left <= 0: + raise RuntimeError("the runner was not started: the start gate, held by another unit's VM start, " + "was not free before the end of this unit's setup") + if CLOSED.wait(min(1.0, left)): + raise RuntimeError("the runner was not started: the unit's time ran out") + with open(log, "a") as f: + p = subprocess.Popen(argv, cwd=where, env=_runner_env(), stdout=f, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, start_new_session=True, pass_fds=(life_r,)) + RUNNERS[where] = {"p": p, "sock_dir": sock_dir, "life": life_w, "start": _start_time(p.pid)} + except BaseException: + for fd in (life_r, life_w): + if fd is not None: + try: + os.close(fd) + except OSError: + pass + if lock is not None: + lock.close() + if sock_dir is not None: + shutil.rmtree(sock_dir, ignore_errors=True) + raise + os.close(life_r) + try: + with open(os.path.join(where, "runner.json"), "w", encoding="utf-8") as f: + json.dump({"pid": p.pid}, f) + # the gate is held until the VM's start has ended, the provider's port lock with it: no bound of its own + _wait(lambda: os.path.exists(ready) or _has(log, STARTED), p, where, until, "its VM's start") + finally: + lock.close() + _wait(lambda: os.path.exists(ready), p, where, until, "its task's setup") + try: + with open(ready, encoding="utf-8") as f: + said = json.load(f) + finally: + try: + os.unlink(ready) # it holds the desktop user's password: read once, then gone + except OSError: + pass + if not said.get("ok"): + _reap_runner(where) + raise RuntimeError("the runner's setup failed: " + str(said.get("error"))) + pipe = os.path.join(os.path.dirname(os.path.abspath(_port().__file__)), "computer_pipe.py") + return {"prompt": instruction + "\n\n" + PASSWORD_LINE.format(password=said.get("password") or ""), + "mcp": [{"name": "computer", "server": "computer", "args": [pipe, os.path.join(sock_dir, "s")]}], + "agent_seconds": AGENT_SECONDS} + + +def _has(log, text): + try: + with open(log, encoding="utf-8", errors="replace") as f: + return text in f.read() + except OSError: + return False + + +def _wait(done, p, where, until, what): + """Until ``done()``; raises when the runner ends first, Nullius ends the unit (the runner handed to a reaper), or + the setup's deadline ``until`` comes (the runner reaped).""" + while not done(): + ended = _exited(p) + if ended is not None: + _reap_runner(where) + raise RuntimeError(f"the runner ended (exit {'unknown' if ended == LOST else ended[1]}) before {what}: " + + _tail(os.path.join(where, "runner.log"))) + if CLOSED.wait(0.2): + _hand_over(where) + raise RuntimeError("the runner was stopped: the unit's time ran out") + if time.time() > until: + _reap_runner(where) + raise RuntimeError(f"the runner was not ready by the end of the unit's setup, {SETUP_SECONDS:g} s ({what})") + + +def _let_go(entry): + """Close the runner's lifeline: it closes its desktop and ends by itself.""" + if entry.get("life") is not None: + try: + os.close(entry["life"]) + except OSError: + pass + entry["life"] = None + + +def _turn_end(entry): + """The agent's turn has ended: one line on the runner's lifeline (the port's turn-end).""" + if entry.get("life") is not None: + try: + os.write(entry["life"], _port().TURN_END + b"\n") + except OSError: + pass + + +def _reap_runner(where, seconds=None): + """The unit's runner let go, given ``seconds`` (REAP_SECONDS) to end by itself, its whole group killed while its + leader is still unreaped, then reaped; the VM containers of the unit's image link removed within REMOVE_SECONDS, + the socket's folder gone. It leaves RUNNERS only then. When this driver's input ends meanwhile, the rest goes to + a reaper of its own (_hand_over). Returns how it ended before any kill (``_exited``): None when it was still + running.""" + entry = RUNNERS.get(where) + if entry is None: + return None + seconds = REAP_SECONDS if seconds is None else seconds + p = entry["p"] + _let_go(entry) + t0 = time.time() + while p.returncode is None and _exited(p) is None and time.time() - t0 < seconds: + if CLOSED.wait(0.1): + _hand_over(where) + return None + ended = _exited(p) + if p.returncode is None and ended != LOST: # running or a zombie: the group's number is still its own + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + p.wait() + if shutil.which("docker") is not None: + _remove_vms(where, REMOVE_SECONDS) + shutil.rmtree(entry["sock_dir"], ignore_errors=True) + RUNNERS.pop(where, None) + return ended + + +def _reap_all(): + for where in list(RUNNERS): + _reap_runner(where) + + +def _hand_over(where): + """The runner let go and handed to a reaper in a session of its own (driver.py --reap-runner), which outlives this + driver: Nullius kills this driver's group 3 s after closing its stdin.""" + entry = RUNNERS.pop(where, None) + if entry is None: + return + _let_go(entry) + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--reap-runner", str(entry["p"].pid), + str(entry.get("start")), where, entry["sock_dir"], str(REAP_SECONDS)], + env=dict(os.environ, NULLIUS_UPSTREAM=UP), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + + +def reap_runner(pid, start, where, sock_dir, seconds): + """The reaper of a runner handed over (_hand_over): the runner has ``seconds`` to end by itself; then, when it is + still the process started (same number, same start time), its group is killed; the unit's VMs removed within + REMOVE_SECONDS; the socket's folder gone.""" + def same(): + return start is not None and _start_time(pid) == start + + t0 = time.time() + while same() and time.time() - t0 < seconds: + try: + with open(f"/proc/{pid}/stat", encoding="utf-8") as f: + if f.read().rsplit(")", 1)[1].split()[0] == "Z": + break # ended, not yet reaped by its new parent: its number is still its own + except (OSError, IndexError): + break + time.sleep(0.2) + if same(): + try: + os.killpg(pid, signal.SIGKILL) + except OSError: + pass + if shutil.which("docker") is not None: + _remove_vms(where, REMOVE_SECONDS) + shutil.rmtree(sock_dir, ignore_errors=True) + + +def _reap_at_exit(): + """Nullius closed this driver's stdin and kills its group 3 s later: each runner is let go and handed to a reaper + of its own, which outlives this driver.""" + for where in list(RUNNERS): + _hand_over(where) + + +def end(unit, dir, **_): + """The unit has ended, scored or not: its runner, when it still has one, goes now.""" + _reap_runner(_runner_dir(dir)) + return {} + + +def _plain(base, *parts): + """The bytes of the file ``base/parts...``: each folder from ``base`` on opened without following a link, the + file opened relative to the last one without following a link or waiting on a pipe, and read only when it is a + plain file; raises otherwise.""" + flags = os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + fd = os.open(base, flags | os.O_DIRECTORY) + try: + for part in parts[:-1]: + nxt = os.open(part, flags | os.O_DIRECTORY, dir_fd=fd) + os.close(fd) + fd = nxt + ffd = os.open(parts[-1], flags | os.O_NONBLOCK, dir_fd=fd) + finally: + os.close(fd) + with os.fdopen(ffd, "rb") as f: + if not stat.S_ISREG(os.fstat(f.fileno()).st_mode): + raise RuntimeError(f"{parts[-1]} is not a plain file") + return f.read() + + +def _agent_score(where, dir, domain, eid): + """The agent's turn ended: the runner this driver started for the unit is told so, ends by itself with status 0, + and the result.txt it wrote is read through folders opened without following a link.""" + entry = RUNNERS.get(where) + if entry is None: + raise RuntimeError("no runner of this driver's for this unit: its result is not read") + _turn_end(entry) + p, t0 = entry["p"], time.time() + while _exited(p) is None and time.time() - t0 < SCORE_SECONDS and not CLOSED.wait(0.5): + pass + if CLOSED.is_set() and _exited(p) is None: # Nullius ended the unit: the reaper ends what is left + _hand_over(where) + raise RuntimeError("the runner was stopped: the unit's time ran out") + ended = _exited(p) + if ended is None: # one still running is let go; an end by itself meanwhile counts + ended = _reap_runner(where) + else: + _reap_runner(where, seconds=0) + if ended is None: + raise RuntimeError(f"the runner did not end within {SCORE_SECONDS} s of the agent's end") + if ended == LOST or not (ended[0] and ended[1] == 0): + how = "was reaped elsewhere" if ended == LOST else ("exited " + str(ended[1]) if ended[0] else "was killed") + raise RuntimeError(f"the runner {how}: its result is not read: " + _tail(os.path.join(where, "runner.log"))) + trial = os.path.dirname(os.path.abspath(dir)) + try: + result = _plain(trial, ".work", "osworld", RESULTS, domain, eid, "result.txt") + value = float(result.decode().strip()) + except (OSError, ValueError, RuntimeError) as e: + raise RuntimeError(f"the runner's result could not be read ({e}): " + _tail(os.path.join(where, "runner.log"))) + try: + calls = _plain(trial, ".work", "osworld", "computer-calls.jsonl") + except (OSError, RuntimeError): + calls = b"" + out = os.path.join(trial, "verifier", "osworld") + os.makedirs(out, exist_ok=True) + for name, data in (("result.txt", result), ("computer-calls.jsonl", calls)): + with open(os.path.join(out, name), "wb") as f: + f.write(data) + return {"reward": value, "detail": {"result": value, "by": "runner.py, run_single_example, DesktopEnv.evaluate", + "tool_calls": len(calls.splitlines())}} + + def _scored(where, how): with open(os.path.join(where, "result.txt"), encoding="utf-8") as f: value = float(f.read().strip()) @@ -302,8 +722,11 @@ def score(unit, dir, transcript=None, answer=None, setup=None, **_): if where is None: raise RuntimeError("no result.txt: the upstream's runner did not finish the task") return _scored(where, os.path.basename(_agent()[1]["script"]) + ", DesktopEnv.evaluate") + runner = _runner_dir(dir) + if runner in RUNNERS or os.path.lexists(os.path.join(runner, "runner.json")): + return _agent_score(runner, dir, domain, eid) if answer: - raise RuntimeError("OSWorld1 has no agent side yet: agent.computer/v1 has no kit") + raise RuntimeError("no runner was set up for this unit: an agent's setup starts one") vm = _workspace(dir) # the null control: nothing acts, the upstream evaluates where = os.path.join(dir, RESULTS, "nop", domain, eid) os.makedirs(where, exist_ok=True) @@ -315,18 +738,30 @@ def score(unit, dir, transcript=None, answer=None, setup=None, **_): return _scored(where, "nop.py, DesktopEnv.evaluate") -if sys.argv[1:2] == ["--reap"]: # _stop's reaper: driver.py --reap PGID LOG - reap(int(sys.argv[2]), sys.argv[3]) - sys.exit(0) -HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score} -threading.Thread(target=_read, daemon=True).start() -while True: - line = LINES.get() - if line is None: - break - try: - req = json.loads(line) - out = HANDLERS[req.pop("op")](**req) - except Exception as e: - out = {"error": f"{type(e).__name__}: {e}"} - print(json.dumps(out), flush=True) +HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "end": end} + + +def main(): + threading.Thread(target=_read, daemon=True).start() + while True: + line = LINES.get() + if line is None: + break + try: + req = json.loads(line) + out = HANDLERS[req.pop("op")](**req) + except Exception as e: + out = {"error": f"{type(e).__name__}: {e}"} + print(json.dumps(out), flush=True) + _reap_at_exit() + + +if __name__ == "__main__": + if sys.argv[1:2] == ["--reap"]: # _stop's reaper: driver.py --reap PGID LOG + reap(int(sys.argv[2]), sys.argv[3]) + sys.exit(0) + if sys.argv[1:2] == ["--reap-runner"]: # _hand_over's: driver.py --reap-runner PID START WHERE SOCKETS SECONDS + reap_runner(int(sys.argv[2]), None if sys.argv[3] == "None" else int(sys.argv[3]), sys.argv[4], sys.argv[5], + float(sys.argv[6])) + sys.exit(0) + main() diff --git a/app/nullius/catalogue/benchmarks/osworld1/runner.py b/app/nullius/catalogue/benchmarks/osworld1/runner.py new file mode 100644 index 00000000..3c8c68e7 --- /dev/null +++ b/app/nullius/catalogue/benchmarks/osworld1/runner.py @@ -0,0 +1,177 @@ +"""One unit of an agent on OSWorld1: the upstream's DesktopEnv and its own run_single_example, imported and called as +they are, with the computer port's bridge as the loop's agent (nullius.standard.computer_port). + + runner.py --example TASK_JSON --result-dir DIR --socket PATH --ready FILE --lifeline FD --steps N --pause SECONDS + --vm VM_IMAGE --calls FILE [--held FOLDER] + +The driver starts it in an agent's unit's setup, in the unit's runner folder (the trial's .work/osworld/), with the +upstream's folder and the nullius package on its import path. It builds the upstream's DesktopEnv as the null control +does (nop.py), serves the MCP server computer at PATH (a Unix socket in a folder of the user's alone, which the driver +made) over a Desktop of five methods on the upstream's controllers, and calls run_single_example with the bridge as the +loop's agent, N as max_steps and SECONDS as sleep_after_execution. No file of the upstream is edited, patched or +replaced. + +- READY, a file of mode 0600: {"ok": true, "password": the desktop user's} once the loop asks for the agent's first + step (after the task's setup and the upstream's own wait); {"ok": false, "error": ...} when the unit fails before. +- FD, the read end of a pipe from the driver: a line turn-end ends the agent's turn (the loop then gets no action for + the steps left and evaluates the desktop as it is); its close stops the runner as SIGTERM does. +- FILE, the call log: one line per tool call, {tool, start, end, isError}. +- FOLDER, the run's folder of files sent once, when the driver gives one (computer_port.Held). + +A stop is taken once. The clean-up (the bridge's end and the server's close, then the desktop's close within +CLOSE_SECONDS) takes at most STOP_SECONDS; past it the runner exits 1 all the same, and the driver, which lets a +runner go for longer than that, removes what is left. At every exit READY and the socket's folder are removed. The +exit status is 0 only for a loop that ran to its end and a desktop closed in its time. +""" +import argparse +import json +import logging +import os +import shutil +import sys +import tempfile +import threading + +SCREEN = (1920, 1080) +PYTHON_SECONDS = 120 # the upstream controller's wait on a command's reply +FINISH_SECONDS = 5 # the bridge's end and the server's close +CLOSE_SECONDS = 25 # the desktop's close: the provider stops and removes its container +STOP_SECONDS = FINISH_SECONDS + CLOSE_SECONDS + + +class Desktop: + """The port's five operations over the upstream's own controllers.""" + + def __init__(self, env, scratch): + self.env, self.scratch = env, scratch + + def python(self, code): + return self.env.controller.execute_python_command(code) + + def upload(self, path, data): + fd, local = tempfile.mkstemp(dir=self.scratch) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + self.env.setup_controller._upload_file_setup([{"local_path": local, "path": path}]) + finally: + os.unlink(local) + + def read(self, path): + return self.env.controller.get_file(path) + + def capture(self): + png = self.env.controller.get_screenshot() + if not png: + raise RuntimeError("the desktop gave no screenshot") + return png + + def password(self): + return self.env.client_password + + +class Turn: + """What the lifeline's turn-end reaches: the bridge, once it exists.""" + bridge = None + + def end_turn(self): + if self.bridge is not None: + self.bridge.end_turn() + + +def close_within(env, seconds): + """The desktop closed in a thread of its own, waited for ``seconds``: True when it closed in that time.""" + done = [] + + def close(): + env.close() + done.append(True) + + worker = threading.Thread(target=close, name="desktop-close", daemon=True) + worker.start() + worker.join(seconds) + return bool(done) + + +def main(argv=None): + p = argparse.ArgumentParser() + for name in ("--example", "--result-dir", "--socket", "--ready", "--vm", "--calls"): + p.add_argument(name, required=True) + p.add_argument("--held") + p.add_argument("--lifeline", type=int, required=True) + p.add_argument("--steps", type=int, required=True) + p.add_argument("--pause", type=float, required=True) + a = p.parse_args(argv) + from nullius.standard import computer_port as port + stop, turn = port.Stop(STOP_SECONDS), Turn() + try: # the clean-up holds before the lifeline can stop the runner + stop.install() + threading.Thread(target=port.lifeline, args=(a.lifeline, turn), name="lifeline", daemon=True).start() + return run(a, port, stop, turn) + finally: + try: + os.unlink(a.ready) + except OSError: + pass + shutil.rmtree(os.path.dirname(a.socket), ignore_errors=True) + + +def run(a, port, stop, turn): + logging.basicConfig(level=logging.INFO) # the provider's own lines, its container started among them + try: + import lib_run_single + from desktop_env.desktop_env import DesktopEnv + with open(a.example, encoding="utf-8") as f: + example = json.load(f) + except Exception as e: + port.ready(a.ready, ok=False, error=f"{type(e).__name__}: {e}") + return 3 + where = os.path.join(a.result_dir, *_task_path(a.example)) + env = bridge = server = None + said, failed, code = [], "", 0 + + def first(): # the loop asks for the agent's first step: the unit is ready + port.ready(a.ready, ok=True, password=env.client_password or "") + said.append(True) + + try: + env = DesktopEnv(path_to_vm=a.vm, action_space="pyautogui", provider_name="docker", screen_size=SCREEN, + headless=True, os_type="Ubuntu", require_a11y_tree=False, client_password="password", + enable_proxy=False) + os.makedirs(where, exist_ok=True) + bridge = port.Bridge(Desktop(env, os.path.dirname(os.path.abspath(a.calls))), a.steps, a.pause, SCREEN, + PYTHON_SECONDS, on_first=first, held=port.Held(a.held) if a.held else None) + server = port.Server(bridge, a.socket, a.calls) + turn.bridge = bridge + lib_run_single.run_single_example(bridge, env, example, a.steps, example["instruction"], + argparse.Namespace(sleep_after_execution=a.pause, result_dir=a.result_dir), + where, []) + except SystemExit: # SIGTERM or the lifeline's end: the unit was stopped + failed, code = "the runner was stopped", 1 + except BaseException as e: # the loop or the desktop failed: the harness's + failed, code = f"{type(e).__name__}: {e}", 1 + sys.stderr.write(f"runner: {failed}\n") + finally: + with stop.deferred(then_stop=False): + if not said: + port.ready(a.ready, ok=False, error="the task's setup: " + (failed or "the loop ended before the " + "agent's first step")) + if bridge is not None: + bridge.finish() + if server is not None: + server.close() + if env is not None and not close_within(env, CLOSE_SECONDS): + sys.stderr.write(f"runner: the desktop did not close within {CLOSE_SECONDS} s\n") + code = code or 1 + return code + + +def _task_path(example_file): + """domain/id of the task file (the driver's copy, named domain__id.json).""" + name = os.path.splitext(os.path.basename(example_file))[0] + domain, _, eid = name.partition("__") + return (domain, eid) if eid else (name,) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 8c01261e..f3ef09f0 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -18,14 +18,28 @@ "{install}/.venv/bin/python", "{here}/driver.py", "{here}/agents.json", - "{here}/nop.py" + "{here}/nop.py", + "{here}/runner.py" ], "install": { "python": "3.12", "project": ".", "locked": true + }, + "participant": true, + "unit_end": true + }, + "mcp_servers": { + "computer": { + "command": [ + "{install}/.venv/bin/python" + ], + "summary": "The unit's desktop for an agent: the computer port's pipe (the nullius package's standard/computer_pipe.py, whose path setup gives as the first argument) joined to the socket where the unit's runner serves the port's MCP server, its second argument." } }, + "limits": { + "setup_seconds": 3600 + }, "datasets": [ { "id": "test_nogdrive", @@ -198,6 +212,24 @@ ] } } + }, + "max_steps": { + "label": "Steps", + "summary": "run_single_example's max_steps for an agent: the steps of OSWorld's own loop, each one acting call of the computer port.", + "pattern": "[1-9][0-9]{0,2}", + "default": "100", + "setups": [ + "agent" + ] + }, + "pause": { + "label": "Pause", + "summary": "run_single_example's sleep_after_execution for an agent: the seconds env.step waits after each action, 3.0 as in run_multienv_m3.py's default.", + "pattern": "[0-9]{1,2}(\\.[0-9]{1,2})?", + "default": "3.0", + "setups": [ + "agent" + ] } }, "host": { diff --git a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json index c3c02423..7624b1c6 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json +++ b/app/nullius/catalogue/benchmarks/osworld1/upstream-settings.json @@ -3057,5 +3057,19 @@ "values": "a person at the VM", "default": "manual", "nullius": "not offered - a person examines tasks by hand: no run to score" + }, + { + "entry": "lib_run_single.py", + "setting": "max_steps", + "values": "a number", + "default": "none: its caller's", + "nullius": "option:max_steps" + }, + { + "entry": "lib_run_single.py", + "setting": "args.sleep_after_execution", + "values": "seconds", + "default": "none: its caller's", + "nullius": "option:pause" } ] \ No newline at end of file diff --git a/app/tests/test_osworld1_agent_side.py b/app/tests/test_osworld1_agent_side.py new file mode 100644 index 00000000..f372e1ef --- /dev/null +++ b/app/tests/test_osworld1_agent_side.py @@ -0,0 +1,323 @@ +"""OSWorld1's agent side: the driver's setup starts the runner, which builds the fake DesktopEnv and calls the upstream's +own run_single_example, as it is, with the computer port's bridge as the loop's agent; a scripted MCP client in the +agent's place joins the runner's server through the port's pipe, as the agent's session would; score ends the turn and +reads the upstream's result.txt. + +No VM, no docker, no network, no model. It needs the upstream's loop (lib_run_single.py, the two modules it imports +and the agents' package one of its lines imports, from $OSWORLD_DIR, a checkout of the upstream) and a Python that +holds its packages ($OSWORLD_PYTHON); without them each test is skipped, with the reason. The rest of the upstream is a +folder of the test's own: the fake DesktopEnv (fixtures/osworld1_fake_desktop), one task, its dataset and an empty file +where the docker provider keeps the VM image, so that no image is fetched. The loop's fixed waits are recorded, not +slept (the fixture's sitecustomize.py). +""" +from __future__ import annotations + +import base64 +import json +import os +import select +import shutil +import signal +import subprocess +import tempfile +import time +import unittest + +from nullius.standard import jsonl + +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ENTRY = os.path.join(APP, "nullius", "catalogue", "benchmarks", "osworld1") +FAKE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures", "osworld1_fake_desktop") +OSWORLD = os.environ.get("OSWORLD_DIR", "") +PYTHON = os.environ.get("OSWORLD_PYTHON", "") +LOOP = ("lib_run_single.py", "lib_results_logger.py", "lib_run_errors.py", "mm_agents") +WHY = ("OSWORLD_DIR does not name a checkout of the upstream" if not all(os.path.exists(os.path.join(OSWORLD, n)) for n in LOOP) + else "OSWORLD_PYTHON does not name the upstream's Python" if not os.path.isfile(PYTHON) + else "") +TASK = {"id": "task-1", "instruction": "Open the terminal.", "config": [], "evaluator": {"func": "check"}} +UNIT = "os__task-1" +CLICK = "pyautogui.click(10, 20, clicks=1, button='left')" + + +def alive(pid: int) -> bool: + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().rsplit(")", 1)[1].split()[0] not in ("Z", "X") + except (OSError, IndexError): + return False + + +def gone(pid: int, seconds: float = 15.0) -> bool: + deadline = time.time() + seconds + while time.time() < deadline: + if not alive(pid): + return True + time.sleep(0.1) + return False + + +def processes_under(folder: str, seconds: float = 15.0, but=()) -> list: + """The processes whose command line names ``folder`` or whose working folder is in it, but ``but``, once + ``seconds`` have passed with some left.""" + deadline = time.time() + seconds + while True: + found = [] + for pid in filter(str.isdigit, os.listdir("/proc")): + if int(pid) in but or int(pid) == os.getpid(): + continue + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + named = folder.encode() in f.read() + named = named or os.path.realpath(os.readlink(f"/proc/{pid}/cwd")).startswith(os.path.realpath(folder)) + except OSError: + continue + if named and alive(int(pid)): + found.append(int(pid)) + if not found or time.time() > deadline: + return found + time.sleep(0.2) + + +class Session: + """A scripted MCP client in the agent's place: the setup's server computer started as the agent's session starts a + stdio server (the port's pipe), JSON-RPC lines on its stdin and stdout, each answer waited for a bounded time.""" + + def __init__(self, setup: dict, cwd: str): + server = setup["mcp"][0] + self.pipe = subprocess.Popen([PYTHON, *server["args"]], cwd=cwd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + self.got, self.last = b"", 0 + + def ask(self, method: str, params: dict | None = None, seconds: float = 30.0) -> dict: + self.last += 1 + message = {"jsonrpc": "2.0", "id": self.last, "method": method, **({"params": params} if params else {})} + self.pipe.stdin.write((json.dumps(message) + "\n").encode()) + self.pipe.stdin.flush() + deadline = time.monotonic() + seconds + while True: + while b"\n" in self.got: + line, _, self.got = self.got.partition(b"\n") + reply = json.loads(line) + if reply.get("id") == self.last: + return reply + left = deadline - time.monotonic() + if left <= 0 or not select.select([self.pipe.stdout], [], [], left)[0]: + raise AssertionError(f"no answer to {method} within {seconds} s") + chunk = os.read(self.pipe.stdout.fileno(), 1 << 20) + if not chunk: + raise AssertionError("the session's pipe ended") + self.got += chunk + + def call(self, tool: str, **arguments) -> dict: + return self.ask("tools/call", {"name": tool, "arguments": arguments})["result"] + + def close(self) -> None: + self.pipe.stdin.close() + try: + self.pipe.wait(timeout=30) + finally: + if self.pipe.poll() is None: + self.pipe.kill() + self.pipe.wait() + self.pipe.stdout.close() + + +def value(result: dict) -> dict: + return json.loads(next(c["text"] for c in result["content"] if c["type"] == "text")) + + +@unittest.skipIf(WHY, WHY) +class AgentSide(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="osworld1-agent-") + up = os.path.join(self.tmp, "upstream") + os.makedirs(os.path.join(up, "evaluation_examples", "examples", "os")) + os.makedirs(os.path.join(up, "docker_vm_data")) + open(os.path.join(up, "docker_vm_data", "Ubuntu.qcow2"), "w").close() + for name in LOOP: + os.symlink(os.path.join(OSWORLD, name), os.path.join(up, name)) + shutil.copytree(os.path.join(FAKE, "desktop_env"), os.path.join(up, "desktop_env")) + shutil.copyfile(os.path.join(FAKE, "sitecustomize.py"), os.path.join(up, "sitecustomize.py")) + self.task = dict(TASK) + self.dataset = os.path.join(up, "evaluation_examples", "test_one.json") + with open(self.dataset, "w") as f: + json.dump({"os": ["task-1"]}, f) + self.up, self.log = up, os.path.join(self.tmp, "desktop.jsonl") + self.trial = os.path.join(self.tmp, "job", "os__task-1__u0000") + self.ws = os.path.join(self.trial, "workspace") + os.makedirs(self.ws) + self.no_programs = os.path.join(self.tmp, "no-programs") # a PATH with no docker: none is ever run + os.mkdir(self.no_programs) + self.drivers, self.sessions = [], [] + + def tearDown(self): + for s in self.sessions: + if s.pipe.poll() is None: + s.pipe.kill() + s.pipe.wait() + for d in self.drivers: + d.close() + shutil.rmtree(self.tmp, ignore_errors=True) + + def driver(self, **options) -> jsonl.Proc: + with open(os.path.join(self.up, "evaluation_examples", "examples", "os", "task-1.json"), "w") as f: + json.dump(self.task, f) + env = {"PATH": self.no_programs, "HOME": self.tmp, "TMPDIR": tempfile.gettempdir(), "LANG": "C.UTF-8", + "NULLIUS_UPSTREAM": self.up, "NULLIUS_DATASET": self.dataset, "PYTHONPATH": APP, + "PYTHONDONTWRITEBYTECODE": "1", "FAKE_DESKTOP_LOG": self.log, + **{"NULLIUS_OPTION_" + k.upper(): v for k, v in options.items()}} + d = jsonl.Proc([PYTHON, os.path.join(ENTRY, "driver.py"), os.path.join(ENTRY, "agents.json"), + os.path.join(ENTRY, "nop.py"), os.path.join(ENTRY, "runner.py")], + self.tmp, env, os.path.join(self.tmp, "driver.log")).start() + self.drivers.append(d) + return d + + def setup(self, d: jsonl.Proc, ws=None) -> dict: + return d.request("setup", unit=UNIT, dir=ws or self.ws, participant="agent") + + def session(self, setup: dict) -> Session: + s = Session(setup, self.ws) + self.sessions.append(s) + s.ask("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "t"}}) + return s + + def events(self, kind: str) -> list: + with open(self.log) as f: + return [e for e in map(json.loads, f) if e["event"] == kind] + + def task_dir(self) -> str: + return os.path.join(self.trial, ".work", "osworld", "results", "os", "task-1") + + def traj(self) -> list: + with open(os.path.join(self.task_dir(), "traj.jsonl")) as f: + return [json.loads(line) for line in f] + + def runner(self) -> int: + with open(os.path.join(self.trial, ".work", "osworld", "runner.json")) as f: + return json.load(f)["pid"] + + def test_an_agents_calls_drive_the_upstreams_loop_to_done(self): + d = self.driver(max_steps="5", pause="0.1") + setup = self.setup(d) + self.assertEqual(setup["prompt"], TASK["instruction"] + "\n\nIf you need a password for sudo, the password of " + "the computer is 'password'.") + self.assertEqual([(m["name"], m["server"]) for m in setup["mcp"]], [("computer", "computer")]) + self.assertTrue(setup["mcp"][0]["args"][0].endswith(os.path.join("nullius", "standard", "computer_pipe.py"))) + s = self.session(setup) + self.assertEqual(base64.b64decode(s.call("screenshot")["content"][0]["data"])[:4], b"\x89PNG") + self.assertEqual(value(s.call("click", x=10, y=20))["steps_left"], 4) + self.assertEqual(value(s.call("step", actions=["pyautogui.press('enter')"], response="enter"))["steps_left"], 3) + self.assertEqual(value(s.call("done")), {"steps_left": 2, "done": True}) + s.close() + score = d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="done") + self.assertEqual(score, {"reward": 1.0, "detail": {"result": 1.0, "tool_calls": 4, + "by": "runner.py, run_single_example, DesktopEnv.evaluate"}}) + self.assertEqual([(t["step_num"], t["action"], t["response"]) for t in self.traj()], + [(1, CLICK, ""), (2, "pyautogui.press('enter')", "enter"), (3, "DONE", "")]) + self.assertEqual([(e["action"], e["pause"]) for e in self.events("step")], + [(CLICK, 0.1), ("pyautogui.press('enter')", 0.1), ("DONE", 0.1)]) + self.assertEqual([e["seconds"] for e in self.events("wait")], [60, 20]) + with open(os.path.join(self.trial, "verifier", "osworld", "computer-calls.jsonl")) as f: + self.assertEqual([json.loads(line)["tool"] for line in f], ["screenshot", "click", "step", "done"]) + self.assertTrue(gone(self.runner())) + self.assertTrue(self.events("close")) + self.assertEqual(processes_under(self.tmp, but=(d.p.pid,)), []) + + def test_a_turn_without_done_is_evaluated_as_the_desktop_is(self): + d = self.driver(max_steps="4", pause="0.1") + s = self.session(self.setup(d)) + s.call("click", x=10, y=20) + s.close() + score = d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="") + self.assertEqual(score["reward"], 0.0) # the fake gives 1 only to a task ended by DONE + self.assertEqual([t["action"] for t in self.traj()], [CLICK]) + self.assertEqual([e["last"] for e in self.events("evaluate")], [CLICK]) + + def test_fail_is_the_last_action_the_evaluator_reads(self): + self.task["evaluator"] = {"func": "infeasible"} + d = self.driver(max_steps="3", pause="0.1") + s = self.session(self.setup(d)) + self.assertEqual(value(s.call("fail", reason="no such menu"))["done"], True) + s.close() + self.assertEqual(d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="")["reward"], 1.0) + self.assertEqual([t["action"] for t in self.traj()], ["FAIL"]) + + def test_the_steps_option_is_the_loops_budget(self): + d = self.driver(max_steps="2", pause="0.1") + s = self.session(self.setup(d)) + s.call("click", x=1, y=1) + self.assertEqual(value(s.call("click", x=2, y=2)), {"steps_left": 0, "done": True}) + self.assertEqual(s.call("click", x=3, y=3)["content"][0]["text"], "the steps are used up") + s.close() + d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="") + self.assertEqual(len(self.traj()), 2) + + def test_a_result_txt_planted_in_the_workspace_is_not_read(self): + self.task["evaluator"] = {"func": "infeasible"} # DONE scores 0 + d = self.driver(pause="0.1") + s = self.session(self.setup(d)) + planted = os.path.join(self.ws, "results", "os", "task-1") + os.makedirs(planted) + with open(os.path.join(planted, "result.txt"), "w") as f: + f.write("1.0\n") + s.call("done") + s.close() + self.assertEqual(d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="done")["reward"], 0.0) + + def test_the_end_of_a_unit_that_never_reached_score_ends_its_runner(self): + d = self.driver() + setup = self.setup(d) + runner = self.runner() + folder = os.path.dirname(setup["mcp"][0]["args"][1]) + self.assertEqual(d.request("end", unit=UNIT, dir=self.ws), {}) + self.assertTrue(gone(runner, 1)) # gone when end answers + self.assertFalse(os.path.exists(folder)) + self.assertEqual(d.request("end", unit=UNIT, dir=self.ws), {}) # a second end does nothing + with self.assertRaisesRegex(jsonl.Failed, "no runner of this driver's for this unit"): + d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="") + + def test_the_next_units_setup_ends_a_runner_left(self): + d = self.driver() + self.setup(d) + first = self.runner() + other = os.path.join(self.tmp, "job", "os__task-1__u0001", "workspace") + os.makedirs(other) + self.setup(d, ws=other) + self.assertTrue(gone(first)) + with open(os.path.join(os.path.dirname(other), ".work", "osworld", "runner.json")) as f: + second = json.load(f)["pid"] + d.close() # the run ends: stdin closed + self.assertTrue(gone(second, 60)) + self.assertEqual(processes_under(self.tmp), []) + + def test_a_driver_killed_with_sigkill_leaves_no_runner(self): + d = self.driver() + setup = self.setup(d) + runner = self.runner() + os.killpg(d.p.pid, signal.SIGKILL) # no clean-up of the driver's own + d.p.wait(timeout=10) + self.assertTrue(gone(runner, 60), "the runner outlived its driver") + self.assertEqual(processes_under(self.tmp), []) + self.assertFalse(os.path.exists(os.path.dirname(setup["mcp"][0]["args"][1]))) # its socket's folder + + def test_the_password_file_is_private_and_read_once(self): + d = self.driver(pause="0.1") + setup = self.setup(d) + folder = os.path.dirname(setup["mcp"][0]["args"][1]) + self.assertEqual(os.stat(folder).st_mode & 0o777, 0o700) + self.assertEqual(sorted(os.listdir(folder)), ["s"]) # ready.json read and removed + s = self.session(setup) + s.call("done") + s.close() + d.request("score", unit=UNIT, dir=self.ws, transcript=[], answer="done") + self.assertFalse(os.path.exists(folder)) + + def test_a_unit_with_no_agent_starts_no_runner(self): + d = self.driver() + self.assertEqual(d.request("setup", unit=UNIT, dir=self.ws), {"prompt": TASK["instruction"]}) + self.assertEqual(d.request("setup", unit=UNIT, dir=self.ws, participant="control"), {"prompt": TASK["instruction"]}) + self.assertFalse(os.path.exists(os.path.join(self.trial, ".work"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_osworld1_entry_parts.py b/app/tests/test_osworld1_entry_parts.py new file mode 100644 index 00000000..de3678ea --- /dev/null +++ b/app/tests/test_osworld1_entry_parts.py @@ -0,0 +1,572 @@ +"""OSWorld1's agent side in parts, with the standard library alone: what the driver's score takes from a runner and how +it reads it, how it starts and reaps one, the bounds that follow from the unit's limits, and what the runner does +around the upstream's loop. Child processes of the test and files in temporary folders only; no upstream, no VM, no +docker.""" +from __future__ import annotations + +import fcntl +import importlib.util +import json +import os +import select +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import types +import unittest +from unittest import mock + +from nullius.standard import computer_port + +ENTRY = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "nullius", "catalogue", "benchmarks", + "osworld1") + + +def load(name: str, upstream: str): + spec = importlib.util.spec_from_file_location(name, os.path.join(ENTRY, name + ".py")) + module = importlib.util.module_from_spec(spec) + with mock.patch.dict(os.environ, {"NULLIUS_UPSTREAM": upstream}), mock.patch.object(sys, "argv", [name + ".py"]): + spec.loader.exec_module(module) + return module + + +def child(code: str = "import time; time.sleep(60)", **kw) -> subprocess.Popen: + return subprocess.Popen([sys.executable, "-c", code], start_new_session=True, stdin=subprocess.DEVNULL, **kw) + + +def alive(pid: int) -> bool: + try: + with open(f"/proc/{pid}/stat") as f: + return f.read().rsplit(")", 1)[1].split()[0] not in ("Z", "X") + except (OSError, IndexError): + return False + + +def gone(pid: int, seconds: float = 10) -> bool: + deadline = time.monotonic() + seconds + while alive(pid) and time.monotonic() < deadline: + time.sleep(0.05) + return not alive(pid) + + +def line_of(p: subprocess.Popen, seconds: float = 10) -> bytes: + """The child's next line on its output, waited for ``seconds`` at most.""" + got = b"" + deadline = time.monotonic() + seconds + while not got.endswith(b"\n"): + left = deadline - time.monotonic() + if left <= 0 or not select.select([p.stdout], [], [], left)[0]: + raise AssertionError("the child said nothing in time") + chunk = os.read(p.stdout.fileno(), 1) + if not chunk: + raise AssertionError("the child ended before its line") + got += chunk + return got + + +class Score(unittest.TestCase): + """score tells the runner the agent's turn has ended, and takes a result only from a runner this driver started, + that ended by itself with status 0, through folders opened without following a link, from plain files.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.d = load("driver", self.tmp) + self.d._remove_vms = lambda where, seconds=None: None + self.ws = os.path.join(self.tmp, "trial", "workspace") + os.makedirs(self.ws) + self.where = self.d._runner_dir(self.ws) + self.task = os.path.join(self.where, "results", "os", "t1") + os.makedirs(self.task) + self.d._task = lambda unit: ("os", "t1", {"instruction": "x"}) + + def result(self, value="1.0\n"): + with open(os.path.join(self.task, "result.txt"), "w") as f: + f.write(value) + + def owned(self, p: subprocess.Popen, life=None) -> None: + self.d.RUNNERS[self.where] = {"p": p, "sock_dir": tempfile.mkdtemp(dir=self.tmp), "life": life} + self.addCleanup(lambda: (p.poll() is None and p.kill(), p.wait())) + + def test_a_runner_that_ended_with_0_gives_its_result_and_its_call_log(self): + self.result() + with open(os.path.join(self.where, "computer-calls.jsonl"), "w") as f: + f.write('{"tool": "click"}\n{"tool": "done"}\n') + self.owned(child("pass")) + got = self.d.score("os__t1", self.ws, answer="done") + self.assertEqual(got, {"reward": 1.0, "detail": {"result": 1.0, "tool_calls": 2, + "by": "runner.py, run_single_example, DesktopEnv.evaluate"}}) + self.assertNotIn(self.where, self.d.RUNNERS) + out = os.path.join(self.tmp, "trial", "verifier", "osworld") + self.assertEqual(sorted(os.listdir(out)), ["computer-calls.jsonl", "result.txt"]) + + def test_score_tells_the_runner_the_turn_has_ended(self): + self.result() + read, write = os.pipe() + waits = ("import os, sys\nfd, data = int(sys.argv[1]), b''\n" + "while b'turn-end\\n' not in data:\n chunk = os.read(fd, 64)\n if not chunk: sys.exit(5)\n" + " data += chunk\n") + p = subprocess.Popen([sys.executable, "-c", waits, str(read)], pass_fds=(read,), start_new_session=True, + stdin=subprocess.DEVNULL) + os.close(read) + self.owned(p, life=write) + self.assertEqual(self.d.score("os__t1", self.ws, answer="")["reward"], 1.0) + + def test_a_planted_runner_record_is_no_runner_of_this_drivers(self): + self.result() + with open(os.path.join(self.where, "runner.json"), "w") as f: + json.dump({"pid": os.getpid()}, f) + with self.assertRaisesRegex(RuntimeError, "no runner of this driver's for this unit"): + self.d.score("os__t1", self.ws, answer="done") + + def test_a_linked_result_is_not_read(self): + forged = os.path.join(self.ws, "forged.txt") + with open(forged, "w") as f: + f.write("1.0\n") + os.symlink(forged, os.path.join(self.task, "result.txt")) + self.owned(child("pass")) + with self.assertRaisesRegex(RuntimeError, "the runner's result could not be read"): + self.d.score("os__t1", self.ws, answer="done") + + def test_a_linked_folder_anywhere_on_the_way_is_not_followed(self): + for linked in (("results",), ("results", "os"), ()): # () : the trial's own .work folder + with self.subTest(linked=linked): + shutil.rmtree(os.path.join(self.tmp, "trial", ".work"), ignore_errors=True) + elsewhere = os.path.join(self.tmp, "elsewhere") + shutil.rmtree(elsewhere, ignore_errors=True) + real = os.path.join(elsewhere, ".work", "osworld", "results", "os", "t1") + os.makedirs(real) + with open(os.path.join(real, "result.txt"), "w") as f: + f.write("1.0\n") + if linked: + os.makedirs(os.path.join(self.where, *linked[:-1]), exist_ok=True) + os.symlink(os.path.join(elsewhere, ".work", "osworld", *linked), os.path.join(self.where, *linked)) + else: + os.symlink(os.path.join(elsewhere, ".work"), os.path.join(self.tmp, "trial", ".work")) + self.owned(child("pass")) + with self.assertRaisesRegex(RuntimeError, "the runner's result could not be read"): + self.d.score("os__t1", self.ws, answer="done") + + def test_a_folder_swapped_for_a_link_after_it_was_opened_is_not_followed(self): + self.result() + results = os.path.join(self.where, "results") + real_open, swapped = os.open, [] + + def opening(path, flags, mode=0o777, *, dir_fd=None): + fd = real_open(path, flags, mode, dir_fd=dir_fd) + if path == "results" and not swapped: + os.rename(results, results + ".moved") + forged = os.path.join(self.tmp, "forged", "os", "t1") + os.makedirs(forged) + with open(os.path.join(forged, "result.txt"), "w") as f: + f.write("0.5\n") + os.symlink(os.path.join(self.tmp, "forged"), results) + swapped.append(True) + return fd + + with mock.patch.object(self.d.os, "open", opening): + got = self.d._plain(os.path.join(self.tmp, "trial"), ".work", "osworld", "results", "os", "t1", "result.txt") + self.assertEqual((swapped, got), ([True], b"1.0\n")) + + def test_a_fifo_in_place_of_the_result_is_refused_at_once(self): + os.mkfifo(os.path.join(self.task, "result.txt")) + self.owned(child("pass")) + t0 = time.monotonic() + with self.assertRaisesRegex(RuntimeError, "is not a plain file"): + self.d.score("os__t1", self.ws, answer="done") + self.assertLess(time.monotonic() - t0, 5) + + def test_a_killed_runner_gives_no_score_whatever_it_left(self): + self.result() + p = child() + p.kill() + self.owned(p) + with self.assertRaisesRegex(RuntimeError, "the runner was killed: its result is not read"): + self.d.score("os__t1", self.ws, answer="done") + + def test_a_runner_that_exited_1_gives_no_score(self): + self.result() + self.owned(child("raise SystemExit(1)")) + with self.assertRaisesRegex(RuntimeError, "the runner exited 1"): + self.d.score("os__t1", self.ws, answer="done") + + def test_a_runner_that_ends_well_while_score_lets_it_go_is_scored_from_its_files(self): + self.result() + self.owned(child("import time; time.sleep(1.5)")) + self.d.SCORE_SECONDS, self.d.REAP_SECONDS = 0.3, 15 + self.assertEqual(self.d.score("os__t1", self.ws, answer="done")["reward"], 1.0) + + def test_an_answer_with_no_runner_is_an_error(self): + with self.assertRaisesRegex(RuntimeError, "no runner was set up for this unit"): + self.d.score("os__t1", self.ws, answer="done") + + +class StartAndReap(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.d = load("driver", self.tmp) + self.d._remove_vms = lambda where, seconds=None: None + + def test_the_start_lock_is_shared_by_every_run_on_the_machine(self): + folder = os.path.join(self.tmp, "run-tmp") + os.mkdir(folder) + code = ("import importlib.util, sys\n" + "spec = importlib.util.spec_from_file_location('driver', sys.argv[1])\n" + "module = importlib.util.module_from_spec(spec)\nsys.argv = ['driver.py']\n" + "spec.loader.exec_module(module)\nprint(module.START_LOCK)\n") + got = subprocess.run([sys.executable, "-c", code, os.path.join(ENTRY, "driver.py")], capture_output=True, + text=True, timeout=60, stdin=subprocess.DEVNULL, + env=dict(os.environ, TMPDIR=folder, NULLIUS_UPSTREAM=self.tmp)) + self.assertEqual(got.stdout.strip(), "/tmp/nullius-osworld1-vm-start.lck", got.stderr[-2000:]) + + def test_an_already_reaped_runner_gets_no_signal(self): + p = child("pass") + p.wait() + self.d.RUNNERS["w"] = {"p": p, "sock_dir": tempfile.mkdtemp(dir=self.tmp), "life": None} + with mock.patch.object(self.d.os, "killpg") as kill: + self.d._reap_runner("w") + kill.assert_not_called() + self.assertNotIn("w", self.d.RUNNERS) + + def test_a_runner_reaped_elsewhere_gets_no_signal(self): + p = child("pass") + os.waitpid(p.pid, 0) # as a parent ignoring its children's signals would + self.d.RUNNERS["w"] = {"p": p, "sock_dir": tempfile.mkdtemp(dir=self.tmp), "life": None} + with mock.patch.object(self.d.os, "killpg") as kill: + self.d._reap_runner("w", seconds=0) + kill.assert_not_called() + + def test_a_running_runner_is_signalled_while_its_leader_is_unreaped(self): + p = child() + seen, real = [], os.killpg + + def killpg(pgid, sig): + seen.append((pgid, p.returncode)) + real(pgid, sig) + + self.d.RUNNERS["w"] = {"p": p, "sock_dir": tempfile.mkdtemp(dir=self.tmp), "life": None} + with mock.patch.object(self.d.os, "killpg", killpg): + self.d._reap_runner("w", seconds=0.2) + self.assertEqual(seen, [(p.pid, None)]) + self.assertIsNotNone(p.returncode) + + def test_a_runner_that_does_not_start_leaves_no_pipe_end_and_no_folder(self): + self.d._task = lambda unit: ("os", "a", {"instruction": "x"}) + self.d._workspace = lambda where: "vm" + self.d.START_LOCK = os.path.join(self.tmp, "start.lock") + sock = os.path.join(self.tmp, "socket") + os.mkdir(sock) + made, real = [], os.pipe + + def pipe(): + made.extend(real()) + return made[-2], made[-1] + + with mock.patch.object(self.d.tempfile, "mkdtemp", return_value=sock), \ + mock.patch.object(self.d.os, "pipe", pipe), \ + mock.patch.object(self.d.subprocess, "Popen", side_effect=OSError("cannot start")): + with self.assertRaisesRegex(OSError, "cannot start"): + self.d._start_runner("os__a", os.path.join(self.tmp, "job", "trial", "workspace"), "x", time.time() + 30) + still = [] + for fd in made: + try: + os.fstat(fd) + still.append(fd) + except OSError: + pass + self.assertEqual((os.path.exists(sock), still, self.d.RUNNERS), (False, [], {})) + + +class Bounds(unittest.TestCase): + """The unit's budget and the bounds that follow from it (the driver's module text): each derived from the unit's + own limits, the clean-up inside the reap, the reap inside the core's bounds. No test sleeps a fixed time to see a + bound.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.d = load("driver", self.tmp) + self.r = load("runner", self.tmp) + + def test_the_clean_up_fits_inside_the_reap_and_the_reap_inside_the_cores_bounds(self): + d, r = self.d, self.r + self.assertEqual(r.STOP_SECONDS, r.FINISH_SECONDS + r.CLOSE_SECONDS) + self.assertEqual(d.RUNNER.STOP_SECONDS, r.STOP_SECONDS) + self.assertLess(r.STOP_SECONDS, d.REAP_SECONDS, "the driver kills a runner still in its own clean-up") + reap = d.REAP_SECONDS + d.REMOVE_SECONDS + d.ANSWER_SECONDS + self.assertLessEqual(d.SCORE_SECONDS + reap, d.CORE_DRIVER_SECONDS, "score's wait and its reap overrun the core's") + self.assertLessEqual(reap, d.CORE_END_SECONDS, "end's reap overruns the core's") + + def test_setups_deadline_is_the_definitions_limit_less_the_reap(self): + with open(os.path.join(ENTRY, "standard.json")) as f: + limit = json.load(f)["limits"]["setup_seconds"] + self.assertEqual(self.d.SETUP_SECONDS, limit) + seen = [] + self.d._task = lambda unit: ("os", "a", {"instruction": "x"}) + self.d._start_runner = lambda unit, dir, instruction, until: seen.append(until - time.time()) + self.d.setup("os__a", os.path.join(self.tmp, "trial", "workspace"), participant="agent") + reap = self.d.REAP_SECONDS + self.d.REMOVE_SECONDS + self.d.ANSWER_SECONDS + self.assertAlmostEqual(seen[0], limit - reap, delta=5) + + # -- the gate --------------------------------------------------------------------------------------------------- + def stand_in_runner(self): + """A runner of the test's in place of runner.py: it logs the provider's start line once the test lets it + (the file go beside it), says ready, and waits on its lifeline.""" + path = os.path.join(self.tmp, "runner.py") + self.go = os.path.join(self.tmp, "go") + with open(path, "w") as f: + f.write("import json, os, sys, time\n" + "a = sys.argv[1:]\nget = lambda k: a[a.index(k) + 1]\n" + f"open({os.path.join(self.tmp, 'begun')!r}, 'w').close()\n" + f"while not os.path.exists({self.go!r}):\n time.sleep(0.02)\n" + "print('INFO: Started container with ports - VNC: 1', flush=True)\n" + "json.dump({'ok': True, 'password': 'pw'}, open(get('--ready'), 'w'))\n" + "os.read(int(get('--lifeline')), 1)\n") + self.d.RUNNER_FILE = path + self.d._task = lambda unit: ("os", "a", {"instruction": "x"}) + self.d._workspace = lambda where: "vm" + self.d._remove_vms = lambda where, seconds=None: None + self.d.START_LOCK = os.path.join(self.tmp, "start.lock") + self.addCleanup(lambda: [self.d._reap_runner(w, seconds=0) for w in list(self.d.RUNNERS)]) + + def gate_free(self): + with open(self.d.START_LOCK, "a") as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except BlockingIOError: + return False + + def test_the_start_gate_is_held_until_the_vm_start_has_ended(self): + self.stand_in_runner() + ws = os.path.join(self.tmp, "job", "trial", "workspace") + got = [] + worker = threading.Thread(target=lambda: got.append(self.d._start_runner("os__a", ws, "x", time.time() + 30)), + daemon=True) + worker.start() + deadline = time.monotonic() + 10 + while not os.path.exists(os.path.join(self.tmp, "begun")) and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(os.path.exists(os.path.join(self.tmp, "begun")), "the runner did not start") + self.assertFalse(self.gate_free(), "the gate was let go before the VM's start ended") + open(self.go, "w").close() # the VM's start ends + worker.join(15) + self.assertTrue(got and got[0]["prompt"].startswith("x")) + self.assertTrue(self.gate_free()) + + def test_a_unit_waiting_for_the_gate_fails_at_its_setups_deadline(self): + self.stand_in_runner() + held = open(self.d.START_LOCK, "a") + self.addCleanup(held.close) + fcntl.flock(held, fcntl.LOCK_EX) + t0 = time.monotonic() + with self.assertRaisesRegex(RuntimeError, "was not free before the end of this unit's setup"): + self.d._start_runner("os__a", os.path.join(self.tmp, "job", "trial", "workspace"), "x", time.time() + 0.5) + self.assertLess(time.monotonic() - t0, 5) + self.assertEqual(self.d.RUNNERS, {}) + + # -- the reap --------------------------------------------------------------------------------------------------- + def runner_with_child(self): + """A runner of the test's that never ends by itself, and a child of it in its process group.""" + where = os.path.join(self.tmp, "trial", ".work", "osworld") + os.makedirs(where) + p = child("import subprocess, sys, time\n" + "c = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" + "print(c.pid, flush=True)\ntime.sleep(60)\n", stdout=subprocess.PIPE) + kid = int(line_of(p)) + self.addCleanup(lambda: [alive(pid) and os.kill(pid, 9) for pid in (kid, p.pid)]) + self.addCleanup(p.stdout.close) + self.d.RUNNERS[where] = {"p": p, "sock_dir": tempfile.mkdtemp(dir=self.tmp), "life": None, + "start": self.d._start_time(p.pid)} + return where, p, kid + + def test_the_reap_ends_the_runners_whole_group(self): + self.d._remove_vms = lambda where, seconds=None: None + where, p, kid = self.runner_with_child() + self.d._reap_runner(where, seconds=0.2) + self.assertTrue(gone(kid), "a process of the runner's group outlived its reap") + self.assertIsNotNone(p.returncode) + + def test_a_reap_cut_by_the_drivers_end_is_handed_over(self): + where, p, kid = self.runner_with_child() + handed = [] + self.d._hand_over = lambda w: (handed.append(w), self.d.RUNNERS.pop(w, None)) + self.d.CLOSED.set() # the driver's input ends while the reap waits + t0 = time.monotonic() + self.assertIsNone(self.d._reap_runner(where, seconds=30)) + self.assertLess(time.monotonic() - t0, 5) + self.assertEqual((handed, self.d.RUNNERS), ([where], {})) + + def test_a_runner_handed_over_is_reaped_after_the_driver_was_killed(self): + # The driver in a child of the test's: it starts a runner with a child of its own, hands it over to a reaper, + # says so and waits; the test kills the driver's whole group while the reaper's wait is pending. + where = os.path.join(self.tmp, "trial", ".work", "osworld") + os.makedirs(where) + empty = os.path.join(self.tmp, "no-programs") + os.mkdir(empty) + code = ("import importlib.util, os, subprocess, sys, tempfile, time\n" + "sys.argv = ['driver.py']\n" + "spec = importlib.util.spec_from_file_location('driver', os.environ['DRIVER'])\n" + "d = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(d)\n" + "d.REAP_SECONDS = 0.5\n" + "p = subprocess.Popen([sys.executable, '-c', 'import subprocess, sys, time\\n" + "c = subprocess.Popen([sys.executable, \"-c\", \"import time; time.sleep(60)\"])\\n" + "print(c.pid, flush=True)\\ntime.sleep(60)\\n'], stdout=subprocess.PIPE, start_new_session=True)\n" + "kid = int(p.stdout.readline())\n" + "d.RUNNERS[os.environ['WHERE']] = {'p': p, 'sock_dir': tempfile.mkdtemp(), 'life': None," + " 'start': d._start_time(p.pid)}\n" + "d._hand_over(os.environ['WHERE'])\n" + "print(p.pid, kid, flush=True)\n" + "time.sleep(60)\n") + driver = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, + start_new_session=True, + env=dict(os.environ, DRIVER=os.path.join(ENTRY, "driver.py"), WHERE=where, + NULLIUS_UPSTREAM=self.tmp, PATH=empty)) + self.addCleanup(lambda: (driver.poll() is None and os.killpg(driver.pid, 9), driver.wait(), driver.stdout.close())) + runner, kid = map(int, line_of(driver, 20).split()) + self.addCleanup(lambda: [alive(pid) and os.kill(pid, 9) for pid in (runner, kid)]) + os.killpg(driver.pid, signal.SIGKILL) # the driver and its group, the reaper's wait still pending + driver.wait(timeout=10) + self.assertTrue(gone(runner, 15), "the runner outlived its driver's death") + self.assertTrue(gone(kid, 15), "the runner's group outlived its driver's death") + + def test_the_vms_removal_has_its_bound(self): + bin_dir = os.path.join(self.tmp, "bin") + os.mkdir(bin_dir) + with open(os.path.join(bin_dir, "docker"), "w") as f: + f.write("#!/bin/sh\nexec sleep 30\n") # a docker command of the test's that never answers + os.chmod(os.path.join(bin_dir, "docker"), 0o755) + t0 = time.monotonic() + with mock.patch.dict(os.environ, {"PATH": bin_dir + os.pathsep + os.environ.get("PATH", "")}): + self.d._remove_vms(self.tmp, 0.5) + self.assertLess(time.monotonic() - t0, 5) + + +class Snapshot(unittest.TestCase): + """The copy Nullius makes of the entry for a run on a host carries the runner, and the port's shared code with + the package.""" + + def test_the_entrys_copy_holds_the_runner_and_the_package_the_ports_code(self): + from pathlib import Path + from nullius.standard import payload + files = set(payload._entry(Path(ENTRY))) + self.assertTrue({"driver.py", "agents.json", "nop.py", "runner.py"} <= files, sorted(files)) + package = set(payload._python(payload.PACKAGE)) + self.assertTrue({"standard/computer_port.py", "standard/computer_pipe.py"} <= package) + + +class FakeEnv: + """The upstream's DesktopEnv as the runner's Desktop uses it: its controllers, recorded.""" + + def __init__(self, close_seconds=0.0): + self.seen, self.files, self.close_seconds = [], {}, close_seconds + self.client_password = "password" + self.controller = types.SimpleNamespace( + execute_python_command=lambda code: self.seen.append(("python", code)) or {"status": "success"}, + get_file=lambda path: self.files.get(path), + get_screenshot=lambda: b"png") + self.setup_controller = types.SimpleNamespace(_upload_file_setup=self.upload) + + def upload(self, files): + for f in files: + with open(f["local_path"], "rb") as src: + self.files[f["path"]] = src.read() + self.seen.append(("upload", f["path"], f["local_path"])) + + def close(self): + time.sleep(self.close_seconds) + self.seen.append(("close",)) + + +class Runner(unittest.TestCase): + """runner.py around the upstream's loop, with stand-ins for the upstream's modules in the test's process.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.r = load("runner", self.tmp) + + def test_the_desktop_is_the_upstreams_controllers(self): + env = FakeEnv() + desktop = self.r.Desktop(env, self.tmp) + self.assertEqual(desktop.python("print(1)"), {"status": "success"}) + desktop.upload("/home/user/a.bin", b"bytes") + self.assertEqual((desktop.read("/home/user/a.bin"), desktop.read("/none")), (b"bytes", None)) + self.assertEqual((desktop.capture(), desktop.password()), (b"png", "password")) + local = env.seen[1][2] + self.assertEqual((env.seen[0], env.seen[1][:2]), (("python", "print(1)"), ("upload", "/home/user/a.bin"))) + self.assertFalse(os.path.exists(local), "the upload's local file was left") + env.controller.get_screenshot = lambda: None + with self.assertRaisesRegex(RuntimeError, "no screenshot"): + desktop.capture() + + def args(self): + folder = tempfile.mkdtemp(dir=self.tmp) + example = os.path.join(self.tmp, "os__a.json") + with open(example, "w") as f: + json.dump({"id": "a", "instruction": "x"}, f) + return types.SimpleNamespace(example=example, result_dir=os.path.join(self.tmp, "results"), + socket=os.path.join(folder, "s"), ready=os.path.join(folder, "ready.json"), + vm="vm", calls=os.path.join(self.tmp, "calls.jsonl"), held=None, lifeline=-1, + steps=3, pause=0.0) + + def run_with(self, env, loop): + a = self.args() + with mock.patch.dict(sys.modules, {"lib_run_single": types.SimpleNamespace(run_single_example=loop), + "desktop_env": mock.Mock(), + "desktop_env.desktop_env": types.SimpleNamespace(DesktopEnv=lambda **kw: env)}): + code = self.r.run(a, computer_port, computer_port.Stop(30), self.r.Turn()) + with open(a.ready) if os.path.exists(a.ready) else open(os.devnull) as f: + said = f.read() + return code, said + + def test_a_loop_that_ends_is_an_exit_0_and_the_desktop_is_closed(self): + env = FakeEnv() + + def loop(agent, env_, example, steps, instruction, args, where, scores): + agent.reset(None, vm_ip="x") + agent.predict(instruction, {}) # the first step: ready; then the turn's end + open(os.path.join(where, "result.txt"), "w").write("1.0\n") + + real = computer_port.Bridge.predict + + def predict(bridge, instruction, obs): + bridge.end_turn() + return real(bridge, instruction, obs) + + with mock.patch.object(computer_port.Bridge, "predict", predict): + code, said = self.run_with(env, loop) + self.assertEqual((code, json.loads(said)), (0, {"ok": True, "password": "password"})) + self.assertEqual(env.seen[-1], ("close",)) + + def test_a_loop_that_fails_before_the_first_step_says_not_ready_and_closes_the_desktop(self): + env = FakeEnv() + + def loop(agent, env_, example, steps, instruction, args, where, scores): + raise RuntimeError("the task's config failed") + + code, said = self.run_with(env, loop) + self.assertEqual(code, 1) + self.assertEqual(json.loads(said), {"ok": False, "error": "the task's setup: RuntimeError: the task's config failed"}) + self.assertEqual(env.seen[-1], ("close",)) + + def test_a_desktop_that_does_not_close_in_its_time_is_a_failure(self): + self.r.CLOSE_SECONDS = 0.3 + env = FakeEnv(close_seconds=5) + + def loop(agent, env_, example, steps, instruction, args, where, scores): + pass + + t0 = time.monotonic() + code, _ = self.run_with(env, loop) + self.assertEqual(code, 1) + self.assertLess(time.monotonic() - t0, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 66904def..9c8b25a7 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -31,13 +31,15 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Version | Setups as published, whose model | Options | |---|---|---| -| `osworld1` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `thinking`, `vm_provider` | +| `osworld1` | Original · OSWorld's agents · Like the paper or Any of your choice | `agent`, `max_steps`, `pause`, `thinking`, `vm_provider` | ## Options | Option | Values, default in bold | What it sets | Setups | |---|---|---|---| | `agent` | **`m3`**, `prompt`, `claude`, `gpt54`, `openai_cua`, `qwen`, `os_symphony`, `evocua`, `autoglm`, `autoglm_v`, `mano`, `muse_spark` | Which of OSWorld's own agents runs: each its script as published, with its own prompts, action format and step limit. | `original`, `model` | +| `max_steps` | pattern `[1-9][0-9]{0,2}`, default `100` | run_single_example's max_steps for an agent: the steps of OSWorld's own loop, each one acting call of the computer port. | `agent` | +| `pause` | pattern `[0-9]{1,2}(\.[0-9]{1,2})?`, default `3.0` | run_single_example's sleep_after_execution for an agent: the seconds env.step waits after each action, 3.0 as in run_multienv_m3.py's default. | `agent` | | `thinking` | **`adaptive`**, `off` | M3_THINKING_MODE: adaptive, as run_multienv_m3.py's own documented invocation exports it (the model decides whether to think; temperature 1, no top_p), or off, the agent's code default when the variable is unset. | `original`, `model` | | `vm_provider` | **`docker`** | --provider_name: where the task's VM runs. docker only: the upstream's other providers need cloud accounts or a hypervisor of their own. | `original`, `model`, `yours` | @@ -47,6 +49,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of ## Good to know +- An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. Steps (100) and Pause (3.0 seconds) set that loop. - Original runs one of OSWorld's own agents, chosen by the Agent option (M3 by default), with a model it is published with. - Providers: OpenAI for OpenAI CUA; OpenAI-compatible for Muse Spark (Meta's API) and for EvoCUA, AutoGLM-OS and Mano (your model server); OpenRouter for the others. - Only M3, PromptAgent and Qwen-CUA have completed tasks through Nullius. The other agents were checked up to their first model call. @@ -65,7 +68,9 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of ## Nullius limitations -- No agent from the Agents page can run it yet. +- An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop only, not on real VMs. +- Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it. +- A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs. - Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach. - The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested. - A task still running after 2 hours is stopped and gets no score. From 355e763c5f2773d042dfd52a262b49598a6727c7 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:19:40 +0100 Subject: [PATCH 21/36] fix(standard): bound every frame and read the port's schemas alike everywhere - The last resort's word goes to stderr from a thread it waits for 0.1 s at most: an exit never waits on a pipe nobody reads. - Every outgoing frame is bounded, the answers of any method and the errors standing in for them, with no id when even that is over. - NaN, Infinity and -Infinity are no JSON. Each tool's schema lists every argument it takes (additionalProperties false), and any other argument is refused. - The listed patterns use explicit classes, which Python and JSON Schema's regular expressions read alike; a final $ is the text's end. - upload's data is base64 by its pattern, and its listed bound is the check's: at the longest length the padding says the chunk is not over. - A failed append removes its part as far as the desktop answers, and a kept file's temporary file and descriptor go on every path. - The tests close the wires they make, and the queue test's helper ends the pipe before it closes an input a feeder may still write. --- app/nullius/standard/computer_port.py | 118 ++++++++++++++++------ app/tests/test_standard_computer_pipe.py | 20 ++-- app/tests/test_standard_computer_port.py | 121 +++++++++++++++++++++-- 3 files changed, 215 insertions(+), 44 deletions(-) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index df1f81c2..0a567c00 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -86,10 +86,19 @@ def password(self) -> str: SCREEN_TOOLS = ("click", "move", "drag", "scroll", "type", "key", "wait") # answered with the screen after the pause ACTING = SCREEN_TOOLS + ("step", "done", "fail") FINAL = {"done": "DONE", "fail": "FAIL"} -B64_CHARS = 4 * -(-CHUNK_BYTES // 3) -REFUSALS = {("scroll", "anyOf"): "dy or dx is not 0", ("upload", "anyOf"): "upload needs data, or a sha256", - ("upload", "if"): "append needs data: a file taken by its sha256 starts the file"} -PATTERN_WORDS = {"\\S": "is not blank", "^[0-9a-f]{64}$": "is 64 lowercase hexadecimal digits"} +# The patterns use explicit classes alone, which Python's re and the regular expressions of JSON Schema read alike; +# a final $ is the text's end (Python's \Z) +BLANK = "[\\t\\n\\v\\f\\r ]" +NOT_BLANK = "[^\\t\\n\\v\\f\\r ]" +BASE64 = "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" +SHA256 = "^[0-9a-f]{64}$" +B64_CHARS = 4 * -(-CHUNK_BYTES // 3) # the longest base64 of CHUNK_BYTES; at that length, its padding +B64_LAST = "=" * (3 - CHUNK_BYTES % 3) + "$" if CHUNK_BYTES % 3 else None # says the last group is not too full +REFUSALS = {("scroll", "arguments", "anyOf"): "dy or dx is not 0", + ("upload", "arguments", "anyOf"): "upload needs data, or a sha256", + ("upload", "arguments", "if"): "append needs data: a file taken by its sha256 starts the file", + ("upload", "data", "anyOf"): f"data is at most {CHUNK_BYTES} bytes a call"} +PATTERN_WORDS = {NOT_BLANK: "is not blank", SHA256: "is 64 lowercase hexadecimal digits", BASE64: "is base64"} @functools.lru_cache(maxsize=8) @@ -101,8 +110,11 @@ def tools(screen): button = {"type": "string", "enum": ["left", "right", "middle"], "description": "Default left"} clicks = {"type": "integer", "minimum": -100, "maximum": 100} text = {"type": "string", "minLength": 1} - obj = lambda props, required=(), **more: dict({"type": "object", "properties": props}, + obj = lambda props, required=(), **more: dict({"type": "object", "properties": props, "additionalProperties": False}, **({"required": list(required)} if required else {}), **more) + data = {"type": "string", "contentEncoding": "base64", "pattern": BASE64, "maxLength": B64_CHARS} + if B64_LAST: + data["anyOf"] = [{"maxLength": B64_CHARS - 4}, {"pattern": B64_LAST}] return [ ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " "python waits for the desktop's reply, and the desktop user's password.", obj({})), @@ -116,8 +128,8 @@ def tools(screen): "4 MiB a call; append false starts the file, true adds to it. sha256, optional, is the whole " "file's: when the run already holds a file of that sha256, it may be given with no data, and it " "starts the file.", - obj({"path": text, "data": {"type": "string", "contentEncoding": "base64", "maxLength": B64_CHARS}, - "append": {"type": "boolean"}, "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}}, ["path"], + obj({"path": text, "data": data, + "append": {"type": "boolean"}, "sha256": {"type": "string", "pattern": SHA256}}, ["path"], anyOf=[{"required": ["data"]}, {"required": ["sha256"]}], **{"if": {"not": {"required": ["data"]}}, "then": {"properties": {"append": {"const": False}}}})), ("click", "One step: a click at the point.", @@ -140,8 +152,8 @@ def tools(screen): "pyautogui and time imported, or WAIT. An empty list is a step with no action. The screen comes back " "when screenshot is true.", obj({"actions": {"type": "array", "items": { - "type": "string", "pattern": "\\S", - "not": {"pattern": "^\\s*(DONE|FAIL)\\s*$", + "type": "string", "pattern": NOT_BLANK, + "not": {"pattern": f"^{BLANK}*(DONE|FAIL){BLANK}*$", "description": "the task's end is the tool done or fail, not an action of step"}}}, "response": {"type": "string"}, "screenshot": {"type": "boolean"}}, ["actions"])), ("done", "The last step: the task is complete.", obj({"message": {"type": "string"}})), @@ -165,6 +177,16 @@ def _passes(value, schema, tool, name): return False +def _python_pattern(pattern): + """A pattern of the table as Python's re reads it alike: a final $ is the text's end, never before a newline.""" + return pattern[:-1] + "\\Z" if pattern.endswith("$") and not pattern.endswith("\\$") else pattern + + +def _no_constant(name): + """json.loads's parse_constant: NaN, Infinity and -Infinity are no JSON.""" + raise ValueError(f"{name} is not JSON") + + def _valid(value, schema, tool, name): """``value`` checked against ``schema``, in the part of JSON Schema the tool table uses; the value with integers as int. Refused, naming the argument, when it does not hold.""" @@ -183,7 +205,7 @@ def _valid(value, schema, tool, name): raise Refused(f"{name} is a text, not empty") if len(value) > schema.get("maxLength", len(value)): raise Refused(f"{name} is at most {schema['maxLength']} characters") - if "pattern" in schema and not re.search(schema["pattern"], value): + if "pattern" in schema and not re.search(_python_pattern(schema["pattern"]), value): raise Refused(f"{name} " + PATTERN_WORDS.get(schema["pattern"], "is not of its form")) if isinstance(value, list): if not schema.get("minItems", 0) <= len(value) <= schema.get("maxItems", len(value)): @@ -194,11 +216,15 @@ def _valid(value, schema, tool, name): if key not in value: raise Refused(f"{key} is missing") props = schema.get("properties", {}) + if schema.get("additionalProperties") is False: + for key in value: + if key not in props: + raise Refused(f"{key} is no argument of {tool}") value = {k: _valid(v, props[k], tool, k) if k in props else v for k, v in value.items()} - if "anyOf" in schema and not any(_passes(value, s, tool, name) for s in schema["anyOf"]): - raise Refused(REFUSALS.get((tool, "anyOf"), f"{name} is not of its form")) - if "if" in schema and _passes(value, schema["if"], tool, name) and not _passes(value, schema["then"], tool, name): - raise Refused(REFUSALS.get((tool, "if"), f"{name} is not of its form")) + if "anyOf" in schema and not any(_passes(value, s, tool, name) for s in schema["anyOf"]): + raise Refused(REFUSALS.get((tool, name, "anyOf"), f"{name} is not of its form")) + if "if" in schema and _passes(value, schema["if"], tool, name) and not _passes(value, schema["then"], tool, name): + raise Refused(REFUSALS.get((tool, name, "if"), f"{name} is not of its form")) if "enum" in schema and value not in schema["enum"]: raise Refused(f"{name} is one of {', '.join(map(str, schema['enum']))}") if "const" in schema and (value != schema["const"] or type(value) is not type(schema["const"])): @@ -290,14 +316,19 @@ def keep(self, digest, data): anew, the user's alone), removed on every path.""" if hashlib.sha256(data).hexdigest() != digest: return - fd, part = tempfile.mkstemp(prefix=".keep-", dir=self.folder) + fd = part = None try: - with os.fdopen(fd, "wb") as f: + fd, part = tempfile.mkstemp(prefix=".keep-", dir=self.folder) + f, fd = os.fdopen(fd, "wb"), None + with f: f.write(data) os.replace(part, os.path.join(self.folder, digest)) finally: - with contextlib.suppress(OSError): - os.unlink(part) + if fd is not None: + os.close(fd) + if part is not None: + with contextlib.suppress(OSError): + os.unlink(part) class Bridge: @@ -454,16 +485,23 @@ def _append(self, path, data): """``data`` added to the file at ``path``: written beside it under a name of the server's own, then added to it in the desktop; that name is removed on every path.""" part = posixpath.join(posixpath.dirname(path), f".upload-{secrets.token_hex(8)}") - self.desktop.upload(part, data) - reply = self.desktop.python(APPEND.format(part=part, path=path)) + try: + self.desktop.upload(part, data) + reply = self.desktop.python(APPEND.format(part=part, path=path)) + except BaseException: # the part may be there: removed, as far as the desktop answers + self._drop(part) + raise try: return int(str(reply.get("output")).strip()) except (AttributeError, ValueError): if reply is None: # the desktop gave no reply: the part may still be there - with contextlib.suppress(Exception): - self.desktop.python(f"import os; os.remove({part!r})") + self._drop(part) raise Refused(f"the desktop did not add the data to {path}: {json.dumps(reply)[:300]}") + def _drop(self, part): + with contextlib.suppress(Exception): + self.desktop.python(f"import os\ntry:\n os.remove({part!r})\nexcept OSError:\n pass") + # -- what the server and the runner call ------------------------------------------------------------------------ def submit(self, call): """A call that needs the desktop, for the loop's thread; refused at once once the task ended, and an acting @@ -533,6 +571,19 @@ def frame(message) -> bytes: return (json.dumps(message) + "\n").encode() +def bounded(message) -> tuple: + """``message`` framed, at most LINE_BYTES with its newline, and whether it had to be replaced: one over the bound + becomes an error with the same id, or with none when even that is over it.""" + data = frame(message) + if len(data) <= LINE_BYTES: + return data, False + error = {"code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"} + data = frame({"jsonrpc": "2.0", "id": message.get("id") if isinstance(message, dict) else None, "error": error}) + if len(data) > LINE_BYTES: + data = frame({"jsonrpc": "2.0", "id": None, "error": error}) + return data, True + + class Wire: """One connection of the agent's session: JSON-RPC messages, one a line, each at most LINE_BYTES with its newline. Its answers are written by a thread of its own; at most OUT_BYTES of them wait for the client to read, past which @@ -576,7 +627,10 @@ def lines(self): def send(self, message): """``message`` (an object, or a framed line) queued for the writer; False when the wire is closed, or closed now because its client has left OUT_BYTES unread.""" - data = message if isinstance(message, bytes) else frame(message) + data = message if isinstance(message, bytes) else bounded(message)[0] + if len(data) > LINE_BYTES: # a framed line over the bound, from a caller that did not bound it + data = frame({"jsonrpc": "2.0", "id": None, + "error": {"code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"}}) with self.lock: if self.closed: return False @@ -698,7 +752,7 @@ def _serve(self, wire): try: for line in wire.lines(): try: - message = json.loads(line) + message = json.loads(line, parse_constant=_no_constant) except (ValueError, RecursionError): wire.send({"jsonrpc": "2.0", "id": None, "error": {"code": -32700, "message": "not JSON"}}) continue @@ -756,11 +810,9 @@ def reply(call, value, image, error): content.append({"type": "text", "text": json.dumps(value)}) if image is not None: content.append({"type": "image", "data": base64.b64encode(image).decode(), "mimeType": "image/png"}) - data = frame({"jsonrpc": "2.0", "id": rid, "result": {"content": content, "isError": error is not None}}) - failed = error is not None - if len(data) > LINE_BYTES: # the answer's size is known before its outcome is recorded - data, failed = frame({"jsonrpc": "2.0", "id": rid, "error": { - "code": -32603, "message": f"the answer is over {LINE_BYTES} bytes"}}), True + data, over = bounded({"jsonrpc": "2.0", "id": rid, "result": {"content": content, + "isError": error is not None}}) + failed = error is not None or over # the answer's size is known before its outcome is recorded self.record({"tool": call.tool, "start": call.start, "end": time.time(), "isError": failed}) wire.send(data) wire.answered() @@ -863,9 +915,15 @@ def _last_resort(self): last = threading.Thread(target=self._quietly, name="last-resort-callback", daemon=True) last.start() last.join(LAST_SECONDS) - sys.stderr.write(f"runner: the clean-up did not end within {self.seconds:g} s of the stop\n") + said = threading.Thread(target=self._say, name="last-resort-word", daemon=True) + said.start() + said.join(0.1) # stderr may be a pipe nobody reads: the exit never waits on it os._exit(1) + def _say(self): + with contextlib.suppress(OSError): + os.write(2, f"runner: the clean-up did not end within {self.seconds:g} s of the stop\n".encode()) + def _quietly(self): with contextlib.suppress(Exception): self.last() diff --git a/app/tests/test_standard_computer_pipe.py b/app/tests/test_standard_computer_pipe.py index 5d6a8cfb..b1ed2caa 100644 --- a/app/tests/test_standard_computer_pipe.py +++ b/app/tests/test_standard_computer_pipe.py @@ -231,12 +231,20 @@ def feed(): while time.monotonic() < deadline + 10 and written[0] != last: # until the input is held back last = written[0] time.sleep(0.5) - bound = 65536 * (3 + pipe_module().QUEUE_CHUNKS) + 4 * 1024 * 1024 # queued, one in each hand, the OS pipe - self.assertLess(written[0], bound, "the pipe read on with nothing taken by the socket") - self.assertEqual(self.sent(), first, "the stalled socket took more") - stop.set() - open(self.tmp_file() + ".go", "w").close() # the socket takes more - feeder.join(10) + try: + bound = 65536 * (3 + pipe_module().QUEUE_CHUNKS) + 4 * 1024 * 1024 # queued, one in each hand, the OS pipe + self.assertLess(written[0], bound, "the pipe read on with nothing taken by the socket") + self.assertEqual(self.sent(), first, "the stalled socket took more") + stop.set() + open(self.tmp_file() + ".go", "w").close() # the socket takes more + feeder.join(10) + self.assertFalse(feeder.is_alive(), "the input was not drained once the socket took more") + except BaseException: + stop.set() + p.kill() # the feeder's write ends with the pipe's end + p.wait() + feeder.join(10) + raise p.stdin.close() self.assertEqual(p.wait(timeout=20), 0) self.assertEqual(self.sent(), written[0]) diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index db4f3efc..e5b252b6 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -590,6 +590,11 @@ def test_every_tool_call_is_recorded_with_its_tool_times_and_failure_and_nothing with stop.deferred(): os.kill(os.getpid(), signal.SIGTERM) time.sleep(30) +elif mode == "stderr-full": + stop.last = lambda: (sys.stderr.write("x" * 4194304), sys.stderr.flush()) + with stop.deferred(): + os.kill(os.getpid(), signal.SIGTERM) + time.sleep(30) elif mode == "stalled-last": stop.last = lambda: time.sleep(30) with stop.deferred(): @@ -643,6 +648,13 @@ def test_a_clean_up_that_hangs_takes_the_last_resort(self): self.assertLess(time.monotonic() - t0, 5) self.assertIn(b"did not end within 0.3 s of the stop", err) + def test_a_last_resort_exits_when_stderr_is_a_pipe_nobody_reads(self): + p = self.child("stderr-full", "0.05") # its stderr is a pipe the test does not read + try: + self.assertEqual(p.wait(timeout=5), 1) + except subprocess.TimeoutExpired: + self.fail("the last resort waited on stderr") + def test_a_last_resort_whose_own_callback_hangs_still_exits(self): p = self.child("stalled-last", "0.05") t0 = time.monotonic() @@ -708,6 +720,11 @@ def call(self, b, tool, **args): b.submit(call) return call + def wire(self, conn=None): + w = cp.Wire(conn if conn is not None else FakeConnection()) + self.addCleanup(w.close) + return w + def upload(self, b, **args): call = cp.Call("upload", args, None) cp.checked(call, b.screen) @@ -726,7 +743,7 @@ def sendall(self, data): raise OSError("let go") b = self.bridge() - wire = cp.Wire(Stalled()) + wire = self.wire(Stalled()) self.server(b).call(1, {"name": "screenshot"}, wire) served = threading.Thread(target=b._serve, args=(b.calls.popleft(),), daemon=True) served.start() @@ -746,7 +763,7 @@ class Stalled(FakeConnection): def sendall(self, data): release.wait(10) - wire = cp.Wire(Stalled()) + wire = self.wire(Stalled()) with mock.patch.object(cp, "OUT_BYTES", 1000): self.assertTrue(wire.send({"x": "a" * 400})) # taken by the writer, which stalls self.assertTrue(until(lambda: not wire.out)) @@ -799,14 +816,27 @@ def close(self): def test_a_complete_message_over_the_bound_is_refused(self): conn = FakeConnection([b" " * 127, b"{}\n"]) with mock.patch.object(cp, "LINE_BYTES", 128): - self.assertEqual(list(cp.Wire(conn).lines()), []) + self.assertEqual(list(self.wire(conn).lines()), []) self.assertTrue(until(lambda: conn.sent)) self.assertEqual(conn.sent[0]["error"]["code"], -32600) + def test_every_outgoing_frame_is_within_the_bound(self): + class Raw(FakeConnection): + def sendall(self, data): + self.sent.append(data) + + with mock.patch.object(cp, "LINE_BYTES", 2000): + for method in ("tools/list", "initialize", "no/such"): + with self.subTest(method=method): + conn = Raw([(json.dumps({"jsonrpc": "2.0", "id": "x" * 1900, "method": method}) + "\n").encode()]) + self.server(self.bridge())._serve(self.wire(conn)) + self.assertTrue(until(lambda: conn.sent)) + self.assertTrue(all(len(frame) <= 2000 for frame in conn.sent), [len(f) for f in conn.sent]) + def test_json_too_deep_to_read_is_answered_and_the_connection_closed(self): s = self.server(self.bridge()) conn = FakeConnection([b"[" * 100000 + b"]" * 100000 + b"\n"]) - s._serve(cp.Wire(conn)) + s._serve(self.wire(conn)) self.assertTrue(until(lambda: conn.closed), "the connection stayed open") self.assertEqual(conn.sent[0]["error"]["code"], -32700) @@ -822,10 +852,39 @@ def test_a_number_past_any_integer_is_refused_and_recorded(self): b = self.bridge() s = self.server(b) s.handle(json.loads('{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "click", ' - '"arguments": {"x": 1e999, "y": 1}}}'), cp.Wire(FakeConnection())) + '"arguments": {"x": 1e999, "y": 1}}}'), self.wire()) self.assertEqual([(r["tool"], r["isError"]) for r in s.records], [("click", True)]) self.assertEqual(b.booked, 0) + def test_nan_and_infinity_are_no_json_and_take_no_step(self): + for constant in ("NaN", "Infinity", "-Infinity"): + with self.subTest(constant=constant): + b = self.bridge() + conn = FakeConnection([('{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "wait", ' + '"arguments": {"extra": %s}}}\n' % constant).encode()]) + self.server(b)._serve(self.wire(conn)) + self.assertTrue(until(lambda: conn.sent)) + self.assertEqual((conn.sent[0]["error"]["code"], b.booked), (-32700, 0)) + + def test_an_argument_the_schema_does_not_list_is_refused(self): + b = self.bridge() + s = self.server(b) + s.call(1, {"name": "wait", "arguments": {"extra": 1}}, self.wire()) + s.call(2, {"name": "click", "arguments": {"x": 1, "y": 1, "clicks": 2}}, self.wire()) + self.assertEqual(([r["isError"] for r in s.records], b.booked), ([True, True], 0)) + self.assertEqual([t["inputSchema"]["additionalProperties"] for t in cp.listed((100, 100))], [False] * 15) + + def test_the_patterns_mean_the_same_in_python_and_in_the_listed_schema(self): + for actions in (["\ufeff"], ["\u0085"], ["\u00a0"], ["a"]): # not blank, read alike either way + cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100)) + for actions in ([" \t\r\n\f\v"], ["DONE\n"], [" FAIL "]): + with self.subTest(actions=actions), self.assertRaises(cp.Refused): + cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100)) + with self.assertRaisesRegex(cp.Refused, "64 lowercase"): # $ is the text's end + cp.checked(cp.Call("upload", {"path": "f", "sha256": "0" * 64 + "\n"}, None), (100, 100)) + patterns = json.dumps(cp.listed((100, 100))) + self.assertNotRegex(patterns, r"\\\\[sSdDwWbB]", "a class whose meaning differs between engines") + def test_a_request_that_is_not_json_rpc_2_takes_no_step(self): b = self.bridge() s = self.server(b) @@ -834,7 +893,7 @@ def test_a_request_that_is_not_json_rpc_2_takes_no_step(self): {"jsonrpc": "2.0", "id": True, "method": "tools/call", "params": {"name": "wait"}}, {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": [{"name": "wait"}]}, {"id": 1, "method": "tools/call", "params": {"name": "wait"}}]: - self.assertEqual(s.handle(m, cp.Wire(FakeConnection()))["error"]["code"], -32600, m) + self.assertEqual(s.handle(m, self.wire())["error"]["code"], -32600, m) self.assertEqual((b.booked, len(b.calls), s.records), (0, 0, [])) def test_calls_against_their_schema_are_refused_before_they_are_queued(self): @@ -843,7 +902,7 @@ def test_calls_against_their_schema_are_refused_before_they_are_queued(self): with self.subTest(tool=tool, args=args): b = self.bridge() s = self.server(b) - s.call(1, {"name": tool, "arguments": args}, cp.Wire(FakeConnection())) + s.call(1, {"name": tool, "arguments": args}, self.wire()) self.assertEqual((b.booked, len(b.calls)), (0, 0)) self.assertEqual(s.records[0]["isError"], True) @@ -867,7 +926,7 @@ def test_each_listed_schema_is_what_the_calls_are_checked_against(self): def test_an_answer_over_the_bound_is_recorded_as_failed(self): b = self.bridge() s, conn = self.server(b), FakeConnection() - s.call(1, {"name": "python", "arguments": {"code": "print('x' * 2000)"}}, cp.Wire(conn)) + s.call(1, {"name": "python", "arguments": {"code": "print('x' * 2000)"}}, self.wire(conn)) with mock.patch.object(cp, "LINE_BYTES", 512): b._serve(b.calls.popleft()) self.assertTrue(until(lambda: conn.sent)) @@ -949,6 +1008,52 @@ def test_what_a_unit_was_keeping_does_not_pass_to_the_next(self): self.assertIsNone(held.get(digest)) self.assertEqual(os.listdir(held.folder), []) + def test_the_listed_bound_of_data_is_the_checks(self): + schema = next(t["inputSchema"] for t in cp.listed((100, 100)) if t["name"] == "upload") + for size in (cp.CHUNK_BYTES - 1, cp.CHUNK_BYTES, cp.CHUNK_BYTES + 1, cp.CHUNK_BYTES + 2): + args = {"path": "f", "data": base64.b64encode(b"a" * size).decode()} + listed = cp._passes(args, schema, "upload", "arguments") + try: + cp.checked(cp.Call("upload", args, None), (100, 100)) + taken = True + except cp.Refused: + taken = False + self.assertEqual((size, listed, taken), (size, size <= cp.CHUNK_BYTES, size <= cp.CHUNK_BYTES)) + for wrong in ("YQ", "YQ=", "Y===", "YWJj\n"): + self.assertFalse(cp._passes({"path": "f", "data": wrong}, schema, "upload", "arguments"), wrong) + + def test_a_transport_failure_during_an_append_leaves_no_part(self): + b = self.bridge() + b64 = lambda d: base64.b64encode(d).decode() + self.upload(b, path="f", data=b64(b"A")) + real, failed = self.desktop.python, [] + + def python(code): + if not failed: + failed.append(code) + raise TimeoutError("the desktop's transport failed") + return real(code) + + self.desktop.python = python + with self.assertRaises(TimeoutError): + self.upload(b, path="f", data=b64(b"B"), append=True) + self.assertEqual(sorted(os.listdir(self.desktop.root)), ["f"]) + + def test_a_stop_as_a_file_is_kept_leaves_no_file_and_no_descriptor(self): + held = self.held() + opened, real = [], cp.tempfile.mkstemp + + def mkstemp(**kw): + opened.append(real(**kw)) + return opened[-1] + + with mock.patch.object(cp.tempfile, "mkstemp", mkstemp), \ + mock.patch.object(cp.os, "fdopen", side_effect=SystemExit(1)), self.assertRaises(SystemExit): + held.keep(hashlib.sha256(b"x").hexdigest(), b"x") + self.assertEqual(os.listdir(held.folder), []) + with self.assertRaises(OSError): # the descriptor was closed + os.fstat(opened[0][0]) + def test_a_file_over_the_bound_is_not_kept(self): held = self.held() digest = hashlib.sha256(b"ABC").hexdigest() From 8413506caba9dc70e4e9b6e80b815df07d21fc28 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:19:40 +0100 Subject: [PATCH 22/36] docs(standard): state that a computer tool takes only the arguments its schema lists --- docs/ports/agent-computer-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 78cac14b..83ab0255 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -42,7 +42,7 @@ An acting call answers once its actions have run and the benchmark's pause has p | Rule | Detail | | --- | --- | | A step | One call of an acting tool; the benchmark's own loop counts these calls | -| Calls | Each call is checked against its tool's schema in `tools/list`, an upload's base64 decoded, before it waits for the desktop. A call that does not hold is refused at once: it takes no step and never reaches the desktop. A message is JSON-RPC 2.0, one a line, at most 24 MiB with its newline; any other is refused | +| Calls | Each call is checked against its tool's schema in `tools/list`, an upload's base64 decoded, before it waits for the desktop. A schema lists every argument its tool takes, and its patterns read alike in any engine. A call that does not hold is refused at once: it takes no step and never reaches the desktop. A message is JSON-RPC 2.0, one a line, at most 24 MiB with its newline; `NaN` and `Infinity` are no JSON, and any other message is refused. Every answer is at most 24 MiB too | | The limits | In `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result | | The step budget | The benchmark's loop stops at its budget. After the last step, every tool but `info` answers with an error, and the benchmark evaluates the desktop as it is. An acting call sent while every step is taken is refused at once | | Model calls | A benchmark may bound the agent's model calls at its step budget | From b1571163c518a302b1203e2fcf8220c4f47e2e66 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:23:52 +0100 Subject: [PATCH 23/36] feat(standard): give the computer port its conformance kit The computer port had a server and the kit a desktop; this joins them so that an agent can prove agent.computer/v1 like any other port. - The kit agent-computer-v1: eight checks, four basic (see, click, type, limit) and four skills (drag, scroll, recover, batch), each decided by the fixture's log, the actions' times and the server's call log. Every check fails on input that came outside an acting call. - driver.py: each unit's desktop and fixture, the port's server over it in the driver's process, and a loop of the kit's own that runs each step's actions and waits the pause. score ends the turn, judges and removes the desktop and the socket. A unit that never reaches score is let go by the next setup or the driver's end; reaper.py, started in a session of its own, removes what a killed driver left. - The reference makes the right calls through the unit's own server; the null control passes nothing. The kit's proof is recorded in its standard.json. - ports.py names the kit; the port's page gets its section, the ports table its row, and the generated agent pages the port's column. - fake_computer_agent.py: a scripted ACP agent that looks and clicks once, for the tests and a scripted conformance. --- .../standard/kits/agent-computer-v1/checks.py | 157 +++++-- .../standard/kits/agent-computer-v1/driver.py | 431 +++++++++++++++++ .../kits/agent-computer-v1/fixture.py | 76 ++- .../standard/kits/agent-computer-v1/reaper.py | 47 ++ .../kits/agent-computer-v1/standard.json | 157 +++++++ app/nullius/standard/ports.py | 2 +- .../fixtures/standard/fake_computer_agent.py | 81 ++++ app/tests/test_computer_kit.py | 2 +- app/tests/test_computer_kit_driver.py | 437 ++++++++++++++++++ app/tests/test_standard_conform.py | 3 +- app/ui_tests/test_agents.py | 2 +- docs/agents/supported/README.md | 22 +- docs/agents/supported/codex.md | 6 +- docs/agents/supported/goose.md | 4 +- docs/agents/supported/jiuwenswarm.md | 6 +- docs/agents/supported/mini-swe-agent.md | 4 +- docs/agents/supported/opencode.md | 4 +- docs/agents/supported/terminus-2.md | 4 +- docs/ports/agent-computer-v1.md | 24 +- docs/standard.md | 2 +- 20 files changed, 1409 insertions(+), 62 deletions(-) create mode 100644 app/nullius/standard/kits/agent-computer-v1/driver.py create mode 100644 app/nullius/standard/kits/agent-computer-v1/reaper.py create mode 100644 app/nullius/standard/kits/agent-computer-v1/standard.json create mode 100644 app/tests/fixtures/standard/fake_computer_agent.py create mode 100644 app/tests/test_computer_kit_driver.py diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index 8ba3ff67..768f6b01 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -1,22 +1,35 @@ -"""The computer kit's first checks: what each one draws, and its verdict. +"""The computer kit's checks: what each one draws, and its verdict. -A verdict is a function of three records and nothing else: the fixture's log -(``fixture.py``: one dict an event, ``kind`` and ``t`` on the desktop's -clock), the actions' times (``desktop.Desktop.act``: one dict an action, -``start`` and ``end`` on the same clock) and the agent's last message. +A verdict is a function of the desktop's records and the server's, and +nothing else: the fixture's log (``fixture.py``: one dict an event, ``kind`` +and ``t`` on the desktop's monotonic clock), the actions' times +(``desktop.Desktop.act``: one dict an action, ``start`` and ``end`` on the +same clock), the agent's last message, and what the server kept of the +agent's calls (``calls``: the call log's lines, ``{tool, start, end, +isError}``; ``past``: the acting calls sent once every step was taken). - ``see``: the last message holds the code the window drew; - ``click``: the log holds a left press on the named button and no press, of any mouse button, on the five others; -- ``type``: the field held exactly the word at the last Enter. - -``click`` and ``type`` pass only when every pointer and key event, Enter in -the field included, lies inside an action's time: input sent any other way, -through the desktop's Python for one, fails them. An event counts as inside -from the action's start to ``SLACK`` seconds after its end, the time the -fixture may need to receive an event the action sent. An action answers only -once that time has passed on the desktop (``desktop.SETTLE``): nothing sent -after it has answered falls in it. +- ``type``: the field held exactly the word at the last Enter; +- ``limit``: six buttons to press with two steps: at least one press on + them, no input after the last action, and no acting call sent once the + steps were taken; +- ``drag``: a left press on the handle, then a release inside the zone; +- ``scroll``: the last message holds the code drawn at the list's end, below + what it shows at first, and the list moved; +- ``recover``: the kit answers the agent's first acting call with an error, + without running it; a later acting call succeeds and the named button is + clicked, as for ``click``; +- ``batch``: each of three fields holds its word, within two steps. + +Every check passes only when every pointer and key event, Enter in a field +included, lies inside an action's time: input sent any other way, through the +desktop's Python for one, fails it. An event counts as inside from the +action's start to ``SLACK`` seconds after its end, the time the fixture may +need to receive an event the action sent. An action answers only once that +time has passed on the desktop (``desktop.SETTLE``): nothing sent after it has +answered falls in it. """ import re import secrets @@ -29,7 +42,22 @@ "UMBRA", "VELVET", "WHEAT", "YEAST") POINTER = ("press", "release", "scroll") KEYS = ("key", "keyup", "enter") -CHECKS = ("see", "click", "type") +CHECKS = ("see", "click", "type", "limit", "drag", "scroll", "recover", "batch") +BASIC = ("see", "click", "type", "limit") +STEPS = {"limit": 2, "batch": 2} # every other check: DEFAULT_STEPS +DEFAULT_STEPS = 8 +ORDER = ("ONE", "TWO", "THREE", "FOUR", "FIVE", "SIX") +FIELDS = ("ONE", "TWO", "THREE") +ACTING = ("click", "move", "drag", "scroll", "type", "key", "wait", "step", "done", "fail") + + +def steps(check): + """The steps a check gives the agent.""" + return STEPS.get(check, DEFAULT_STEPS) + + +def code(rng): + return "".join(rng.choice(ALPHABET) for _ in range(CODE_LENGTH)) def word(rng=None, length=8): @@ -43,14 +71,22 @@ def draw(check, rng=None): expects: ``(values, expected)``. Nothing the verdict expects is in the values but what the screen shows.""" rng = rng or secrets.SystemRandom() - if check == "see": - code = "".join(rng.choice(ALPHABET) for _ in range(CODE_LENGTH)) - return {"screen": "see", "code": code}, {"code": code} - if check == "click": + if check in ("see", "scroll"): + c = code(rng) + return {"screen": check, "code": c}, {"code": c} + if check in ("click", "recover"): labels = rng.sample(LABELS, 6) return {"screen": "click", "labels": labels}, {"target": rng.choice(labels), "labels": labels} if check == "type": return {"screen": "type"}, {"word": word(rng)} + if check == "limit": + labels = list(ORDER) + rng.shuffle(labels) + return {"screen": "click", "labels": labels}, {"labels": list(ORDER)} + if check == "drag": + return {"screen": "drag"}, {} + if check == "batch": + return {"screen": "batch", "labels": list(FIELDS)}, {"words": {f: word(rng) for f in FIELDS}} raise ValueError(f"no such check: {check!r}") @@ -80,12 +116,26 @@ def _input_rule(events, actions): return "" -def see(expected, events, actions, message): - code = expected["code"] - return _verdict("" if code in _flat(message) else "the last message does not hold the code") +def _first(*whys): + return next((w for w in whys if w), "") + + +def _boxes(events): + """Where the fixture drew its targets: its ``ready`` record's boxes.""" + ready = next((e for e in events if e.get("kind") == "ready"), None) + return (ready or {}).get("boxes") or {} -def click(expected, events, actions, message): +def _in(box, e): + return bool(box) and box[0] <= e.get("x", -1) < box[2] and box[1] <= e.get("y", -1) < box[3] + + +def see(expected, events, actions, message, calls=(), past=0): + return _verdict(_first("" if expected["code"] in _flat(message) else "the last message does not hold the code", + _input_rule(events, actions))) + + +def click(expected, events, actions, message, calls=(), past=0): target = expected["target"] presses = [e for e in events if e.get("kind") == "press"] if not any(e.get("hit") == target and e.get("button") == 1 for e in presses): @@ -96,7 +146,7 @@ def click(expected, events, actions, message): return _verdict(_input_rule(events, actions)) -def type_word(expected, events, actions, message): +def type_word(expected, events, actions, message, calls=(), past=0): entered = [e for e in events if e.get("kind") == "enter"] if not entered: return _verdict("Enter was never pressed in the field") @@ -105,9 +155,62 @@ def type_word(expected, events, actions, message): return _verdict(_input_rule(events, actions)) -VERDICTS = {"see": see, "click": click, "type": type_word} +def limit(expected, events, actions, message, calls=(), past=0): + labels = set(expected["labels"]) + if not any(e.get("kind") == "press" and e.get("hit") in labels for e in events): + return _verdict("no press on the buttons") + last = max((a["end"] for a in actions), default=None) + late = [e for e in events if e.get("kind") in POINTER + KEYS and (last is None or e.get("t", -1.0) > last + SLACK)] + if late: + return _verdict(f"{len(late)} input event(s) after the last step") + if past: + return _verdict(f"{past} acting call(s) sent once every step was taken") + return _verdict(_input_rule(events, actions)) + + +def drag(expected, events, actions, message, calls=(), past=0): + zone = _boxes(events).get("zone") + pressed = None + for e in events: + if e.get("kind") == "press" and e.get("button") == 1: + pressed = e.get("hit") == "handle" + elif e.get("kind") == "release" and e.get("button") == 1 and pressed: + if _in(zone, e): + return _verdict(_input_rule(events, actions)) + pressed = None + return _verdict("no press on the handle released inside the zone") + + +def scroll(expected, events, actions, message, calls=(), past=0): + moved = any(e.get("kind") == "moved" and e.get("top", 0) > 0 for e in events) + return _verdict(_first("" if expected["code"] in _flat(message) else "the last message does not hold the code", + "" if moved else "the list never moved", _input_rule(events, actions))) + + +def recover(expected, events, actions, message, calls=(), past=0): + acting = [c for c in calls if c.get("tool") in ACTING] + failed = next((i for i, c in enumerate(acting) if c.get("isError")), None) + if failed is None: + return _verdict("no acting call was answered with an error") + if not any(not c.get("isError") for c in acting[failed + 1:]): + return _verdict("no acting call succeeded after the error") + return click(expected, events, actions, message) + + +def batch(expected, events, actions, message, calls=(), past=0): + last = {} + for e in events: + if e.get("kind") == "value": + last[e.get("field")] = e.get("value") + wrong = [f for f, w in expected["words"].items() if last.get(f) != w] + return _verdict(_first(f"the field(s) {', '.join(wrong)} do not hold their word" if wrong else "", + _input_rule(events, actions))) + + +VERDICTS = {"see": see, "click": click, "type": type_word, "limit": limit, "drag": drag, "scroll": scroll, + "recover": recover, "batch": batch} -def verdict(check, expected, events, actions, message): +def verdict(check, expected, events, actions, message, calls=(), past=0): """``{passed, why}`` for one check; ``why`` is empty when it passed.""" - return VERDICTS[check](expected, events, actions, message) + return VERDICTS[check](expected, events, actions, message, calls, past) diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py new file mode 100644 index 00000000..e2d29f84 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -0,0 +1,431 @@ +"""Kit for agent.computer/v1: eight checks on a small real desktop, decided by the desktop's own records. + +Basic: ``see``, ``click``, ``type``, ``limit``. Skills: ``drag``, ``scroll``, ``recover``, ``batch`` +(``checks.py`` gives each one's screen and verdict). + +**A unit.** ``setup`` starts a desktop of the kit's image (``desktop.py``) and the fixture on it, then, in this +driver's process, the port's server over that desktop (``computer_port.Bridge`` and ``Server``, its socket in a +folder of this user's alone) and the kit's loop: a thread that asks the bridge for a step, runs its actions with +``Desktop.act`` and waits the pause after each, as a desktop benchmark's own loop does. The session gets the +server through ``computer_pipe.py`` (``mcp_servers.computer``). ``score`` ends the turn, waits for the loop, +reads the fixture's log, the actions' times and the call log, judges, and removes the desktop and the socket. +``oracle`` is the reference: the right calls, sent to the unit's own server as a client. + +**What is left when a unit never reaches ``score``.** Nothing: a new ``setup`` ends the unit before it, and the +end of the driver's input (the run is over, or stopped) ends the one still open. A driver that is killed cannot +do that, so it keeps a reaper (``reaper.py``), started in a session of its own, told of each container and +socket folder as it is made and as it goes: when the driver is gone, however it went, the reaper removes what is +left and exits. Each container also carries the job's label, which a run on the container backend sweeps. +""" +import atexit +import base64 +import importlib.util +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time + +from nullius.standard import computer_port as port +from nullius.standard import container +from nullius.standard.jsonl import serve + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +desktop = _load("nullius_kit_desktop", "desktop.py") +checks = _load("nullius_kit_checks", "checks.py") + +UNITS = checks.CHECKS +PAUSE = 1.0 # after each action +PYTHON_SECONDS = 90 # one python call's reply +AGENT_SECONDS = 600 # the agent's turn +LOOP_SECONDS = PYTHON_SECONDS + desktop.SETTLE + PAUSE + 30 # the loop's end, once the turn has ended +USE = "Use the tools of the MCP server named computer, given to this session: they show and drive a desktop. " +DONE = " Then reply with the word done." + + +def prompt(unit, expected): + if unit == "see": + return USE + "The screen shows a code of twelve characters. Reply with the code and nothing else." + if unit in ("click", "recover"): + return USE + f"Click the button labelled {expected['target']} on the screen, once." + DONE + if unit == "type": + return USE + f"Click the text field on the screen, type the word {expected['word']} in it and press Enter." + DONE + if unit == "limit": + return (USE + "Click each of the six buttons on the screen, one click each, in this order: " + + ", ".join(expected["labels"]) + "." + DONE) + if unit == "drag": + return USE + "Drag the blue square labelled MOVE into the green box labelled HERE and let it go inside the box." + DONE + if unit == "scroll": + return (USE + "The list on the screen ends with a code of twelve characters, below what it shows at first. " + "Scroll the list to find it, and reply with the code and nothing else.") + if unit == "batch": + w = expected["words"] + return (USE + "Fill the three text fields on the screen: " + ", ".join(f"{w[f]} in the field labelled {f}" + for f in checks.FIELDS) + "." + DONE) + raise ValueError(f"no unit {unit!r}") + + +class Reaper: + """The reaper's stdin, one line a thing made or let go; started at the first thing made.""" + + def __init__(self): + self.p, self.lock = None, threading.Lock() + + def tell(self, line): + with self.lock: + if self.p is None: + self.p = subprocess.Popen([sys.executable, os.path.join(HERE, "reaper.py"), container.DOCKER], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, env=container.client_env( + {"PATH": os.environ.get("PATH", os.defpath)}), + text=True, start_new_session=True) + try: + self.p.stdin.write(line + "\n") + self.p.stdin.flush() + except OSError: + pass + + def close(self): + with self.lock: + if self.p is not None: + try: + self.p.stdin.close() + except OSError: + pass + try: + self.p.wait(timeout=10) # with nothing named it ends at once; else it ends on its own + except subprocess.TimeoutExpired: + pass + self.p = None + + +REAPER = Reaper() + + +class KitDesktop: + """The five operations the port's bridge calls, over the unit's desktop.""" + + def __init__(self, d): + self.d = d + + def python(self, code): + return self.d.python(code, PYTHON_SECONDS) + + def upload(self, path, data): + self.d.upload(path, data) + + def read(self, path): + try: + return self.d.read(path) + except desktop.Refused as e: # the agent's error, said as the port says it + raise port.Refused(str(e)) + + def capture(self): + return self.d.capture() + + def password(self): + return self.d.password() + + +class KitBridge(port.Bridge): + """The port's bridge, counting the acting calls sent once every step was taken (``past``).""" + + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.past = 0 + + def submit(self, call): + if call.tool in port.ACTING: + with self.change: + if self.booked >= self.steps: + self.past += 1 + super().submit(call) + + +class Unit: + """One unit's desktop, server and loop.""" + + def __init__(self, unit, dir): + self.unit, self.trial = unit, os.path.dirname(os.path.abspath(dir)) + self.values, self.expected = checks.draw(unit) + self.steps = checks.steps(unit) + self.log = os.path.join(self.trial, "verifier", "computer-calls.jsonl") + self.desktop = self.bridge = self.server = self.loop = self.folder = self.boxes = None + self.final, self.error, self.failed_one = "", None, False + + def start(self, job): + os.makedirs(os.path.dirname(self.log), exist_ok=True) + if os.path.exists(self.log): + os.remove(self.log) + image = desktop.image(os.path.dirname(self.log)) + name = f"nullius-desktop-{os.urandom(6).hex()}" + REAPER.tell(f"+container {name}") # before it exists: a kill during the start is covered + self.desktop = desktop.Desktop.start(image, name, job=job) + self.desktop.start_fixture(self.values) + self.boxes = self.desktop.events()[0].get("boxes") or {} + self.folder = tempfile.mkdtemp(prefix="nullius-computer-") + REAPER.tell(f"+folder {self.folder}") + self.bridge = KitBridge(KitDesktop(self.desktop), self.steps, PAUSE, desktop.SCREEN, PYTHON_SECONDS) + self.server = port.Server(self.bridge, self.socket(), log=self.log) + self.loop = threading.Thread(target=self.run_loop, name=f"kit-loop-{self.unit}", daemon=True) + self.loop.start() + + def socket(self): + return os.path.join(self.folder, "s") + + def run_loop(self): + """The kit's own loop: a step from the bridge, its actions run, the pause after each; the budget is the + loop's. ``recover``: the first acting call with input is answered with an error, not run.""" + try: + for _ in range(self.steps): + response, actions = self.bridge.predict("", None) + actions = iter(actions) + stop = False + try: + for action in actions: + if action in ("DONE", "FAIL"): + self.final, stop = response or "", True + break + if action == "WAIT": + time.sleep(PAUSE) + continue + if self.unit == "recover" and not self.failed_one: + self.failed_one = True + break # not run: the call is answered with an error + self.desktop.act(action, PYTHON_SECONDS) + time.sleep(PAUSE) + finally: + close = getattr(actions, "close", None) + if close is not None: + close() + if stop: + break + except Exception as e: # the desktop failed: the unit says so at score + self.error = f"{type(e).__name__}: {e}" + finally: + self.bridge.finish() + + def end(self, wait=LOOP_SECONDS): + """The turn's end, the loop's (``wait`` seconds at most), the server's: what the verdict reads is + complete.""" + if self.bridge is not None: + self.bridge.end_turn() + if self.loop is not None: + self.loop.join(wait) + if self.loop.is_alive() and self.error is None: + self.error = f"the kit's loop did not end within {wait:g} s of the turn's end" + if self.server is not None: + self.server.close() + self.server = None + + def calls(self): + out = [] + try: + with open(self.log, encoding="utf-8") as f: + for line in f: + out.append(json.loads(line)) + except (OSError, ValueError): + pass + return out + + def remove(self): + """The unit's desktop and socket folder gone; an error is said once both were tried.""" + failed = None + if self.desktop is not None: + try: + self.desktop.remove() + REAPER.tell(f"-container {self.desktop.name}") + except desktop.DesktopError as e: + failed = e + self.desktop = None + if self.folder is not None: + shutil.rmtree(self.folder, ignore_errors=True) + REAPER.tell(f"-folder {self.folder}") + self.folder = None + if failed is not None: + raise failed + + def close(self): + """The unit ended without its score: everything it holds let go, the loop not waited for (its desktop + goes from under it).""" + try: + self.end(wait=1) + finally: + self.remove() + + +OPEN = {} # unit -> Unit, while it is set up and not scored +OPEN_LOCK = threading.Lock() + + +def close_all(): + with OPEN_LOCK: + held = list(OPEN.values()) + OPEN.clear() + for u in held: + try: + u.close() + except Exception as e: + sys.stderr.write(f"computer kit: {u.unit} could not be let go: {e}\n") + + +def job_of(dir): + """The job a unit's folder belongs to: //workspace.""" + return os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(dir)))) or None + + +def units(dataset, **_): + return {"units": list(UNITS)} + + +def setup(unit, dir, **_): + if unit not in UNITS: + raise ValueError(f"no unit {unit!r}") + close_all() # one unit at a time in a driver: the one before never reached score + u = Unit(unit, dir) + with OPEN_LOCK: + OPEN[unit] = u + try: + u.start(job_of(dir)) + except BaseException: + with OPEN_LOCK: + OPEN.pop(unit, None) + u.close() + raise + return {"prompt": prompt(unit, u.expected), "mcp": [{"name": "computer", "server": "computer", "args": [u.socket()]}], + "agent_seconds": AGENT_SECONDS} + + +def score(unit, dir, transcript, answer, **_): + with OPEN_LOCK: + u = OPEN.pop(unit, None) + if u is None: + raise ValueError(f"{unit} was not set up") + try: + u.end() + events = u.desktop.events() + actions = list(u.desktop.actions) + calls = u.calls() + finally: + u.remove() + message = "\n".join(x for x in (answer or "", u.final) if x) + v = checks.verdict(unit, u.expected, events, actions, message, calls, u.bridge.past) + passed = v["passed"] and u.error is None + detail = {"said": (answer or "")[:200], "steps": u.steps, "steps_used": u.bridge.used, "past_budget": u.bridge.past, + "calls": [c.get("tool") for c in calls], "errors": sum(1 for c in calls if c.get("isError")), + "actions": len(actions), "events": sum(1 for e in events if e.get("kind") in checks.POINTER + checks.KEYS)} + why = v["why"] or u.error + if why: + detail["why"] = why + return {"reward": 1.0 if passed else 0.0, "passed": passed, "detail": detail} + + +class Client: + """An MCP client over the unit's socket: one call at a time, its answer waited for.""" + + def __init__(self, path, timeout=AGENT_SECONDS): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(timeout) + self.sock.connect(path) + self.reader, self.n = self.sock.makefile("rb"), 0 + self.ask("initialize", {"protocolVersion": port.PROTOCOL, "capabilities": {}, + "clientInfo": {"name": "reference", "version": "1"}}) + + def ask(self, method, params): + self.n += 1 + self.sock.sendall(port.frame({"jsonrpc": "2.0", "id": self.n, "method": method, "params": params})) + while True: + line = self.reader.readline() + if not line: + raise ConnectionError("the server closed the connection") + m = json.loads(line) + if m.get("id") == self.n: + return m + + def call(self, tool, arguments=None): + """``(isError, value)``: the result's JSON text read, its image as bytes.""" + r = self.ask("tools/call", {"name": tool, "arguments": arguments or {}})["result"] + value = image = None + for c in r["content"]: + if c["type"] == "image": + image = base64.b64decode(c["data"]) + elif value is None: + try: + value = json.loads(c["text"]) + except ValueError: + value = c["text"] + return r["isError"], image if tool == "screenshot" else value + + def close(self): + self.reader.close() + self.sock.close() + + +def centre(box): + return (box[0] + box[2]) // 2, (box[1] + box[3]) // 2 + + +def oracle(unit, dir, **_): + """The reference: the calls a correct agent makes, sent to the unit's own server, through its desktop, to the + same scorer; the answer they lead to.""" + with OPEN_LOCK: + u = OPEN.get(unit) + if u is None: + raise ValueError(f"{unit} was not set up") + e, b, answer = u.expected, u.boxes, "done" + c = Client(u.socket()) + try: + c.call("info") + if unit in ("see", "scroll"): + if unit == "scroll": + x, y = centre(b["list"]) + c.call("scroll", {"x": x, "y": y, "dy": 100}) + c.call("screenshot") + answer = e["code"] + elif unit in ("click", "recover"): + x, y = centre(b[e["target"]]) + for _ in range(2 if unit == "recover" else 1): # recover: the first is answered with an error + c.call("click", {"x": x, "y": y}) + elif unit == "type": + x, y = centre(b["field"]) + c.call("click", {"x": x, "y": y}) + c.call("type", {"text": e["word"]}) + c.call("key", {"keys": ["enter"]}) + elif unit == "limit": + for label in e["labels"][:u.steps]: + x, y = centre(b[label]) + c.call("click", {"x": x, "y": y}) + elif unit == "drag": + (x, y), (tx, ty) = centre(b["handle"]), centre(b["zone"]) + c.call("drag", {"x": x, "y": y, "to_x": tx, "to_y": ty}) + elif unit == "batch": + actions = [] + for f in checks.FIELDS: + x, y = centre(b[f]) + actions += [f"pyautogui.click({x}, {y})", f"pyautogui.write({e['words'][f]!r})"] + c.call("step", {"actions": actions}) + finally: + c.close() + return {"answer": answer} + + +def main(): + atexit.register(REAPER.close) + atexit.register(close_all) # runs first: the units go, then the reaper is let go with nothing + serve({"units": units, "setup": setup, "score": score, "oracle": oracle}) + + +if __name__ == "__main__": + main() diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py index 864e5c12..66b728b1 100644 --- a/app/nullius/standard/kits/agent-computer-v1/fixture.py +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -5,7 +5,12 @@ - ``{"screen": "see", "code": "..."}``: the code, drawn large; - ``{"screen": "click", "labels": [six words]}``: six buttons, each labelled; -- ``{"screen": "type"}``: a text field. +- ``{"screen": "type"}``: a text field; +- ``{"screen": "drag"}``: a handle labelled MOVE and a zone labelled HERE; +- ``{"screen": "scroll", "code": "..."}``: a list of words with the code at + its end, below what the list shows at first; +- ``{"screen": "batch", "labels": [three words]}``: three text fields, each + labelled. Codes and labels are drawn as bitmaps from the glyphs below: no widget holds their text, and Tk's ``send`` is removed, so no other program can read them @@ -15,10 +20,13 @@ home, a folder only ``fixture`` reads, one JSON object a line: ``kind``, the time ``t`` on the desktop's monotonic clock (the one an action's times are read on), the point or the key, and ``hit``, what it -landed on (a button's label, ``field`` or ``background``). The first line is +landed on (a button's label, ``field``, a field's label, ``handle``, ``zone``, +``list`` or ``background``). The first line is ``ready``, with where each target is on the screen, written once the window is mapped at the screen's size and holds the focus; Enter in the field writes -``enter`` with the field's text. +``enter`` with the field's text; each key let go in a labelled field writes +``value`` with its text; each turn of the wheel that moves the list writes +``moved`` with the list's top, a fraction of its length. The program leaves the caller once its window is up: it prints ``ready`` and the window lives on, on its own, until the desktop ends. @@ -32,6 +40,8 @@ LOG = os.path.join(HOME, "events.jsonl") ERRORS = os.path.join(HOME, "fixture.err") TITLE = "Check" +SCREENS = ("see", "click", "type", "drag", "scroll", "batch") +LIST_WORDS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "MAPLE", "NORTH") BACKGROUND = "#eceff1" INK = "#111111" @@ -127,6 +137,64 @@ def _click(self): b.grid(row=i // 3, column=i % 3, padx=60, pady=60) self.names[str(b)] = label + def _drag(self): + zone = self.tk.Frame(self.root, width=360, height=240, background="#ffffff", highlightthickness=6, + highlightbackground="#2e7d32") + zone.place(relx=0.72, rely=0.5, anchor="center") + img = bitmap(self.tk, zone, "HERE", 5, ink="#2e7d32", paper="#ffffff") + self.images.append(img) + self.tk.Label(zone, image=img, background="#ffffff", borderwidth=0).place(relx=0.5, rely=0.5, anchor="center") + self.names[str(zone)] = "zone" + img = bitmap(self.tk, self.root, "MOVE", 4, ink="#ffffff", paper="#1565c0") + self.images.append(img) + handle = self.tk.Label(self.root, image=img, background="#1565c0", borderwidth=0, padx=16, pady=16) + handle.place(relx=0.25, rely=0.5, anchor="center") + self.names[str(handle)] = "handle" + + def follow(e): # the handle goes where the pointer drags it + handle.place(relx=0, rely=0, x=e.x_root - self.root.winfo_rootx(), y=e.y_root - self.root.winfo_rooty(), + anchor="center") + handle.bind("", follow, add="+") + + def _scroll(self): + rows, step, scale = 40, 60, 4 + c = self.tk.Canvas(self.root, width=760, height=540, background="#ffffff", highlightthickness=2, + highlightbackground="#90a4ae", yscrollincrement=20) + c.place(relx=0.5, rely=0.5, anchor="center") + for i in range(rows): + img = bitmap(self.tk, c, LIST_WORDS[i % len(LIST_WORDS)], scale, paper="#ffffff") + self.images.append(img) + c.create_image(40, 40 + i * step, image=img, anchor="nw") + img = bitmap(self.tk, c, self.values["code"], scale, paper="#ffffff") + self.images.append(img) + c.create_image(40, 40 + rows * step + 40, image=img, anchor="nw") + c.configure(scrollregion=(0, 0, 760, 40 + rows * step + 160)) + self.names[str(c)] = "list" + + def turn(units): + def moved(e): + before = c.yview()[0] + c.yview_scroll(units, "units") + if c.yview()[0] != before: + self.write("moved", top=round(c.yview()[0], 4)) + return moved + c.bind("", turn(-3), add="+") + c.bind("", turn(3), add="+") + c.bind("", lambda e: turn(-3 if e.delta > 0 else 3)(e), add="+") + + def _batch(self): + frame = self.tk.Frame(self.root, background=BACKGROUND) + frame.place(relx=0.5, rely=0.5, anchor="center") + for i, label in enumerate(self.values["labels"]): + img = bitmap(self.tk, frame, label, 4) + self.images.append(img) + self.tk.Label(frame, image=img, background=BACKGROUND, borderwidth=0).grid(row=i, column=0, padx=30, pady=30, + sticky="e") + field = self.tk.Entry(frame, font=("DejaVu Sans", 24), width=20, relief="sunken", borderwidth=3) + field.grid(row=i, column=1, padx=30, pady=30) + self.names[str(field)] = label + field.bind("", lambda e, f=field, n=label: self.write("value", field=n, value=f.get()), add="+") + def _type(self): self.field = self.tk.Entry(self.root, font=("DejaVu Sans", 28), width=24, relief="sunken", borderwidth=3) self.field.place(relx=0.5, rely=0.5, anchor="center") @@ -189,7 +257,7 @@ def up(self, seconds=10.0): def main(): values = json.loads(sys.stdin.read()) - if values.get("screen") not in ("see", "click", "type"): + if values.get("screen") not in SCREENS: sys.exit(f"fixture: no such screen: {values.get('screen')!r}") r, w = os.pipe() if os.fork(): # the caller's side: wait for the window, then return diff --git a/app/nullius/standard/kits/agent-computer-v1/reaper.py b/app/nullius/standard/kits/agent-computer-v1/reaper.py new file mode 100644 index 00000000..9a9794b5 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/reaper.py @@ -0,0 +1,47 @@ +"""The computer kit's reaper: what a driver made, removed once the driver is gone, however it went. + + reaper.py ENGINE + +The kit's driver starts it at its first unit, in a session of its own (a run that kills the driver's process +group does not kill it), and keeps its stdin open. Each line names what the driver made or let go: +``+container NAME``, ``-container NAME``, ``+folder PATH``, ``-folder PATH``. Once its stdin ends (the driver +ended, closed it, or was killed), it removes each container still named, with ENGINE (``docker`` or +``podman``), and each folder still named, then exits. Standard library only. +""" +import shutil +import subprocess +import sys + +REMOVE_SECONDS = 120 + + +def held(lines): + """What the lines leave named: {"container": [...], "folder": [...]}, in the order made.""" + out = {"container": [], "folder": []} + for line in lines: + sign, _, rest = line.rstrip("\n").partition(" ") + kind, what = sign[1:], rest + if sign[:1] not in "+-" or kind not in out or not what: + continue + if sign[0] == "+" and what not in out[kind]: + out[kind].append(what) + elif sign[0] == "-" and what in out[kind]: + out[kind].remove(what) + return out + + +def main(argv): + engine = argv[1] + left = held(sys.stdin) + for name in left["container"]: + try: + subprocess.run([engine, "rm", "-f", name], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=REMOVE_SECONDS) + except (OSError, subprocess.SubprocessError): + pass + for path in left["folder"]: + shutil.rmtree(path, ignore_errors=True) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json new file mode 100644 index 00000000..72456259 --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -0,0 +1,157 @@ +{ + "schema": "nullius.benchmark/v1", + "key": "agent-computer-v1", + "kit_version": 1, + "label": "Kit: agent.computer/v1", + "summary": "Eight checks on a small real desktop, reached through the computer port's MCP server: four basic - read the screen, click, type, stop at the step budget - and four skills - drag, scroll, recover from an error, several actions in one step. The desktop's own records decide each one.", + "family": "agent", + "needs": [ + "agent.computer/v1" + ], + "basic": [ + "see", + "click", + "type", + "limit" + ], + "skills": { + "drag": "drags a handle into a marked zone", + "scroll": "scrolls a list to read what it shows at first below its edge", + "recover": "reads an acting call's error and acts again", + "batch": "sends several actions in one step" + }, + "driver": { + "command": [ + "python3", + "{here}/driver.py" + ] + }, + "mcp_servers": { + "computer": { + "command": [ + "python3", + "{here}/../../computer_pipe.py" + ], + "summary": "The unit's desktop: a pipe to the socket where the kit's driver serves the computer port over it; the socket is setup's argument." + } + }, + "environments": { + "desktop": { + "summary": "The kit's desktop, which the driver starts for each unit: a virtual X display, a window manager, a session bus, the accessibility bus, Python with Tk, pyautogui and Pillow, and the kit's fixture. No agent runs in it but through the port.", + "dockerfile": "desktop.Dockerfile", + "files": [ + "desktop-requirements.txt", + "fixture.py", + "session.py" + ] + } + }, + "datasets": [ + { + "id": "kit", + "units": 8 + } + ], + "units": { + "noun": "check", + "nouns": "checks" + }, + "controls": { + "nop": { + "source": "nullius", + "from": "Nullius: no agent runs; the desktop is left as setup made it, and the turn ends with no call." + }, + "oracle": { + "source": "nullius", + "from": "The calls a correct agent makes, sent by the kit's driver to the unit's own computer server, run on its desktop." + } + }, + "score": { + "kind": "pass", + "says": "the desktop's records show the task done, every pointer and key event inside an acting call, and the final message holds what the screen showed" + }, + "limits": { + "turn_seconds": 600 + }, + "check_details": { + "see": { + "label": "Read the screen", + "task": "Read a code of twelve characters drawn on the screen.", + "passes": "The final message holds the code, and no input came outside an acting call." + }, + "click": { + "label": "Click a button", + "task": "Click the named one of six buttons.", + "passes": "The desktop logged a left press on that button, none on the others, and every input inside an acting call." + }, + "type": { + "label": "Type a word", + "task": "Type a given word in a text field and press Enter.", + "passes": "The field held exactly the word at the last Enter, and every key came inside an acting call." + }, + "limit": { + "label": "Stop at the budget", + "task": "Click six buttons in order with a budget of two steps.", + "passes": "At least one button was pressed, no input came after the last step, and no acting call was sent once the steps were used up." + }, + "drag": { + "label": "Drag", + "task": "Drag a handle into a marked zone.", + "passes": "A left press on the handle was released inside the zone, inside acting calls." + }, + "scroll": { + "label": "Scroll", + "task": "Scroll a list to read a code below what it shows at first.", + "passes": "The final message holds the code, and the list moved." + }, + "recover": { + "label": "Recover from an error", + "task": "Click a named button; the first acting call is answered with an error.", + "passes": "An acting call succeeded after the error, and the button was clicked as in the click check." + }, + "batch": { + "label": "Several actions in one step", + "task": "Fill three text fields with a budget of two steps.", + "passes": "Each field holds its word, every key inside an acting call." + } + }, + "proven": { + "kit": { + "units": { + "units": 8, + "passed": "8 ids, the same twice, unique", + "on": "2026-11-16" + }, + "leak": { + "units": 8, + "looked_for": 2, + "short": 6, + "passed": "8/8", + "note": "6 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-16" + }, + "nop": { + "passed": "0/8", + "value": 0.0, + "expect": { + "at_most": 0.0 + }, + "units": 8, + "job": "2026-11-16__12-16-15__agent-computer-v1__kit__prove-nop", + "source": "nullius", + "on": "2026-11-16" + }, + "oracle": { + "passed": "8/8", + "value": 1.0, + "expect": { + "at_least": 1.0 + }, + "units": 8, + "job": "2026-11-16__12-16-15__agent-computer-v1__kit__prove-oracle", + "source": "nullius", + "on": "2026-11-16" + } + } + } +} diff --git a/app/nullius/standard/ports.py b/app/nullius/standard/ports.py index 98e5eb51..91d1785e 100644 --- a/app/nullius/standard/ports.py +++ b/app/nullius/standard/ports.py @@ -56,7 +56,7 @@ "label": "Computer use", "family": "agent", "summary": "A desktop the agent sees in screenshots and drives with mouse and keyboard actions.", - "kit": None, + "kit": "agent-computer-v1", }, "cap.memory/v1": { "label": "Memory", diff --git a/app/tests/fixtures/standard/fake_computer_agent.py b/app/tests/fixtures/standard/fake_computer_agent.py new file mode 100644 index 00000000..1081349b --- /dev/null +++ b/app/tests/fixtures/standard/fake_computer_agent.py @@ -0,0 +1,81 @@ +"""A fake ACP agent for the computer kit's tests: no model; whatever the prompt, it looks and clicks once. + +At each prompt it starts the session's MCP server named computer (stdio, as given), lists its tools, takes a +screenshot, clicks at a fixed point (``FAKE_CLICK``, "x,y", the screen's centre by default), and says what the +click answered. Its own calls are the proof that an ACP session reaches the server through the pipe. Started +with the word ``hang``, it never ends its turn after the click: a run can be stopped during the unit. +""" +import json +import os +import subprocess +import sys +import threading +import time + +HANG = sys.argv[1:] == ["hang"] +lock = threading.Lock() +sessions = {} + + +def send(m): + with lock: + sys.stdout.write(json.dumps(m) + "\n") + sys.stdout.flush() + + +def say(sid, text): + send({"jsonrpc": "2.0", "method": "session/update", "params": {"sessionId": sid, "update": { + "sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": text}}}}) + + +def turn(mid, sid): + servers = [m for m in sessions[sid] if m.get("name") == "computer"] + if not servers: + say(sid, "no computer server") + send({"jsonrpc": "2.0", "id": mid, "result": {"stopReason": "end_turn"}}) + return + m = servers[0] + env = dict(os.environ, **{e["name"]: e["value"] for e in m.get("env") or []}) + p = subprocess.Popen([m["command"]] + list(m.get("args") or []), stdin=subprocess.PIPE, stdout=subprocess.PIPE, + text=True, env=env) + + def rpc(rid, method, params): + p.stdin.write(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}) + "\n") + p.stdin.flush() + while True: + r = json.loads(p.stdout.readline()) + if r.get("id") == rid: + return r + + rpc(1, "initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "fake", "version": "1"}}) + p.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n") + tools = [t["name"] for t in rpc(2, "tools/list", {})["result"]["tools"]] + shot = rpc(3, "tools/call", {"name": "screenshot", "arguments": {}})["result"] + x, y = (int(v) for v in os.environ.get("FAKE_CLICK", "960,540").split(",")) + click = rpc(4, "tools/call", {"name": "click", "arguments": {"x": x, "y": y}})["result"] + p.stdin.close() + p.wait(timeout=30) + said = next((c["text"] for c in click["content"] if c["type"] == "text"), "") + say(sid, f"tools {len(tools)}; screenshot {'failed' if shot['isError'] else 'taken'}; click {said}") + while HANG: + time.sleep(3600) + send({"jsonrpc": "2.0", "id": mid, "result": {"stopReason": "end_turn"}}) + + +for line in sys.stdin: + m = json.loads(line) + if "method" not in m: + continue + meth, params = m["method"], m.get("params") or {} + if meth == "initialize": + send({"jsonrpc": "2.0", "id": m["id"], "result": {"protocolVersion": 1, "agentCapabilities": {}}}) + elif meth == "session/new": + sid = "s%d" % (len(sessions) + 1) + sessions[sid] = params.get("mcpServers") or [] + send({"jsonrpc": "2.0", "id": m["id"], "result": {"sessionId": sid}}) + elif meth == "session/prompt": + threading.Thread(target=turn, args=(m["id"], params["sessionId"]), daemon=True).start() + elif meth == "session/cancel": + pass + elif "id" in m: + send({"jsonrpc": "2.0", "id": m["id"], "error": {"code": -32601, "message": "no " + meth}}) diff --git a/app/tests/test_computer_kit.py b/app/tests/test_computer_kit.py index e52f7e3f..394626e3 100644 --- a/app/tests/test_computer_kit.py +++ b/app/tests/test_computer_kit.py @@ -140,7 +140,7 @@ def test_each_check_draws_from_its_own_values_and_expects_what_the_screen_shows( self.assertEqual(values, {"screen": "type"}) # the word is the prompt's, never the screen's self.assertRegex(exp["word"], r"^[a-z]{8}$") with self.assertRaises(ValueError): - checks.draw("drag") + checks.draw("hover") def test_every_character_drawn_has_a_glyph_of_five_by_seven(self): for ch in checks.ALPHABET + "".join(checks.LABELS): diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py new file mode 100644 index 00000000..3178f856 --- /dev/null +++ b/app/tests/test_computer_kit_driver.py @@ -0,0 +1,437 @@ +"""The computer kit's eight checks and its driver: each verdict on hand-written records; the driver's wiring +(the port's server and the kit's loop over a desktop, the turn's end at score, the budget, the reference, a unit +let go without its score, the reaper) over a fake desktop, with no container; the docker tier runs the eight +checks on real desktops, the reference passing each and the null control none.""" +from __future__ import annotations + +import importlib.util +import json +import os +import re +import secrets +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from unittest import mock + +from nullius.standard import computer_port as port +from nullius.standard import container, definitions, ports +from tests import docker_tier + +KIT = os.path.join(definitions.KITS, "agent-computer-v1") +APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _module(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(KIT, file)) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +driver = _module("computer_kit_driver_under_test", "driver.py") +checks, desktop = driver.checks, driver.desktop +reaper = _module("computer_kit_reaper_under_test", "reaper.py") +ACTIONS = [{"start": 100.0, "end": 101.0}, {"start": 110.0, "end": 111.0}] + + +def ev(kind, t, **kw): + return {"kind": kind, "t": t, **kw} + + +def press(t, hit, button=1, x=10, y=10): + return [ev("press", t, button=button, x=x, y=y, hit=hit), ev("release", t + 0.05, button=button, x=x, y=y, hit=hit)] + + +# -- the verdicts ------------------------------------------------------------------------------------------------------ +LIMIT = {"labels": list(checks.ORDER)} + + +class NewVerdicts(unittest.TestCase): + def test_limit_passes_on_presses_within_the_steps_and_nothing_after(self): + events = press(100.2, "ONE") + press(110.2, "TWO") + self.assertEqual(checks.verdict("limit", LIMIT, events, ACTIONS, ""), {"passed": True, "why": ""}) + + def test_limit_fails_without_a_press_with_input_after_the_last_step_or_a_call_past_the_budget(self): + self.assertIn("no press", checks.verdict("limit", LIMIT, [], ACTIONS, "")["why"]) + late = press(100.2, "ONE") + press(112.0, "THREE") + self.assertIn("after the last step", checks.verdict("limit", LIMIT, late, ACTIONS, "")["why"]) + self.assertIn("once every step was taken", + checks.verdict("limit", LIMIT, press(100.2, "ONE"), ACTIONS, "", (), 1)["why"]) + + def test_drag_passes_on_a_press_on_the_handle_released_inside_the_zone(self): + ready = ev("ready", 99.0, boxes={"handle": [0, 0, 50, 50], "zone": [500, 500, 700, 700]}) + good = [ready, ev("press", 100.2, button=1, hit="handle", x=20, y=20), + ev("release", 100.8, button=1, hit="handle", x=600, y=600)] + self.assertTrue(checks.verdict("drag", {}, good, ACTIONS, "")["passed"]) + outside_zone = good[:2] + [ev("release", 100.8, button=1, hit="handle", x=400, y=600)] + self.assertFalse(checks.verdict("drag", {}, outside_zone, ACTIONS, "")["passed"]) + not_handle = [ready, ev("press", 100.2, button=1, hit="background", x=20, y=20), + ev("release", 100.8, button=1, hit="background", x=600, y=600)] + self.assertFalse(checks.verdict("drag", {}, not_handle, ACTIONS, "")["passed"]) + + def test_scroll_passes_on_the_code_once_the_list_moved(self): + exp = {"code": "ACDE34679HJK"} + moved = [ev("scroll", 100.2, button=5, hit="list"), ev("moved", 100.21, top=0.4)] + self.assertTrue(checks.verdict("scroll", exp, moved, ACTIONS, "ACDE34679HJK")["passed"]) + self.assertIn("never moved", checks.verdict("scroll", exp, moved[:1], ACTIONS, "ACDE34679HJK")["why"]) + self.assertIn("does not hold", checks.verdict("scroll", exp, moved, ACTIONS, "")["why"]) + + def test_recover_passes_on_a_success_after_the_error_and_the_click(self): + exp = {"target": "OCEAN", "labels": ["OCEAN", "APPLE"]} + calls = [{"tool": "screenshot", "isError": False}, {"tool": "click", "isError": True}, + {"tool": "click", "isError": False}] + events = press(110.2, "OCEAN") + self.assertTrue(checks.verdict("recover", exp, events, ACTIONS, "", calls)["passed"]) + self.assertIn("no acting call was answered with an error", + checks.verdict("recover", exp, events, ACTIONS, "", calls[2:])["why"]) + self.assertIn("succeeded after", checks.verdict("recover", exp, events, ACTIONS, "", calls[:2])["why"]) + self.assertFalse(checks.verdict("recover", exp, [], ACTIONS, "", calls)["passed"]) + + def test_batch_passes_when_each_field_holds_its_word(self): + exp = {"words": {"ONE": "kotabiru", "TWO": "mafodafi", "THREE": "sanepilo"}} + events = [ev("value", 100.1 + i / 10, field=f, value=w) for i, (f, w) in enumerate(exp["words"].items())] + self.assertTrue(checks.verdict("batch", exp, events, ACTIONS, "")["passed"]) + wrong = events + [ev("value", 100.5, field="TWO", value="mafodaf")] + self.assertIn("TWO", checks.verdict("batch", exp, wrong, ACTIONS, "")["why"]) + + def test_every_check_fails_on_input_outside_every_action(self): + passing = { + "see": ({"code": "ACDE34679HJK"}, [], "ACDE34679HJK", ()), + "click": ({"target": "OCEAN", "labels": ["OCEAN"]}, press(100.2, "OCEAN"), "", ()), + "type": ({"word": "kotabiru"}, [ev("enter", 100.3, value="kotabiru")], "", ()), + "limit": (LIMIT, press(100.2, "ONE"), "", ()), + "drag": ({}, [ev("ready", 99, boxes={"zone": [500, 500, 700, 700]}), ev("press", 100.2, button=1, hit="handle"), + ev("release", 100.8, button=1, hit="handle", x=600, y=600)], "", ()), + "scroll": ({"code": "ACDE34679HJK"}, [ev("moved", 100.3, top=0.5)], "ACDE34679HJK", ()), + "recover": ({"target": "OCEAN", "labels": ["OCEAN"]}, press(110.2, "OCEAN"), "", + ({"tool": "click", "isError": True}, {"tool": "click", "isError": False})), + "batch": ({"words": {"ONE": "a"}}, [ev("value", 100.2, field="ONE", value="a")], "", ()), + } + self.assertEqual(sorted(passing), sorted(checks.CHECKS)) + for check, (exp, events, message, calls) in passing.items(): + with self.subTest(check=check): + self.assertTrue(checks.verdict(check, exp, events, ACTIONS, message, calls)["passed"]) + stray = events + [ev("key", 105.0, keysym="a", char="a", hit="background")] + self.assertFalse(checks.verdict(check, exp, stray, ACTIONS, message, calls)["passed"]) + + def test_each_check_draws_its_own_screen(self): + for check in checks.CHECKS: + values, exp = checks.draw(check) + self.assertIn(values["screen"], ("see", "click", "type", "drag", "scroll", "batch"), check) + for label in values.get("labels") or []: + self.assertTrue(re.fullmatch(r"[A-Z]+", label), label) + self.assertEqual(sorted(checks.draw("limit")[0]["labels"]), sorted(checks.ORDER)) + self.assertEqual((checks.steps("limit"), checks.steps("batch"), checks.steps("see")), (2, 2, 8)) + + +# -- the kit's definition --------------------------------------------------------------------------------------------- +class Definition(unittest.TestCase): + def test_the_port_names_the_kit_and_the_kit_its_checks(self): + kit = definitions.kit("agent.computer/v1") + self.assertEqual(ports.PORTS["agent.computer/v1"]["kit"], "agent-computer-v1") + self.assertEqual(kit["key"], "agent-computer-v1") + self.assertEqual(definitions.problems({k: v for k, v in kit.items() if not k.startswith("_")}, "benchmark"), []) + self.assertEqual(list(kit["check_details"]), list(checks.CHECKS)) + self.assertEqual(kit["basic"], list(checks.BASIC)) + self.assertEqual(set(kit["skills"]), set(checks.CHECKS) - set(checks.BASIC)) + self.assertEqual(kit["environments"]["desktop"], dict(desktop.KIT["environments"]["desktop"], + summary=kit["environments"]["desktop"]["summary"])) + pipe = os.path.normpath(kit["mcp_servers"]["computer"]["command"][1].replace("{here}", KIT)) + self.assertEqual(pipe, os.path.join(os.path.dirname(os.path.dirname(KIT)), "computer_pipe.py")) + self.assertTrue(os.path.isfile(pipe)) + + +# -- the driver over a fake desktop ------------------------------------------------------------------------------------- +class FakeDesktop: + """A desktop with no container: the fixture's boxes, and the input each click, write or key of an action (or + of a python call) would send, logged at the time it runs, as the fixture would.""" + made = [] + + def __init__(self, name, values): + self.name, self.values, self.actions, self.log, self.removed = name, values, [], [], False + labels = values.get("labels") or [] + boxes = {label: [100 + 300 * i, 100, 300 + 300 * i, 200] for i, label in enumerate(labels)} + boxes.update(field=[600, 500, 1200, 560], handle=[200, 500, 260, 560], zone=[1300, 400, 1700, 700], + list=[600, 300, 1300, 800]) + self.boxes = boxes + self.log.append({"kind": "ready", "t": time.monotonic(), "boxes": boxes}) + FakeDesktop.made.append(self) + + def hit(self, x, y): + return next((k for k, b in self.boxes.items() if b[0] <= x < b[2] and b[1] <= y < b[3]), "background") + + def _input(self, code): + t = time.monotonic() + for x, y in re.findall(r"pyautogui\.click\((\d+), (\d+)", code): + h = self.hit(int(x), int(y)) + self.log += [{"kind": "press", "t": t, "button": 1, "x": int(x), "y": int(y), "hit": h}, + {"kind": "release", "t": t, "button": 1, "x": int(x), "y": int(y), "hit": h}] + for text in re.findall(r"pyautogui\.write\('([a-z]+)'", code): + self.log += [{"kind": "key", "t": t, "keysym": c, "char": c, "hit": "field"} for c in text] + if "hotkey('enter')" in code: + self.log.append({"kind": "enter", "t": t, "value": "".join(e["char"] for e in self.log if e["kind"] == "key")}) + + def act(self, code, seconds=None): + start = time.monotonic() + self._input(code) + self.actions.append({"start": start, "end": time.monotonic()}) + time.sleep(0.01) + return {"start": start, "end": self.actions[-1]["end"], "reply": {"status": "success", "returncode": 0}} + + def python(self, code, seconds=None): + self._input(code) + return {"status": "success", "output": "", "error": "", "returncode": 0} + + def start_fixture(self, values): + pass + + def events(self): + return list(self.log) + + def capture(self): + return b"\x89PNG\r\n\x1a\nfake" + + def upload(self, path, data, append=False): + return len(data) + + def read(self, path): + return None + + def password(self): + return "" + + def remove(self): + self.removed = True + + +class Wiring(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.told = [] + values = {} + + def start(image, name, *, job=None, seconds=None): + return FakeDesktop(name, values["now"]) + + def draw(check, rng=None): + got = self._draw(check) + values["now"] = got[0] + return got + self._draw = checks.draw + self.enterContext(mock.patch.object(driver.desktop, "image", return_value="img")) + self.enterContext(mock.patch.object(driver.desktop.Desktop, "start", side_effect=start)) + self.enterContext(mock.patch.object(driver.checks, "draw", side_effect=draw)) + self.enterContext(mock.patch.object(driver.REAPER, "tell", side_effect=self.told.append)) + self.enterContext(mock.patch.object(driver, "PAUSE", 0.01)) + self.addCleanup(driver.close_all) + + def unit(self, name): + ws = os.path.join(self.tmp, "job", name, "workspace") + os.makedirs(ws) + return ws, driver.setup(name, ws) + + def test_a_unit_is_served_scored_and_let_go(self): + ws, s = self.unit("click") + self.assertEqual(s["mcp"], [{"name": "computer", "server": "computer", "args": [s["mcp"][0]["args"][0]]}]) + self.assertEqual(s["agent_seconds"], 600) + u = driver.OPEN["click"] + self.assertIn(u.expected["target"], s["prompt"]) + sock = s["mcp"][0]["args"][0] + c = driver.Client(sock) + self.assertEqual(c.ask("tools/list", {})["result"]["tools"][0]["name"], "info") + x, y = driver.centre(u.boxes[u.expected["target"]]) + self.assertEqual(c.call("click", {"x": x, "y": y})[0], False) + c.close() + got = driver.score("click", ws, [], "done") + self.assertTrue(got["passed"], got) + self.assertEqual(got["detail"]["calls"], ["click"]) + self.assertFalse(os.path.exists(sock)) # the socket and its folder gone + self.assertTrue(FakeDesktop.made[-1].removed) + self.assertEqual([l.split()[0] for l in self.told], ["+container", "+folder", "-container", "-folder"]) + self.assertNotIn("click", driver.OPEN) + + def test_the_turns_end_at_score_ends_the_loop_and_judges_what_is_there(self): + ws, s = self.unit("click") + u = driver.OPEN["click"] + t = time.monotonic() + got = driver.score("click", ws, [], "") # no call: the null control + self.assertLess(time.monotonic() - t, 5) + self.assertFalse(got["passed"]) + self.assertFalse(u.loop.is_alive()) + self.assertEqual(got["detail"]["steps_used"], 0) + + def test_a_client_that_types_through_python_fails(self): + ws, s = self.unit("type") + u = driver.OPEN["type"] + c = driver.Client(s["mcp"][0]["args"][0]) + c.call("python", {"code": f"import pyautogui; pyautogui.write({u.expected['word']!r}); pyautogui.hotkey('enter')"}) + c.close() + got = driver.score("type", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertIn("outside every action", got["detail"]["why"]) + + def test_a_client_that_acts_past_its_budget_fails(self): + ws, s = self.unit("limit") + u = driver.OPEN["limit"] + c = driver.Client(s["mcp"][0]["args"][0]) + answers = [c.call("click", {"x": x, "y": y})[0] for x, y in + (driver.centre(u.boxes[label]) for label in checks.ORDER[:3])] + c.close() + self.assertEqual(answers, [False, False, True]) # the third is refused + got = driver.score("limit", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertEqual(got["detail"]["past_budget"], 1) + self.assertIn("once every step was taken", got["detail"]["why"]) + + def test_the_reference_passes_click_type_limit_and_recover(self): + for unit in ("click", "type", "limit", "recover"): + with self.subTest(unit=unit): + ws, s = self.unit(unit) + answer = driver.oracle(unit, ws)["answer"] + got = driver.score(unit, ws, [], answer) + self.assertTrue(got["passed"], got) + self.assertEqual(got["detail"]["errors"], 1) # recover: the first click answered with an error + + def test_a_unit_never_scored_is_let_go_by_the_next_setup_and_by_the_drivers_end(self): + ws, s = self.unit("see") + first = FakeDesktop.made[-1] + self.unit("click") + self.assertTrue(first.removed) + self.assertFalse(os.path.exists(s["mcp"][0]["args"][0])) + self.assertEqual(list(driver.OPEN), ["click"]) + driver.close_all() # the driver's input ended + self.assertTrue(FakeDesktop.made[-1].removed) + self.assertEqual(driver.OPEN, {}) + + +class Reaper(unittest.TestCase): + def test_what_is_still_named_is_removed_once_the_driver_is_gone(self): + self.assertEqual(reaper.held(["+container a\n", "+folder /x\n", "+container b\n", "-container a\n", "junk\n"]), + {"container": ["b"], "folder": ["/x"]}) + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + engine, said, folder = os.path.join(tmp, "engine"), os.path.join(tmp, "said"), os.path.join(tmp, "left") + os.makedirs(folder) + with open(engine, "w") as f: + f.write(f"#!/bin/sh\necho \"$@\" >> {said}\n") + os.chmod(engine, 0o755) + p = subprocess.Popen([sys.executable, os.path.join(KIT, "reaper.py"), engine], stdin=subprocess.PIPE, text=True, + start_new_session=True) + p.stdin.write(f"+container one\n+container two\n-container one\n+folder {folder}\n") + p.stdin.close() # the driver is gone + self.assertEqual(p.wait(timeout=30), 0) + with open(said) as f: + self.assertEqual(f.read(), "rm -f two\n") + self.assertFalse(os.path.exists(folder)) + + def test_the_driver_tells_the_reaper_before_the_container_exists(self): + order = [] + with mock.patch.object(driver.REAPER, "tell", side_effect=lambda line: order.append(line)), \ + mock.patch.object(driver.desktop, "image", return_value="img"), \ + mock.patch.object(driver.desktop.Desktop, "start", + side_effect=lambda *a, **k: order.append("start") or (_ for _ in ()).throw( + desktop.DesktopError("no engine"))): + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + ws = os.path.join(tmp, "job", "see", "workspace") + os.makedirs(ws) + with self.assertRaises(desktop.DesktopError): + driver.setup("see", ws) + self.assertTrue(order[0].startswith("+container nullius-desktop-")) + self.assertEqual(order[1], "start") + self.assertEqual(driver.OPEN, {}) + + +# -- the docker tier: the eight checks on real desktops -------------------------------------------------------------- +def why_no_desktop() -> str: + if not os.environ.get(docker_tier.VAR): + return "the docker tier runs with parallel.py --docker" + return container.unusable() or "" + + +@unittest.skipIf(why_no_desktop(), why_no_desktop() or "") +class DockerKit(unittest.TestCase): + """The driver's ops on real desktops, one unit at a time.""" + + @classmethod + def setUpClass(cls): + desktop.image() + + def setUp(self): + self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.addCleanup(driver.close_all) + + def unit(self, name): + ws = os.path.join(self.tmp, "job", f"{name}-{secrets.token_hex(3)}", "workspace") + os.makedirs(ws) + return ws, driver.setup(name, ws) + + def test_the_reference_passes_every_check_and_the_null_control_none(self): + for unit in checks.CHECKS: + with self.subTest(unit=unit, control="oracle"): + ws, _ = self.unit(unit) + answer = driver.oracle(unit, ws)["answer"] + got = driver.score(unit, ws, [], answer) + self.assertTrue(got["passed"], got) + with self.subTest(unit=unit, control="nop"): + ws, _ = self.unit(unit) + self.assertFalse(driver.score(unit, ws, [], "")["passed"]) + + def test_scripted_clients_that_type_through_python_or_act_past_their_budget_fail(self): + ws, s = self.unit("type") + u = driver.OPEN["type"] + c = driver.Client(s["mcp"][0]["args"][0]) + x, y = driver.centre(u.boxes["field"]) + c.call("python", {"code": f"import pyautogui\npyautogui.click({x}, {y})\npyautogui.write({u.expected['word']!r})\n" + "pyautogui.press('enter')"}) + c.close() + got = driver.score("type", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertIn("outside every action", got["detail"]["why"]) + ws, s = self.unit("limit") + u = driver.OPEN["limit"] + c = driver.Client(s["mcp"][0]["args"][0]) + for label in checks.ORDER[:3]: + x, y = driver.centre(u.boxes[label]) + c.call("click", {"x": x, "y": y}) + c.close() + got = driver.score("limit", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertEqual(got["detail"]["past_budget"], 1) + + def test_a_driver_killed_during_a_unit_leaves_no_container_socket_or_process(self): + ws = os.path.join(self.tmp, "job", "see-kill", "workspace") + os.makedirs(ws) + env = dict(os.environ, PYTHONPATH=APP, PYTHONDONTWRITEBYTECODE="1") + p = subprocess.Popen([sys.executable, os.path.join(KIT, "driver.py")], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + text=True, env=env, start_new_session=True) + try: + p.stdin.write(json.dumps({"op": "setup", "unit": "see", "dir": ws}) + "\n") + p.stdin.flush() + reply = json.loads(p.stdout.readline()) + finally: + os.killpg(p.pid, signal.SIGKILL) # as a run kills a driver's group + p.wait() + sock = reply["mcp"][0]["args"][0] + self.assertTrue(sock.endswith("/s")) + for _ in range(120): + left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", "label=" + container.LABEL + "=job", + "--filter", "name=nullius-desktop-"], capture_output=True, text=True, timeout=60) + if left.returncode == 0 and not left.stdout.split() and not os.path.exists(os.path.dirname(sock)): + break + time.sleep(0.5) + self.assertEqual(left.returncode, 0, left.stderr) + self.assertEqual(left.stdout.split(), []) + self.assertFalse(os.path.exists(os.path.dirname(sock))) + procs = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True, timeout=30).stdout + self.assertNotIn(os.path.join(KIT, "reaper.py"), procs) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_conform.py b/app/tests/test_standard_conform.py index cb61ee16..827aea7e 100644 --- a/app/tests/test_standard_conform.py +++ b/app/tests/test_standard_conform.py @@ -66,7 +66,8 @@ def test_a_kit_that_fails_takes_the_claim_back(self): def test_candidates_follow_the_family_and_the_form(self): a = definitions.get("agent", "fake", self.root) - self.assertEqual(conform.candidates(a), ["agent.task/v1", "agent.workspace/v1", "agent.tools/v1", "agent.conversation/v1", "agent.search/v1"]) + self.assertEqual(conform.candidates(a), ["agent.task/v1", "agent.workspace/v1", "agent.tools/v1", "agent.conversation/v1", "agent.search/v1", + "agent.computer/v1"]) cmd = dict(a, start={"command": ["x", "{prompt}"]}) self.assertEqual(conform.candidates(cmd), ["agent.task/v1", "agent.workspace/v1", "agent.search/v1"]) self.assertEqual(conform.candidates(definitions.get("capability", "fakemem", self.root)), ["cap.memory/v1"]) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index ae4d363e..2202e344 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -198,7 +198,7 @@ def test_an_agents_page_has_its_ports_and_what_it_runs(self): self.assertIn("Fake agent", card) self.assertIn(">1/1", card) self.assertIn('Proven', card) # agent.task/v1's record - self.assertEqual(card.count("no kit yet"), 2) # agent.browser/v1 and agent.computer/v1 + self.assertEqual(card.count("no kit yet"), 1) # agent.browser/v1 self.assertIn('href="/new-run?standard=fakebench&other=fake"', card) # a fitting pair opens its recipe self.assertNotIn("fakememo", card) # a capability benchmark is not an agent's diff --git a/docs/agents/supported/README.md b/docs/agents/supported/README.md index 396b2815..2954dc74 100644 --- a/docs/agents/supported/README.md +++ b/docs/agents/supported/README.md @@ -31,20 +31,20 @@ Every agent in the catalogue, the ports each version has proven, and the benchma | `agent.computer/v1` | Computer use | A desktop the agent sees in screenshots and drives with mouse and keyboard actions. | - + - - - - - - - - - + + + + + + + + +
AgentPortsTools, the skills of agent.tools/v1
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
AgentPortsTools, the skills of agent.tools/v1
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
codex@2.2.2✓✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✓✓✓✓✓
goose✓✓✓✓✓✓✓✓
jiuwenswarm@0.2.7✓✓✓✓–✓✓✓
jiuwenswarm@0.2.8✓✓✓✓✓✓✓✓
mini-swe-agent–✓––––––
opencode✓✓✓✓✓✓✓✓
reference✓✓✓✓–✓✓✓
terminus-2–✓––––––
codex@2.2.2✓✓✓✓✓–✓✓✓
codex@0.16.0✓✓✓✓✓–✓✓✓
goose✓✓✓✓✓–✓✓✓
jiuwenswarm@0.2.7✓✓✓✓––✓✓✓
jiuwenswarm@0.2.8✓✓✓✓✓–✓✓✓
mini-swe-agent–✓–––––––
opencode✓✓✓✓✓–✓✓✓
reference✓✓✓✓––✓✓✓
terminus-2–✓–––––––
-- ✓ proven by `cli conform`, – not proven. An agent that misses a skill still has the port. Browser and Computer use have no kit yet, so no agent is tested on them. +- ✓ proven by `cli conform`, – not proven. An agent that misses a skill still has the port. Browser has no kit yet, so no agent is tested on it. ### Benchmarks each agent fits diff --git a/docs/agents/supported/codex.md b/docs/agents/supported/codex.md index f2e30756..291cfcaa 100644 --- a/docs/agents/supported/codex.md +++ b/docs/agents/supported/codex.md @@ -17,10 +17,10 @@ OpenAI's coding agent. ## Versions - + - - + +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
codex@2.2.2 (default)2.2.2✓✓✓✓✓✓✓✓
codex@0.16.00.16.0✓✓✓✓✓✓✓✓
codex@2.2.2 (default)2.2.2✓✓✓✓✓–✓✓✓
codex@0.16.00.16.0✓✓✓✓✓–✓✓✓
- `key@version`: a revision of the definition and driver; a recipe pins one, the builder offers the default. diff --git a/docs/agents/supported/goose.md b/docs/agents/supported/goose.md index 4ac61db7..452dab7c 100644 --- a/docs/agents/supported/goose.md +++ b/docs/agents/supported/goose.md @@ -16,9 +16,9 @@ The AAIF's open-source agent, formerly Block's. ## Versions - + - +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
goose1.53.0✓✓✓✓✓✓✓✓
goose1.53.0✓✓✓✓✓–✓✓✓
## Options diff --git a/docs/agents/supported/jiuwenswarm.md b/docs/agents/supported/jiuwenswarm.md index 0a536b16..24fd32f8 100644 --- a/docs/agents/supported/jiuwenswarm.md +++ b/docs/agents/supported/jiuwenswarm.md @@ -17,10 +17,10 @@ openJiuwen's agent. ## Versions - + - - + +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
jiuwenswarm@0.2.70.2.7✓✓✓✓–✓✓✓
jiuwenswarm@0.2.8 (default)0.2.8✓✓✓✓✓✓✓✓
jiuwenswarm@0.2.70.2.7✓✓✓✓––✓✓✓
jiuwenswarm@0.2.8 (default)0.2.8✓✓✓✓✓–✓✓✓
- `key@version`: a revision of the definition and driver; a recipe pins one, the builder offers the default. diff --git a/docs/agents/supported/mini-swe-agent.md b/docs/agents/supported/mini-swe-agent.md index dec313f7..cd151797 100644 --- a/docs/agents/supported/mini-swe-agent.md +++ b/docs/agents/supported/mini-swe-agent.md @@ -16,9 +16,9 @@ Princeton's 100-line agent, bash only. ## Versions - + - +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
mini-swe-agent2.4.6–✓––––––
mini-swe-agent2.4.6–✓–––––––
## Options diff --git a/docs/agents/supported/opencode.md b/docs/agents/supported/opencode.md index 34e1e610..d574fb0f 100644 --- a/docs/agents/supported/opencode.md +++ b/docs/agents/supported/opencode.md @@ -16,9 +16,9 @@ The open-source terminal coding agent. ## Versions - + - +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
opencode1.18.35✓✓✓✓✓✓✓✓
opencode1.18.35✓✓✓✓✓–✓✓✓
## Options diff --git a/docs/agents/supported/terminus-2.md b/docs/agents/supported/terminus-2.md index 26469d36..d5179c95 100644 --- a/docs/agents/supported/terminus-2.md +++ b/docs/agents/supported/terminus-2.md @@ -16,9 +16,9 @@ Harbor's terminal agent, used by Terminal-Bench. ## Versions - + - +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchdecoyschainretry
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
terminus-20.24.0–✓––––––
terminus-20.24.0–✓–––––––
## Options diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 78cac14b..02a93059 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -71,4 +71,26 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li ## The kit -None yet. Until it exists, no agent offers this port. +`cli conform AGENT --port agent.computer/v1` runs the kit `agent-computer-v1`: eight checks, each on a desktop of its own, reached through this port's server. A program decides each check from the desktop's own records and the server's call log. + +| Check | Kind | The task | Passes when | +| --- | --- | --- | --- | +| `see` | basic | Read a code of twelve characters drawn on the screen | The final message holds the code | +| `click` | basic | Click the named one of six buttons | A left press on that button, and no press on the five others | +| `type` | basic | Type a word in a text field and press Enter | The field holds exactly the word at the last Enter | +| `limit` | basic | Click six buttons with a budget of two steps | A button was pressed, no input came after the last step, and no acting call was sent once the steps were used up | +| `drag` | skill | Drag a handle into a marked zone | A left press on the handle, released inside the zone | +| `scroll` | skill | Scroll a list to read a code below what it shows at first | The final message holds the code, and the list moved | +| `recover` | skill | Click a named button; the first acting call is answered with an error and not run | An acting call succeeds after the error, and the button is clicked as in `click` | +| `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | + +Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. + +| Piece | Where | +| --- | --- | +| The desktop | A container of the kit's image, `desktop.Dockerfile` on Nullius's base image, built the first time a unit needs it: an X display at 1920 by 1080, a window manager, a session bus, the accessibility bus, Python with Tk, `pyautogui` and Pillow. The agent's Python and its actions run as its user; the kit's fixture runs as another, its log out of that user's reach | +| The server and the loop | In the kit's driver: the port's server over the unit's desktop, and a loop of the kit's own that runs each step's actions and waits the pause | +| What a stop leaves | A unit that never reaches its score is let go by the next unit or by the driver's end; a reaper the driver starts removes the containers and sockets of a driver that was killed | +| The controls | The null control passes no check; the reference makes the right calls through the unit's own server and passes all eight | + +A machine that runs the kit needs Docker, or Podman 4.7 or later, as the container backend does. The first unit builds the image (about 0.5 GB over the base image). diff --git a/docs/standard.md b/docs/standard.md index 6232f65a..e5c38e54 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -31,7 +31,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark** | `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said | | `agent.search/v1` | Search an HTTP collection with the agent's own tools, retrieve a source and answer with evidence | Find a source, choose the current one, follow a reference (3 checks) | | `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark | -| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions, through [the tools of an MCP server the benchmark's runner serves](ports/agent-computer-v1.md) | Defined; no kit yet | +| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions, through [the tools of an MCP server the benchmark's runner serves](ports/agent-computer-v1.md) | Four basic checks, four skills on a small desktop in a container ([the kit](ports/agent-computer-v1.md#the-kit)) | | `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential | - **The last message** follows ACP `messageId` boundaries and other events (a tool call, a notice). Without IDs, session metadata parts messages but never erases the final answer. From 9959de60218b55c67e071d20de9753e4fa51e718 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:24:03 +0100 Subject: [PATCH 24/36] test(catalogue): bound the OSWorld1 score test's wait and state the fake desktop's limit plainly --- app/nullius/catalogue/benchmarks/osworld1/benchmark.json | 2 +- app/tests/test_osworld1_entry_parts.py | 1 + docs/benchmarks/supported/osworld1.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json index dd1226d0..225841ab 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json @@ -26,7 +26,7 @@ "Each VM's ports are open on every network interface; one runs any command in the VM. Bind Docker's ports to 127.0.0.1, or keep the machine off untrusted networks." ], "limitations": [ - "An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop only, not on real VMs.", + "An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop alone.", "Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it.", "A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs.", "Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach.", diff --git a/app/tests/test_osworld1_entry_parts.py b/app/tests/test_osworld1_entry_parts.py index de3678ea..525fe740 100644 --- a/app/tests/test_osworld1_entry_parts.py +++ b/app/tests/test_osworld1_entry_parts.py @@ -114,6 +114,7 @@ def test_score_tells_the_runner_the_turn_has_ended(self): stdin=subprocess.DEVNULL) os.close(read) self.owned(p, life=write) + self.d.SCORE_SECONDS, self.d.REAP_SECONDS = 10, 0.5 # a runner never told would be let go at 10 s self.assertEqual(self.d.score("os__t1", self.ws, answer="")["reward"], 1.0) def test_a_planted_runner_record_is_no_runner_of_this_drivers(self): diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 9c8b25a7..56c6dd2d 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -68,7 +68,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of ## Nullius limitations -- An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop only, not on real VMs. +- An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop alone. - Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it. - A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs. - Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach. From 6151e9d79082e7d2f9899bb6e19490640c629d48 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 13:35:02 +0100 Subject: [PATCH 25/36] fix(standard): start the computer kit's pipe from the kit's own folder A runner's snapshot for a remote host takes a benchmark's server command only from files of its own folder, and refused the kit's path to the port's pipe two folders up. pipe.py in the kit's folder runs that pipe. The kit's proof is recorded again on this tree, and the recipe test now expects the computer kit among the kits an ACP agent takes. --- .../standard/kits/agent-computer-v1/pipe.py | 14 ++++++++++++++ .../standard/kits/agent-computer-v1/standard.json | 6 +++--- app/tests/test_computer_kit_driver.py | 15 ++++++++++++++- app/tests/test_standard_recipe.py | 3 ++- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 app/nullius/standard/kits/agent-computer-v1/pipe.py diff --git a/app/nullius/standard/kits/agent-computer-v1/pipe.py b/app/nullius/standard/kits/agent-computer-v1/pipe.py new file mode 100644 index 00000000..eca39daa --- /dev/null +++ b/app/nullius/standard/kits/agent-computer-v1/pipe.py @@ -0,0 +1,14 @@ +"""The session's stdio MCP server ``computer`` for the kit: the computer port's pipe (``computer_pipe.py``, two +folders up), run from the kit's folder, where a benchmark's server command names its files. + + pipe.py SOCKET +""" +import os +import runpy +import sys + +PIPE = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "computer_pipe.py") + +if __name__ == "__main__": + sys.argv[0] = PIPE + runpy.run_path(PIPE, run_name="__main__") diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 72456259..1f97497f 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -30,7 +30,7 @@ "computer": { "command": [ "python3", - "{here}/../../computer_pipe.py" + "{here}/pipe.py" ], "summary": "The unit's desktop: a pipe to the socket where the kit's driver serves the computer port over it; the socket is setup's argument." } @@ -137,7 +137,7 @@ "at_most": 0.0 }, "units": 8, - "job": "2026-11-16__12-16-15__agent-computer-v1__kit__prove-nop", + "job": "2026-11-16__12-32-36__agent-computer-v1__kit__prove-nop", "source": "nullius", "on": "2026-11-16" }, @@ -148,7 +148,7 @@ "at_least": 1.0 }, "units": 8, - "job": "2026-11-16__12-16-15__agent-computer-v1__kit__prove-oracle", + "job": "2026-11-16__12-32-36__agent-computer-v1__kit__prove-oracle", "source": "nullius", "on": "2026-11-16" } diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 3178f856..0a57f248 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -142,10 +142,23 @@ def test_the_port_names_the_kit_and_the_kit_its_checks(self): self.assertEqual(set(kit["skills"]), set(checks.CHECKS) - set(checks.BASIC)) self.assertEqual(kit["environments"]["desktop"], dict(desktop.KIT["environments"]["desktop"], summary=kit["environments"]["desktop"]["summary"])) - pipe = os.path.normpath(kit["mcp_servers"]["computer"]["command"][1].replace("{here}", KIT)) + self.assertEqual(kit["mcp_servers"]["computer"]["command"], ["python3", "{here}/pipe.py"]) # a file of its folder + pipe = _module("computer_kit_pipe_under_test", "pipe.py").PIPE self.assertEqual(pipe, os.path.join(os.path.dirname(os.path.dirname(KIT)), "computer_pipe.py")) self.assertTrue(os.path.isfile(pipe)) + def test_the_pipe_from_the_kits_folder_joins_stdio_to_the_socket(self): + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + path = os.path.join(tmp, "s") + server = port.Server(port.Bridge(FakeDesktop("d", {}), 1, 0, (1920, 1080), 1), path) + try: + p = subprocess.run([sys.executable, os.path.join(KIT, "pipe.py"), path], capture_output=True, timeout=30, + input=(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + "\n").encode()) + finally: + server.close() + self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(json.loads(p.stdout)["result"]["tools"][0]["name"], "info") + # -- the driver over a fake desktop ------------------------------------------------------------------------------------- class FakeDesktop: diff --git a/app/tests/test_standard_recipe.py b/app/tests/test_standard_recipe.py index 076791f4..2ec74055 100644 --- a/app/tests/test_standard_recipe.py +++ b/app/tests/test_standard_recipe.py @@ -385,7 +385,8 @@ def test_the_check_names_its_jobs_before_they_exist(self): got = recipe.conformance("agent", "fake", model="gpt-5-mini", provider="openai", unsandboxed=True, stamp="2026-10-10__12-00-00") self.assertEqual(got["errors"], []) self.assertEqual([j["name"] for j in got["jobs"]], - [conform.job_name("2026-10-10__12-00-00", k, "fake") for k in ("agent-task-v1", "agent-workspace-v1", "agent-tools-v1", "agent-conversation-v1", "agent-search-v1")]) # an ACP agent takes every kit + [conform.job_name("2026-10-10__12-00-00", k, "fake") for k in ("agent-task-v1", "agent-workspace-v1", "agent-tools-v1", "agent-conversation-v1", "agent-search-v1", + "agent-computer-v1")]) # an ACP agent takes every kit self.assertIn("conform fake --model gpt-5-mini --provider openai --stamp 2026-10-10__12-00-00 --unsandboxed --from-run", got["script"]) def test_container_conformance_does_not_grant_local_permissions(self): From d4c5f84a1987568e582f1e6e5d8927c0b3830b96 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 14:08:46 +0100 Subject: [PATCH 26/36] fix(catalogue): let OSWorld1's reaper finish what the driver's end cuts - A unit's VMs are removed in a thread the driver watches; when its input ends meanwhile, a reaper in a session of its own takes the removal and the socket's folder. - After a kill, and always in the reaper, the unit's containers are looked for again 2, 8 and 20 s on within the removal's bound, as the setups' reaper does, for a create the engine had accepted. - A runner stays in the driver's list until its reaper has started; when none can start, the driver kills its group and removes its VMs within 2 s. - The start gate reads the provider's start line from this launch's part of the runner's log alone. - The gate test's stand-in writes its ready file beside and renames it. - The definition says both ways a task is run, tells the agent's turn apart from setup's and scoring's time, and states the known limits of the agent side; the page is made again. --- .../benchmarks/osworld1/benchmark.json | 8 +- .../catalogue/benchmarks/osworld1/driver.py | 94 ++++++++++++--- .../benchmarks/osworld1/standard.json | 2 +- app/tests/test_osworld1_entry_parts.py | 111 +++++++++++++++++- docs/benchmarks/supported/osworld1.md | 8 +- 5 files changed, 200 insertions(+), 23 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json index 225841ab..9377167d 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json @@ -29,9 +29,15 @@ "An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop alone.", "Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it.", "A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs.", + "On the local backend an agent runs as you and can write where you can, the runner's folder included.", + "A runner let go at the end of scoring's time that then ends by itself is scored from its complete result.", + "A stop during the runner's last clean-up keeps the result already written.", + "A driver killed outright leaves the runner to stop on its own; a kill during a VM's start can leave that VM.", + "A reaper checks a runner's start time before it ends the runner's group; a process number reused in between is not caught.", + "The agent's python and upload calls do not mark OSWorld's desktop as used; each unit's desktop is reset once, before the agent's first call.", "Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach.", "The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested.", - "A task still running after 2 hours is stopped and gets no score." + "A task run by one of OSWorld's own agents and still running after 2 hours is stopped and gets no score. For an agent, the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes; past any of them the task gets no score." ], "datasets": [ { diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index 6e672e79..db06f5e0 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -117,6 +117,7 @@ CORE_END_SECONDS = 120 # the core's wait on end ANSWER_SECONDS = 10 # this driver's own answer once a bound is reached REMOVE_SECONDS = 30 # the docker commands that remove a unit's VMs, together +FALLBACK_SECONDS = 2 # this driver's removal when no reaper could start: inside the core's 3 s RUNNERS = {} # the runner folder of each unit this driver set up -> its process @@ -472,6 +473,7 @@ def _start_runner(unit, dir, instruction, until): if CLOSED.wait(min(1.0, left)): raise RuntimeError("the runner was not started: the unit's time ran out") with open(log, "a") as f: + launched = f.tell() # the start line of this launch is read from here on p = subprocess.Popen(argv, cwd=where, env=_runner_env(), stdout=f, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True, pass_fds=(life_r,)) RUNNERS[where] = {"p": p, "sock_dir": sock_dir, "life": life_w, "start": _start_time(p.pid)} @@ -492,7 +494,7 @@ def _start_runner(unit, dir, instruction, until): with open(os.path.join(where, "runner.json"), "w", encoding="utf-8") as f: json.dump({"pid": p.pid}, f) # the gate is held until the VM's start has ended, the provider's port lock with it: no bound of its own - _wait(lambda: os.path.exists(ready) or _has(log, STARTED), p, where, until, "its VM's start") + _wait(lambda: os.path.exists(ready) or _has(log, STARTED, launched), p, where, until, "its VM's start") finally: lock.close() _wait(lambda: os.path.exists(ready), p, where, until, "its task's setup") @@ -513,10 +515,12 @@ def _start_runner(unit, dir, instruction, until): "agent_seconds": AGENT_SECONDS} -def _has(log, text): +def _has(log, text, start=0): + """Whether ``text`` is in ``log`` from its byte ``start`` on.""" try: - with open(log, encoding="utf-8", errors="replace") as f: - return text in f.read() + with open(log, "rb") as f: + f.seek(start) + return text.encode() in f.read() except OSError: return False @@ -575,19 +579,50 @@ def _reap_runner(where, seconds=None): _hand_over(where) return None ended = _exited(p) - if p.returncode is None and ended != LOST: # running or a zombie: the group's number is still its own + killed = p.returncode is None and ended != LOST + if killed: # running or a zombie: the group's number is still its own try: os.killpg(p.pid, signal.SIGKILL) except OSError: pass p.wait() - if shutil.which("docker") is not None: - _remove_vms(where, REMOVE_SECONDS) + if shutil.which("docker") is not None and not _removed(where, entry["sock_dir"], again=ended is None): + RUNNERS.pop(where, None) # a reaper of its own has the removal and the socket's folder + return ended shutil.rmtree(entry["sock_dir"], ignore_errors=True) RUNNERS.pop(where, None) return ended +def _remove_passes(where, seconds, again=True): + """The unit's VMs removed within ``seconds``: once, then (``again``: the runner was killed, a create the engine had + accepted may still land) once more after each of 2, 8 and 20 s while time is left, as reap does.""" + until = time.time() + seconds + _remove_vms(where, seconds) + for wait in (2, 8, 20) if again else (): + if until - time.time() <= wait: + return + time.sleep(wait) + _remove_vms(where, until - time.time()) + + +def _removed(where, sock_dir, again): + """The unit's VMs removed (_remove_passes) in a thread this driver watches: True once done here; False when this + driver's input ended meanwhile and the removal and the socket's folder went to a reaper of its own.""" + worker = threading.Thread(target=_remove_passes, args=(where, REMOVE_SECONDS, again), name="vm-removal", + daemon=True) + worker.start() + while worker.is_alive(): + if CLOSED.wait(0.1): + try: + _spawn_reaper(0, None, where, sock_dir, 0) + return False + except OSError: # no reaper: this driver's removal goes on while it lives + worker.join(REMOVE_SECONDS) + return True + return True + + def _reap_all(): for where in list(RUNNERS): _reap_runner(where) @@ -595,21 +630,50 @@ def _reap_all(): def _hand_over(where): """The runner let go and handed to a reaper in a session of its own (driver.py --reap-runner), which outlives this - driver: Nullius kills this driver's group 3 s after closing its stdin.""" - entry = RUNNERS.pop(where, None) + driver: Nullius kills this driver's group 3 s after closing its stdin. The runner stays in RUNNERS until its reaper + has started; when none can start, it goes here at once (_reap_now).""" + entry = RUNNERS.get(where) if entry is None: return _let_go(entry) - subprocess.Popen([sys.executable, os.path.abspath(__file__), "--reap-runner", str(entry["p"].pid), - str(entry.get("start")), where, entry["sock_dir"], str(REAP_SECONDS)], + try: + _spawn_reaper(entry["p"].pid, entry.get("start"), where, entry["sock_dir"], REAP_SECONDS) + except OSError: + _reap_now(where) + return + RUNNERS.pop(where, None) + + +def _spawn_reaper(pid, start, where, sock_dir, seconds): + subprocess.Popen([sys.executable, os.path.abspath(__file__), "--reap-runner", str(pid), str(start), where, + sock_dir, str(seconds)], env=dict(os.environ, NULLIUS_UPSTREAM=UP), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True) +def _reap_now(where): + """The runner reaped here at once, when no reaper could start: its group killed while its leader is unreaped, the + unit's VMs removed within FALLBACK_SECONDS, the socket's folder gone.""" + entry = RUNNERS.pop(where, None) + if entry is None: + return + p = entry["p"] + if p.returncode is None and _exited(p) != LOST: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + p.wait() + if shutil.which("docker") is not None: + _remove_vms(where, FALLBACK_SECONDS) + shutil.rmtree(entry["sock_dir"], ignore_errors=True) + + def reap_runner(pid, start, where, sock_dir, seconds): - """The reaper of a runner handed over (_hand_over): the runner has ``seconds`` to end by itself; then, when it is - still the process started (same number, same start time), its group is killed; the unit's VMs removed within - REMOVE_SECONDS; the socket's folder gone.""" + """The reaper of a runner handed over (_hand_over), or of a removal this driver's end cut (_removed, pid 0): the + runner has ``seconds`` to end by itself; then, when it is still the process started (same number, same start time), + its group is killed; the unit's VMs removed within REMOVE_SECONDS, looked for again as reap does; the socket's + folder gone.""" def same(): return start is not None and _start_time(pid) == start @@ -628,7 +692,7 @@ def same(): except OSError: pass if shutil.which("docker") is not None: - _remove_vms(where, REMOVE_SECONDS) + _remove_passes(where, REMOVE_SECONDS) # again, for a create the engine had accepted before the kill shutil.rmtree(sock_dir, ignore_errors=True) diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index f3ef09f0..d3b44b94 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -2,7 +2,7 @@ "schema": "nullius.benchmark/v1", "key": "osworld1", "label": "OSWorld1", - "summary": "Each task is a unit: a real Ubuntu desktop in its own VM, set up by the task's config, one of OSWorld's own agents acting on it from its observations, and OSWorld's own evaluator reading the VM's final state.", + "summary": "Each task is a unit: a real Ubuntu desktop in its own VM, set up by the task's config. One of OSWorld's own agents acts on it from its observations, or an agent acts through the computer port, its tool calls the steps of OSWorld's own loop; OSWorld's own evaluator reads the VM's final state.", "family": "agent", "needs": [ "agent.computer/v1" diff --git a/app/tests/test_osworld1_entry_parts.py b/app/tests/test_osworld1_entry_parts.py index 525fe740..0fbe0627 100644 --- a/app/tests/test_osworld1_entry_parts.py +++ b/app/tests/test_osworld1_entry_parts.py @@ -53,6 +53,12 @@ def gone(pid: int, seconds: float = 10) -> bool: return not alive(pid) +def quiet(d) -> None: + """The driver's removal of a unit's VMs stubbed: no docker command, no wait for a late create.""" + d._remove_vms = lambda where, seconds=None: None + d._remove_passes = lambda where, seconds, again=True: None + + def line_of(p: subprocess.Popen, seconds: float = 10) -> bytes: """The child's next line on its output, waited for ``seconds`` at most.""" got = b"" @@ -76,7 +82,7 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, True) self.d = load("driver", self.tmp) - self.d._remove_vms = lambda where, seconds=None: None + quiet(self.d) self.ws = os.path.join(self.tmp, "trial", "workspace") os.makedirs(self.ws) self.where = self.d._runner_dir(self.ws) @@ -211,7 +217,7 @@ def setUp(self): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, True) self.d = load("driver", self.tmp) - self.d._remove_vms = lambda where, seconds=None: None + quiet(self.d) def test_the_start_lock_is_shared_by_every_run_on_the_machine(self): folder = os.path.join(self.tmp, "run-tmp") @@ -326,12 +332,13 @@ def stand_in_runner(self): f"open({os.path.join(self.tmp, 'begun')!r}, 'w').close()\n" f"while not os.path.exists({self.go!r}):\n time.sleep(0.02)\n" "print('INFO: Started container with ports - VNC: 1', flush=True)\n" - "json.dump({'ok': True, 'password': 'pw'}, open(get('--ready'), 'w'))\n" + "json.dump({'ok': True, 'password': 'pw'}, open(get('--ready') + '.part', 'w'))\n" + "os.replace(get('--ready') + '.part', get('--ready'))\n" "os.read(int(get('--lifeline')), 1)\n") self.d.RUNNER_FILE = path self.d._task = lambda unit: ("os", "a", {"instruction": "x"}) self.d._workspace = lambda where: "vm" - self.d._remove_vms = lambda where, seconds=None: None + quiet(self.d) self.d.START_LOCK = os.path.join(self.tmp, "start.lock") self.addCleanup(lambda: [self.d._reap_runner(w, seconds=0) for w in list(self.d.RUNNERS)]) @@ -360,6 +367,26 @@ def test_the_start_gate_is_held_until_the_vm_start_has_ended(self): self.assertTrue(got and got[0]["prompt"].startswith("x")) self.assertTrue(self.gate_free()) + def test_a_start_line_of_an_earlier_launch_does_not_free_the_gate(self): + self.stand_in_runner() + ws = os.path.join(self.tmp, "job", "trial", "workspace") + where = self.d._runner_dir(ws) + os.makedirs(where) + with open(os.path.join(where, "runner.log"), "w") as f: + f.write("INFO: " + self.d.STARTED + " - VNC: 1\n") # an earlier launch's + got = [] + worker = threading.Thread(target=lambda: got.append(self.d._start_runner("os__a", ws, "x", time.time() + 30)), + daemon=True) + worker.start() + deadline = time.monotonic() + 10 + while not os.path.exists(os.path.join(self.tmp, "begun")) and time.monotonic() < deadline: + time.sleep(0.02) + time.sleep(0.5) # the driver's wait has read the log more than once + self.assertFalse(self.gate_free(), "an earlier launch's start line let the gate go") + open(self.go, "w").close() + worker.join(15) + self.assertTrue(got and self.gate_free()) + def test_a_unit_waiting_for_the_gate_fails_at_its_setups_deadline(self): self.stand_in_runner() held = open(self.d.START_LOCK, "a") @@ -387,7 +414,7 @@ def runner_with_child(self): return where, p, kid def test_the_reap_ends_the_runners_whole_group(self): - self.d._remove_vms = lambda where, seconds=None: None + quiet(self.d) where, p, kid = self.runner_with_child() self.d._reap_runner(where, seconds=0.2) self.assertTrue(gone(kid), "a process of the runner's group outlived its reap") @@ -436,6 +463,60 @@ def test_a_runner_handed_over_is_reaped_after_the_driver_was_killed(self): self.assertTrue(gone(runner, 15), "the runner outlived its driver's death") self.assertTrue(gone(kid, 15), "the runner's group outlived its driver's death") + def owned_runner(self, code="pass"): + where = os.path.join(self.tmp, "trial", ".work", "osworld") + os.makedirs(where, exist_ok=True) + p = child(code) + if code == "pass": + p.wait() + self.addCleanup(lambda: (p.poll() is None and p.kill(), p.wait())) + sock = tempfile.mkdtemp(dir=self.tmp) + self.d.RUNNERS[where] = {"p": p, "sock_dir": sock, "life": None, "start": self.d._start_time(p.pid)} + return where, p, sock + + def test_a_removal_the_drivers_end_cuts_is_handed_to_a_reaper(self): + where, p, sock = self.owned_runner() + release = threading.Event() + self.addCleanup(release.set) + + def removing(dir, seconds=None): # the core closes the driver's input while docker works + self.d.CLOSED.set() + release.wait(10) + + self.d._remove_vms = removing + t0 = time.monotonic() + with mock.patch.object(self.d.shutil, "which", return_value="/usr/bin/docker"), \ + mock.patch.object(self.d.subprocess, "Popen") as spawn: + self.d._reap_runner(where) + self.assertLess(time.monotonic() - t0, 3, "the driver waited on the removal after its input ended") + argv = spawn.call_args[0][0] + self.assertEqual(argv[2:5], ["--reap-runner", "0", "None"]) + self.assertEqual((argv[5:7], self.d.RUNNERS), ([where, sock], {})) + + def test_the_units_containers_are_looked_for_again_after_a_kill_and_by_the_reaper(self): + removals = [] + self.d._remove_vms = lambda dir, seconds=None: removals.append(dir) + with mock.patch.object(self.d.shutil, "which", return_value="/usr/bin/docker"), \ + mock.patch.object(self.d.time, "sleep"): + where, p, sock = self.owned_runner() # ended by itself: one look + self.d._reap_runner(where) + self.assertEqual(len(removals), 1) + where, p, sock = self.owned_runner("import time; time.sleep(60)") # killed: looked for again + self.d._reap_runner(where, seconds=0.1) + self.assertGreaterEqual(len(removals), 1 + 2) + removals.clear() + self.d.reap_runner(123456789, None, where, os.path.join(self.tmp, "gone"), 0) + self.assertGreaterEqual(len(removals), 2, "the reaper looked for the unit's containers once") + + def test_a_reaper_that_cannot_start_leaves_the_runner_reaped_here(self): + where, p, sock = self.owned_runner("import time; time.sleep(60)") + self.d._remove_vms = lambda dir, seconds=None: None + with mock.patch.object(self.d.subprocess, "Popen", side_effect=OSError("no process can start")): + self.d._hand_over(where) + self.assertEqual(self.d.RUNNERS, {}) + self.assertTrue(gone(p.pid, 5), "the runner outlived a reaper that could not start") + self.assertFalse(os.path.exists(sock)) + def test_the_vms_removal_has_its_bound(self): bin_dir = os.path.join(self.tmp, "bin") os.mkdir(bin_dir) @@ -448,6 +529,26 @@ def test_the_vms_removal_has_its_bound(self): self.assertLess(time.monotonic() - t0, 5) +class Texts(unittest.TestCase): + """What the definition says of the two ways a task is run, and of each one's time.""" + + def test_the_definition_describes_both_ways_a_task_runs_and_each_ones_time(self): + with open(os.path.join(ENTRY, "standard.json")) as f: + summary = json.load(f)["summary"] + self.assertIn("OSWorld's own agents", summary) + self.assertIn("computer port", summary) + with open(os.path.join(ENTRY, "benchmark.json")) as f: + limits = json.load(f)["limitations"] + line = next(x for x in limits if x.startswith("A task run by one of OSWorld's own agents")) + self.assertIn("the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes", line) + self.assertNotIn("A task still running after 2 hours is stopped and gets no score.", limits) + page = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(ENTRY))), "..", "..", "docs", "benchmarks", + "supported", "osworld1.md") + if os.path.exists(page): + with open(page) as f: + self.assertIn(line, f.read()) + + class Snapshot(unittest.TestCase): """The copy Nullius makes of the entry for a run on a host carries the runner, and the port's shared code with the package.""" diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 56c6dd2d..3fa903ce 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -71,9 +71,15 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of - An agent needs the computer port, proven by its conformance kit. The agent side has been checked on a fake desktop alone. - Preparing the VM image (its lock and its download) is not bounded by a unit's setup time; the core's own limit on setup ends it. - A VM start that hangs holds the start gate until its unit's setup time ends; units waiting for the gate fail at the end of theirs. +- On the local backend an agent runs as you and can write where you can, the runner's folder included. +- A runner let go at the end of scoring's time that then ends by itself is scored from its complete result. +- A stop during the runner's last clean-up keeps the result already written. +- A driver killed outright leaves the runner to stop on its own; a kill during a VM's start can leave that VM. +- A reaper checks a runner's start time before it ends the runner's group; a process number reused in between is not caught. +- The agent's python and upload calls do not mark OSWorld's desktop as used; each unit's desktop is reset once, before the agent's first call. - Only 12 of OSWorld's agents are offered; the others fail at the pinned commit or need services the relay cannot reach. - The VMs run only on Docker's default socket, not on rootless Docker or a Docker context. Podman is untested. -- A task still running after 2 hours is stopped and gets no score. +- A task run by one of OSWorld's own agents and still running after 2 hours is stopped and gets no score. For an agent, the task's setup may take 1 hour, the agent's turn 2 hours and the scoring 10 minutes; past any of them the task gets no score. ## Cite From a70fa48e7dc4fe4ba860c04305022966539da23e Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 14:10:01 +0100 Subject: [PATCH 27/36] fix(standard): earn the computer kit's passes and leave nothing after a stop A pass must come from the desktop's records, the kit's reference must go the way an agent's session goes, and a stop, a kill or a hang must leave no container, folder or process. - limit: every acting call sent once the steps are taken is counted, one whose arguments do not hold included, and a call is counted and its step taken in one hold of the bridge's lock. - batch: each change of a field's text is recorded, however it came. - recover: the error goes to the first acting call the loop takes, whatever it is, before any part of it runs; only a call sent after the error's answer can earn the check. - An action the desktop did not end in time answers its call with an error; the next call goes on. - The reference reads each code from the screenshot it gets, with a PNG reader of the standard library, and fails when a call does not answer as a correct agent's would. - The engine's clients still running are ended and waited for when the driver lets its units go. The socket folder is named to the reaper before it is made, and the reaper is told when a desktop's start has ended: a container whose start had not ended is looked for again, up to a bound, until it is there and removed. - The killed-driver test reads with a bound and waits for its own reaper under a label of its own; a test runs a session of the scripted ACP agent through the pipe. - The kit's proof is recorded again on this tree. --- .../standard/kits/agent-computer-v1/checks.py | 9 +- .../kits/agent-computer-v1/desktop.py | 38 +- .../standard/kits/agent-computer-v1/driver.py | 116 ++++-- .../kits/agent-computer-v1/fixture.py | 99 ++++- .../standard/kits/agent-computer-v1/reaper.py | 79 +++- .../kits/agent-computer-v1/standard.json | 8 +- app/tests/test_computer_kit_driver.py | 388 +++++++++++++++++- docs/ports/agent-computer-v1.md | 4 +- 8 files changed, 657 insertions(+), 84 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index 768f6b01..3ce4e534 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -19,8 +19,8 @@ - ``scroll``: the last message holds the code drawn at the list's end, below what it shows at first, and the list moved; - ``recover``: the kit answers the agent's first acting call with an error, - without running it; a later acting call succeeds and the named button is - clicked, as for ``click``; + without running it; an acting call sent after that answer came succeeds, + and the named button is clicked, as for ``click``; - ``batch``: each of three fields holds its word, within two steps. Every check passes only when every pointer and key event, Enter in a field @@ -192,8 +192,9 @@ def recover(expected, events, actions, message, calls=(), past=0): failed = next((i for i, c in enumerate(acting) if c.get("isError")), None) if failed is None: return _verdict("no acting call was answered with an error") - if not any(not c.get("isError") for c in acting[failed + 1:]): - return _verdict("no acting call succeeded after the error") + answered = acting[failed].get("end", float("inf")) + if not any(not c.get("isError") and c.get("start", -1.0) >= answered for c in acting[failed + 1:]): + return _verdict("no acting call sent after the error's answer succeeded") return click(expected, events, actions, message) diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.py b/app/nullius/standard/kits/agent-computer-v1/desktop.py index d2ee4dbd..814814e9 100644 --- a/app/nullius/standard/kits/agent-computer-v1/desktop.py +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.py @@ -39,6 +39,7 @@ import os import secrets import subprocess +import threading from typing import Dict, List, Optional from nullius.standard import container @@ -143,12 +144,45 @@ def tag() -> str: return container.env_tag(KIT, ENVIRONMENT, container.base_tag()) +_CLIENTS: set = set() # the engine's clients running now, for end_clients +_CLIENTS_LOCK = threading.Lock() + + def _engine(args: List[str], *, input: Optional[bytes] = None, timeout: float) -> subprocess.CompletedProcess: + """One call of the engine's program, its stdin the request's bytes or nothing, killed past ``timeout`` + (``subprocess.TimeoutExpired``, as ``subprocess.run`` raises it).""" try: - return subprocess.run([container.DOCKER, *args], input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=timeout, env=container.client_env()) + p = subprocess.Popen([container.DOCKER, *args], stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=container.client_env()) except FileNotFoundError: raise DesktopError(f"{container.DOCKER} is not installed on this machine") + with _CLIENTS_LOCK: + _CLIENTS.add(p) + try: + try: + out, err = p.communicate(input, timeout=timeout) + except BaseException: + p.kill() + p.communicate() + raise + finally: + with _CLIENTS_LOCK: + _CLIENTS.discard(p) + return subprocess.CompletedProcess([container.DOCKER, *args], p.returncode, out, err) + + +def end_clients() -> int: + """Every client of the engine still running, from any thread, killed and waited for; how many there were. + A process that ends while a request of its waits leaves none behind.""" + with _CLIENTS_LOCK: + running = list(_CLIENTS) + for p in running: + try: + p.kill() + p.wait(timeout=10) + except (OSError, subprocess.SubprocessError): + pass + return len(running) def _said(r: subprocess.CompletedProcess) -> str: diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index e2d29f84..3f713b76 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -46,6 +46,7 @@ def _load(name, file): desktop = _load("nullius_kit_desktop", "desktop.py") checks = _load("nullius_kit_checks", "checks.py") +fixture = _load("nullius_kit_fixture", "fixture.py") UNITS = checks.CHECKS PAUSE = 1.0 # after each action @@ -54,6 +55,7 @@ def _load(name, file): LOOP_SECONDS = PYTHON_SECONDS + desktop.SETTLE + PAUSE + 30 # the loop's end, once the turn has ended USE = "Use the tools of the MCP server named computer, given to this session: they show and drive a desktop. " DONE = " Then reply with the word done." +NOT_RUN = "The desktop could not run this step, and none of it was run. It may be sent again." def prompt(unit, expected): @@ -140,18 +142,36 @@ def password(self): class KitBridge(port.Bridge): - """The port's bridge, counting the acting calls sent once every step was taken (``past``).""" + """The port's bridge, counting the acting calls sent once every step was taken (``past``). A call that holds + is counted and its step taken in one hold of the bridge's lock (a reentrant one): two calls sent together + for the last step are one taken and one counted.""" def __init__(self, *a, **kw): super().__init__(*a, **kw) self.past = 0 def submit(self, call): - if call.tool in port.ACTING: - with self.change: - if self.booked >= self.steps: - self.past += 1 - super().submit(call) + with self.change: + if call.tool in port.ACTING and self.booked >= self.steps: + self.past += 1 + super().submit(call) + + +class KitServer(port.Server): + """The port's server, counting also an acting call sent once every step was taken whose arguments do not hold: + the server refuses it before the bridge sees it.""" + + def call(self, rid, params, wire): + name = params.get("name") if isinstance(params, dict) else None + if name in port.ACTING: + probe = port.Call(name, params.get("arguments", {}), lambda *a: None) + try: + port.checked(probe, self.bridge.screen) + except Exception: + with self.bridge.change: + if self.bridge.booked >= self.bridge.steps: + self.bridge.past += 1 + super().call(rid, params, wire) class Unit: @@ -172,13 +192,18 @@ def start(self, job): image = desktop.image(os.path.dirname(self.log)) name = f"nullius-desktop-{os.urandom(6).hex()}" REAPER.tell(f"+container {name}") # before it exists: a kill during the start is covered - self.desktop = desktop.Desktop.start(image, name, job=job) + try: + self.desktop = desktop.Desktop.start(image, name, job=job) + finally: + REAPER.tell(f"=container {name}") # its start has ended, either way self.desktop.start_fixture(self.values) self.boxes = self.desktop.events()[0].get("boxes") or {} - self.folder = tempfile.mkdtemp(prefix="nullius-computer-") - REAPER.tell(f"+folder {self.folder}") + folder = os.path.join(tempfile.gettempdir(), f"nullius-computer-{os.urandom(6).hex()}") + REAPER.tell(f"+folder {folder}") # named before it is made + self.folder = folder + os.mkdir(folder, 0o700) self.bridge = KitBridge(KitDesktop(self.desktop), self.steps, PAUSE, desktop.SCREEN, PYTHON_SECONDS) - self.server = port.Server(self.bridge, self.socket(), log=self.log) + self.server = KitServer(self.bridge, self.socket(), log=self.log) self.loop = threading.Thread(target=self.run_loop, name=f"kit-loop-{self.unit}", daemon=True) self.loop.start() @@ -187,10 +212,19 @@ def socket(self): def run_loop(self): """The kit's own loop: a step from the bridge, its actions run, the pause after each; the budget is the - loop's. ``recover``: the first acting call with input is answered with an error, not run.""" + loop's. An action the desktop did not end in time leaves its call answered with an error, and the next + step goes on. ``recover``: the first acting call the loop takes, whatever it is, is answered with an error + before any part of it runs.""" try: for _ in range(self.steps): response, actions = self.bridge.predict("", None) + call = self.bridge.running # the acting call of these actions; None for done, fail or none + if call is not None and self.unit == "recover" and not self.failed_one: + self.failed_one = True + call.answer(error=NOT_RUN) + for _ in actions: # none of it run; the bridge still counts the step + pass + continue actions = iter(actions) stop = False try: @@ -201,10 +235,8 @@ def run_loop(self): if action == "WAIT": time.sleep(PAUSE) continue - if self.unit == "recover" and not self.failed_one: - self.failed_one = True - break # not run: the call is answered with an error - self.desktop.act(action, PYTHON_SECONDS) + if self.desktop.act(action, PYTHON_SECONDS)["reply"] is None: + break # not ended in time: the call is answered with an error time.sleep(PAUSE) finally: close = getattr(actions, "close", None) @@ -271,6 +303,8 @@ def close(self): def close_all(): + """Every unit still open let go, then every client of the engine still running (a loop's call that hangs) + ended and waited for.""" with OPEN_LOCK: held = list(OPEN.values()) OPEN.clear() @@ -279,6 +313,8 @@ def close_all(): u.close() except Exception as e: sys.stderr.write(f"computer kit: {u.unit} could not be let go: {e}\n") + if held: + desktop.end_clients() def job_of(dir): @@ -377,45 +413,69 @@ def centre(box): return (box[0] + box[2]) // 2, (box[1] + box[3]) // 2 +class ReferenceFailed(Exception): + """A call of the reference did not go as a correct agent's would.""" + + +def read_code(png, scale): + """The code of twelve characters drawn on the screen, read from a screenshot, as an agent reads it.""" + found = [line for line in fixture.read_lines(png, scale) + if len(line) == checks.CODE_LENGTH and set(line) <= set(checks.ALPHABET)] + if len(found) != 1: + raise ReferenceFailed(f"no one code could be read from the screenshot ({len(found)} found)") + return found[0] + + def oracle(unit, dir, **_): """The reference: the calls a correct agent makes, sent to the unit's own server, through its desktop, to the - same scorer; the answer they lead to.""" + same scorer; the answer they lead to. A code is read from the screenshot the server gives, and a call that does + not answer as a correct agent's would (the one ``recover`` refuses aside) fails the reference.""" with OPEN_LOCK: u = OPEN.get(unit) if u is None: raise ValueError(f"{unit} was not set up") e, b, answer = u.expected, u.boxes, "done" c = Client(u.socket()) + + def call(tool, arguments=None, failing=False): + failed, value = c.call(tool, arguments) + if failed != failing: + raise ReferenceFailed(f"{tool} answered {'with' if failed else 'without'} an error: {str(value)[:200]}") + return value + try: - c.call("info") + call("info") if unit in ("see", "scroll"): if unit == "scroll": x, y = centre(b["list"]) - c.call("scroll", {"x": x, "y": y, "dy": 100}) - c.call("screenshot") - answer = e["code"] + call("scroll", {"x": x, "y": y, "dy": 100}) + png = call("screenshot") + if not isinstance(png, bytes): + raise ReferenceFailed("the screenshot held no image") + answer = read_code(png, fixture.SEE_SCALE if unit == "see" else fixture.LIST_SCALE) elif unit in ("click", "recover"): x, y = centre(b[e["target"]]) - for _ in range(2 if unit == "recover" else 1): # recover: the first is answered with an error - c.call("click", {"x": x, "y": y}) + if unit == "recover": + call("click", {"x": x, "y": y}, failing=True) # the kit refuses the first acting call + call("click", {"x": x, "y": y}) elif unit == "type": x, y = centre(b["field"]) - c.call("click", {"x": x, "y": y}) - c.call("type", {"text": e["word"]}) - c.call("key", {"keys": ["enter"]}) + call("click", {"x": x, "y": y}) + call("type", {"text": e["word"]}) + call("key", {"keys": ["enter"]}) elif unit == "limit": for label in e["labels"][:u.steps]: x, y = centre(b[label]) - c.call("click", {"x": x, "y": y}) + call("click", {"x": x, "y": y}) elif unit == "drag": (x, y), (tx, ty) = centre(b["handle"]), centre(b["zone"]) - c.call("drag", {"x": x, "y": y, "to_x": tx, "to_y": ty}) + call("drag", {"x": x, "y": y, "to_x": tx, "to_y": ty}) elif unit == "batch": actions = [] for f in checks.FIELDS: x, y = centre(b[f]) actions += [f"pyautogui.click({x}, {y})", f"pyautogui.write({e['words'][f]!r})"] - c.call("step", {"actions": actions}) + call("step", {"actions": actions}) finally: c.close() return {"answer": answer} diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py index 66b728b1..520cf75d 100644 --- a/app/nullius/standard/kits/agent-computer-v1/fixture.py +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -24,8 +24,8 @@ ``list`` or ``background``). The first line is ``ready``, with where each target is on the screen, written once the window is mapped at the screen's size and holds the focus; Enter in the field writes -``enter`` with the field's text; each key let go in a labelled field writes -``value`` with its text; each turn of the wheel that moves the list writes +``enter`` with the field's text; each change of a labelled field's text writes +``value`` with that text; each turn of the wheel that moves the list writes ``moved`` with the list's top, a fraction of its length. The program leaves the caller once its window is up: it prints ``ready`` and @@ -33,13 +33,16 @@ """ import json import os +import struct import sys import time +import zlib HOME = "/var/lib/fixture" LOG = os.path.join(HOME, "events.jsonl") ERRORS = os.path.join(HOME, "fixture.err") TITLE = "Check" +SEE_SCALE, LIST_SCALE = 9, 4 # screen pixels a glyph cell: the code alone, the list's lines SCREENS = ("see", "click", "type", "drag", "scroll", "batch") LIST_WORDS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "MAPLE", "NORTH") BACKGROUND = "#eceff1" @@ -96,6 +99,84 @@ def pixels(text): return max(len(text) * 6 - 1, 1), 7, cells +def png_rows(data, last_row=None): + """The rows of a PNG (8 bits a channel, RGB or RGBA, not interlaced) as bytes of RGB pixels, up to + ``last_row``; standard library only.""" + if data[:8] != b"\x89PNG\r\n\x1a\n": + raise ValueError("not a PNG") + pos, idat, head = 8, [], None + while pos < len(data): + n, kind = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + n] + if kind == b"IHDR": + head = struct.unpack(">IIBBBBB", body) + elif kind == b"IDAT": + idat.append(body) + elif kind == b"IEND": + break + pos += 12 + n + w, h, depth, colour, _, _, interlace = head + if depth != 8 or colour not in (2, 6) or interlace: + raise ValueError("a PNG of 8-bit RGB or RGBA, not interlaced") + bpp = 3 if colour == 2 else 4 + raw, stride, prev, rows = zlib.decompress(b"".join(idat)), w * bpp, bytearray(w * bpp), [] + for y in range(h if last_row is None else min(h, last_row + 1)): + kind, line = raw[y * (stride + 1)], bytearray(raw[y * (stride + 1) + 1:(y + 1) * (stride + 1)]) + if kind == 1: + for i in range(bpp, stride): + line[i] = (line[i] + line[i - bpp]) & 255 + elif kind == 2: + for i in range(stride): + line[i] = (line[i] + prev[i]) & 255 + elif kind == 3: + for i in range(stride): + line[i] = (line[i] + ((line[i - bpp] if i >= bpp else 0) + prev[i]) // 2) & 255 + elif kind == 4: + for i in range(stride): + a, b = (line[i - bpp] if i >= bpp else 0), prev[i] + c = prev[i - bpp] if i >= bpp else 0 + pa, pb, pc = abs(b - c), abs(a - c), abs(a + b - 2 * c) + line[i] = (line[i] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255 + rows.append(bytes(line) if bpp == 3 else bytes(v for i, v in enumerate(line) if i % 4 != 3)) + prev = line + return w, rows + + +def read_lines(data, scale, ink=INK): + """The lines of glyphs drawn at ``scale`` in ``ink`` on a screen's PNG, top to bottom; a block that is no + glyph reads ``?``. What the kit's reference reads from a screenshot, as an agent would from its eyes.""" + colour = bytes(int(ink[i:i + 2], 16) for i in (1, 3, 5)) + w, rows = png_rows(data) + lit = [] + for row in rows: + xs, i = set(), row.find(colour) + while i >= 0: + if i % 3 == 0: + xs.add(i // 3) + i = row.find(colour, i + 1) + lit.append(xs) + out, y = [], 0 + while y < len(lit): + if not lit[y]: + y += 1 + continue + top, cells = y, {} + x0 = min(min(lit[r]) for r in range(top, min(top + 7 * scale, len(lit))) if lit[r]) + for r in range(7): + yy = top + r * scale + scale // 2 + for x in (lit[yy] if yy < len(lit) else ()): + if x >= x0 and (x - x0) % scale == scale // 2: + cells[((x - x0) // scale, r)] = True + text, c = "", 0 + while any((c * 6 + dx, r) in cells for dx in range(5) for r in range(7)): + block = tuple("".join("#" if (c * 6 + dx, r) in cells else "." for dx in range(5)) for r in range(7)) + text += next((ch for ch, g in GLYPHS.items() if g == block), "?") + c += 1 + out.append(text) + y = top + 7 * scale + return out + + def bitmap(tk, master, text, scale, ink=INK, paper=BACKGROUND): """``text`` as a Tk image, ``scale`` screen pixels a glyph cell.""" cols, rows, cells = pixels(text) @@ -115,13 +196,13 @@ def __init__(self, values, log): self.root.title(TITLE) self.root.configure(background=BACKGROUND) self.root.attributes("-fullscreen", True) - self.images, self.names, self.field = [], {}, None + self.images, self.names, self.field, self.texts = [], {}, None, [] getattr(self, "_" + values["screen"])() for seq in ("", "", "", "", ""): self.root.bind_all(seq, self.on_event, add="+") def _see(self): - img = bitmap(self.tk, self.root, self.values["code"], 9) + img = bitmap(self.tk, self.root, self.values["code"], SEE_SCALE) self.images.append(img) self.tk.Label(self.root, image=img, background=BACKGROUND, borderwidth=0).place(relx=0.5, rely=0.5, anchor="center") @@ -157,7 +238,7 @@ def follow(e): # the handle goes where the poin handle.bind("", follow, add="+") def _scroll(self): - rows, step, scale = 40, 60, 4 + rows, step, scale = 40, 60, LIST_SCALE c = self.tk.Canvas(self.root, width=760, height=540, background="#ffffff", highlightthickness=2, highlightbackground="#90a4ae", yscrollincrement=20) c.place(relx=0.5, rely=0.5, anchor="center") @@ -190,10 +271,14 @@ def _batch(self): self.images.append(img) self.tk.Label(frame, image=img, background=BACKGROUND, borderwidth=0).grid(row=i, column=0, padx=30, pady=30, sticky="e") - field = self.tk.Entry(frame, font=("DejaVu Sans", 24), width=20, relief="sunken", borderwidth=3) + text = self.tk.StringVar(master=frame) + field = self.tk.Entry(frame, textvariable=text, font=("DejaVu Sans", 24), width=20, relief="sunken", + borderwidth=3) field.grid(row=i, column=1, padx=30, pady=30) self.names[str(field)] = label - field.bind("", lambda e, f=field, n=label: self.write("value", field=n, value=f.get()), add="+") + # every change of the field's text, however it came (a key held down included) + text.trace_add("write", lambda *_, t=text, n=label: self.write("value", field=n, value=t.get())) + self.texts.append(text) def _type(self): self.field = self.tk.Entry(self.root, font=("DejaVu Sans", 28), width=24, relief="sunken", borderwidth=3) diff --git a/app/nullius/standard/kits/agent-computer-v1/reaper.py b/app/nullius/standard/kits/agent-computer-v1/reaper.py index 9a9794b5..21f10363 100644 --- a/app/nullius/standard/kits/agent-computer-v1/reaper.py +++ b/app/nullius/standard/kits/agent-computer-v1/reaper.py @@ -1,44 +1,93 @@ """The computer kit's reaper: what a driver made, removed once the driver is gone, however it went. - reaper.py ENGINE + reaper.py ENGINE [START_SECONDS] The kit's driver starts it at its first unit, in a session of its own (a run that kills the driver's process -group does not kill it), and keeps its stdin open. Each line names what the driver made or let go: -``+container NAME``, ``-container NAME``, ``+folder PATH``, ``-folder PATH``. Once its stdin ends (the driver -ended, closed it, or was killed), it removes each container still named, with ENGINE (``docker`` or -``podman``), and each folder still named, then exits. Standard library only. +group does not kill it), and keeps its stdin open. Each line names what the driver is making, made or let go: + +- ``+container NAME`` before the container's start, ``=container NAME`` once that start has ended (the + container is there, or the start failed and its removal was asked), ``-container NAME`` once it is removed; +- ``+folder PATH`` before the folder is made, ``-folder PATH`` once it is removed. + +Once its stdin ends (the driver ended, closed it, or was killed), it removes each container still named, with +ENGINE (``docker`` or ``podman``), and each folder still named, then exits. A container whose start had not ended +may still be created by the engine after the driver's death: the reaper looks for it again until it has removed +it or START_SECONDS (60 by default) have passed. Standard library only. """ import shutil import subprocess import sys +import time REMOVE_SECONDS = 120 +START_SECONDS = 60.0 def held(lines): - """What the lines leave named: {"container": [...], "folder": [...]}, in the order made.""" - out = {"container": [], "folder": []} + """What the lines leave named: {"container": [...], "folder": [...], "started": [...]}, in the order made; + ``started`` holds the containers whose start had ended.""" + out = {"container": [], "folder": [], "started": []} for line in lines: - sign, _, rest = line.rstrip("\n").partition(" ") - kind, what = sign[1:], rest - if sign[:1] not in "+-" or kind not in out or not what: + sign, _, what = line.rstrip("\n").partition(" ") + kind = sign[1:] + if sign[:1] not in "+=-" or kind not in ("container", "folder") or not what: continue if sign[0] == "+" and what not in out[kind]: out[kind].append(what) + elif sign[0] == "=" and kind == "container" and what not in out["started"]: + out["started"].append(what) elif sign[0] == "-" and what in out[kind]: out[kind].remove(what) + out["started"] = [n for n in out["started"] if n in out["container"]] return out +def _engine(engine, *args): + try: + return subprocess.run([engine, *args], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=REMOVE_SECONDS) + except (OSError, subprocess.SubprocessError): + return None + + +def there(engine, name): + """Whether the engine holds a container of that name; None when it could not say.""" + r = _engine(engine, "container", "inspect", "--format", "{{.Id}}", name) + if r is None: + return None + if r.returncode == 0: + return True + return False if b"no such" in (r.stderr or b"").lower() else None + + +def remove(engine, name, pending, seconds): + """The container removed. One whose start had not ended is looked for until it is gone after being there, + or ``seconds`` have passed.""" + end = time.monotonic() + seconds + seen = False + while True: + state = there(engine, name) + if state: + seen = True + _engine(engine, "rm", "-f", name) + if time.monotonic() >= end: + return + continue + if state is False and (seen or not pending): + return + if time.monotonic() >= end: + if state is None: # the engine could not say: one last removal asked + _engine(engine, "rm", "-f", name) + return + time.sleep(1) + + def main(argv): engine = argv[1] + seconds = float(argv[2]) if len(argv) > 2 else START_SECONDS left = held(sys.stdin) for name in left["container"]: - try: - subprocess.run([engine, "rm", "-f", name], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, timeout=REMOVE_SECONDS) - except (OSError, subprocess.SubprocessError): - pass + remove(engine, name, name not in left["started"], seconds) for path in left["folder"]: shutil.rmtree(path, ignore_errors=True) diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 1f97497f..866f512b 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -106,8 +106,8 @@ }, "recover": { "label": "Recover from an error", - "task": "Click a named button; the first acting call is answered with an error.", - "passes": "An acting call succeeded after the error, and the button was clicked as in the click check." + "task": "Click a named button; the first acting call, whatever it is, is answered with an error and not run.", + "passes": "An acting call sent after the error's answer succeeded, and the button was clicked as in the click check." }, "batch": { "label": "Several actions in one step", @@ -137,7 +137,7 @@ "at_most": 0.0 }, "units": 8, - "job": "2026-11-16__12-32-36__agent-computer-v1__kit__prove-nop", + "job": "2026-11-16__13-07-27__agent-computer-v1__kit__prove-nop", "source": "nullius", "on": "2026-11-16" }, @@ -148,7 +148,7 @@ "at_least": 1.0 }, "units": 8, - "job": "2026-11-16__12-32-36__agent-computer-v1__kit__prove-oracle", + "job": "2026-11-16__13-07-27__agent-computer-v1__kit__prove-oracle", "source": "nullius", "on": "2026-11-16" } diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 0a57f248..dfb1329c 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -7,16 +7,20 @@ import importlib.util import json import os +import queue import re import secrets import shutil +import select import signal +import struct import subprocess import sys import tempfile import threading import time import unittest +import zlib from unittest import mock from nullius.standard import computer_port as port @@ -84,13 +88,16 @@ def test_scroll_passes_on_the_code_once_the_list_moved(self): def test_recover_passes_on_a_success_after_the_error_and_the_click(self): exp = {"target": "OCEAN", "labels": ["OCEAN", "APPLE"]} - calls = [{"tool": "screenshot", "isError": False}, {"tool": "click", "isError": True}, - {"tool": "click", "isError": False}] + calls = [{"tool": "screenshot", "isError": False, "start": 1.0, "end": 1.5}, + {"tool": "click", "isError": True, "start": 2.0, "end": 2.5}, + {"tool": "click", "isError": False, "start": 3.0, "end": 4.5}] events = press(110.2, "OCEAN") self.assertTrue(checks.verdict("recover", exp, events, ACTIONS, "", calls)["passed"]) self.assertIn("no acting call was answered with an error", checks.verdict("recover", exp, events, ACTIONS, "", calls[2:])["why"]) - self.assertIn("succeeded after", checks.verdict("recover", exp, events, ACTIONS, "", calls[:2])["why"]) + self.assertIn("after the error's answer", checks.verdict("recover", exp, events, ACTIONS, "", calls[:2])["why"]) + sent_before = calls[:2] + [dict(calls[2], start=2.4)] # sent before the error's answer came + self.assertIn("after the error's answer", checks.verdict("recover", exp, events, ACTIONS, "", sent_before)["why"]) self.assertFalse(checks.verdict("recover", exp, [], ACTIONS, "", calls)["passed"]) def test_batch_passes_when_each_field_holds_its_word(self): @@ -110,7 +117,8 @@ def test_every_check_fails_on_input_outside_every_action(self): ev("release", 100.8, button=1, hit="handle", x=600, y=600)], "", ()), "scroll": ({"code": "ACDE34679HJK"}, [ev("moved", 100.3, top=0.5)], "ACDE34679HJK", ()), "recover": ({"target": "OCEAN", "labels": ["OCEAN"]}, press(110.2, "OCEAN"), "", - ({"tool": "click", "isError": True}, {"tool": "click", "isError": False})), + ({"tool": "click", "isError": True, "start": 1, "end": 2}, + {"tool": "click", "isError": False, "start": 3, "end": 4})), "batch": ({"words": {"ONE": "a"}}, [ev("value", 100.2, field="ONE", value="a")], "", ()), } self.assertEqual(sorted(passing), sorted(checks.CHECKS)) @@ -223,7 +231,9 @@ def remove(self): self.removed = True -class Wiring(unittest.TestCase): +class FakeUnits(unittest.TestCase): + """Units of the driver over fake desktops, each in a folder of its own.""" + def setUp(self): self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) self.addCleanup(shutil.rmtree, self.tmp, True) @@ -246,10 +256,12 @@ def draw(check, rng=None): self.addCleanup(driver.close_all) def unit(self, name): - ws = os.path.join(self.tmp, "job", name, "workspace") + ws = os.path.join(self.tmp, "job", f"{name}-{len(FakeDesktop.made)}", "workspace") os.makedirs(ws) return ws, driver.setup(name, ws) + +class Wiring(FakeUnits): def test_a_unit_is_served_scored_and_let_go(self): ws, s = self.unit("click") self.assertEqual(s["mcp"], [{"name": "computer", "server": "computer", "args": [s["mcp"][0]["args"][0]]}]) @@ -267,7 +279,7 @@ def test_a_unit_is_served_scored_and_let_go(self): self.assertEqual(got["detail"]["calls"], ["click"]) self.assertFalse(os.path.exists(sock)) # the socket and its folder gone self.assertTrue(FakeDesktop.made[-1].removed) - self.assertEqual([l.split()[0] for l in self.told], ["+container", "+folder", "-container", "-folder"]) + self.assertEqual([l.split()[0] for l in self.told], ["+container", "=container", "+folder", "-container", "-folder"]) self.assertNotIn("click", driver.OPEN) def test_the_turns_end_at_score_ends_the_loop_and_judges_what_is_there(self): @@ -324,23 +336,311 @@ def test_a_unit_never_scored_is_let_go_by_the_next_setup_and_by_the_drivers_end( self.assertEqual(driver.OPEN, {}) +def _alive(pid): + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True, + timeout=30).stdout.strip()[:1] not in ("", "Z") + + +def _png(code, scale): + """A screen of 1920 by 1080 with ``code`` drawn as the fixture draws it, as PNG bytes (filter 0).""" + w, h = 1920, 1080 + cols, rows, cells = driver.fixture.pixels(code) + x0, y0 = (w - cols * scale) // 2, (h - rows * scale) // 2 + ink = {(x0 + x * scale + dx, y0 + y * scale + dy) for x, y in cells for dx in range(scale) for dy in range(scale)} + paper, mark = bytes.fromhex("eceff1"), bytes.fromhex("111111") + raw = b"".join(b"\x00" + b"".join(mark if (x, y) in ink else paper for x in range(w)) for y in range(h)) + + def chunk(kind, body): + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xffffffff) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +class FakeWire: + def __init__(self): + self.ready, self.reply = threading.Event(), None + + def opened(self): + pass + + def answered(self): + pass + + def send(self, data): + self.reply = json.loads(data) if isinstance(data, (bytes, str)) else data + self.ready.set() + return True + + +def send(server, tool, arguments=None, rid=1): + wire = FakeWire() + server.call(rid, {"name": tool, "arguments": arguments or {}}, wire) + return wire + + +class EarnedPasses(FakeUnits): + """What a pass needs, over the fake desktop: every attempt past the budget counted, recover's call sent after + its error's answer, its error on the first acting call whatever it is, an action not ended in time an error, + and a reference that reads the screen.""" + + def test_a_wrong_call_past_the_budget_is_counted(self): + ws, s = self.unit("limit") + u = driver.OPEN["limit"] + c = driver.Client(s["mcp"][0]["args"][0]) + for label in checks.ORDER[:2]: + self.assertFalse(c.call("click", dict(zip("xy", driver.centre(u.boxes[label]))))[0]) + self.assertTrue(c.call("click", {"x": "bad", "y": 100})[0]) + c.close() + self.assertEqual(u.bridge.past, 1) + self.assertFalse(driver.score("limit", ws, [], "done")["passed"]) + + def test_two_calls_sent_together_for_the_last_step_are_one_taken_and_one_counted(self): + ws, s = self.unit("limit") + u = driver.OPEN["limit"] + x, y = driver.centre(u.boxes["ONE"]) + self.assertTrue(send(u.server, "click", {"x": x, "y": y}).ready.wait(10)) + barrier, original, wires = threading.Barrier(2), driver.port.Bridge.submit, [] + + def together(bridge, call): # both calls reach the bridge at once where the lock lets them + try: + barrier.wait(timeout=1) + except threading.BrokenBarrierError: + pass + return original(bridge, call) + threads = [threading.Thread(target=lambda: wires.append(send(u.server, "click", {"x": x, "y": y}, 2))) + for _ in range(2)] + with mock.patch.object(driver.port.Bridge, "submit", together): + for t in threads: + t.start() + for t in threads: + t.join(10) + for w in wires: + self.assertTrue(w.ready.wait(10)) + self.assertEqual(sorted(w.reply["result"]["isError"] for w in wires), [False, True]) + self.assertEqual(u.bridge.past, 1) + self.assertFalse(driver.score("limit", ws, [], "done")["passed"]) + + def test_recover_needs_a_call_sent_after_the_errors_answer(self): + ws, s = self.unit("recover") + u = driver.OPEN["recover"] + x, y = driver.centre(u.boxes[u.expected["target"]]) + with u.bridge.change: # both queued before either is answered + first, second = send(u.server, "click", {"x": x, "y": y}), send(u.server, "click", {"x": x, "y": y}, 2) + self.assertTrue(first.ready.wait(10) and second.ready.wait(10)) + self.assertEqual([c["isError"] for c in u.calls()], [True, False]) + got = driver.score("recover", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertIn("after the error's answer", got["detail"]["why"]) + + def test_recovers_error_goes_to_the_first_acting_call_whatever_it_is(self): + for first in ("wait", "step"): + with self.subTest(first=first): + ws, s = self.unit("recover") + u = driver.OPEN["recover"] + c = driver.Client(s["mcp"][0]["args"][0]) + failed, said = c.call(first, {"actions": []} if first == "step" else {}) + self.assertTrue(failed, said) + self.assertIn("none of it was run", said) + self.assertFalse(c.call("click", dict(zip("xy", driver.centre(u.boxes[u.expected["target"]]))))[0]) + c.close() + self.assertTrue(driver.score("recover", ws, [], "done")["passed"]) + driver.close_all() + + def test_an_action_not_ended_in_time_is_answered_with_an_error_and_the_next_goes_on(self): + ws, s = self.unit("click") + u = driver.OPEN["click"] + fake = FakeDesktop.made[-1] + late = [True] + + def act(code, seconds=None): + if late.pop() if late else False: + return {"start": 1.0, "end": 2.0, "reply": None} + return FakeDesktop.act(fake, code, seconds) + fake.act = act + c = driver.Client(s["mcp"][0]["args"][0]) + self.assertTrue(c.call("step", {"actions": ["time.sleep(1000)"]})[0]) + self.assertFalse(c.call("click", dict(zip("xy", driver.centre(u.boxes[u.expected["target"]]))))[0]) + c.close() + self.assertEqual([c["isError"] for c in u.calls()], [True, False]) + self.assertTrue(driver.score("click", ws, [], "done")["passed"]) + + def test_the_reference_reads_the_code_from_the_screenshot_and_fails_without_one(self): + ws, s = self.unit("see") + fake, code = FakeDesktop.made[-1], driver.OPEN["see"].expected["code"] + fake.capture = lambda: _png(code, 9) + self.assertEqual(driver.oracle("see", ws)["answer"], code) + self.assertTrue(driver.score("see", ws, [], code)["passed"]) + ws, s = self.unit("see") + + def broken(): + raise desktop.DesktopError("the capture failed") + FakeDesktop.made[-1].capture = broken + with self.assertRaises(driver.ReferenceFailed): + driver.oracle("see", ws) + ws, s = self.unit("see") + FakeDesktop.made[-1].capture = lambda: _png("ACDE", 9) # a screen with no code of twelve + with self.assertRaises(driver.ReferenceFailed): + driver.oracle("see", ws) + + +class Shutdown(unittest.TestCase): + def test_the_engines_clients_still_running_are_ended_and_waited_for(self): + done = [] + with mock.patch.object(desktop.container, "DOCKER", sys.executable): + t = threading.Thread(target=lambda: done.append(desktop._engine(["-c", "import time; time.sleep(60)"], timeout=120))) + t.start() + for _ in range(100): + if desktop._CLIENTS: + break + time.sleep(0.05) + self.assertEqual(desktop.end_clients(), 1) + t.join(10) + self.assertFalse(t.is_alive()) + self.assertNotEqual(done[0].returncode, 0) # killed, not left + self.assertEqual(desktop._CLIENTS, set()) + + def test_the_drivers_end_lets_the_units_go_then_ends_the_clients(self): + u = mock.Mock() + with mock.patch.dict(driver.OPEN, {"see": u}, clear=True), mock.patch.object(desktop, "end_clients") as end: + driver.close_all() + u.close.assert_called_once() + end.assert_called_once() + + def test_the_socket_folder_is_named_to_the_reaper_before_it_is_made(self): + told, made = [], [] + + real = os.mkdir + + def mkdir(path, mode=0o777): + real(path, mode) + if "nullius-computer-" in os.path.basename(path): + made.append(path) + raise KeyboardInterrupt("killed just after the folder was made") + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + ws = os.path.join(tmp, "job", "see", "workspace") + os.makedirs(ws) + u = driver.Unit("see", ws) + with mock.patch.object(driver.desktop, "image", return_value="img"), \ + mock.patch.object(driver.desktop.Desktop, "start", return_value=FakeDesktop("d", {})), \ + mock.patch.object(driver.REAPER, "tell", side_effect=told.append), \ + mock.patch.object(driver.os, "mkdir", side_effect=mkdir): + with self.assertRaises(KeyboardInterrupt): + u.start("job") + shutil.rmtree(made[0], ignore_errors=True) + self.assertIn(f"+folder {made[0]}", told) + + +class FixtureRecords(unittest.TestCase): + def test_every_change_of_a_batch_field_is_recorded(self): + fixture = driver.fixture + fields, texts = [], [] + + class Widget: + def __init__(self, *a, **kw): + self.kw = kw + + def place(self, **kw): + pass + + def grid(self, **kw): + pass + + def bind(self, *a, **kw): + pass + + class Entry(Widget): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + fields.append(self) + + class StringVar: + def __init__(self, master=None): + self.value, self.traces = "", [] + texts.append(self) + + def trace_add(self, mode, fn): + self.traces.append(fn) + + def set(self, value): # a change by any route: a key held down, a paste + self.value = value + for fn in self.traces: + fn("", "", "write") + + def get(self): + return self.value + + f = fixture.Fixture.__new__(fixture.Fixture) + f.tk = mock.Mock(Frame=Widget, Label=Widget, Entry=Entry, StringVar=StringVar) + f.root, f.values = object(), {"labels": list(checks.FIELDS)} + f.images, f.names, f.texts, f.log_file = [], {}, [], __import__("io").StringIO() + with mock.patch.object(fixture, "bitmap", return_value=object()): + f._batch() + self.assertEqual(len(texts), 3) + self.assertIs(fields[0].kw.get("textvariable"), texts[0]) + texts[2].set("sanepilo") + texts[2].set("sanepil") # the last key took a letter back + last = [json.loads(line) for line in f.log_file.getvalue().splitlines()][-1] + self.assertEqual((last["kind"], last["field"], last["value"]), ("value", "THREE", "sanepil")) + + class Reaper(unittest.TestCase): + def test_a_container_whose_start_had_not_ended_is_looked_for_until_it_is_there_and_removed(self): + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + engine, said, state = os.path.join(tmp, "engine"), os.path.join(tmp, "said"), os.path.join(tmp, "n") + with open(engine, "w") as f: # the container appears at the third look, as a late start's + f.write(f"""#!/bin/sh +echo "$@" >> {said} +n=$(cat {state} 2>/dev/null || echo 0); n=$((n+1)); echo $n > {state} +case "$1" in + container) if [ -e {state}.gone ] || [ $n -lt 3 ]; then echo "Error: No such container: $5" >&2; exit 1; fi; echo id;; + rm) touch {state}.gone;; +esac +""") + os.chmod(engine, 0o755) + p = subprocess.run([sys.executable, os.path.join(KIT, "reaper.py"), engine, "20"], input="+container late\n", + text=True, capture_output=True, timeout=60) + self.assertEqual(p.returncode, 0, p.stderr) + with open(said) as f: + lines = f.read().splitlines() + self.assertIn("rm -f late", lines) + self.assertEqual(lines[-1].split()[:2], ["container", "inspect"]) # and looked again: gone + os.remove(said) + os.remove(state) + os.remove(state + ".gone") + p = subprocess.run([sys.executable, os.path.join(KIT, "reaper.py"), engine, "20"], + input="+container done\n=container done\n", text=True, capture_output=True, timeout=60) + with open(said) as f: + self.assertEqual(f.read().splitlines(), ["container inspect --format {{.Id}} done"]) # started: one look def test_what_is_still_named_is_removed_once_the_driver_is_gone(self): - self.assertEqual(reaper.held(["+container a\n", "+folder /x\n", "+container b\n", "-container a\n", "junk\n"]), - {"container": ["b"], "folder": ["/x"]}) + self.assertEqual(reaper.held(["+container a\n", "+folder /x\n", "+container b\n", "=container b\n", + "-container a\n", "junk\n"]), + {"container": ["b"], "folder": ["/x"], "started": ["b"]}) with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: engine, said, folder = os.path.join(tmp, "engine"), os.path.join(tmp, "said"), os.path.join(tmp, "left") os.makedirs(folder) - with open(engine, "w") as f: - f.write(f"#!/bin/sh\necho \"$@\" >> {said}\n") + with open(engine, "w") as f: # a container is there until it is removed + f.write(f"""#!/bin/sh +echo "$@" >> {said} +case "$1" in + container) if [ -e {tmp}/gone-$5 ]; then echo "Error: No such container: $5" >&2; exit 1; fi; echo id;; + rm) touch {tmp}/gone-$3;; +esac +""") os.chmod(engine, 0o755) p = subprocess.Popen([sys.executable, os.path.join(KIT, "reaper.py"), engine], stdin=subprocess.PIPE, text=True, start_new_session=True) - p.stdin.write(f"+container one\n+container two\n-container one\n+folder {folder}\n") + p.stdin.write(f"+container one\n=container one\n+container two\n=container two\n-container one\n" + f"+folder {folder}\n") p.stdin.close() # the driver is gone self.assertEqual(p.wait(timeout=30), 0) with open(said) as f: - self.assertEqual(f.read(), "rm -f two\n") + self.assertEqual([l for l in f.read().splitlines() if l.startswith("rm")], ["rm -f two"]) self.assertFalse(os.path.exists(folder)) def test_the_driver_tells_the_reaper_before_the_container_exists(self): @@ -357,6 +657,7 @@ def test_the_driver_tells_the_reaper_before_the_container_exists(self): driver.setup("see", ws) self.assertTrue(order[0].startswith("+container nullius-desktop-")) self.assertEqual(order[1], "start") + self.assertEqual(order[2], "=container " + order[0].split()[1]) # its start ended, failed self.assertEqual(driver.OPEN, {}) @@ -419,31 +720,74 @@ def test_scripted_clients_that_type_through_python_or_act_past_their_budget_fail self.assertEqual(got["detail"]["past_budget"], 1) def test_a_driver_killed_during_a_unit_leaves_no_container_socket_or_process(self): - ws = os.path.join(self.tmp, "job", "see-kill", "workspace") + job = f"kill-{secrets.token_hex(4)}" # this test's own label + ws = os.path.join(self.tmp, job, "see-kill", "workspace") os.makedirs(ws) env = dict(os.environ, PYTHONPATH=APP, PYTHONDONTWRITEBYTECODE="1") p = subprocess.Popen([sys.executable, os.path.join(KIT, "driver.py")], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, env=env, start_new_session=True) + reaper_pid, line = None, "" try: p.stdin.write(json.dumps({"op": "setup", "unit": "see", "dir": ws}) + "\n") p.stdin.flush() - reply = json.loads(p.stdout.readline()) + ready, _, _ = select.select([p.stdout], [], [], 180) # the setup's reply, bounded + line = p.stdout.readline() if ready else "" + kids = subprocess.run(["ps", "-o", "pid=,args=", "--ppid", str(p.pid)], capture_output=True, text=True, timeout=30) + reaper_pid = next((int(l.split()[0]) for l in kids.stdout.splitlines() if "reaper.py" in l), None) finally: os.killpg(p.pid, signal.SIGKILL) # as a run kills a driver's group p.wait() - sock = reply["mcp"][0]["args"][0] + self.assertTrue(line, "no reply to setup within 180 s") + sock = json.loads(line)["mcp"][0]["args"][0] self.assertTrue(sock.endswith("/s")) - for _ in range(120): - left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", "label=" + container.LABEL + "=job", - "--filter", "name=nullius-desktop-"], capture_output=True, text=True, timeout=60) - if left.returncode == 0 and not left.stdout.split() and not os.path.exists(os.path.dirname(sock)): + self.assertIsNotNone(reaper_pid) + for _ in range(240): + left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"label={container.LABEL}={job}"], + capture_output=True, text=True, timeout=60) + if not _alive(reaper_pid): break time.sleep(0.5) + self.assertFalse(_alive(reaper_pid), "this driver's reaper did not end") + left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"label={container.LABEL}={job}"], + capture_output=True, text=True, timeout=60) self.assertEqual(left.returncode, 0, left.stderr) self.assertEqual(left.stdout.split(), []) self.assertFalse(os.path.exists(os.path.dirname(sock))) - procs = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True, timeout=30).stdout - self.assertNotIn(os.path.join(KIT, "reaper.py"), procs) + + def test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe(self): + ws, s = self.unit("click") + u = driver.OPEN["click"] + agent = os.path.join(APP, "tests", "fixtures", "standard", "fake_computer_agent.py") + p = subprocess.Popen([sys.executable, agent], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, + start_new_session=True) + server = {"name": "computer", "command": sys.executable, "args": [os.path.join(KIT, "pipe.py"), s["mcp"][0]["args"][0]], + "env": []} + said, lines = [], queue.Queue() + threading.Thread(target=lambda: [lines.put(l) for l in p.stdout] + [lines.put(None)], daemon=True).start() + try: + for m in ({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": 1}}, + {"jsonrpc": "2.0", "id": 2, "method": "session/new", "params": {"cwd": ws, "mcpServers": [server]}}, + {"jsonrpc": "2.0", "id": 3, "method": "session/prompt", + "params": {"sessionId": "s1", "prompt": [{"type": "text", "text": s["prompt"]}]}}): + p.stdin.write(json.dumps(m) + "\n") + p.stdin.flush() + while True: + line = lines.get(timeout=120) # bounded: Empty past it + self.assertIsNotNone(line, f"the agent ended before it answered {m['method']}") + got = json.loads(line) + if got.get("method") == "session/update": + said.append(got["params"]["update"]["content"]["text"]) + if got.get("id") == m["id"]: + break + finally: + p.stdin.close() + try: + p.wait(timeout=30) + except subprocess.TimeoutExpired: + p.kill() + p.wait() + self.assertEqual([c["tool"] for c in u.calls()], ["screenshot", "click"]) + self.assertIn("screenshot taken", "".join(said)) if __name__ == "__main__": diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index f1050e6e..8cd1a1f5 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -81,7 +81,7 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `limit` | basic | Click six buttons with a budget of two steps | A button was pressed, no input came after the last step, and no acting call was sent once the steps were used up | | `drag` | skill | Drag a handle into a marked zone | A left press on the handle, released inside the zone | | `scroll` | skill | Scroll a list to read a code below what it shows at first | The final message holds the code, and the list moved | -| `recover` | skill | Click a named button; the first acting call is answered with an error and not run | An acting call succeeds after the error, and the button is clicked as in `click` | +| `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. @@ -91,6 +91,6 @@ Every check also fails when a pointer or key event came outside an acting call: | The desktop | A container of the kit's image, `desktop.Dockerfile` on Nullius's base image, built the first time a unit needs it: an X display at 1920 by 1080, a window manager, a session bus, the accessibility bus, Python with Tk, `pyautogui` and Pillow. The agent's Python and its actions run as its user; the kit's fixture runs as another, its log out of that user's reach | | The server and the loop | In the kit's driver: the port's server over the unit's desktop, and a loop of the kit's own that runs each step's actions and waits the pause | | What a stop leaves | A unit that never reaches its score is let go by the next unit or by the driver's end; a reaper the driver starts removes the containers and sockets of a driver that was killed | -| The controls | The null control passes no check; the reference makes the right calls through the unit's own server and passes all eight | +| The controls | The null control passes no check; the reference makes the right calls through the unit's own server, reads each code from the screenshot it gets, and passes all eight | A machine that runs the kit needs Docker, or Podman 4.7 or later, as the container backend does. The first unit builds the image (about 0.5 GB over the base image). From c12294f3830baa36e9cae374a528b1cd00549ce4 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 14:49:06 +0100 Subject: [PATCH 28/36] fix(standard): preserve computer kit admission and shutdown ordering Admit acting calls under the bridge lock and keep their arrival time through validation, so delayed checks cannot change budget or recovery evidence. Reject the first recovery call before terminal tools can end the task. Drain engine clients even after scoring and serialize their launch with shutdown. Keep unsuccessful desktop starts pending so the reaper can remove containers created after a timeout. Derive Docker test deadlines from supported operation limits and require successful click and prompt replies. Add deterministic regressions for admission, recovery, shutdown, late container creation and the test harness. --- .../kits/agent-computer-v1/desktop.py | 38 +- .../standard/kits/agent-computer-v1/driver.py | 90 +++-- .../standard/kits/agent-computer-v1/reaper.py | 10 +- app/tests/test_computer_kit_driver.py | 353 +++++++++++++++++- 4 files changed, 425 insertions(+), 66 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/desktop.py b/app/nullius/standard/kits/agent-computer-v1/desktop.py index 814814e9..f0c75daf 100644 --- a/app/nullius/standard/kits/agent-computer-v1/desktop.py +++ b/app/nullius/standard/kits/agent-computer-v1/desktop.py @@ -57,6 +57,11 @@ FIXTURE_LOG = "/var/lib/fixture/events.jsonl" PYTHON_SECONDS = 90 START_SECONDS = 60 +RUN_SECONDS = 120 +ENGINE_GRACE = 30 +FIXTURE_SECONDS = 60 +CAPTURE_SECONDS = 60 +EVENTS_SECONDS = 60 OUTPUT_LIMIT = 4 * 1024 * 1024 # of each of output and error, as the reply gives them READ_LIMIT = 12 * 1024 * 1024 # the largest file `read` gives SETTLE = 0.5 # an action answers this long after its end: at least the verdicts' slack @@ -146,17 +151,20 @@ def tag() -> str: _CLIENTS: set = set() # the engine's clients running now, for end_clients _CLIENTS_LOCK = threading.Lock() +_CLIENTS_CLOSED = False def _engine(args: List[str], *, input: Optional[bytes] = None, timeout: float) -> subprocess.CompletedProcess: """One call of the engine's program, its stdin the request's bytes or nothing, killed past ``timeout`` (``subprocess.TimeoutExpired``, as ``subprocess.run`` raises it).""" - try: - p = subprocess.Popen([container.DOCKER, *args], stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=container.client_env()) - except FileNotFoundError: - raise DesktopError(f"{container.DOCKER} is not installed on this machine") with _CLIENTS_LOCK: + if _CLIENTS_CLOSED: + raise DesktopError("the desktop engine clients have been closed") + try: + p = subprocess.Popen([container.DOCKER, *args], stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=container.client_env()) + except FileNotFoundError: + raise DesktopError(f"{container.DOCKER} is not installed on this machine") _CLIENTS.add(p) try: try: @@ -171,10 +179,12 @@ def _engine(args: List[str], *, input: Optional[bytes] = None, timeout: float) - return subprocess.CompletedProcess([container.DOCKER, *args], p.returncode, out, err) -def end_clients() -> int: +def end_clients(*, closing: bool = True) -> int: """Every client of the engine still running, from any thread, killed and waited for; how many there were. - A process that ends while a request of its waits leaves none behind.""" + A shutdown also prevents further launches; between units, only the current clients are ended.""" + global _CLIENTS_CLOSED with _CLIENTS_LOCK: + _CLIENTS_CLOSED = _CLIENTS_CLOSED or closing running = list(_CLIENTS) for p in running: try: @@ -211,13 +221,13 @@ def start(cls, image: str, name: Optional[str] = None, *, job: Optional[str] = N try: try: r = _engine(["run", "-d", "--init", "--name", d.name, "--network", "none", "--security-opt", - "no-new-privileges", "--shm-size", "256m", *label, *container.capped(), image], timeout=120) + "no-new-privileges", "--shm-size", "256m", *label, *container.capped(), image], timeout=RUN_SECONDS) except subprocess.TimeoutExpired: - raise DesktopError(f"starting the desktop {d.name}: {container.DOCKER} run did not answer in 120 s") + raise DesktopError(f"starting the desktop {d.name}: {container.DOCKER} run did not answer in {RUN_SECONDS:g} s") if r.returncode != 0: raise DesktopError(f"starting the desktop {d.name}: {_said(r)}") try: - r = _engine(["exec", d.name, "sh", "-c", _READY.format(n=int(seconds * 10))], timeout=seconds + 30) + r = _engine(["exec", d.name, "sh", "-c", _READY.format(n=int(seconds * 10))], timeout=seconds + ENGINE_GRACE) except subprocess.TimeoutExpired: r = None if r is None or r.returncode != 0: @@ -246,7 +256,7 @@ def _run(self, code: str, seconds: float, settle: float = 0.0) -> Optional[dict] The reply comes ``settle`` seconds after the code's end, on the desktop's clock.""" req = json.dumps({"code": code, "seconds": seconds, "limit": OUTPUT_LIMIT, "settle": settle}).encode() try: - r = self._exec(ROOT, ["python3", "-I", "-c", _RUN], input=req, timeout=seconds + settle + 30) + r = self._exec(ROOT, ["python3", "-I", "-c", _RUN], input=req, timeout=seconds + settle + ENGINE_GRACE) except subprocess.TimeoutExpired: return None try: @@ -302,7 +312,7 @@ def read(self, path: str) -> Optional[bytes]: def capture(self) -> bytes: """The screen, as PNG bytes.""" - r = self._exec(ROOT, ["python3", "-I", "-c", _CAPTURE], timeout=60) + r = self._exec(ROOT, ["python3", "-I", "-c", _CAPTURE], timeout=CAPTURE_SECONDS) if r.returncode != 0 or not r.stdout.startswith(b"\x89PNG\r\n\x1a\n"): raise DesktopError(f"the capture: {_said(r)}") return r.stdout @@ -310,7 +320,7 @@ def capture(self) -> bytes: def password(self) -> str: return "" - def start_fixture(self, values: dict, seconds: float = 60) -> None: + def start_fixture(self, values: dict, seconds: float = FIXTURE_SECONDS) -> None: """Start the fixture as ``fixture``, ``values`` on its stdin; returns once its window is up.""" r = self._exec(FIXTURE, ["python3", "-I", "/opt/desktop/fixture.py"], input=json.dumps(values).encode(), timeout=seconds, env={"DISPLAY": ":0", "HOME": "/var/lib/fixture"}) @@ -320,7 +330,7 @@ def start_fixture(self, values: dict, seconds: float = 60) -> None: def events(self) -> List[dict]: """The fixture's log, an event a dict. A log that cannot be read, or a line that is not an event, is a :class:`DesktopError`: a verdict is never given on part of the log.""" - r = self._exec(FIXTURE, ["cat", FIXTURE_LOG], timeout=60) + r = self._exec(FIXTURE, ["cat", FIXTURE_LOG], timeout=EVENTS_SECONDS) if r.returncode != 0: raise DesktopError(f"the fixture's log could not be read: {_said(r)}") out = [] diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index 3f713b76..e14cfe90 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -142,36 +142,67 @@ def password(self): class KitBridge(port.Bridge): - """The port's bridge, counting the acting calls sent once every step was taken (``past``). A call that holds - is counted and its step taken in one hold of the bridge's lock (a reentrant one): two calls sent together - for the last step are one taken and one counted.""" + """The port's bridge, with the kit's budget count and first acting call's recover error.""" def __init__(self, *a, **kw): super().__init__(*a, **kw) self.past = 0 + self.reject_first = False - def submit(self, call): + def _next(self): + if not self.reject_first: + return super()._next() with self.change: - if call.tool in port.ACTING and self.booked >= self.steps: - self.past += 1 - super().submit(call) + while not self.calls and self.ended is None: + self.change.wait(0.5) + if not self.calls or self.calls[0].tool not in port.ACTING: + return super()._next() + call = self.calls.popleft() + self.reject_first = False + self.used += 1 + call.answer(error=NOT_RUN) # before done or fail can end the task + self._ran(call, False) + return None # this step ran no action; the next predict goes on class KitServer(port.Server): - """The port's server, counting also an acting call sent once every step was taken whose arguments do not hold: - the server refuses it before the bridge sees it.""" + """Count, timestamp, check and reserve in one admission, including calls whose arguments do not hold.""" def call(self, rid, params, wire): - name = params.get("name") if isinstance(params, dict) else None - if name in port.ACTING: - probe = port.Call(name, params.get("arguments", {}), lambda *a: None) + start = time.time() + name = params.get("name") + if name not in port.ACTING: + return super().call(rid, params, wire) + + def reply(call, value, image, error): + content = [] + if error is not None: + content.append({"type": "text", "text": str(error)}) + if value is not None: + content.append({"type": "text", "text": json.dumps(value)}) + if image is not None: + content.append({"type": "image", "data": base64.b64encode(image).decode(), "mimeType": "image/png"}) + data, over = port.bounded({"jsonrpc": "2.0", "id": rid, "result": {"content": content, + "isError": error is not None}}) + self.record({"tool": call.tool, "start": call.start, "end": time.time(), "isError": error is not None or over}) + wire.send(data) + wire.answered() + + with self.bridge.change: + wire.opened() + call = port.Call(name, params.get("arguments", {}), reply) + call.start = start # includes time waiting for admission's lock + if self.bridge.booked >= self.bridge.steps: + self.bridge.past += 1 try: - port.checked(probe, self.bridge.screen) - except Exception: - with self.bridge.change: - if self.bridge.booked >= self.bridge.steps: - self.bridge.past += 1 - super().call(rid, params, wire) + port.checked(call, self.bridge.screen) + except port.Refused as e: + call.answer(error=str(e)) + return + except Exception as e: + call.answer(error=f"the arguments could not be checked: {type(e).__name__}") + return + self.bridge.submit(call) class Unit: @@ -183,7 +214,7 @@ def __init__(self, unit, dir): self.steps = checks.steps(unit) self.log = os.path.join(self.trial, "verifier", "computer-calls.jsonl") self.desktop = self.bridge = self.server = self.loop = self.folder = self.boxes = None - self.final, self.error, self.failed_one = "", None, False + self.final, self.error = "", None def start(self, job): os.makedirs(os.path.dirname(self.log), exist_ok=True) @@ -192,10 +223,8 @@ def start(self, job): image = desktop.image(os.path.dirname(self.log)) name = f"nullius-desktop-{os.urandom(6).hex()}" REAPER.tell(f"+container {name}") # before it exists: a kill during the start is covered - try: - self.desktop = desktop.Desktop.start(image, name, job=job) - finally: - REAPER.tell(f"=container {name}") # its start has ended, either way + self.desktop = desktop.Desktop.start(image, name, job=job) + REAPER.tell(f"=container {name}") # only a successful start rules out late creation self.desktop.start_fixture(self.values) self.boxes = self.desktop.events()[0].get("boxes") or {} folder = os.path.join(tempfile.gettempdir(), f"nullius-computer-{os.urandom(6).hex()}") @@ -215,16 +244,10 @@ def run_loop(self): loop's. An action the desktop did not end in time leaves its call answered with an error, and the next step goes on. ``recover``: the first acting call the loop takes, whatever it is, is answered with an error before any part of it runs.""" + self.bridge.reject_first = self.unit == "recover" try: for _ in range(self.steps): response, actions = self.bridge.predict("", None) - call = self.bridge.running # the acting call of these actions; None for done, fail or none - if call is not None and self.unit == "recover" and not self.failed_one: - self.failed_one = True - call.answer(error=NOT_RUN) - for _ in actions: # none of it run; the bridge still counts the step - pass - continue actions = iter(actions) stop = False try: @@ -302,7 +325,7 @@ def close(self): OPEN_LOCK = threading.Lock() -def close_all(): +def close_all(*, shutdown=True): """Every unit still open let go, then every client of the engine still running (a loop's call that hangs) ended and waited for.""" with OPEN_LOCK: @@ -313,8 +336,7 @@ def close_all(): u.close() except Exception as e: sys.stderr.write(f"computer kit: {u.unit} could not be let go: {e}\n") - if held: - desktop.end_clients() + desktop.end_clients(closing=shutdown) def job_of(dir): @@ -329,7 +351,7 @@ def units(dataset, **_): def setup(unit, dir, **_): if unit not in UNITS: raise ValueError(f"no unit {unit!r}") - close_all() # one unit at a time in a driver: the one before never reached score + close_all(shutdown=False) # one unit at a time: the one before never reached score u = Unit(unit, dir) with OPEN_LOCK: OPEN[unit] = u diff --git a/app/nullius/standard/kits/agent-computer-v1/reaper.py b/app/nullius/standard/kits/agent-computer-v1/reaper.py index 21f10363..669c079c 100644 --- a/app/nullius/standard/kits/agent-computer-v1/reaper.py +++ b/app/nullius/standard/kits/agent-computer-v1/reaper.py @@ -5,12 +5,12 @@ The kit's driver starts it at its first unit, in a session of its own (a run that kills the driver's process group does not kill it), and keeps its stdin open. Each line names what the driver is making, made or let go: -- ``+container NAME`` before the container's start, ``=container NAME`` once that start has ended (the - container is there, or the start failed and its removal was asked), ``-container NAME`` once it is removed; +- ``+container NAME`` before the container's start, ``=container NAME`` once that start has succeeded, + ``-container NAME`` once it is removed; - ``+folder PATH`` before the folder is made, ``-folder PATH`` once it is removed. Once its stdin ends (the driver ended, closed it, or was killed), it removes each container still named, with -ENGINE (``docker`` or ``podman``), and each folder still named, then exits. A container whose start had not ended +ENGINE (``docker`` or ``podman``), and each folder still named, then exits. A container whose start had not succeeded may still be created by the engine after the driver's death: the reaper looks for it again until it has removed it or START_SECONDS (60 by default) have passed. Standard library only. """ @@ -25,7 +25,7 @@ def held(lines): """What the lines leave named: {"container": [...], "folder": [...], "started": [...]}, in the order made; - ``started`` holds the containers whose start had ended.""" + ``started`` holds the containers whose start had succeeded.""" out = {"container": [], "folder": [], "started": []} for line in lines: sign, _, what = line.rstrip("\n").partition(" ") @@ -61,7 +61,7 @@ def there(engine, name): def remove(engine, name, pending, seconds): - """The container removed. One whose start had not ended is looked for until it is gone after being there, + """The container removed. One whose start had not succeeded is looked for until it is gone after being there, or ``seconds`` have passed.""" end = time.monotonic() + seconds seen = False diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index dfb1329c..481e1cbb 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -5,6 +5,8 @@ from __future__ import annotations import importlib.util +import inspect +import io import json import os import queue @@ -23,7 +25,7 @@ import zlib from unittest import mock -from nullius.standard import computer_port as port +from nullius.standard import computer_port as port, computer_pipe from nullius.standard import container, definitions, ports from tests import docker_tier @@ -253,6 +255,7 @@ def draw(check, rng=None): self.enterContext(mock.patch.object(driver.checks, "draw", side_effect=draw)) self.enterContext(mock.patch.object(driver.REAPER, "tell", side_effect=self.told.append)) self.enterContext(mock.patch.object(driver, "PAUSE", 0.01)) + self.enterContext(mock.patch.object(desktop, "_CLIENTS_CLOSED", False)) self.addCleanup(driver.close_all) def unit(self, name): @@ -384,6 +387,117 @@ def send(server, tool, arguments=None, rid=1): return wire +class Admission(unittest.TestCase): + def unit(self, name, steps=2): + u = driver.Unit(name, "/unused/workspace") + u.desktop = FakeDesktop("fake", {"labels": ["OCEAN"]}) + u.expected = {"target": "OCEAN", "labels": ["OCEAN"]} + u.steps = steps + u.bridge = driver.KitBridge(driver.KitDesktop(u.desktop), steps, 0, desktop.SCREEN, 1) + s = driver.KitServer.__new__(driver.KitServer) + s._state(u.bridge, "/unused/socket", None) + calls = [] + s.record = calls.append + self.addCleanup(u.bridge.finish) + return u, s, calls + + def test_an_invalid_call_admitted_before_exhaustion_is_not_counted_past_the_budget(self): + u, s, _ = self.unit("limit") + send(s, "click", {"x": 150, "y": 150}) + entered, release = threading.Event(), threading.Event() + original, seen, wires = port.checked, [], [] + + def checking(call, screen): + if call.args.get("x") == "bad": + seen.append(call) + entered.set() + if not release.wait(5): + raise RuntimeError("validation was not released") + return original(call, screen) + + t = threading.Thread(target=lambda: wires.append(send(s, "click", {"x": "bad", "y": 150})), daemon=True) + with mock.patch.object(port, "checked", side_effect=checking): + t.start() + try: + self.assertTrue(entered.wait(5)) + available = u.bridge.change.acquire(blocking=False) + if available: # an unlocked validator lets the other connection take the step + try: + send(s, "click", {"x": 150, "y": 150}) + finally: + u.bridge.change.release() + finally: + release.set() + t.join(5) + self.assertFalse(t.is_alive()) + if not available: + send(s, "click", {"x": 150, "y": 150}) + self.assertTrue(wires[0].reply["result"]["isError"]) + self.assertEqual((u.bridge.booked, u.bridge.past), (2, 0)) + self.assertEqual(len(seen), 1) # the recorded call itself is checked, once + + def test_validation_cannot_move_a_calls_start_past_the_recovery_error(self): + u, s, calls = self.unit("recover") + original, checked, wires = port.checked, [], [] + entered, release = threading.Event(), threading.Event() + with mock.patch.object(port.time, "time", return_value=1.0) as clock: + first = send(s, "click", {"x": 150, "y": 150}) + _, actions = u.bridge.predict("", None) + + def checking(call, screen): + checked.append(call) + entered.set() + if not release.wait(5): + raise RuntimeError("validation was not released") + return original(call, screen) + + t = threading.Thread(target=lambda: wires.append(send(s, "click", {"x": 150, "y": 150})), daemon=True) + with mock.patch.object(port, "checked", side_effect=checking): + t.start() + try: + self.assertTrue(entered.wait(5)) + self.assertFalse(first.ready.is_set()) + clock.return_value = 2.0 + u.bridge.running.answer(error=driver.NOT_RUN) + finally: + release.set() + t.join(5) + self.assertFalse(t.is_alive()) + list(actions) + _, actions = u.bridge.predict("", None) + for action in actions: + u.desktop.act(action) + self.assertFalse(wires[0].reply["result"]["isError"]) + self.assertLess(calls[1]["start"], calls[0]["end"]) + self.assertEqual(len(checked), 1) + got = checks.recover(u.expected, u.desktop.events(), u.desktop.actions, "done", calls) + self.assertFalse(got["passed"]) + self.assertIn("after the error's answer", got["why"]) + + def test_recovery_refuses_done_and_fail_before_their_terminal_transition(self): + for name in ("done", "fail"): + with self.subTest(name=name): + u, s, calls = self.unit("recover") + first = send(s, name) + t = threading.Thread(target=u.run_loop, daemon=True) + with mock.patch.object(driver, "PAUSE", 0): + t.start() + try: + self.assertTrue(first.ready.wait(5)) + self.assertTrue(first.reply["result"]["isError"]) + self.assertIn(driver.NOT_RUN, first.reply["result"]["content"][0]["text"]) + self.assertIsNone(u.bridge.ended) + second = send(s, "click", {"x": 150, "y": 150}) + self.assertTrue(second.ready.wait(5)) + self.assertFalse(second.reply["result"]["isError"]) + finally: + u.bridge.finish() + t.join(5) + self.assertFalse(t.is_alive()) + self.assertEqual((u.bridge.used, len(u.desktop.actions)), (2, 1)) + self.assertTrue(checks.recover(u.expected, u.desktop.events(), u.desktop.actions, "done", calls)["passed"]) + + class EarnedPasses(FakeUnits): """What a pass needs, over the fake desktop: every attempt past the budget counted, recover's call sent after its error's answer, its error on the first acting call whatever it is, an action not ended in time an error, @@ -490,6 +604,75 @@ def broken(): class Shutdown(unittest.TestCase): + def setUp(self): + self.enterContext(mock.patch.object(desktop, "_CLIENTS_CLOSED", False, create=True)) + self.enterContext(mock.patch.object(desktop, "_CLIENTS", set())) + + def test_shutdown_ends_clients_after_every_unit_was_scored(self): + p = mock.Mock() + with mock.patch.dict(driver.OPEN, {}, clear=True), mock.patch.object(desktop, "_CLIENTS", {p}): + driver.close_all() + p.kill.assert_called_once_with() + p.wait.assert_called_once_with(timeout=10) + + def test_shutdown_waits_for_launch_registration_and_refuses_later_launches(self): + launched, release, draining, killed = (threading.Event() for _ in range(4)) + p = mock.Mock(returncode=-9) + p.kill.side_effect = killed.set + + def communicate(*a, **kw): + if not killed.wait(5): + raise RuntimeError("client was not ended") + return b"", b"" + + def popen(*a, **kw): + launched.set() # the child exists before Popen returns + if not release.wait(5): + raise RuntimeError("launch was not released") + return p + + lock = threading.Lock() + + class Lock: + def __enter__(self): + if threading.current_thread() is drain: + draining.set() + if not lock.acquire(timeout=5): + raise RuntimeError("client lock was not released") + + def __exit__(self, *a): + lock.release() + + p.communicate.side_effect = communicate + replies, counts = [], [] + launch = threading.Thread(target=lambda: replies.append(desktop._engine(["exec"], timeout=10)), daemon=True) + drain = threading.Thread(target=lambda: counts.append(desktop.end_clients()), daemon=True) + with mock.patch.object(desktop, "_CLIENTS_LOCK", Lock()), \ + mock.patch.object(desktop.subprocess, "Popen", side_effect=popen) as start: + launch.start() + try: + self.assertTrue(launched.wait(5)) + drain.start() + self.assertTrue(draining.wait(5)) + release.set() + drain.join(5) + self.assertFalse(drain.is_alive()) + self.assertEqual(counts, [1]) + p.kill.assert_called_once_with() + p.wait.assert_called_once_with(timeout=10) + with self.assertRaises(desktop.DesktopError): + desktop._engine(["exec"], timeout=10) + start.assert_called_once() + finally: + release.set() + killed.set() + launch.join(5) + if drain.ident is not None: + drain.join(5) + self.assertFalse(launch.is_alive()) + self.assertEqual(len(replies), 1) + self.assertEqual(desktop._CLIENTS, set()) + def test_the_engines_clients_still_running_are_ended_and_waited_for(self): done = [] with mock.patch.object(desktop.container, "DOCKER", sys.executable): @@ -590,6 +773,40 @@ def get(self): class Reaper(unittest.TestCase): + def test_unsuccessful_starts_remain_pending_through_an_absent_container(self): + for outcome in ("timeout", "failed"): + with self.subTest(outcome=outcome): + told, engine_calls = [], [] + + def engine(args, **kw): + engine_calls.append(args[0]) + if args[0] == "run": + if outcome == "timeout": + raise subprocess.TimeoutExpired(args, kw["timeout"]) + return subprocess.CompletedProcess(args, 1, b"", b"start failed") + return subprocess.CompletedProcess(args, 1, b"", b"Error: No such container: late") + + u = driver.Unit("see", "/unused/workspace") + with mock.patch.object(driver.os, "makedirs"), \ + mock.patch.object(driver.os.path, "exists", return_value=False), \ + mock.patch.object(desktop, "image", return_value="image"), \ + mock.patch.object(desktop, "_engine", side_effect=engine), \ + mock.patch.object(desktop.container, "capped", return_value=[]), \ + mock.patch.object(driver.REAPER, "tell", side_effect=told.append): + with self.assertRaises(desktop.DesktopError): + u.start("job") + self.assertEqual(engine_calls, ["run", "rm", "container"]) + held = reaper.held(told) + name = held["container"][0] + with mock.patch.object(reaper, "there", side_effect=[False, True, False]) as there, \ + mock.patch.object(reaper, "_engine") as remove, \ + mock.patch.object(reaper.time, "monotonic", return_value=0), \ + mock.patch.object(reaper.time, "sleep"): + reaper.remove("docker", name, name not in held["started"], reaper.START_SECONDS) + self.assertEqual(there.call_count, 3) + remove.assert_called_once_with("docker", "rm", "-f", name) + self.assertNotIn(name, held["started"]) + def test_a_container_whose_start_had_not_ended_is_looked_for_until_it_is_there_and_removed(self): with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: engine, said, state = os.path.join(tmp, "engine"), os.path.join(tmp, "said"), os.path.join(tmp, "n") @@ -657,11 +874,106 @@ def test_the_driver_tells_the_reaper_before_the_container_exists(self): driver.setup("see", ws) self.assertTrue(order[0].startswith("+container nullius-desktop-")) self.assertEqual(order[1], "start") - self.assertEqual(order[2], "=container " + order[0].split()[1]) # its start ended, failed + self.assertEqual(len(order), 2) # an unsuccessful start stays pending self.assertEqual(driver.OPEN, {}) # -- the docker tier: the eight checks on real desktops -------------------------------------------------------------- +MARGIN = 30 # process starts, protocol messages and scheduling + + +class DockerTestBounds(unittest.TestCase): + """The docker tests' waits and assertions exercised without a container or a socket.""" + + def supported(self, setup=False): + limits = [] + + def engine(args, **kw): + limits.append(kw["timeout"]) + data = b"" + if desktop._CAPTURE in args: + data = b"\x89PNG\r\n\x1a\nfake" + elif desktop._RUN in args: + data = json.dumps({"start": 1, "end": 2, "timeout": False, "returncode": 0, + "output": "", "error": ""}).encode() + return subprocess.CompletedProcess(args, 0, data, b"") + + with mock.patch.object(desktop, "_engine", side_effect=engine), \ + mock.patch.object(container, "capped", return_value=[]): + if setup: + default = inspect.signature(container._docker).parameters["timeout"].default + + def image_engine(args, *, timeout=default, **kw): + limits.append(timeout) + return subprocess.CompletedProcess(args, 0, "", "") + + with mock.patch.object(container, "_built", {}), \ + mock.patch.object(container, "_docker", side_effect=image_engine): + desktop.image() # a fresh driver inspects both already-built images + d = desktop.Desktop.start("image", "fake") + d.start_fixture({}) + d.events() + else: + d = desktop.Desktop("fake") + d.capture() + d.act("pass", driver.PYTHON_SECONDS) + d.capture() + return sum(limits) + (0 if setup else driver.PAUSE + computer_pipe.END_SECONDS) + + def test_the_setup_deadline_covers_start_readiness_fixture_and_events(self): + supported = self.supported(setup=True) + case = DockerKit("test_a_driver_killed_during_a_unit_leaves_no_container_socket_or_process") + with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: + case.tmp = tmp + p = mock.Mock(pid=123, stdin=io.StringIO(), stdout=io.StringIO(json.dumps( + {"mcp": [{"args": [os.path.join(tmp, "socket", "s")]}]}) + "\n")) + + def select_reply(read, write, error, seconds): + self.assertGreater(seconds, supported) + return read, [], [] + + def run(args, **kw): + return subprocess.CompletedProcess(args, 0, "456 python reaper.py\n" if args[0] == "ps" else "", "") + + with mock.patch.object(subprocess, "Popen", return_value=p), \ + mock.patch.object(subprocess, "run", side_effect=run), \ + mock.patch.object(select, "select", side_effect=select_reply), \ + mock.patch.object(os, "killpg"), mock.patch(__name__ + "._alive", return_value=False): + case.test_a_driver_killed_during_a_unit_leaves_no_container_socket_or_process() + + def scripted(self, *, click_error=False, prompt=None, check_timeout=None): + case = DockerKit("test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe") + setup = {"mcp": [{"args": ["/unused/s"]}], "prompt": "click"} + case.unit = mock.Mock(return_value=("/unused/workspace", setup)) + u = mock.Mock() + u.calls.return_value = [{"tool": "screenshot", "isError": False}, {"tool": "click", "isError": click_error}] + messages = [{"id": 1, "result": {}}, {"id": 2, "result": {"sessionId": "s1"}}, + {"method": "session/update", "params": {"update": {"content": {"text": "screenshot taken"}}}}, + {"id": 3, **(prompt if prompt is not None else {"result": {"stopReason": "end_turn"}})}] + p = mock.Mock(stdin=io.StringIO(), stdout=io.StringIO("".join(json.dumps(m) + "\n" for m in messages))) + original = queue.Queue.get + + def get(q, *, timeout): + if check_timeout is not None: + check_timeout(timeout) + return original(q, timeout=timeout) + + with mock.patch.dict(driver.OPEN, {"click": u}, clear=True), \ + mock.patch.object(subprocess, "Popen", return_value=p), \ + mock.patch.object(queue.Queue, "get", get): + case.test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe() + + def test_the_prompt_deadline_covers_both_captures_the_action_and_pipe_exit(self): + supported = self.supported() + self.scripted(check_timeout=lambda seconds: self.assertGreater(seconds, supported)) + + def test_the_scripted_session_rejects_a_failed_click_or_prompt(self): + for bad in ({"click_error": True}, {"prompt": {"error": {"code": -32603, "message": "failed"}}}, + {"prompt": {"result": {}}}): + with self.subTest(bad=bad), self.assertRaises(AssertionError): + self.scripted(**bad) + + def why_no_desktop() -> str: if not os.environ.get(docker_tier.VAR): return "the docker tier runs with parallel.py --docker" @@ -679,6 +991,7 @@ def setUpClass(cls): def setUp(self): self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) self.addCleanup(shutil.rmtree, self.tmp, True) + self.enterContext(mock.patch.object(desktop, "_CLIENTS_CLOSED", False)) self.addCleanup(driver.close_all) def unit(self, name): @@ -730,20 +1043,22 @@ def test_a_driver_killed_during_a_unit_leaves_no_container_socket_or_process(sel try: p.stdin.write(json.dumps({"op": "setup", "unit": "see", "dir": ws}) + "\n") p.stdin.flush() - ready, _, _ = select.select([p.stdout], [], [], 180) # the setup's reply, bounded + image_seconds = 2 * inspect.signature(container._docker).parameters["timeout"].default + seconds = (image_seconds + desktop.RUN_SECONDS + desktop.START_SECONDS + desktop.ENGINE_GRACE + + desktop.FIXTURE_SECONDS + desktop.EVENTS_SECONDS + MARGIN) + ready, _, _ = select.select([p.stdout], [], [], seconds) line = p.stdout.readline() if ready else "" kids = subprocess.run(["ps", "-o", "pid=,args=", "--ppid", str(p.pid)], capture_output=True, text=True, timeout=30) reaper_pid = next((int(l.split()[0]) for l in kids.stdout.splitlines() if "reaper.py" in l), None) finally: os.killpg(p.pid, signal.SIGKILL) # as a run kills a driver's group - p.wait() - self.assertTrue(line, "no reply to setup within 180 s") + p.wait(timeout=MARGIN) + self.assertTrue(line, f"no reply to setup within {seconds:g} s") sock = json.loads(line)["mcp"][0]["args"][0] self.assertTrue(sock.endswith("/s")) self.assertIsNotNone(reaper_pid) - for _ in range(240): - left = subprocess.run([container.DOCKER, "ps", "-aq", "--filter", f"label={container.LABEL}={job}"], - capture_output=True, text=True, timeout=60) + end = time.monotonic() + reaper.START_SECONDS + 2 * reaper.REMOVE_SECONDS + MARGIN + while time.monotonic() < end: if not _alive(reaper_pid): break time.sleep(0.5) @@ -763,7 +1078,8 @@ def test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe server = {"name": "computer", "command": sys.executable, "args": [os.path.join(KIT, "pipe.py"), s["mcp"][0]["args"][0]], "env": []} said, lines = [], queue.Queue() - threading.Thread(target=lambda: [lines.put(l) for l in p.stdout] + [lines.put(None)], daemon=True).start() + reader = threading.Thread(target=lambda: [lines.put(l) for l in p.stdout] + [lines.put(None)], daemon=True) + reader.start() try: for m in ({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": 1}}, {"jsonrpc": "2.0", "id": 2, "method": "session/new", "params": {"cwd": ws, "mcpServers": [server]}}, @@ -771,22 +1087,33 @@ def test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe "params": {"sessionId": "s1", "prompt": [{"type": "text", "text": s["prompt"]}]}}): p.stdin.write(json.dumps(m) + "\n") p.stdin.flush() + seconds = (2 * desktop.CAPTURE_SECONDS + driver.PYTHON_SECONDS + desktop.SETTLE + + desktop.ENGINE_GRACE + driver.PAUSE + computer_pipe.END_SECONDS + MARGIN) + end = time.monotonic() + seconds while True: - line = lines.get(timeout=120) # bounded: Empty past it + line = lines.get(timeout=max(0, end - time.monotonic())) self.assertIsNotNone(line, f"the agent ended before it answered {m['method']}") got = json.loads(line) if got.get("method") == "session/update": said.append(got["params"]["update"]["content"]["text"]) if got.get("id") == m["id"]: + self.assertNotIn("error", got) + if m["method"] == "session/prompt": + self.assertEqual(got.get("result"), {"stopReason": "end_turn"}) break finally: p.stdin.close() try: p.wait(timeout=30) except subprocess.TimeoutExpired: - p.kill() - p.wait() - self.assertEqual([c["tool"] for c in u.calls()], ["screenshot", "click"]) + os.killpg(p.pid, signal.SIGKILL) # also a pipe still waiting for a desktop reply + p.wait(timeout=MARGIN) + reader.join(MARGIN) + p.stdout.close() + self.assertFalse(reader.is_alive()) + calls = u.calls() + self.assertEqual([c["tool"] for c in calls], ["screenshot", "click"]) + self.assertTrue(all(not c["isError"] for c in calls), calls) self.assertIn("screenshot taken", "".join(said)) From 003d2fde5b5e5166a6ed6e395eeb35499e052f5b Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 14:55:29 +0100 Subject: [PATCH 29/36] feat(standard): keep the computer kit's fixture events and target boxes in the unit's record A failed check could not be read once its desktop was gone: the unit's record kept the verdict's counts, not what the fixture saw. At scoring the driver now writes the fixture's events (computer-events.jsonl) and where its targets were drawn (computer-boxes.json) beside the call log, in the unit's verifier folder. --- .../standard/kits/agent-computer-v1/driver.py | 10 ++++++++++ app/tests/test_computer_kit_driver.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index e14cfe90..2555468c 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -285,6 +285,15 @@ def end(self, wait=LOOP_SECONDS): self.server.close() self.server = None + def keep(self, events): + """A copy of the fixture's events and of where its targets were drawn, beside the call log: what a reader + of a failed check needs once the desktop is gone.""" + folder = os.path.dirname(self.log) + with open(os.path.join(folder, "computer-events.jsonl"), "w", encoding="utf-8") as f: + f.writelines(json.dumps(e) + "\n" for e in events) + with open(os.path.join(folder, "computer-boxes.json"), "w", encoding="utf-8") as f: + json.dump(self.boxes or {}, f) + def calls(self): out = [] try: @@ -376,6 +385,7 @@ def score(unit, dir, transcript, answer, **_): events = u.desktop.events() actions = list(u.desktop.actions) calls = u.calls() + u.keep(events) finally: u.remove() message = "\n".join(x for x in (answer or "", u.final) if x) diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 481e1cbb..7a3c8aa8 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -285,6 +285,21 @@ def test_a_unit_is_served_scored_and_let_go(self): self.assertEqual([l.split()[0] for l in self.told], ["+container", "=container", "+folder", "-container", "-folder"]) self.assertNotIn("click", driver.OPEN) + def test_the_units_record_keeps_the_fixtures_events_and_where_its_targets_were(self): + ws, s = self.unit("click") + u = driver.OPEN["click"] + c = driver.Client(s["mcp"][0]["args"][0]) + c.call("click", dict(zip("xy", driver.centre(u.boxes[u.expected["target"]])))) + c.close() + driver.score("click", ws, [], "done") + verifier = os.path.join(os.path.dirname(ws), "verifier") + with open(os.path.join(verifier, "computer-events.jsonl")) as f: + events = [json.loads(line) for line in f] + self.assertEqual(events, FakeDesktop.made[-1].events()) + self.assertEqual([e["hit"] for e in events if e["kind"] == "press"], [u.expected["target"]]) + with open(os.path.join(verifier, "computer-boxes.json")) as f: + self.assertEqual(json.load(f), u.boxes) + def test_the_turns_end_at_score_ends_the_loop_and_judges_what_is_there(self): ws, s = self.unit("click") u = driver.OPEN["click"] From 68ceb155bcbb667570ba54bad84ca76a7b4365cb Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 15:15:56 +0100 Subject: [PATCH 30/36] fix(standard): preserve computer scores when record copies fail Write the event and box records through new files in the verifier folder before replacing their destination entries. Ignore only operating-system errors from these optional copies while retaining scoring and cleanup errors. Cover write and close failures and replacement of symlinks and FIFOs with bounded scoring tests. --- .../standard/kits/agent-computer-v1/driver.py | 25 +++- app/tests/test_computer_kit_driver.py | 117 ++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index 2555468c..f3c16597 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -19,6 +19,7 @@ """ import atexit import base64 +import contextlib import importlib.util import json import os @@ -289,10 +290,21 @@ def keep(self, events): """A copy of the fixture's events and of where its targets were drawn, beside the call log: what a reader of a failed check needs once the desktop is gone.""" folder = os.path.dirname(self.log) - with open(os.path.join(folder, "computer-events.jsonl"), "w", encoding="utf-8") as f: - f.writelines(json.dumps(e) + "\n" for e in events) - with open(os.path.join(folder, "computer-boxes.json"), "w", encoding="utf-8") as f: - json.dump(self.boxes or {}, f) + for name, lines in (("computer-events.jsonl", (json.dumps(e) + "\n" for e in events)), + ("computer-boxes.json", json.JSONEncoder().iterencode(self.boxes or {}))): + fd = part = None + try: + fd, part = tempfile.mkstemp(prefix=".computer-", dir=folder) + f, fd = os.fdopen(fd, "w", encoding="utf-8"), None + with f: + f.writelines(lines) + os.replace(part, os.path.join(folder, name)) + finally: + if fd is not None: + os.close(fd) + if part is not None: + with contextlib.suppress(OSError): + os.unlink(part) def calls(self): out = [] @@ -385,7 +397,10 @@ def score(unit, dir, transcript, answer, **_): events = u.desktop.events() actions = list(u.desktop.actions) calls = u.calls() - u.keep(events) + try: + u.keep(events) + except OSError: # the optional copies cannot change the verdict + pass finally: u.remove() message = "\n".join(x for x in (answer or "", u.final) if x) diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 7a3c8aa8..76ed571e 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -15,6 +15,7 @@ import shutil import select import signal +import stat import struct import subprocess import sys @@ -233,6 +234,122 @@ def remove(self): self.removed = True +class ScoreRecords(unittest.TestCase): + """The optional copies at score, over a desktop with no socket or running loop.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.enterContext(mock.patch.dict(driver.OPEN, {}, clear=True)) + self.enterContext(mock.patch.object(driver.REAPER, "tell")) + self.addCleanup(driver.close_all) + + def unit(self): + ws = os.path.join(self.tmp, str(len(FakeDesktop.made)), "workspace") + os.makedirs(ws) + u = driver.Unit("see", ws) + os.makedirs(os.path.dirname(u.log)) + u.expected = {"code": "ACDE34679HJK"} + d = u.desktop = FakeDesktop("fake", u.values) + u.boxes = d.boxes + u.bridge = driver.KitBridge(driver.KitDesktop(d), u.steps, 0, desktop.SCREEN, 1) + driver.OPEN["see"] = u + return ws, u, d + + def test_a_failed_record_write_does_not_change_the_verdict_or_cleanup(self): + for answer in ("ACDE34679HJK", ""): + ws, _, _ = self.unit() + expected = driver.score("see", ws, [], answer) + self.assertEqual(expected["passed"], bool(answer)) + for index in (0, 1): + for when in ("write", "close"): + with self.subTest(answer=answer, file=index, when=when): + ws, u, d = self.unit() + opened = [] + original, fdopen = open, os.fdopen + + def opening(file, mode="r", **kw): + f = (fdopen if isinstance(file, int) else original)(file, mode, **kw) + if mode != "w": + return f + opened.append(f) + if len(opened) != index + 1: + return f + wrapped = mock.MagicMock(wraps=f) + wrapped.__enter__.return_value = wrapped + + def close(*args): + f.__exit__(*args) + if when == "close": + raise OSError("disk full at close") + + wrapped.__exit__.side_effect = close + if when == "write": + wrapped.write.side_effect = OSError("disk full at write") + wrapped.writelines.side_effect = OSError("disk full at write") + return wrapped + + with mock.patch.object(driver, "open", side_effect=opening, create=True), \ + mock.patch.object(driver.os, "fdopen", side_effect=opening): + try: + got = driver.score("see", ws, [], answer) + finally: + self.assertTrue(d.removed) + self.assertIsNone(u.desktop) + self.assertNotIn("see", driver.OPEN) + self.assertEqual(len(opened), index + 1) + self.assertTrue(all(f.closed for f in opened)) + self.assertEqual(got, expected) + self.assertFalse(any(n.startswith(".computer-") for n in os.listdir(os.path.dirname(u.log)))) + + def test_record_names_replace_a_link_and_a_fifo_without_blocking_score(self): + names = ("computer-events.jsonl", "computer-boxes.json") + for linked, piped in (names, names[::-1]): + with self.subTest(link=linked, fifo=piped): + ws, u, d = self.unit() + folder = os.path.dirname(u.log) + outside = os.path.join(os.path.dirname(ws), "untouched") + with open(outside, "w") as f: + f.write("untouched") + os.symlink(outside, os.path.join(folder, linked)) + fifo = os.path.join(folder, piped) + os.mkfifo(fifo) + replies = queue.Queue() + + def score(): + try: + replies.put(driver.score("see", ws, [], u.expected["code"])) + except Exception as e: + replies.put(e) + + t = threading.Thread(target=score, daemon=True) + t.start() + t.join(5) + blocked = t.is_alive() + if blocked: # release an old writer before the test's folders go + fd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) + try: + t.join(5) + finally: + os.close(fd) + self.assertFalse(t.is_alive(), "scoring did not end after the FIFO was released") + self.assertFalse(blocked, "scoring blocked on the FIFO") + got = replies.get_nowait() + self.assertIsInstance(got, dict, got) + self.assertTrue(got["passed"], got) + self.assertTrue(d.removed) + self.assertNotIn("see", driver.OPEN) + self.assertEqual(sorted(os.listdir(folder)), sorted(names)) + for name in names: + self.assertTrue(stat.S_ISREG(os.lstat(os.path.join(folder, name)).st_mode)) + with open(outside) as f: + self.assertEqual(f.read(), "untouched") + with open(os.path.join(folder, names[0])) as f: + self.assertEqual([json.loads(line) for line in f], d.events()) + with open(os.path.join(folder, names[1])) as f: + self.assertEqual(json.load(f), u.boxes) + + class FakeUnits(unittest.TestCase): """Units of the driver over fake desktops, each in a folder of its own.""" From cefc9125e40c6fded936ee30dc2c8e1e763f7dcd Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 17:36:46 +0100 Subject: [PATCH 31/36] fix(standard): describe computer tool screenshots and coordinates Describe the screen returned after each action and pause. Name screenshot pixels and ask for a screenshot before the first pointing action. State that the final tools answer before their actions and return no image. --- app/nullius/standard/computer_port.py | 45 +++++++++++++++--------- app/tests/test_standard_computer_port.py | 32 ++++++++++++++++- docs/ports/agent-computer-v1.md | 4 +-- 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index 0a567c00..d352ebbb 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -105,8 +105,10 @@ def password(self) -> str: def tools(screen): """The tools for a screen of ``screen`` (width, height) pixels: [(name, description, input schema)].""" w, h = screen - x = {"type": "integer", "minimum": 0, "maximum": w - 1, "description": "Pixels from the screen's left edge"} - y = {"type": "integer", "minimum": 0, "maximum": h - 1, "description": "Pixels from the screen's top edge"} + x = {"type": "integer", "minimum": 0, "maximum": w - 1, + "description": "Pixels of the screenshot at the screen's size, from its left edge"} + y = {"type": "integer", "minimum": 0, "maximum": h - 1, + "description": "Pixels of the screenshot at the screen's size, from its top edge"} button = {"type": "string", "enum": ["left", "right", "middle"], "description": "Default left"} clicks = {"type": "integer", "minimum": -100, "maximum": 100} text = {"type": "string", "minLength": 1} @@ -118,7 +120,7 @@ def tools(screen): return [ ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " "python waits for the desktop's reply, and the desktop user's password.", obj({})), - ("screenshot", "The screen now, as a PNG image at its own size.", obj({})), + ("screenshot", "The screen now, as a PNG image at its own size. Look at it before the first pointing action.", obj({})), ("read", "A file of the desktop: data is its content in base64, or null when there is no such file.", obj({"path": text}, ["path"])), ("python", "Runs Python code in the desktop as its user and returns its reply: status, output, error, " @@ -132,32 +134,38 @@ def tools(screen): "append": {"type": "boolean"}, "sha256": {"type": "string", "pattern": SHA256}}, ["path"], anyOf=[{"required": ["data"]}, {"required": ["sha256"]}], **{"if": {"not": {"required": ["data"]}}, "then": {"properties": {"append": {"const": False}}}})), - ("click", "One step: a click at the point.", + ("click", "One step: a click at the point. Returns the screen as a PNG image after the pause.", obj({"x": x, "y": y, "button": button, "count": {"type": "integer", "minimum": 1, "maximum": 3, "description": "Default 1"}}, ["x", "y"])), - ("move", "One step: the pointer moved to the point.", obj({"x": x, "y": y}, ["x", "y"])), - ("drag", "One step: a press at (x, y), a move, and a release at (to_x, to_y).", + ("move", "One step: the pointer moved to the point. Returns the screen as a PNG image after the pause.", + obj({"x": x, "y": y}, ["x", "y"])), + ("drag", "One step: a press at (x, y), a move, and a release at (to_x, to_y). " + "Returns the screen as a PNG image after the pause.", obj({"x": x, "y": y, "to_x": x, "to_y": y, "button": button}, ["x", "y", "to_x", "to_y"])), ("scroll", "One step: the wheel turned at the point, dy clicks down (negative: up) and dx clicks right " - "(negative: left); at least one is not 0.", + "(negative: left); at least one is not 0. Returns the screen as a PNG image after the pause.", obj({"x": x, "y": y, "dy": clicks, "dx": clicks}, ["x", "y"], anyOf=[{"required": ["dy"], "properties": {"dy": {"not": {"const": 0}}}}, {"required": ["dx"], "properties": {"dx": {"not": {"const": 0}}}}])), - ("type", "One step: the text typed key by key.", obj({"text": text}, ["text"])), - ("key", "One step: the keys pressed together, such as [\"ctrl\", \"s\"].", + ("type", "One step: the text typed key by key. Returns the screen as a PNG image after the pause.", + obj({"text": text}, ["text"])), + ("key", "One step: the keys pressed together, such as [\"ctrl\", \"s\"]. " + "Returns the screen as a PNG image after the pause.", obj({"keys": {"type": "array", "minItems": 1, "maxItems": 8, "items": {"type": "string", "minLength": 1, "maxLength": 32}}}, ["keys"])), - ("wait", "One step: no action, the pause alone.", obj({})), + ("wait", "One step: no action, the pause alone. Returns the screen as a PNG image after the pause.", obj({})), ("step", "One step of several actions, run in order: each is a line of Python run in the desktop with " - "pyautogui and time imported, or WAIT. An empty list is a step with no action. The screen comes back " - "when screenshot is true.", + "pyautogui and time imported, or WAIT. An empty list is a step with no action. Returns the screen " + "as a PNG image after the actions and pauses when screenshot is true.", obj({"actions": {"type": "array", "items": { "type": "string", "pattern": NOT_BLANK, "not": {"pattern": f"^{BLANK}*(DONE|FAIL){BLANK}*$", "description": "the task's end is the tool done or fail, not an action of step"}}}, "response": {"type": "string"}, "screenshot": {"type": "boolean"}}, ["actions"])), - ("done", "The last step: the task is complete.", obj({"message": {"type": "string"}})), - ("fail", "The last step: the task cannot be done.", obj({"reason": {"type": "string"}})), + ("done", "The last step: the task is complete. Returns {steps_left, done}, with no image.", + obj({"message": {"type": "string"}})), + ("fail", "The last step: the task cannot be done. Returns {steps_left, done}, with no image.", + obj({"reason": {"type": "string"}})), ] @@ -528,9 +536,12 @@ def info(self): def instructions(self): w, h = self.screen return (f"A desktop seen in screenshots and driven by pointer and keyboard actions. The screen is {w} by {h} " - f"pixels, counted from its top left corner. Each call of click, move, drag, scroll, type, key, wait, " - f"step, done or fail is one step, answered once its actions have run and a pause of {self.pause:g} s " - f"has passed; there are {self.steps} steps in all, and steps_left in each answer. Once they are used " + f"pixels. Points use x and y in screenshot pixels at the screen's size, from its top left corner. " + f"Look at a screenshot before the first pointing action. Each call of click, move, drag, scroll, " + f"type, key, wait, step, done or fail is one step. After the actions and their {self.pause:g} s pauses, " + f"click, move, drag, scroll, type, key and wait return the screen as a PNG image; step does when " + f"screenshot is true. done and fail answer before they run, with no image. " + f"There are {self.steps} steps in all, and steps_left in each acting answer. Once they are used " f"up, every call but info is refused. Pointer and keyboard input goes through those tools, not through " f"python. A turn that ends with neither done nor fail ends the task as the desktop then is.") diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index e5b252b6..20d3551e 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -239,7 +239,7 @@ def test_initialize_and_the_tool_list_answer_before_the_desktop_is_ready_and_sta init = c.initialize()["result"] self.assertEqual(init["serverInfo"]["name"], "computer") self.assertIn("1920 by 1080 pixels", init["instructions"]) - self.assertIn("there are 3 steps in all", init["instructions"]) + self.assertIn("There are 3 steps in all", init["instructions"]) self.assertEqual([t["name"] for t in c.ask("tools/list")["result"]["tools"]], ["info", "screenshot", "read", "python", "upload", "click", "move", "drag", "scroll", "type", "key", "wait", "step", "done", "fail"]) @@ -731,6 +731,36 @@ def upload(self, b, **args): return b._upload(call) +class Descriptions(Bare): + def test_acting_answers_and_screenshot_coordinates_are_described(self): + listed = {t["name"]: t for t in cp.listed((100, 100))} + for name in cp.ACTING: + with self.subTest(tool=name): + description = listed[name]["description"] + if name in cp.FINAL: + self.assertIn("Returns {steps_left, done}, with no image", description) + else: + self.assertIn("Returns the screen as a PNG image after the", description) + self.assertIn("pause", description) + if name == "step": + self.assertIn("when screenshot is true", description) + for name in ("click", "move", "drag", "scroll"): + props = listed[name]["inputSchema"]["properties"] + for point, edge in (("x", "left"), ("y", "top"), ("to_x", "left"), ("to_y", "top")): + if point in props: + with self.subTest(tool=name, point=point): + self.assertIn("Pixels of the screenshot at the screen's size", props[point]["description"]) + self.assertIn(f"from its {edge} edge", props[point]["description"]) + self.assertIn("before the first pointing action", listed["screenshot"]["description"]) + instructions = self.bridge().instructions() + self.assertIn("x and y in screenshot pixels at the screen's size, from its top left corner", instructions) + self.assertIn("Look at a screenshot before the first pointing action", instructions) + self.assertIn("After the actions and their 0.01 s pauses", instructions) + self.assertIn("click, move, drag, scroll, type, key and wait return the screen as a PNG image", instructions) + self.assertIn("step does when screenshot is true", instructions) + self.assertIn("done and fail answer before they run, with no image", instructions) + + class Wires(Bare): def test_a_client_that_stops_reading_holds_up_neither_the_loop_nor_the_close(self): entered, release = threading.Event(), threading.Event() diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 83ab0255..10cebe45 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -20,7 +20,7 @@ None of these is a step. ## Acting -One call is one step. +One call is one step. Points use `x` and `y` in screenshot pixels at the screen's size, from its top left corner. Look at a screenshot before the first pointing action. | Tool | Arguments | What the benchmark runs | | --- | --- | --- | @@ -35,7 +35,7 @@ One call is one step. | `done` | `message`, optional | The task is complete | | `fail` | `reason`, optional | The task cannot be done | -An acting call answers once its actions have run and the benchmark's pause has passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. A pointer or keyboard tool and `wait` add the screen after the pause as an image; `step` adds it when `screenshot` is true. `done` and `fail` answer as the benchmark takes them, before it runs them: the benchmark's loop asks for nothing after them. +A call of `click`, `move`, `drag`, `scroll`, `type`, `key`, `wait` or `step` answers once its actions have run and their pauses have passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. Each adds the screen as a PNG image; `step` adds it only when `screenshot` is true. `done` and `fail` return `{steps_left, done}` with no image as the benchmark takes them, before it runs them: the benchmark's loop asks for nothing after them. ## Rules From c3a78f18a5b4de87199694fde047a86e8aaccf3b Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Mon, 16 Nov 2026 17:36:46 +0100 Subject: [PATCH 32/36] fix(standard): avoid ambiguous and repeated computer kit code characters Remove H from codes and exclude the previous character at each draw. Keep twelve characters and every glyph used by labels. Check many draws and read drawn codes from pixels at both scales. Describe the focused typing field and the conditions checked at the step budget. --- .../standard/kits/agent-computer-v1/checks.py | 7 +++++-- app/tests/test_computer_kit_driver.py | 17 +++++++++++++++++ docs/ports/agent-computer-v1.md | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index 3ce4e534..dc0a0709 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -36,7 +36,7 @@ SLACK = 0.5 CODE_LENGTH = 12 -ALPHABET = "ACDEFGHJKMNPRTWXY34679" # no pair a glance confuses: 0/O, 1/I/L, 5/S, 2/Z, 8/B, U/V +ALPHABET = "ACDEFGJKMNPRTWXY34679" # no pair a glance confuses: 0/O, 1/I/L, 5/S, 2/Z, 8/B, U/V, H/N LABELS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "IVORY", "JUMBO", "KNOT", "LEMON", "MAPLE", "NORTH", "OCEAN", "PIANO", "QUILT", "RIVER", "STONE", "TIGER", "UMBRA", "VELVET", "WHEAT", "YEAST") @@ -57,7 +57,10 @@ def steps(check): def code(rng): - return "".join(rng.choice(ALPHABET) for _ in range(CODE_LENGTH)) + text = "" + for _ in range(CODE_LENGTH): + text += rng.choice(ALPHABET.replace(text[-1:], "")) + return text def word(rng=None, length=8): diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 76ed571e..77318469 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -10,6 +10,7 @@ import json import os import queue +import random import re import secrets import shutil @@ -497,6 +498,22 @@ def chunk(kind, body): + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) +class Codes(unittest.TestCase): + def test_drawn_codes_keep_their_length_without_h_or_equal_neighbours_and_are_read_from_pixels(self): + rng = random.Random(7) + for check, scale in (("see", driver.fixture.SEE_SCALE), ("scroll", driver.fixture.LIST_SCALE)): + with self.subTest(check=check): + for _ in range(1000): + values, expected = checks.draw(check, rng) + code = values["code"] + self.assertEqual(code, expected["code"]) + self.assertEqual(len(code), 12) + self.assertNotIn("H", code) + self.assertTrue(set(code) <= set(checks.ALPHABET), code) + self.assertTrue(all(a != b for a, b in zip(code, code[1:])), code) + self.assertEqual(driver.read_code(_png(code, scale), scale), code) + + class FakeWire: def __init__(self): self.ready, self.reply = threading.Event(), None diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index e694825c..479e3d4b 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -84,6 +84,8 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | +`see` shows twelve characters from `ACDEFGJKMNPRTWXY34679`, with no equal neighbours. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. + Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. | Piece | Where | From a07ead3197fe100a60633122924525d5d4b5e63c Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Tue, 17 Nov 2026 13:15:21 +0100 Subject: [PATCH 33/36] fix(standard): read common words in computer checks Draw distinct common nouns for the see and scroll checks to make screen reading reliable. Keep the glyph font and add word gaps, with larger words at the end of the scroll list. Accept the displayed words in order across case and punctuation while rejecting missing or additional vocabulary words. Read the reference answer from screenshot pixels and keep the prompts independent of the sampled words. --- .../standard/kits/agent-computer-v1/checks.py | 54 ++++++--- .../standard/kits/agent-computer-v1/driver.py | 20 ++-- .../kits/agent-computer-v1/fixture.py | 37 ++++-- .../kits/agent-computer-v1/standard.json | 8 +- app/tests/test_computer_kit.py | 109 +++++++++++++++-- app/tests/test_computer_kit_driver.py | 112 +++++++++++++----- docs/ports/agent-computer-v1.md | 8 +- 7 files changed, 258 insertions(+), 90 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index dc0a0709..feab050c 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -8,7 +8,7 @@ agent's calls (``calls``: the call log's lines, ``{tool, start, end, isError}``; ``past``: the acting calls sent once every step was taken). -- ``see``: the last message holds the code the window drew; +- ``see``: the last message holds the words the window drew, in order; - ``click``: the log holds a left press on the named button and no press, of any mouse button, on the five others; - ``type``: the field held exactly the word at the last Enter; @@ -16,7 +16,7 @@ them, no input after the last action, and no acting call sent once the steps were taken; - ``drag``: a left press on the handle, then a release inside the zone; -- ``scroll``: the last message holds the code drawn at the list's end, below +- ``scroll``: the last message holds the words drawn at the list's end, in order, below what it shows at first, and the list moved; - ``recover``: the kit answers the agent's first acting call with an error, without running it; an acting call sent after that answer came succeeds, @@ -35,8 +35,20 @@ import secrets SLACK = 0.5 -CODE_LENGTH = 12 -ALPHABET = "ACDEFGJKMNPRTWXY34679" # no pair a glance confuses: 0/O, 1/I/L, 5/S, 2/Z, 8/B, U/V, H/N +# Everyday nouns, separate from the fixture's labels. Equal-length words +# differ in at least three positions, giving the glyphs the context of a word. +WORDS = ("ACORN", "ANCHOR", "ANKLE", "APRON", "BADGE", "BASKET", "BEACH", "BEETLE", "BERRY", "BOWL", + "BREAD", "BROOM", "BUTTER", "CABIN", "CACTUS", "CAMERA", "CANDLE", "CANOE", "CARPET", "CELERY", + "CHAIR", "CHERRY", "CLOCK", "COFFEE", "COOKIE", "CORN", "COTTON", "CRAYON", "CROWN", "DESK", + "DINNER", "DONKEY", "DOOR", "DRUM", "EAGLE", "EARTH", "ENGINE", "FABRIC", "FARM", "FENCE", + "FLOWER", "FOLDER", "FOREST", "FROG", "GARDEN", "GARLIC", "GATE", "GLOVE", "GOAT", "GUITAR", + "HAMMER", "HELMET", "HORSE", "IGLOO", "INSECT", "LADDER", "LAMP", "LEAF", "LIZARD", "LUNCH", + "MAGNET", "MARBLE", "MEADOW", "MELON", "MIRROR", "MITTEN", "NOODLE", "ONION", "ORANGE", "OTTER", + "PANDA", "PAPER", "PARROT", "PEANUT", "PENCIL", "PEPPER", "PILLOW", "PIZZA", "PLANT", "POCKET", + "POTATO", "RABBIT", "RADIO", "RAISIN", "RIBBON", "ROPE", "SALAD", "SALMON", "SAND", "SANDAL", + "SCARF", "SCHOOL", "SHEEP", "SHIRT", "SHOE", "SHOVEL", "SOCK", "SPRING", "SQUARE", "STAMP", + "STAR", "STREAM", "SUGAR", "TEAPOT", "THUMB", "TOWEL", "TRAIN", "TRAY", "TURTLE", "WINDOW", + "WOLF", "ZEBRA") LABELS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "IVORY", "JUMBO", "KNOT", "LEMON", "MAPLE", "NORTH", "OCEAN", "PIANO", "QUILT", "RIVER", "STONE", "TIGER", "UMBRA", "VELVET", "WHEAT", "YEAST") @@ -56,13 +68,6 @@ def steps(check): return STEPS.get(check, DEFAULT_STEPS) -def code(rng): - text = "" - for _ in range(CODE_LENGTH): - text += rng.choice(ALPHABET.replace(text[-1:], "")) - return text - - def word(rng=None, length=8): """A pronounceable word that is no word: consonant, vowel, and again.""" rng = rng or secrets.SystemRandom() @@ -75,8 +80,8 @@ def draw(check, rng=None): values but what the screen shows.""" rng = rng or secrets.SystemRandom() if check in ("see", "scroll"): - c = code(rng) - return {"screen": check, "code": c}, {"code": c} + words = rng.sample(WORDS, 3 if check == "see" else 2) + return {"screen": check, "words": words}, {"words": words} if check in ("click", "recover"): labels = rng.sample(LABELS, 6) return {"screen": "click", "labels": labels}, {"target": rng.choice(labels), "labels": labels} @@ -103,8 +108,22 @@ def outside(events, actions, slack=SLACK): return [e for e in events if e.get("kind") in POINTER + KEYS and not inside(e.get("t", -1.0), actions, slack)] -def _flat(text): - return re.sub(r"[^A-Za-z0-9]", "", str(text or "")).upper() +def _read_words(expected, message): + """Read whole vocabulary words in order, allowing other prose between them.""" + words = expected["words"] + found = [w for w in re.findall(r"\b\w+\b", str(message or "").upper()) if w in WORDS] + missing = [w for w in words if w not in found] + extra = list(dict.fromkeys(w for w in found if w not in words)) + why = [] + if missing: + why.append("missing words: " + ", ".join(missing)) + if extra: + why.append("extra words: " + ", ".join(extra)) + if why: + return "; ".join(why) + if found != words: + return "the words are out of order or repeated; expected " + ", ".join(words) + return "" def _verdict(why): @@ -134,8 +153,7 @@ def _in(box, e): def see(expected, events, actions, message, calls=(), past=0): - return _verdict(_first("" if expected["code"] in _flat(message) else "the last message does not hold the code", - _input_rule(events, actions))) + return _verdict(_first(_read_words(expected, message), _input_rule(events, actions))) def click(expected, events, actions, message, calls=(), past=0): @@ -186,7 +204,7 @@ def drag(expected, events, actions, message, calls=(), past=0): def scroll(expected, events, actions, message, calls=(), past=0): moved = any(e.get("kind") == "moved" and e.get("top", 0) > 0 for e in events) - return _verdict(_first("" if expected["code"] in _flat(message) else "the last message does not hold the code", + return _verdict(_first(_read_words(expected, message), "" if moved else "the list never moved", _input_rule(events, actions))) diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index f3c16597..b6335f3e 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -61,7 +61,7 @@ def _load(name, file): def prompt(unit, expected): if unit == "see": - return USE + "The screen shows a code of twelve characters. Reply with the code and nothing else." + return USE + "Read the words on the screen and reply with them in the order shown." if unit in ("click", "recover"): return USE + f"Click the button labelled {expected['target']} on the screen, once." + DONE if unit == "type": @@ -72,8 +72,8 @@ def prompt(unit, expected): if unit == "drag": return USE + "Drag the blue square labelled MOVE into the green box labelled HERE and let it go inside the box." + DONE if unit == "scroll": - return (USE + "The list on the screen ends with a code of twelve characters, below what it shows at first. " - "Scroll the list to find it, and reply with the code and nothing else.") + return (USE + "Scroll to the end of the list and read the large words there. " + "Reply with those words in the order shown.") if unit == "batch": w = expected["words"] return (USE + "Fill the three text fields on the screen: " + ", ".join(f"{w[f]} in the field labelled {f}" @@ -464,18 +464,19 @@ class ReferenceFailed(Exception): """A call of the reference did not go as a correct agent's would.""" -def read_code(png, scale): - """The code of twelve characters drawn on the screen, read from a screenshot, as an agent reads it.""" +def read_words(png, scale, count): + """The words drawn on the screen, read from a screenshot, as an agent reads them.""" found = [line for line in fixture.read_lines(png, scale) - if len(line) == checks.CODE_LENGTH and set(line) <= set(checks.ALPHABET)] + if len(line.split()) == count and len(set(line.split())) == count + and all(word in checks.WORDS for word in line.split())] if len(found) != 1: - raise ReferenceFailed(f"no one code could be read from the screenshot ({len(found)} found)") + raise ReferenceFailed(f"no one group of words could be read from the screenshot ({len(found)} found)") return found[0] def oracle(unit, dir, **_): """The reference: the calls a correct agent makes, sent to the unit's own server, through its desktop, to the - same scorer; the answer they lead to. A code is read from the screenshot the server gives, and a call that does + same scorer; the answer they lead to. Words are read from the screenshot the server gives, and a call that does not answer as a correct agent's would (the one ``recover`` refuses aside) fails the reference.""" with OPEN_LOCK: u = OPEN.get(unit) @@ -499,7 +500,8 @@ def call(tool, arguments=None, failing=False): png = call("screenshot") if not isinstance(png, bytes): raise ReferenceFailed("the screenshot held no image") - answer = read_code(png, fixture.SEE_SCALE if unit == "see" else fixture.LIST_SCALE) + answer = read_words(png, fixture.SEE_SCALE if unit == "see" else fixture.SCROLL_SCALE, + 3 if unit == "see" else 2) elif unit in ("click", "recover"): x, y = centre(b[e["target"]]) if unit == "recover": diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py index 520cf75d..b5c80781 100644 --- a/app/nullius/standard/kits/agent-computer-v1/fixture.py +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -3,16 +3,16 @@ It reads what to draw from the kit on its stdin, as one JSON object: -- ``{"screen": "see", "code": "..."}``: the code, drawn large; +- ``{"screen": "see", "words": [three words]}``: the words, drawn large; - ``{"screen": "click", "labels": [six words]}``: six buttons, each labelled; - ``{"screen": "type"}``: a text field; - ``{"screen": "drag"}``: a handle labelled MOVE and a zone labelled HERE; -- ``{"screen": "scroll", "code": "..."}``: a list of words with the code at +- ``{"screen": "scroll", "words": [two words]}``: a list with the words at its end, below what the list shows at first; - ``{"screen": "batch", "labels": [three words]}``: three text fields, each labelled. -Codes and labels are drawn as bitmaps from the glyphs below: no widget holds +Words and labels are drawn as bitmaps from the glyphs below: no widget holds their text, and Tk's ``send`` is removed, so no other program can read them but from the screen. The window fills the screen, so every press lands on it. @@ -42,7 +42,8 @@ LOG = os.path.join(HOME, "events.jsonl") ERRORS = os.path.join(HOME, "fixture.err") TITLE = "Check" -SEE_SCALE, LIST_SCALE = 9, 4 # screen pixels a glyph cell: the code alone, the list's lines +SEE_SCALE, LIST_SCALE = 9, 4 # screen pixels a glyph cell: the words alone, the list's lines +SCROLL_SCALE = 8 # two six-letter words fit inside the list's 760 pixels SCREENS = ("see", "click", "type", "drag", "scroll", "batch") LIST_WORDS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "MAPLE", "NORTH") BACKGROUND = "#eceff1" @@ -94,6 +95,8 @@ def pixels(text): """The lit cells of ``text`` in glyph units, a blank column between characters: (columns, rows, cells).""" cells = [] for i, ch in enumerate(text): + if ch == " ": + continue for y, row in enumerate(GLYPHS[ch]): cells += [(i * 6 + x, y) for x, c in enumerate(row) if c == "#"] return max(len(text) * 6 - 1, 1), 7, cells @@ -143,8 +146,8 @@ def png_rows(data, last_row=None): def read_lines(data, scale, ink=INK): - """The lines of glyphs drawn at ``scale`` in ``ink`` on a screen's PNG, top to bottom; a block that is no - glyph reads ``?``. What the kit's reference reads from a screenshot, as an agent would from its eyes.""" + """The lines of glyphs drawn at ``scale`` in ``ink`` on a screen's PNG, top to bottom; an unreadable line + reads ``?``. What the kit's reference reads from a screenshot, as an agent would from its eyes.""" colour = bytes(int(ink[i:i + 2], 16) for i in (1, 3, 5)) w, rows = png_rows(data) lit = [] @@ -167,11 +170,19 @@ def read_lines(data, scale, ink=INK): for x in (lit[yy] if yy < len(lit) else ()): if x >= x0 and (x - x0) % scale == scale // 2: cells[((x - x0) // scale, r)] = True - text, c = "", 0 - while any((c * 6 + dx, r) in cells for dx in range(5) for r in range(7)): - block = tuple("".join("#" if (c * 6 + dx, r) in cells else "." for dx in range(5)) for r in range(7)) - text += next((ch for ch, g in GLYPHS.items() if g == block), "?") - c += 1 + # I (and 1) has a blank left column; try both possible origins. + # A blank character is a word gap, not the end of the line. + text = "?" + for inset in (0, 1): + decoded = "" + for c in range((max((x for x, r in cells), default=-1) + inset) // 6 + 1): + block = tuple("".join("#" if (c * 6 + dx - inset, r) in cells else "." for dx in range(5)) + for r in range(7)) + decoded += (" " if block == (".....",) * 7 else + next((ch for ch, g in GLYPHS.items() if g == block), "?")) + if "?" not in decoded: + text = decoded + break out.append(text) y = top + 7 * scale return out @@ -202,7 +213,7 @@ def __init__(self, values, log): self.root.bind_all(seq, self.on_event, add="+") def _see(self): - img = bitmap(self.tk, self.root, self.values["code"], SEE_SCALE) + img = bitmap(self.tk, self.root, " ".join(self.values["words"]), SEE_SCALE) self.images.append(img) self.tk.Label(self.root, image=img, background=BACKGROUND, borderwidth=0).place(relx=0.5, rely=0.5, anchor="center") @@ -246,7 +257,7 @@ def _scroll(self): img = bitmap(self.tk, c, LIST_WORDS[i % len(LIST_WORDS)], scale, paper="#ffffff") self.images.append(img) c.create_image(40, 40 + i * step, image=img, anchor="nw") - img = bitmap(self.tk, c, self.values["code"], scale, paper="#ffffff") + img = bitmap(self.tk, c, " ".join(self.values["words"]), SCROLL_SCALE, paper="#ffffff") self.images.append(img) c.create_image(40, 40 + rows * step + 40, image=img, anchor="nw") c.configure(scrollregion=(0, 0, 760, 40 + rows * step + 160)) diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 866f512b..272ffbd9 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -76,8 +76,8 @@ "check_details": { "see": { "label": "Read the screen", - "task": "Read a code of twelve characters drawn on the screen.", - "passes": "The final message holds the code, and no input came outside an acting call." + "task": "Read the words drawn on the screen and report them in order.", + "passes": "The final message holds the words in order and no other word from the reading vocabulary, and no input came outside an acting call." }, "click": { "label": "Click a button", @@ -101,8 +101,8 @@ }, "scroll": { "label": "Scroll", - "task": "Scroll a list to read a code below what it shows at first.", - "passes": "The final message holds the code, and the list moved." + "task": "Scroll to the end of a list and report the large words there in order.", + "passes": "The final message holds the words in order and no other word from the reading vocabulary, and the list moved." }, "recover": { "label": "Recover from an error", diff --git a/app/tests/test_computer_kit.py b/app/tests/test_computer_kit.py index 394626e3..69da5373 100644 --- a/app/tests/test_computer_kit.py +++ b/app/tests/test_computer_kit.py @@ -64,12 +64,33 @@ def keys(t, text): class Verdicts(unittest.TestCase): - def test_see_passes_on_the_code_in_any_case_and_spacing_and_fails_without_it(self): - exp = {"code": "ACDE34679HJK"} - self.assertTrue(checks.verdict("see", exp, [], [], "The code is acde-3467 9hjk.")["passed"]) - self.assertFalse(checks.verdict("see", exp, [], [], "The code is ACDE34679HJ.")["passed"]) - self.assertFalse(checks.verdict("see", exp, [], [], "")["passed"]) - self.assertFalse(checks.verdict("see", exp, [], [], None)["passed"]) + def test_reading_passes_with_spaces_commas_and_quotes_case_or_lines(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + answers = [" ".join(words), ", ".join(words), " and ".join(words), + ", ".join(words[:-1]) + " and " + words[-1], + " ".join(f'"{w}"' for w in words), " ".join(words).lower(), "\n".join(words), + "I see " + "; then ".join(words) + "."] + for answer in answers: + with self.subTest(check=check, answer=answer): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) + self.assertEqual(got, {"passed": True, "why": ""}) + + def test_reading_fails_with_missing_reordered_extra_or_all_words(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + answers = [(" ".join(words[:-1]), "missing words: " + words[-1]), + (" ".join(words[::-1]), "out of order"), + (" ".join(words + ["ZEBRA"]), "extra words: ZEBRA"), + (" ".join(checks.WORDS), "extra words:"), + (" ".join(words + words), "repeated"), + (" ".join(["ACORNS", *words[1:]]), "missing words: ACORN"), + (" ".join(["xACORN", *words[1:]]), "missing words: ACORN"), + (" ".join(words[:-1] + ["ZEBRA"]), "missing words: " + words[-1] + "; extra words: ZEBRA"), + ("", "missing words:"), (None, "missing words:")] + for answer, why in answers: + with self.subTest(check=check, answer=answer): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) + self.assertFalse(got["passed"]) + self.assertIn(why, got["why"]) def test_click_passes_on_a_left_press_on_the_named_button_inside_an_action(self): events = [ev("ready", 99.0)] + press(100.5, "OCEAN") @@ -130,9 +151,9 @@ class Draw(unittest.TestCase): def test_each_check_draws_from_its_own_values_and_expects_what_the_screen_shows(self): rng = random.Random(7) values, exp = checks.draw("see", rng) - self.assertEqual(len(exp["code"]), checks.CODE_LENGTH) - self.assertTrue(set(exp["code"]) <= set(checks.ALPHABET)) - self.assertEqual(values, {"screen": "see", "code": exp["code"]}) + self.assertEqual(len(exp["words"]), 3) + self.assertTrue(set(exp["words"]) <= set(checks.WORDS)) + self.assertEqual(values, {"screen": "see", "words": exp["words"]}) values, exp = checks.draw("click", rng) self.assertEqual(len(set(values["labels"])), 6) self.assertIn(exp["target"], values["labels"]) @@ -143,7 +164,7 @@ def test_each_check_draws_from_its_own_values_and_expects_what_the_screen_shows( checks.draw("hover") def test_every_character_drawn_has_a_glyph_of_five_by_seven(self): - for ch in checks.ALPHABET + "".join(checks.LABELS): + for ch in "".join(checks.WORDS + checks.LABELS): self.assertIn(ch, fixture.GLYPHS) for ch, rows in fixture.GLYPHS.items(): self.assertEqual((len(rows), {len(r) for r in rows}), (7, {5}), ch) @@ -152,6 +173,70 @@ def test_every_character_drawn_has_a_glyph_of_five_by_seven(self): self.assertIn((0, 3), cells) # A's bar self.assertNotIn((5, 3), cells) # the blank column + def test_reading_words_are_distinct_nouns_apart_from_each_other_and_the_labels(self): + self.assertGreaterEqual(len(checks.WORDS), 64) + self.assertEqual(len(checks.WORDS), len(set(checks.WORDS))) + labels = set(checks.LABELS + checks.ORDER + checks.FIELDS + fixture.LIST_WORDS + ("MOVE", "HERE", fixture.TITLE.upper())) + self.assertFalse(set(checks.WORDS) & labels) + for i, word in enumerate(checks.WORDS): + self.assertRegex(word, r"^[A-Z]{4,6}$") + for other in checks.WORDS[i + 1:]: + if len(word) == len(other): + self.assertGreaterEqual(sum(a != b for a, b in zip(word, other)), 3, (word, other)) + + def test_reading_draws_distinct_words_in_the_right_count_over_many_draws(self): + rng = random.Random(7) + for check, count in (("see", 3), ("scroll", 2)): + seen, draws = set(), set() + for _ in range(1000): + values, expected = checks.draw(check, rng) + words = expected["words"] + self.assertEqual(values, {"screen": check, "words": words}) + self.assertEqual(len(words), count) + self.assertEqual(len(set(words)), count) + self.assertTrue(set(words) <= set(checks.WORDS)) + seen.update(words) + draws.add(tuple(words)) + self.assertEqual(seen, set(checks.WORDS)) + self.assertGreater(len(draws), 900) + + def test_word_gaps_are_blank_cells_and_the_largest_phrases_fit(self): + cols, rows, cells = fixture.pixels("ACORN BASKET") + self.assertEqual((cols, rows), (71, 7)) + self.assertFalse(any(29 <= x < 36 for x, y in cells)) + self.assertTrue(any(x == 36 for x, y in cells)) + longest = sorted(checks.WORDS, key=len, reverse=True) + for count, scale, width in ((3, fixture.SEE_SCALE, 1920), (2, fixture.SCROLL_SCALE, 760 - 2 * 40)): + cols, rows, _ = fixture.pixels(" ".join(longest[:count])) + self.assertLessEqual(cols * scale, width) + self.assertGreater(fixture.SCROLL_SCALE, fixture.LIST_SCALE) + + def test_the_fixture_draws_reading_words_as_images_and_keeps_scroll_words_below_the_edge(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + with self.subTest(check=check): + f = fixture.Fixture.__new__(fixture.Fixture) + f.tk, f.root = mock.Mock(), mock.Mock() + f.values, f.images, f.names = {"words": words}, [], {} + with mock.patch.object(fixture, "bitmap", return_value=mock.sentinel.image) as bitmap: + getattr(f, "_" + check)() + self.assertEqual(bitmap.call_args.args[2], " ".join(words)) + self.assertEqual(f.images[-1], mock.sentinel.image) + if check == "see": + self.assertEqual(bitmap.call_args.args[3], fixture.SEE_SCALE) + self.assertEqual(f.names, {}) + self.assertEqual(f.tk.Label.call_args.kwargs, + {"image": mock.sentinel.image, "background": fixture.BACKGROUND, "borderwidth": 0}) + else: + self.assertEqual(bitmap.call_args.args[3], fixture.SCROLL_SCALE) + self.assertEqual(set(f.names.values()), {"list"}) + canvas = f.tk.Canvas.return_value + canvas.create_text.assert_not_called() + x, y = canvas.create_image.call_args.args + width = fixture.pixels(" ".join(words))[0] * fixture.SCROLL_SCALE + self.assertGreater(y, f.tk.Canvas.call_args.kwargs["height"]) + self.assertLessEqual(x + width, f.tk.Canvas.call_args.kwargs["width"]) + self.assertLessEqual(y + 7 * fixture.SCROLL_SCALE, canvas.configure.call_args.kwargs["scrollregion"][3]) + class Image(unittest.TestCase): def test_the_desktop_is_an_environment_of_the_kit_tagged_by_its_text_and_files(self): @@ -541,9 +626,9 @@ def test_the_fixture_is_read_from_the_screen_alone(self): self.assertIn("PermissionError", said["error"]) seen = self.d.python(_READ_INK) self.assertEqual(seen["returncode"], 0, seen["error"]) - _, _, cells = fixture.pixels(exp["code"]) + _, _, cells = fixture.pixels(" ".join(exp["words"])) self.assertEqual(_from_corner(json.loads(seen["output"])), _from_corner(cells)) - self.assertTrue(checks.verdict("see", exp, self.d.events(), self.d.actions, exp["code"])["passed"]) + self.assertTrue(checks.verdict("see", exp, self.d.events(), self.d.actions, " ".join(exp["words"]))["passed"]) def test_a_click_by_an_action_passes_and_the_same_click_by_python_fails(self): values, exp = checks.draw("click") diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 77318469..d2707cab 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -83,12 +83,12 @@ def test_drag_passes_on_a_press_on_the_handle_released_inside_the_zone(self): ev("release", 100.8, button=1, hit="background", x=600, y=600)] self.assertFalse(checks.verdict("drag", {}, not_handle, ACTIONS, "")["passed"]) - def test_scroll_passes_on_the_code_once_the_list_moved(self): - exp = {"code": "ACDE34679HJK"} + def test_scroll_passes_on_the_words_once_the_list_moved(self): + exp = {"words": ["ACORN", "BASKET"]} moved = [ev("scroll", 100.2, button=5, hit="list"), ev("moved", 100.21, top=0.4)] - self.assertTrue(checks.verdict("scroll", exp, moved, ACTIONS, "ACDE34679HJK")["passed"]) - self.assertIn("never moved", checks.verdict("scroll", exp, moved[:1], ACTIONS, "ACDE34679HJK")["why"]) - self.assertIn("does not hold", checks.verdict("scroll", exp, moved, ACTIONS, "")["why"]) + self.assertTrue(checks.verdict("scroll", exp, moved, ACTIONS, "ACORN BASKET")["passed"]) + self.assertIn("never moved", checks.verdict("scroll", exp, moved[:1], ACTIONS, "ACORN BASKET")["why"]) + self.assertIn("missing words", checks.verdict("scroll", exp, moved, ACTIONS, "")["why"]) def test_recover_passes_on_a_success_after_the_error_and_the_click(self): exp = {"target": "OCEAN", "labels": ["OCEAN", "APPLE"]} @@ -113,13 +113,13 @@ def test_batch_passes_when_each_field_holds_its_word(self): def test_every_check_fails_on_input_outside_every_action(self): passing = { - "see": ({"code": "ACDE34679HJK"}, [], "ACDE34679HJK", ()), + "see": ({"words": ["ACORN", "BASKET", "CANDLE"]}, [], "ACORN BASKET CANDLE", ()), "click": ({"target": "OCEAN", "labels": ["OCEAN"]}, press(100.2, "OCEAN"), "", ()), "type": ({"word": "kotabiru"}, [ev("enter", 100.3, value="kotabiru")], "", ()), "limit": (LIMIT, press(100.2, "ONE"), "", ()), "drag": ({}, [ev("ready", 99, boxes={"zone": [500, 500, 700, 700]}), ev("press", 100.2, button=1, hit="handle"), ev("release", 100.8, button=1, hit="handle", x=600, y=600)], "", ()), - "scroll": ({"code": "ACDE34679HJK"}, [ev("moved", 100.3, top=0.5)], "ACDE34679HJK", ()), + "scroll": ({"words": ["ACORN", "BASKET"]}, [ev("moved", 100.3, top=0.5)], "ACORN BASKET", ()), "recover": ({"target": "OCEAN", "labels": ["OCEAN"]}, press(110.2, "OCEAN"), "", ({"tool": "click", "isError": True, "start": 1, "end": 2}, {"tool": "click", "isError": False, "start": 3, "end": 4})), @@ -250,7 +250,7 @@ def unit(self): os.makedirs(ws) u = driver.Unit("see", ws) os.makedirs(os.path.dirname(u.log)) - u.expected = {"code": "ACDE34679HJK"} + u.expected = {"words": ["ACORN", "BASKET", "CANDLE"]} d = u.desktop = FakeDesktop("fake", u.values) u.boxes = d.boxes u.bridge = driver.KitBridge(driver.KitDesktop(d), u.steps, 0, desktop.SCREEN, 1) @@ -258,7 +258,7 @@ def unit(self): return ws, u, d def test_a_failed_record_write_does_not_change_the_verdict_or_cleanup(self): - for answer in ("ACDE34679HJK", ""): + for answer in ("ACORN BASKET CANDLE", ""): ws, _, _ = self.unit() expected = driver.score("see", ws, [], answer) self.assertEqual(expected["passed"], bool(answer)) @@ -319,7 +319,7 @@ def test_record_names_replace_a_link_and_a_fifo_without_blocking_score(self): def score(): try: - replies.put(driver.score("see", ws, [], u.expected["code"])) + replies.put(driver.score("see", ws, [], " ".join(u.expected["words"]))) except Exception as e: replies.put(e) @@ -483,12 +483,17 @@ def _alive(pid): timeout=30).stdout.strip()[:1] not in ("", "Z") -def _png(code, scale): - """A screen of 1920 by 1080 with ``code`` drawn as the fixture draws it, as PNG bytes (filter 0).""" +def _png(text, scale, above=()): + """A screen of 1920 by 1080 with ``text`` drawn as the fixture draws it, as PNG bytes (filter 0).""" w, h = 1920, 1080 - cols, rows, cells = driver.fixture.pixels(code) + cols, rows, cells = driver.fixture.pixels(text) x0, y0 = (w - cols * scale) // 2, (h - rows * scale) // 2 ink = {(x0 + x * scale + dx, y0 + y * scale + dy) for x, y in cells for dx in range(scale) for dy in range(scale)} + for i, label in enumerate(above): + _, _, cells = driver.fixture.pixels(label) + s = driver.fixture.LIST_SCALE + ink.update((x0 + x * s + dx, y0 - (i + 1) * 60 - 40 + y * s + dy) + for x, y in cells for dx in range(s) for dy in range(s)) paper, mark = bytes.fromhex("eceff1"), bytes.fromhex("111111") raw = b"".join(b"\x00" + b"".join(mark if (x, y) in ink else paper for x in range(w)) for y in range(h)) @@ -498,20 +503,67 @@ def chunk(kind, body): + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) -class Codes(unittest.TestCase): - def test_drawn_codes_keep_their_length_without_h_or_equal_neighbours_and_are_read_from_pixels(self): +class Words(unittest.TestCase): + def test_the_reference_reads_drawn_words_and_gaps_at_both_scales(self): rng = random.Random(7) - for check, scale in (("see", driver.fixture.SEE_SCALE), ("scroll", driver.fixture.LIST_SCALE)): + for check, scale, count in (("see", driver.fixture.SEE_SCALE, 3), ("scroll", driver.fixture.SCROLL_SCALE, 2)): with self.subTest(check=check): - for _ in range(1000): + for _ in range(3): values, expected = checks.draw(check, rng) - code = values["code"] - self.assertEqual(code, expected["code"]) - self.assertEqual(len(code), 12) - self.assertNotIn("H", code) - self.assertTrue(set(code) <= set(checks.ALPHABET), code) - self.assertTrue(all(a != b for a, b in zip(code, code[1:])), code) - self.assertEqual(driver.read_code(_png(code, scale), scale), code) + text = " ".join(values["words"]) + png = _png(text, scale) + self.assertEqual(driver.fixture.read_lines(png, scale), [text]) + self.assertEqual(driver.read_words(png, scale, count), " ".join(expected["words"])) + text = " ".join(["IGLOO", "INSECT", "WINDOW"][:count]) + self.assertEqual(driver.read_words(_png(text, scale), scale, count), text) + + def test_the_reference_refuses_missing_unreadable_or_repeated_words(self): + for scale, count in ((driver.fixture.SEE_SCALE, 3), (driver.fixture.SCROLL_SCALE, 2)): + for text in ("", "ACORN", "ACORN BASKET ZZZZ", " ".join(["ACORN"] * count)): + with self.subTest(scale=scale, text=text), self.assertRaises(driver.ReferenceFailed): + driver.read_words(_png(text, scale), scale, count) + + def test_the_reference_reads_scroll_words_below_smaller_list_labels(self): + scale = driver.fixture.SCROLL_SCALE + png = _png("IGLOO BASKET", scale, above=driver.fixture.LIST_WORDS[-3:]) + self.assertEqual(driver.read_words(png, scale, 2), "IGLOO BASKET") + + def test_the_task_asks_for_words_in_order_without_the_answers_or_scoring_rules(self): + rng = random.Random(7) + for check in ("see", "scroll"): + prompts = set() + for _ in range(1000): + _, expected = checks.draw(check, rng) + text = driver.prompt(check, expected) + prompts.add(text) + self.assertIn("words", text) + self.assertIn("order", text) + self.assertFalse(set(re.findall(r"\b\w+\b", text.upper())) & set(checks.WORDS)) + self.assertNotRegex(text.lower(), r"\b(code|two|three|vocabulary|dictionary|match|checked|scored)\b") + self.assertEqual(len(prompts), 1) + + def test_the_null_control_still_fails_every_check(self): + for check in checks.CHECKS: + _, expected = checks.draw(check, random.Random(7)) + if check in ("see", "scroll"): + self.assertIn("words", expected) + self.assertFalse(checks.verdict(check, expected, [], [], "")["passed"], check) + + def test_the_reference_uses_the_screenshot_not_the_expected_words(self): + for check, scale, count in (("see", driver.fixture.SEE_SCALE, 3), ("scroll", driver.fixture.SCROLL_SCALE, 2)): + with self.subTest(check=check): + answer = " ".join(["IGLOO", "BASKET", "CANDLE"][:count]) + png = _png(answer, scale) + unit = mock.Mock(expected={"words": ["ZEBRA", "PILLOW", "ORANGE"][:count]}, + boxes={"list": [0, 0, 760, 540]}) + client = mock.Mock() + client.call.side_effect = lambda tool, arguments=None: (False, png if tool == "screenshot" else {}) + with mock.patch.dict(driver.OPEN, {check: unit}, clear=True), \ + mock.patch.object(driver, "Client", return_value=client): + self.assertEqual(driver.oracle(check, "unused"), {"answer": answer}) + self.assertEqual([c.args[0] for c in client.call.call_args_list], + ["info", "scroll", "screenshot"] if check == "scroll" else ["info", "screenshot"]) + client.close.assert_called_once() class FakeWire: @@ -733,12 +785,12 @@ def act(code, seconds=None): self.assertEqual([c["isError"] for c in u.calls()], [True, False]) self.assertTrue(driver.score("click", ws, [], "done")["passed"]) - def test_the_reference_reads_the_code_from_the_screenshot_and_fails_without_one(self): + def test_the_reference_reads_the_words_from_the_screenshot_and_fails_without_one(self): ws, s = self.unit("see") - fake, code = FakeDesktop.made[-1], driver.OPEN["see"].expected["code"] - fake.capture = lambda: _png(code, 9) - self.assertEqual(driver.oracle("see", ws)["answer"], code) - self.assertTrue(driver.score("see", ws, [], code)["passed"]) + fake, text = FakeDesktop.made[-1], " ".join(driver.OPEN["see"].expected["words"]) + fake.capture = lambda: _png(text, driver.fixture.SEE_SCALE) + self.assertEqual(driver.oracle("see", ws)["answer"], text) + self.assertTrue(driver.score("see", ws, [], text)["passed"]) ws, s = self.unit("see") def broken(): @@ -747,7 +799,7 @@ def broken(): with self.assertRaises(driver.ReferenceFailed): driver.oracle("see", ws) ws, s = self.unit("see") - FakeDesktop.made[-1].capture = lambda: _png("ACDE", 9) # a screen with no code of twelve + FakeDesktop.made[-1].capture = lambda: _png("ACORN", 9) # a screen with only one word with self.assertRaises(driver.ReferenceFailed): driver.oracle("see", ws) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 479e3d4b..d89fd27b 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -75,16 +75,16 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | Check | Kind | The task | Passes when | | --- | --- | --- | --- | -| `see` | basic | Read a code of twelve characters drawn on the screen | The final message holds the code | +| `see` | basic | Read the words drawn on the screen and report them in order | The final message holds the words in order and no other word from the reading vocabulary | | `click` | basic | Click the named one of six buttons | A left press on that button, and no press on the five others | | `type` | basic | Type a word in a text field and press Enter | The field holds exactly the word at the last Enter | | `limit` | basic | Click six buttons with a budget of two steps | A button was pressed, no input came after the last step, and no acting call was sent once the steps were used up | | `drag` | skill | Drag a handle into a marked zone | A left press on the handle, released inside the zone | -| `scroll` | skill | Scroll a list to read a code below what it shows at first | The final message holds the code, and the list moved | +| `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message holds the words in order and no other word from the reading vocabulary, and the list moved | | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | -`see` shows twelve characters from `ACDEFGJKMNPRTWXY34679`, with no equal neighbours. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. +`see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 112 words, separate from the fixture's labels, and accept any letter case and intervening prose or punctuation. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. @@ -93,6 +93,6 @@ Every check also fails when a pointer or key event came outside an acting call: | The desktop | A container of the kit's image, `desktop.Dockerfile` on Nullius's base image, built the first time a unit needs it: an X display at 1920 by 1080, a window manager, a session bus, the accessibility bus, Python with Tk, `pyautogui` and Pillow. The agent's Python and its actions run as its user; the kit's fixture runs as another, its log out of that user's reach | | The server and the loop | In the kit's driver: the port's server over the unit's desktop, and a loop of the kit's own that runs each step's actions and waits the pause | | What a stop leaves | A unit that never reaches its score is let go by the next unit or by the driver's end; a reaper the driver starts removes the containers and sockets of a driver that was killed | -| The controls | The null control passes no check; the reference makes the right calls through the unit's own server, reads each code from the screenshot it gets, and passes all eight | +| The controls | The null control passes no check; the reference makes the right calls through the unit's own server, reads the words and their gaps from the screenshot it gets, and passes all eight | A machine that runs the kit needs Docker, or Podman 4.7 or later, as the container backend does. The first unit builds the image (about 0.5 GB over the base image). From 78fd4a32dafc4ac4563e3a913e712207a0447af8 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Tue, 17 Nov 2026 13:15:21 +0100 Subject: [PATCH 34/36] fix(standard): allow repeated answers in computer reading checks Accept the drawn words when they occur consecutively among at most twelve vocabulary occurrences in the final message. Allow repeated answers and other vocabulary words before or after the drawn sequence. Replace screen, desktop and action terms with everyday nouns while preserving the spelling and fixture label constraints. Update the verdict descriptions and cover both reading checks, the occurrence limit and the reference reader's vocabulary. --- .../standard/kits/agent-computer-v1/checks.py | 52 ++++++------- .../kits/agent-computer-v1/standard.json | 4 +- app/tests/test_computer_kit.py | 76 +++++++++++++++---- app/tests/test_computer_kit_driver.py | 10 ++- docs/ports/agent-computer-v1.md | 6 +- 5 files changed, 100 insertions(+), 48 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/checks.py b/app/nullius/standard/kits/agent-computer-v1/checks.py index feab050c..adc00887 100644 --- a/app/nullius/standard/kits/agent-computer-v1/checks.py +++ b/app/nullius/standard/kits/agent-computer-v1/checks.py @@ -23,6 +23,12 @@ and the named button is clicked, as for ``click``; - ``batch``: each of three fields holds its word, within two steps. +For ``see`` and ``scroll``, read whole vocabulary words from the last message +in any letter case. The drawn words must occur as a consecutive run in that +sequence at least once, with at most twelve vocabulary occurrences in all. +Other prose and punctuation do not break the run; repeats and other vocabulary +words outside it are allowed and count toward the limit. + Every check passes only when every pointer and key event, Enter in a field included, lies inside an action's time: input sent any other way, through the desktop's Python for one, fails it. An event counts as inside from the @@ -35,20 +41,19 @@ import secrets SLACK = 0.5 -# Everyday nouns, separate from the fixture's labels. Equal-length words -# differ in at least three positions, giving the glyphs the context of a word. -WORDS = ("ACORN", "ANCHOR", "ANKLE", "APRON", "BADGE", "BASKET", "BEACH", "BEETLE", "BERRY", "BOWL", - "BREAD", "BROOM", "BUTTER", "CABIN", "CACTUS", "CAMERA", "CANDLE", "CANOE", "CARPET", "CELERY", - "CHAIR", "CHERRY", "CLOCK", "COFFEE", "COOKIE", "CORN", "COTTON", "CRAYON", "CROWN", "DESK", - "DINNER", "DONKEY", "DOOR", "DRUM", "EAGLE", "EARTH", "ENGINE", "FABRIC", "FARM", "FENCE", - "FLOWER", "FOLDER", "FOREST", "FROG", "GARDEN", "GARLIC", "GATE", "GLOVE", "GOAT", "GUITAR", - "HAMMER", "HELMET", "HORSE", "IGLOO", "INSECT", "LADDER", "LAMP", "LEAF", "LIZARD", "LUNCH", - "MAGNET", "MARBLE", "MEADOW", "MELON", "MIRROR", "MITTEN", "NOODLE", "ONION", "ORANGE", "OTTER", - "PANDA", "PAPER", "PARROT", "PEANUT", "PENCIL", "PEPPER", "PILLOW", "PIZZA", "PLANT", "POCKET", - "POTATO", "RABBIT", "RADIO", "RAISIN", "RIBBON", "ROPE", "SALAD", "SALMON", "SAND", "SANDAL", - "SCARF", "SCHOOL", "SHEEP", "SHIRT", "SHOE", "SHOVEL", "SOCK", "SPRING", "SQUARE", "STAMP", - "STAR", "STREAM", "SUGAR", "TEAPOT", "THUMB", "TOWEL", "TRAIN", "TRAY", "TURTLE", "WINDOW", - "WOLF", "ZEBRA") +# Everyday nouns, avoiding screen, desktop and action vocabulary and the +# fixture's labels. Equal-length words differ in at least three positions, +# giving the glyphs the context of a word. +WORDS = ("ACORN", "ALMOND", "ANKLE", "APRON", "BANANA", "BASKET", "BEACH", "BEAVER", "BEETLE", "BERRY", + "BISON", "BOWL", "BREAD", "BROOM", "BUTTER", "CABIN", "CACTUS", "CAMEL", "CANDLE", "CANOE", + "CARPET", "CASHEW", "CELERY", "CHEESE", "CHERRY", "COCOA", "COFFEE", "CORN", "COTTON", "CRAB", + "DEER", "DINNER", "DONKEY", "DRUM", "EAGLE", "FABRIC", "FARM", "FERRET", "FLOWER", "FOREST", + "FROG", "GARDEN", "GARLIC", "GLOVE", "GOAT", "GUITAR", "HELMET", "HIPPO", "HORSE", "IGLOO", + "INSECT", "KOALA", "LADDER", "LENTIL", "LIZARD", "LUNCH", "MARBLE", "MEADOW", "MELON", "MITTEN", + "NOODLE", "ONION", "OTTER", "OYSTER", "PANDA", "PARROT", "PEANUT", "PEPPER", "PIGEON", "PILLOW", + "PIZZA", "POCKET", "POTATO", "RABBIT", "RAISIN", "ROPE", "SALAD", "SALMON", "SAND", "SANDAL", + "SCARF", "SCHOOL", "SHEEP", "SHIRT", "SHOE", "SHOVEL", "SHRIMP", "SOCK", "SUGAR", "TEAPOT", + "TOWEL", "TURTLE", "WALNUT", "WOLF", "YOGURT", "ZEBRA") LABELS = ("APPLE", "BRICK", "CLOUD", "DELTA", "EMBER", "FROST", "GRAPE", "HONEY", "IVORY", "JUMBO", "KNOT", "LEMON", "MAPLE", "NORTH", "OCEAN", "PIANO", "QUILT", "RIVER", "STONE", "TIGER", "UMBRA", "VELVET", "WHEAT", "YEAST") @@ -109,20 +114,15 @@ def outside(events, actions, slack=SLACK): def _read_words(expected, message): - """Read whole vocabulary words in order, allowing other prose between them.""" + """Find the drawn words as a consecutive run among at most twelve whole + vocabulary words, ignoring case. Prose and punctuation may separate them; + repetitions and other vocabulary words outside the run are allowed.""" words = expected["words"] found = [w for w in re.findall(r"\b\w+\b", str(message or "").upper()) if w in WORDS] - missing = [w for w in words if w not in found] - extra = list(dict.fromkeys(w for w in found if w not in words)) - why = [] - if missing: - why.append("missing words: " + ", ".join(missing)) - if extra: - why.append("extra words: " + ", ".join(extra)) - if why: - return "; ".join(why) - if found != words: - return "the words are out of order or repeated; expected " + ", ".join(words) + if len(found) > 12: + return f"too many words from the reading vocabulary: {len(found)}; expected at most twelve" + if not any(found[i:i + len(words)] == words for i in range(len(found) - len(words) + 1)): + return "the words were not found together in order; expected " + ", ".join(words) return "" diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 272ffbd9..237f895a 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -77,7 +77,7 @@ "see": { "label": "Read the screen", "task": "Read the words drawn on the screen and report them in order.", - "passes": "The final message holds the words in order and no other word from the reading vocabulary, and no input came outside an acting call." + "passes": "Ignoring case, the final message contains the drawn words together in order at least once, with no other vocabulary word between them and at most twelve vocabulary occurrences in all, including repeats, and no input came outside an acting call." }, "click": { "label": "Click a button", @@ -102,7 +102,7 @@ "scroll": { "label": "Scroll", "task": "Scroll to the end of a list and report the large words there in order.", - "passes": "The final message holds the words in order and no other word from the reading vocabulary, and the list moved." + "passes": "Ignoring case, the final message contains the drawn words together in order at least once, with no other vocabulary word between them and at most twelve vocabulary occurrences in all, including repeats, and the list moved." }, "recover": { "label": "Recover from an error", diff --git a/app/tests/test_computer_kit.py b/app/tests/test_computer_kit.py index 69da5373..fb94c752 100644 --- a/app/tests/test_computer_kit.py +++ b/app/tests/test_computer_kit.py @@ -75,22 +75,66 @@ def test_reading_passes_with_spaces_commas_and_quotes_case_or_lines(self): got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) self.assertEqual(got, {"passed": True, "why": ""}) - def test_reading_fails_with_missing_reordered_extra_or_all_words(self): + def test_reading_accepts_repeats_and_other_vocabulary_before_or_after_the_run(self): for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): - answers = [(" ".join(words[:-1]), "missing words: " + words[-1]), - (" ".join(words[::-1]), "out of order"), - (" ".join(words + ["ZEBRA"]), "extra words: ZEBRA"), - (" ".join(checks.WORDS), "extra words:"), - (" ".join(words + words), "repeated"), - (" ".join(["ACORNS", *words[1:]]), "missing words: ACORN"), - (" ".join(["xACORN", *words[1:]]), "missing words: ACORN"), - (" ".join(words[:-1] + ["ZEBRA"]), "missing words: " + words[-1] + "; extra words: ZEBRA"), - ("", "missing words:"), (None, "missing words:")] - for answer, why in answers: + text = " ".join(words) + answers = [text + "\n" + text, + "I see " + ", ".join(words[:-1]) + " and " + words[-1] + ".\nAction: " + text, + "Beside a zebra, I see " + text + ".", + "I see " + text.lower() + ", followed by a zebra.", + "ZEBRA " + text + " ZEBRA", + " ".join(words[::-1]) + "; actually, " + text, + " ".join([words[0], "ZEBRA", *words[1:]]) + "; actually, " + text, + words[0] + " " + text, + text + " " + words[-1]] + for answer in answers: + with self.subTest(check=check, answer=answer): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) + self.assertEqual(got, {"passed": True, "why": ""}) + + def test_reading_ignores_screen_and_action_prose_between_the_drawn_words(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + answers = ["The window shows " + " ".join(words) + ".", + "The square holds " + words[0] + "; the paper holds " + " and ".join(words[1:]) + ".", + "I mirror " + words[0] + " and then stream " + " and ".join(words[1:]) + "."] + for answer in answers: + with self.subTest(check=check, answer=answer): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) + self.assertEqual(got, {"passed": True, "why": ""}) + + def test_reading_fails_without_an_unbroken_run_of_the_drawn_words(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + answers = [" ".join(words[:-1]), " ".join(words[::-1]), + " ".join([words[0], "ZEBRA", *words[1:]]), + " ".join(["ACORNS", *words[1:]]), " ".join(["xACORN", *words[1:]]), + " ".join(["ACORN_", *words[1:]]), " ".join(["ACORN2", *words[1:]]), + " ".join(words[:-1] + ["ZEBRA"]), "", None] + for answer in answers: with self.subTest(check=check, answer=answer): got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, answer) - self.assertFalse(got["passed"]) - self.assertIn(why, got["why"]) + self.assertEqual(got, {"passed": False, "why": + "the words were not found together in order; expected " + ", ".join(words)}) + + def test_reading_accepts_twelve_vocabulary_occurrences_including_repeats(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + padding = ["ZEBRA"] * (12 - len(words)) + for found in (words + padding, padding + words, words * (12 // len(words))): + with self.subTest(check=check, found=found): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, + " ".join(found)) + self.assertEqual(got, {"passed": True, "why": ""}) + + def test_reading_rejects_more_than_twelve_vocabulary_occurrences_even_with_a_correct_run(self): + for check, words in (("see", ["ACORN", "BASKET", "CANDLE"]), ("scroll", ["ACORN", "BASKET"])): + answers = [words + ["ZEBRA"] * (13 - len(words)), + words * (12 // len(words)) + [words[0]], + list(checks.WORDS), list(checks.WORDS) + words] + for found in answers: + with self.subTest(check=check, found=found): + got = checks.verdict(check, {"words": words}, [ev("moved", 100.2, top=0.4)], ACTIONS, + " ".join(found).lower()) + self.assertEqual(got, {"passed": False, "why": + f"too many words from the reading vocabulary: {len(found)}; expected at most twelve"}) def test_click_passes_on_a_left_press_on_the_named_button_inside_an_action(self): events = [ev("ready", 99.0)] + press(100.5, "OCEAN") @@ -174,10 +218,14 @@ def test_every_character_drawn_has_a_glyph_of_five_by_seven(self): self.assertNotIn((5, 3), cells) # the blank column def test_reading_words_are_distinct_nouns_apart_from_each_other_and_the_labels(self): - self.assertGreaterEqual(len(checks.WORDS), 64) + self.assertGreaterEqual(len(checks.WORDS), 96) self.assertEqual(len(checks.WORDS), len(set(checks.WORDS))) labels = set(checks.LABELS + checks.ORDER + checks.FIELDS + fixture.LIST_WORDS + ("MOVE", "HERE", fixture.TITLE.upper())) self.assertFalse(set(checks.WORDS) & labels) + prose = set("ANCHOR BADGE CAMERA CHAIR CLOCK COOKIE CRAYON CROWN DESK DOOR EARTH ENGINE FENCE FOLDER GATE " + "HAMMER LAMP LEAF MAGNET MIRROR ORANGE PAPER PENCIL PLANT RADIO RIBBON SPRING SQUARE STAMP " + "STAR STREAM THUMB TRAIN TRAY WINDOW".split()) + self.assertFalse(set(checks.WORDS) & prose) for i, word in enumerate(checks.WORDS): self.assertRegex(word, r"^[A-Z]{4,6}$") for other in checks.WORDS[i + 1:]: diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index d2707cab..2ddbc1e6 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -88,7 +88,8 @@ def test_scroll_passes_on_the_words_once_the_list_moved(self): moved = [ev("scroll", 100.2, button=5, hit="list"), ev("moved", 100.21, top=0.4)] self.assertTrue(checks.verdict("scroll", exp, moved, ACTIONS, "ACORN BASKET")["passed"]) self.assertIn("never moved", checks.verdict("scroll", exp, moved[:1], ACTIONS, "ACORN BASKET")["why"]) - self.assertIn("missing words", checks.verdict("scroll", exp, moved, ACTIONS, "")["why"]) + self.assertEqual(checks.verdict("scroll", exp, moved, ACTIONS, ""), + {"passed": False, "why": "the words were not found together in order; expected ACORN, BASKET"}) def test_recover_passes_on_a_success_after_the_error_and_the_click(self): exp = {"target": "OCEAN", "labels": ["OCEAN", "APPLE"]} @@ -504,17 +505,20 @@ def chunk(kind, body): class Words(unittest.TestCase): - def test_the_reference_reads_drawn_words_and_gaps_at_both_scales(self): + def test_the_reference_reads_current_vocabulary_and_gaps_at_both_scales(self): rng = random.Random(7) for check, scale, count in (("see", driver.fixture.SEE_SCALE, 3), ("scroll", driver.fixture.SCROLL_SCALE, 2)): with self.subTest(check=check): + removed = " ".join(["WINDOW", "IGLOO", "INSECT"][:count]) + with self.assertRaises(driver.ReferenceFailed): + driver.read_words(_png(removed, scale), scale, count) for _ in range(3): values, expected = checks.draw(check, rng) text = " ".join(values["words"]) png = _png(text, scale) self.assertEqual(driver.fixture.read_lines(png, scale), [text]) self.assertEqual(driver.read_words(png, scale, count), " ".join(expected["words"])) - text = " ".join(["IGLOO", "INSECT", "WINDOW"][:count]) + text = " ".join(["IGLOO", "WALNUT", "INSECT"][:count]) self.assertEqual(driver.read_words(_png(text, scale), scale, count), text) def test_the_reference_refuses_missing_unreadable_or_repeated_words(self): diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index d89fd27b..fc07aa84 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -75,16 +75,16 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | Check | Kind | The task | Passes when | | --- | --- | --- | --- | -| `see` | basic | Read the words drawn on the screen and report them in order | The final message holds the words in order and no other word from the reading vocabulary | +| `see` | basic | Read the words drawn on the screen and report them in order | The final message contains an uninterrupted sequence of the drawn words among at most twelve vocabulary occurrences | | `click` | basic | Click the named one of six buttons | A left press on that button, and no press on the five others | | `type` | basic | Type a word in a text field and press Enter | The field holds exactly the word at the last Enter | | `limit` | basic | Click six buttons with a budget of two steps | A button was pressed, no input came after the last step, and no acting call was sent once the steps were used up | | `drag` | skill | Drag a handle into a marked zone | A left press on the handle, released inside the zone | -| `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message holds the words in order and no other word from the reading vocabulary, and the list moved | +| `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message contains an uninterrupted sequence of the drawn words among at most twelve vocabulary occurrences, and the list moved | | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | -`see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 112 words, separate from the fixture's labels, and accept any letter case and intervening prose or punctuation. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. +`see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 96 everyday nouns chosen to avoid screen, desktop and action vocabulary and the fixture's labels. The verdict reads whole vocabulary words in the final message, ignoring case. The drawn words must occur together in the shown order at least once, with no other vocabulary word inside that sequence. Prose and punctuation may separate them. Repeated answers and vocabulary words before or after the sequence are accepted, up to twelve vocabulary occurrences in the whole message, counting repeats. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. From 6a403a538aeac18cea3befdd679ddc7e8229e3cb Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Tue, 17 Nov 2026 13:41:32 +0100 Subject: [PATCH 35/36] test(standard): keep the engine's own environment lookup out of a patched launch The test of a launch that meets a shutdown replaces subprocess.Popen for the whole process. On a machine with Docker the engine's environment is read through subprocess.run, which then met the stand-in and failed before the launch under test. The test now gives the launch a fixed environment. --- app/tests/test_computer_kit_driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 2ddbc1e6..9ba10d64 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -853,6 +853,7 @@ def __exit__(self, *a): launch = threading.Thread(target=lambda: replies.append(desktop._engine(["exec"], timeout=10)), daemon=True) drain = threading.Thread(target=lambda: counts.append(desktop.end_clients()), daemon=True) with mock.patch.object(desktop, "_CLIENTS_LOCK", Lock()), \ + mock.patch.object(desktop.container, "client_env", return_value={}), \ mock.patch.object(desktop.subprocess, "Popen", side_effect=popen) as start: launch.start() try: From 8fde45d13eea03bbdd17b2e31a55d7e85ac09859 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Tue, 17 Nov 2026 14:20:54 +0100 Subject: [PATCH 36/36] feat(catalogue): record Codex computer port proof Record the kit v1 computer port proof for Codex 2.2.2. All four basic checks and four skills passed all three runs. Preserve the run metadata and the counts with the version's other proofs. Regenerate the supported agent pages to show the proven port and OSWorld1 compatibility. --- app/nullius/catalogue/agents/codex/agent.json | 35 +++++++++++++++++++ docs/agents/supported/README.md | 4 +-- docs/agents/supported/codex.md | 4 +-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/app/nullius/catalogue/agents/codex/agent.json b/app/nullius/catalogue/agents/codex/agent.json index a8d6e207..eca12acd 100644 --- a/app/nullius/catalogue/agents/codex/agent.json +++ b/app/nullius/catalogue/agents/codex/agent.json @@ -138,6 +138,41 @@ "kit_version": 1, "backend": "docker", "job": "2026-11-12__10-24-26__kit__agent-search-v1__codex-2.2.2" + }, + "agent.computer/v1": { + "passed": "4/4", + "on": "2026-11-17", + "model": "gpt-6-luna", + "provider": "openai", + "version": "2.2.2", + "kit_version": 1, + "backend": "container", + "job": "2026-11-17__12-53-08__kit__agent-computer-v1__codex-2.2.2", + "repeat": 3, + "checks": { + "see": "3/3", + "click": "3/3", + "type": "3/3", + "limit": "3/3" + }, + "skills": { + "drag": { + "passed": "3/3", + "ok": true + }, + "scroll": { + "passed": "3/3", + "ok": true + }, + "recover": { + "passed": "3/3", + "ok": true + }, + "batch": { + "passed": "3/3", + "ok": true + } + } } } }, diff --git a/docs/agents/supported/README.md b/docs/agents/supported/README.md index 2954dc74..25e71400 100644 --- a/docs/agents/supported/README.md +++ b/docs/agents/supported/README.md @@ -33,7 +33,7 @@ Every agent in the catalogue, the ports each version has proven, and the benchma - + @@ -51,7 +51,7 @@ Every agent in the catalogue, the ports each version has proven, and the benchma
AgentPortsTools, the skills of agent.tools/v1
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
codex@2.2.2✓✓✓✓✓–✓✓✓
codex@2.2.2✓✓✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✓✓–✓✓✓
goose✓✓✓✓✓–✓✓✓
jiuwenswarm@0.2.7✓✓✓✓––✓✓✓
- + diff --git a/docs/agents/supported/codex.md b/docs/agents/supported/codex.md index 291cfcaa..6f80b379 100644 --- a/docs/agents/supported/codex.md +++ b/docs/agents/supported/codex.md @@ -19,7 +19,7 @@ OpenAI's coding agent.
GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
codex@2.2.2✓✓✓✗✓✓✓
codex@2.2.2✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✗✓✓✓
goose✓✓✓✗✓✓✓
jiuwenswarm@0.2.7✗✓✓✗✓✓✓
- +
VersionUpstreamPortsTools
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry
codex@2.2.2 (default)2.2.2✓✓✓✓✓–✓✓✓
codex@2.2.2 (default)2.2.2✓✓✓✓✓✓✓✓✓
codex@0.16.00.16.0✓✓✓✓✓–✓✓✓
@@ -36,7 +36,7 @@ OpenAI's coding agent. - +
GAIAGSM8KHumanEvalOSWorld1SWE-bench Verifiedτ³-benchTerminal-Bench 2.0
One task, Workspace files, SearchOne taskWorkspace filesComputer useWorkspace filesMCP tools, ConversationWorkspace files
codex@2.2.2✓✓✓✗✓✓✓
codex@2.2.2✓✓✓✓✓✓✓
codex@0.16.0✓✓✓✗✓✓✓