Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,11 @@ and its resources. The source layout stays under `app/`.
tools kit, an MCP server `mcp_kit`) - the definitions it reads are in the catalogue:
a benchmark's `standard.json` and `driver` in its folder (τ³-bench's also the MCP
server of its domains' tools, `mcp_server`: the upstream's own, run on the machine; OSWorld1's also its null control's
agent, `nop`), `catalogue/agents/<key>/`
agent, `nop`; OSWorld2's also `nop`, `api_logs` (each image of its API logs kept once), `websites` (its mock
websites: the checks before a VM start, Overleaf's lock and reset, the stack's command line), `gitlab_server` (its
GitLab: the check before a VM start, its tasks' lock, the stack's command line), `proxy` (a
tinyproxy of a unit's own for the tasks that ask for a proxy) and, in `startup/`,
`sitecustomize`: its VMs' disk write limit in every process its driver starts), `catalogue/agents/<key>/`
(the reference `agent`; JiuwenSwarm's `agent` and mini-swe-agent's `run`, the wrappers that start them) and `catalogue/capabilities/<key>/` (the keyword-memory `capability`); `workspace` (and
`workspace_cli`, its shell command),
`cli` and `__main__` (the unified `nullius` and `python -m nullius` commands),
Expand Down
83 changes: 83 additions & 0 deletions app/nullius/catalogue/benchmarks/osworld2/agents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
{
"about": "The upstream's agents the driver runs, by the agent option: each its own script at the tested commit, as published, on one task. argv is what its scripts/bash wrapper passes (run_multienv_m3.sh, run_multienv_muse_spark.sh), less what Nullius sets for every unit (common: the docker provider, the VM image, one environment, the image's password as the wrapper passes it, the task's meta file, the unit's results) and less the aws provider's --region and --use_public_ip and the wrapper's one --specific_task_id. M3 also gets the relay's key as --api_key, since its script reads MINIMAX_ANTHROPIC_BASE_URL and MINIMAX_API_KEY before ANTHROPIC_*, and a .env file in the upstream (each script calls load_dotenv) could set them. Muse Spark gets the reasoning effort and output budget of its published run's args.json (max, 262144) in place of its wrapper's (high, 8192), and MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT, since its count of input tokens before each call asks the endpoint for a path OpenRouter does not serve. env points each script's model client at the unit's relay too, and its simulated user's, asked by the name the documented invocation gives it (OSWORLD_USER_SIM_MODEL), which the Original setup sends on as a side model. {model} is the model asked, {base_url} the unit's relay (ending /v1), {base} the same without /v1 (the Anthropic client adds /v1/messages), {key} the relay's placeholder, {vm} the VM image, {up} the upstream, {meta} the unit's task file, {results} the unit's results folder, {examples} the upstream's evaluation_examples, {max_steps} the max_steps option's value. models are the ones each agent is published with, by the names its code knows; model_argv adds what its documented invocation passes for one of them.",
"common": [
"--provider_name",
"docker",
"--path_to_vm",
"{vm}",
"--headless",
"--num_envs",
"1",
"--test_config_base_dir",
"{examples}",
"--test_all_meta_path",
"{meta}",
"--result_dir",
"{results}",
"--client_password",
"osworld-public-evaluation"
],
"agents": {
"m3": {
"script": "scripts/python/run_multienv_m3.py",
"models": [
"MiniMax-M3"
],
"argv": [
"--action_space",
"pyautogui",
"--observation_type",
"screenshot",
"--max_tokens",
"8192",
"--max_steps",
"{max_steps}",
"--model",
"{model}",
"--base_url",
"{base}",
"--api_key",
"{key}"
],
"env": {
"ANTHROPIC_BASE_URL": "{base}",
"ANTHROPIC_API_KEY": "{key}",
"OSWORLD_USER_SIM_PROVIDER": "anthropic",
"OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6",
"OSWORLD_USER_SIM_BASE_URL": "{base}",
"OSWORLD_USER_SIM_API_KEY": "{key}"
}
},
"muse_spark": {
"script": "scripts/python/run_multienv_muse_spark.py",
"models": [
"muse-spark-1.3"
],
"argv": [
"--action_space",
"pyautogui",
"--observation_type",
"screenshot",
"--max_tokens",
"262144",
"--reasoning_effort",
"max",
"--max_steps",
"{max_steps}",
"--model",
"{model}",
"--base_url",
"{base_url}",
"--save_model_eval_raw_info"
],
"env": {
"MUSE_SPARK_API_KEY": "{key}",
"MUSE_SPARK_SKIP_INPUT_TOKEN_PREFLIGHT": "1",
"OSWORLD_USER_SIM_PROVIDER": "anthropic",
"OSWORLD_USER_SIM_MODEL": "claude-sonnet-4-6",
"OSWORLD_USER_SIM_BASE_URL": "{base}",
"OSWORLD_USER_SIM_API_KEY": "{key}"
}
}
}
}
232 changes: 232 additions & 0 deletions app/nullius/catalogue/benchmarks/osworld2/api_logs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
"""OSWorld2's api_logs with each image in them kept once.

The upstream's M3 agent logs each model call's request with its screenshots as
base64, so one image repeats across files. pack(folder) writes each image once
as images/<sha256>.<ext> and puts that path in its place; restore(folder) gives
each file back byte for byte. Each write is a temporary file renamed over its
name, so a failure leaves every file whole and running again changes nothing.

python3 api_logs.py pack|restore FOLDER... prints what it did, as JSON."""
import base64
import binascii
import contextlib
import fcntl
import hashlib
import json
import os
import re
import stat
import sys
import tempfile

IMAGES = "images"
TEMPORARY = ".nullius-tmp"
SHORTEST = 256
VALUE = rb'(?:(?<=: )|(?<=\[)|(?<=, ))"'
BASE64 = re.compile(VALUE + rb'([A-Za-z0-9+/]{%d,}={0,2})"' % SHORTEST)
MARK = b'{"_packed_images": "' + IMAGES.encode() + b'", '
LINK = re.compile(VALUE + IMAGES.encode() + rb'/([0-9a-f]{64}\.(?:jpg|png|gif|webp))"')


def _kind(data):
if data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return "webp"
for magic, ext in ((b"\xff\xd8\xff", "jpg"), (b"\x89PNG\r\n\x1a\n", "png"), (b"GIF87a", "gif"), (b"GIF89a", "gif")):
if data.startswith(magic):
return ext
return None


def _image(text):
"""The image a base64 value holds, as (its file name, its bytes), or None."""
if len(text) % 4:
return None
try:
data = base64.b64decode(text, validate=True)
except (binascii.Error, ValueError):
return None
ext = _kind(data)
if ext is None or base64.b64encode(data) != text:
return None
return f"{hashlib.sha256(data).hexdigest()}.{ext}", data


def _expand(text, image):
"""A packed file's text as it was, with ``image`` giving a file name's bytes."""
return b"{" + LINK.sub(lambda m: b'"' + base64.b64encode(image(m.group(1).decode())) + b'"', text[len(MARK):])


def _read(path, folder=None):
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=folder)
with os.fdopen(fd, "rb") as f:
st = os.fstat(f.fileno())
if not stat.S_ISREG(st.st_mode):
raise OSError(f"{path} is not a plain file")
return f.read(), st


def _write(path, data, like):
fd, tmp = tempfile.mkstemp(prefix="." + os.path.basename(path) + ".", suffix=TEMPORARY, dir=os.path.dirname(path))
try:
with os.fdopen(fd, "wb") as f:
f.write(data)
f.flush()
os.fchmod(f.fileno(), stat.S_IMODE(like.st_mode))
os.fsync(f.fileno())
os.utime(tmp, ns=(like.st_atime_ns, like.st_mtime_ns))
os.replace(tmp, path)
except BaseException:
with contextlib.suppress(OSError):
os.unlink(tmp)
raise


def _sync(folder):
fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)


def _held(path, name, folder=None):
"""The bytes of ``path``, a plain file holding the image ``name`` names; None for anything else."""
try:
data = _read(path, folder)[0]
except OSError:
return None
return data if f"{hashlib.sha256(data).hexdigest()}.{_kind(data)}" == name else None


def _inner(fd, name):
return os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)


def _tidy(fd):
for name in os.listdir(fd):
if name.startswith(".") and name.endswith(TEMPORARY):
with contextlib.suppress(OSError):
os.unlink(name, dir_fd=fd)


@contextlib.contextmanager
def _alone(folder):
"""The folder locked for this process, cleared of a stopped one's half-written files; None when another holds it."""
fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
yield None
return
_tidy(fd)
with contextlib.suppress(OSError):
images = _inner(fd, IMAGES)
try:
_tidy(images)
finally:
os.close(images)
yield fd
finally:
os.close(fd)


def _files(folder):
for name in sorted(os.listdir(folder)):
if name.endswith(".json") and not name.startswith("."):
path = os.path.join(folder, name)
try:
yield path, *_read(path)
except OSError:
continue


def _images(folder):
images = os.path.join(folder, IMAGES)
with contextlib.suppress(FileExistsError):
os.mkdir(images)
if not stat.S_ISDIR(os.lstat(images).st_mode):
raise OSError(f"{images} is not a folder")
return images


def pack(folder):
done = dict(files=0, packed=0, images=0, written=0, before=0, after=0)
with _alone(folder) as mine:
if mine is None:
return dict(done, busy=True)
images, kept = None, set()
for path, raw, st in _files(folder):
done["files"] += 1
done["before"] += len(raw)
found = {}

def link(m):
image = _image(m.group(1))
if image is None:
return m.group(0)
found[image[0]] = image[1]
return b'"' + f"{IMAGES}/{image[0]}".encode() + b'"'

packed = raw if not raw.startswith(b'{"') or raw.startswith(MARK) or LINK.search(raw) \
else MARK + BASE64.sub(link, raw[1:])
if not found or _expand(packed, found.__getitem__) != raw:
done["after"] += len(raw)
continue
images = images or _images(folder)
new = [name for name in found if name not in kept and _held(os.path.join(images, name), name) is None]
for name in new:
_write(os.path.join(images, name), found[name], st)
if new:
_sync(images)
kept.update(found)
done["written"] += len(new)
_write(path, packed, st)
done["packed"] += 1
done["after"] += len(packed)
if done["packed"]:
_sync(folder)
done["images"] = len(kept)
return done


def restore(folder):
"""A file whose image is missing or does not match its name stays packed."""
done = dict(restored=0, failed=0)
with _alone(folder) as mine, contextlib.ExitStack() as stack:
if mine is None:
return dict(done, busy=True)
try:
images = _inner(mine, IMAGES)
stack.callback(os.close, images)
except OSError:
images = None
cache = {}

def image(name):
if name not in cache:
cache[name] = None if images is None else _held(name, name, images)
if cache[name] is None:
raise ValueError(f"no image {name}")
return cache[name]

for path, raw, st in _files(folder):
if not raw.startswith(MARK):
continue
try:
text = _expand(raw, image)
except ValueError:
done["failed"] += 1
continue
_write(path, text, st)
done["restored"] += 1
if done["restored"]:
_sync(folder)
return done


if __name__ == "__main__":
act = {"pack": pack, "restore": restore}.get(sys.argv[1] if len(sys.argv) > 2 else "")
if act is None:
sys.exit("usage: api_logs.py pack|restore FOLDER...")
print(json.dumps({folder: act(folder) for folder in sys.argv[2:]}))
Loading
Loading