Skip to content
10 changes: 7 additions & 3 deletions app/nullius/standard/acp.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@
from typing import Dict, List, Optional

PROTOCOL_VERSION = 1
# How long an agent has to answer each of the two requests that set it up: together, the limit of
# a unit's agent setup (``run``).
INITIALIZE_SECONDS = 120
SESSION_SECONDS = 180


class AgentFailed(Exception):
Expand Down Expand Up @@ -135,7 +139,7 @@ def start(self) -> "Client":
self.info = self.request("initialize", {"protocolVersion": PROTOCOL_VERSION,
"clientCapabilities": {"fs": {"readTextFile": self.fs, "writeTextFile": self.fs},
"terminal": False, "session": {"notices": {}}},
"clientInfo": {"name": "nullius", "version": "1"}}, timeout=120)
"clientInfo": {"name": "nullius", "version": "1"}}, timeout=INITIALIZE_SECONDS)
return self

def _send(self, msg: dict):
Expand Down Expand Up @@ -242,11 +246,11 @@ def new_session(self, cwd: str, mcp: Optional[List[dict]] = None) -> str:
"env": [{"name": k, "value": v} for k, v in (s.get("env") or {}).items()]} for s in mcp or []]
params = {"cwd": os.path.abspath(cwd), "mcpServers": servers}
try:
res = self.request("session/new", params, timeout=180)
res = self.request("session/new", params, timeout=SESSION_SECONDS)
except AgentFailed as e:
if "auth" not in str(e).lower() or not self.authenticate():
raise
res = self.request("session/new", params, timeout=180)
res = self.request("session/new", params, timeout=SESSION_SECONDS)
sid = res["sessionId"]
self.roots[sid] = os.path.abspath(cwd)
return sid
Expand Down
30 changes: 28 additions & 2 deletions app/nullius/standard/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,8 @@ def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[d
def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float,
produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict],
trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "",
image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None),
image: str = "", setup_end: Optional[float] = None, agent_ready: Optional[float] = None,
agent_unready: Optional[float] = None, checking: tuple = (None, None),
serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None,
retries: int = 0, retryable: bool = True) -> dict:
"""Write one unit's trial and return its result. ``serving``: the run's
Expand All @@ -472,12 +473,20 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st
benchmark's environment the unit asked for (none: the base image), and
``image``, the one it ran in on the container backend - names only.
``setup_end``: when the unit's environment was ready and the agent began
(its setup, relay and container from ``t0``); ``checking``: the check's
(its setup, relay and container from ``t0``); ``agent_ready``: when the
agent, started and in its session, was first asked - its setup from
``setup_end``, its run from there; ``agent_unready``: when its start ended
with no session (it never answered, it exited) - its setup ends there and
it has no run, as Harbor writes a failed agent setup; ``checking``: the check's
start and end (its own step in the container, then the driver's score).
``retryable``: False for an exception a further try meets again
whatever its name (``retries``), which its record then says."""
os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True)
os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True)
if produce and produce[0] and agent_ready: # its run starts once it is set up: the trace's window too
produce = (agent_ready, produce[1])
elif agent_unready: # it never got a session: no run to time
produce = (None, None)
tot = _usage_total(calls)
# the pages read a task's name after its last slash: a unit's own slashes
# (a unit id like "set/12") would leave only its last part
Expand All @@ -486,6 +495,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st
"agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""),
"model_info": {"name": model}},
"environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None,
"agent_setup": {"started_at": iso(setup_end), "finished_at": iso(agent_ready or agent_unready)}
if setup_end and (agent_ready or agent_unready) else None,
"agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None,
"verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None,
"agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"],
Expand Down Expand Up @@ -583,6 +594,21 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str
"ModelNotFound")


FEED_PHASES = (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier"))


def feed_durations(res: dict) -> Optional[dict]:
"""The phases a trial recorded, in the live feed's words and in seconds, read from its result so
the board and the trial's page say the same (Harbor's writer sends its four the same way). A
phase it has none of is left out; None when it recorded none."""
out = {}
for phase, key in FEED_PHASES:
window = res.get(key) or {}
if window.get("started_at") and window.get("finished_at"):
out[phase] = round((datetime.fromisoformat(window["finished_at"]) - datetime.fromisoformat(window["started_at"])).total_seconds(), 3)
return out or None


def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict:
"""A unit's end in the live feed's words (``nullius.runs.live``): its
outcome, with the reward (a pass benchmark) or the score, and the error's
Expand Down
56 changes: 39 additions & 17 deletions app/nullius/standard/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1202,6 +1202,12 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non
if self.feed is not None:
self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout)

def over_acp(self) -> bool:
"""Whether the unit's other is an agent driven over ACP: the one kind with a setup of its own,
its start and its session. A command agent starts and runs in one call, an upstream setup and a
control are the driver's, and a capability's process answers nothing until its first request."""
return self.other["_kind"] == "agent" and "command" not in self.other["start"]

def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]:
if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()):
return None # the endpoint refused the run: nothing more is begun
Expand All @@ -1223,7 +1229,8 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]:
self.progress.emit("unit", "start", unit=unit)
self.phase(name, unit, "env_setup", None if self.upstream else self.phase_limit("setup", DRIVER_SECONDS))
produce, score, exc, traj, logs = (None, None), None, None, None, {}
p0, checking = None, (None, None) # when the agent began; the check's start and end
p0, p1, checking = None, None, (None, None) # when the agent began and ended; the check's start and end
clock: dict = {} # an agent's start: "ready" when its session opened, "failed" when it ended with none
setup: dict = {}
transcript: List[dict] = []
rl, environment, box, changes = None, None, None, None
Expand Down Expand Up @@ -1268,21 +1275,27 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]:
if setup.get("prepare") is not None:
self.prepare(setup["prepare"], box, trial)
with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts
p0 = time.time()
self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control"
else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None)
try:
with self.agent_internet(setup, box, rl, trial):
if self.other["_kind"] == "control":
answer = self.control_answer(unit, ws, box, name)
elif self.upstream:
answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl)
elif self.other["_kind"] == "agent":
answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name)
else:
answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl)
p0 = time.time() # the task's door is open: the environment is ready, the agent begins
try:
if self.over_acp(): # its start and session are its setup: its limit begins once it is in one
self.phase(name, unit, "agent_setup", acp.INITIALIZE_SECONDS + acp.SESSION_SECONDS)
else:
self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control"
else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None)
if self.other["_kind"] == "control":
answer = self.control_answer(unit, ws, box, name)
elif self.upstream:
answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl)
elif self.other["_kind"] == "agent":
answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name, clock=clock)
else:
answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl)
finally:
p1 = time.time() # before the door closes
finally:
produce = (p0, time.time())
produce = (p0, p1 if p1 is not None else time.time())
if box is not None: # what the agent (or the control) left in the container, before the check
changes = self.box_changes(box)
cut = self.cap_reached(rl, tag)
Expand Down Expand Up @@ -1359,7 +1372,8 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]:
traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")])
res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit,
t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls,
setup_end=p0 or time.time(), checking=checking,
setup_end=p0 or time.time(), agent_ready=clock.get("ready"), agent_unready=clock.get("failed"),
checking=checking,
trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})),
environment=environment or "", retries=retry, retryable=retry_ok, serving=getattr(self, "serving", None),
# who served the unit: its agent's calls and its judge's and simulated user's routes
Expand All @@ -1374,7 +1388,7 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]:
self.feed.emit("trial.end", name, task=unit, agent=self.other["key"],
**job.feed_outcome(self.bench["score"]["kind"], score, exc),
**{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS},
durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None)
durations=job.feed_durations(res))
self.note_refusal(exc)
if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None:
with self._said:
Expand Down Expand Up @@ -2071,13 +2085,16 @@ def agent_limit(self, setup: dict) -> float:
return float(own)
return self.turn_seconds

def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name=""):
def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name="", clock=None):
"""The agent over ACP, its session's ``cwd`` the unit's workspace - in
an own image (``box``), the container's working directory, which this
machine does not see. Files are served (``acp.Client(fs=)``) only to an
agent running here: one in a container has its workspace mounted, and
serving it from this machine would let it swap a folder for a link
between the check and the open and reach any file of the user's."""
between the check and the open and reach any file of the user's.
``clock`` gets when its setup ended: ``ready``, its session open, or
``failed``, its start over with none (it never answered, it exited, the
run was stopped)."""
a = self.other
env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options)
log = os.path.join(os.path.dirname(ws), "agent", "agent.log")
Expand All @@ -2098,10 +2115,13 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name
except TimeoutError as e:
# the agent never answered its start: this unit's setup failed, not the run
raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})")
if clock is not None:
clock["ready"] = time.time() # started and in its session: its setup ends, its run begins
prompt = setup["prompt"]
answer = ""
# one limit for the agent's whole work on the unit, every turn and the simulated user's between them
limit = self.agent_limit(setup)
self.phase(name, unit, "agent_run", limit)
deadline = time.monotonic() + limit

def left() -> float:
Expand Down Expand Up @@ -2153,6 +2173,8 @@ def left() -> float:
except jsonl.Failed as e:
raise _Ended("VerifierError", "the benchmark's next turn: " + str(e), retryable=not isinstance(e, jsonl.NoReply))
finally:
if clock is not None and "ready" not in clock:
clock["failed"] = time.time() # its start ended with no session: all of it was its setup
c.close()
raw = "".join(json.dumps(e) + "\n" for e in events)
return answer, job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {"acp.jsonl": raw}
Expand Down
Loading
Loading