diff --git a/app/nullius/standard/acp.py b/app/nullius/standard/acp.py index 464d7c0..534b300 100644 --- a/app/nullius/standard/acp.py +++ b/app/nullius/standard/acp.py @@ -43,6 +43,10 @@ from typing import Dict, List, Optional PROTOCOL_VERSION = 1 +# How long an agent has to answer each of the two requests that set it up: together, the limit of +# a unit's agent setup (``run``). +INITIALIZE_SECONDS = 120 +SESSION_SECONDS = 180 class AgentFailed(Exception): @@ -135,7 +139,7 @@ def start(self) -> "Client": self.info = self.request("initialize", {"protocolVersion": PROTOCOL_VERSION, "clientCapabilities": {"fs": {"readTextFile": self.fs, "writeTextFile": self.fs}, "terminal": False, "session": {"notices": {}}}, - "clientInfo": {"name": "nullius", "version": "1"}}, timeout=120) + "clientInfo": {"name": "nullius", "version": "1"}}, timeout=INITIALIZE_SECONDS) return self def _send(self, msg: dict): @@ -242,11 +246,11 @@ def new_session(self, cwd: str, mcp: Optional[List[dict]] = None) -> str: "env": [{"name": k, "value": v} for k, v in (s.get("env") or {}).items()]} for s in mcp or []] params = {"cwd": os.path.abspath(cwd), "mcpServers": servers} try: - res = self.request("session/new", params, timeout=180) + res = self.request("session/new", params, timeout=SESSION_SECONDS) except AgentFailed as e: if "auth" not in str(e).lower() or not self.authenticate(): raise - res = self.request("session/new", params, timeout=180) + res = self.request("session/new", params, timeout=SESSION_SECONDS) sid = res["sessionId"] self.roots[sid] = os.path.abspath(cwd) return sid diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 04ca51a..4949c38 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -461,7 +461,8 @@ def served_total(results: List[dict], also: Optional[dict] = None) -> Optional[d def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None), + image: str = "", setup_end: Optional[float] = None, agent_ready: Optional[float] = None, + agent_unready: Optional[float] = None, checking: tuple = (None, None), serving: Optional[Dict[str, List[str]]] = None, served_calls: Optional[List[dict]] = None, retries: int = 0, retryable: bool = True) -> dict: """Write one unit's trial and return its result. ``serving``: the run's @@ -472,12 +473,20 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began - (its setup, relay and container from ``t0``); ``checking``: the check's + (its setup, relay and container from ``t0``); ``agent_ready``: when the + agent, started and in its session, was first asked - its setup from + ``setup_end``, its run from there; ``agent_unready``: when its start ended + with no session (it never answered, it exited) - its setup ends there and + it has no run, as Harbor writes a failed agent setup; ``checking``: the check's start and end (its own step in the container, then the driver's score). ``retryable``: False for an exception a further try meets again whatever its name (``retries``), which its record then says.""" os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True) os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True) + if produce and produce[0] and agent_ready: # its run starts once it is set up: the trace's window too + produce = (agent_ready, produce[1]) + elif agent_unready: # it never got a session: no run to time + produce = (None, None) tot = _usage_total(calls) # the pages read a task's name after its last slash: a unit's own slashes # (a unit id like "set/12") would leave only its last part @@ -486,6 +495,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""), "model_info": {"name": model}}, "environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None, + "agent_setup": {"started_at": iso(setup_end), "finished_at": iso(agent_ready or agent_unready)} + if setup_end and (agent_ready or agent_unready) else None, "agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None, "verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None, "agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"], @@ -583,6 +594,21 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str "ModelNotFound") +FEED_PHASES = (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier")) + + +def feed_durations(res: dict) -> Optional[dict]: + """The phases a trial recorded, in the live feed's words and in seconds, read from its result so + the board and the trial's page say the same (Harbor's writer sends its four the same way). A + phase it has none of is left out; None when it recorded none.""" + out = {} + for phase, key in FEED_PHASES: + window = res.get(key) or {} + if window.get("started_at") and window.get("finished_at"): + out[phase] = round((datetime.fromisoformat(window["finished_at"]) - datetime.fromisoformat(window["started_at"])).total_seconds(), 3) + return out or None + + def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict: """A unit's end in the live feed's words (``nullius.runs.live``): its outcome, with the reward (a pass benchmark) or the score, and the error's diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 3d9f0f6..7518e9f 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -1202,6 +1202,12 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non if self.feed is not None: self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout) + def over_acp(self) -> bool: + """Whether the unit's other is an agent driven over ACP: the one kind with a setup of its own, + its start and its session. A command agent starts and runs in one call, an upstream setup and a + control are the driver's, and a capability's process answers nothing until its first request.""" + return self.other["_kind"] == "agent" and "command" not in self.other["start"] + def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()): return None # the endpoint refused the run: nothing more is begun @@ -1223,7 +1229,8 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: self.progress.emit("unit", "start", unit=unit) self.phase(name, unit, "env_setup", None if self.upstream else self.phase_limit("setup", DRIVER_SECONDS)) produce, score, exc, traj, logs = (None, None), None, None, None, {} - p0, checking = None, (None, None) # when the agent began; the check's start and end + p0, p1, checking = None, None, (None, None) # when the agent began and ended; the check's start and end + clock: dict = {} # an agent's start: "ready" when its session opened, "failed" when it ended with none setup: dict = {} transcript: List[dict] = [] rl, environment, box, changes = None, None, None, None @@ -1268,21 +1275,27 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) with self.unit_vm(ws, home, trial, setup): # boat: the VM up and given its files before the agent's time starts - p0 = time.time() - self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control" - else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None) try: with self.agent_internet(setup, box, rl, trial): - if self.other["_kind"] == "control": - answer = self.control_answer(unit, ws, box, name) - elif self.upstream: - answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) - elif self.other["_kind"] == "agent": - answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) - else: - answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + p0 = time.time() # the task's door is open: the environment is ready, the agent begins + try: + if self.over_acp(): # its start and session are its setup: its limit begins once it is in one + self.phase(name, unit, "agent_setup", acp.INITIALIZE_SECONDS + acp.SESSION_SECONDS) + else: + self.phase(name, unit, "agent_run", self.agent_limit(setup) if self.other["_kind"] != "control" + else self.phase_limit("agent", DRIVER_SECONDS) if self.other["key"] == "oracle" else None) + if self.other["_kind"] == "control": + answer = self.control_answer(unit, ws, box, name) + elif self.upstream: + answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) + elif self.other["_kind"] == "agent": + answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name, clock=clock) + else: + answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + finally: + p1 = time.time() # before the door closes finally: - produce = (p0, time.time()) + produce = (p0, p1 if p1 is not None else time.time()) if box is not None: # what the agent (or the control) left in the container, before the check changes = self.box_changes(box) cut = self.cap_reached(rl, tag) @@ -1359,7 +1372,8 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")]) res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit, t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, - setup_end=p0 or time.time(), checking=checking, + setup_end=p0 or time.time(), agent_ready=clock.get("ready"), agent_unready=clock.get("failed"), + checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), environment=environment or "", retries=retry, retryable=retry_ok, serving=getattr(self, "serving", None), # who served the unit: its agent's calls and its judge's and simulated user's routes @@ -1374,7 +1388,7 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: self.feed.emit("trial.end", name, task=unit, agent=self.other["key"], **job.feed_outcome(self.bench["score"]["kind"], score, exc), **{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS}, - durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None) + durations=job.feed_durations(res)) self.note_refusal(exc) if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None: with self._said: @@ -2071,13 +2085,16 @@ def agent_limit(self, setup: dict) -> float: return float(own) return self.turn_seconds - def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name=""): + def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name="", clock=None): """The agent over ACP, its session's ``cwd`` the unit's workspace - in an own image (``box``), the container's working directory, which this machine does not see. Files are served (``acp.Client(fs=)``) only to an agent running here: one in a container has its workspace mounted, and serving it from this machine would let it swap a folder for a link - between the check and the open and reach any file of the user's.""" + between the check and the open and reach any file of the user's. + ``clock`` gets when its setup ended: ``ready``, its session open, or + ``failed``, its start over with none (it never answered, it exited, the + run was stopped).""" a = self.other env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options) log = os.path.join(os.path.dirname(ws), "agent", "agent.log") @@ -2098,10 +2115,13 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name except TimeoutError as e: # the agent never answered its start: this unit's setup failed, not the run raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})") + if clock is not None: + clock["ready"] = time.time() # started and in its session: its setup ends, its run begins prompt = setup["prompt"] answer = "" # one limit for the agent's whole work on the unit, every turn and the simulated user's between them limit = self.agent_limit(setup) + self.phase(name, unit, "agent_run", limit) deadline = time.monotonic() + limit def left() -> float: @@ -2153,6 +2173,8 @@ def left() -> float: except jsonl.Failed as e: raise _Ended("VerifierError", "the benchmark's next turn: " + str(e), retryable=not isinstance(e, jsonl.NoReply)) finally: + if clock is not None and "ready" not in clock: + clock["failed"] = time.time() # its start ended with no session: all of it was its setup c.close() raw = "".join(json.dumps(e) + "\n" for e in events) return answer, job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {"acp.jsonl": raw} diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index bd8a3dd..f23ef69 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -12,6 +12,7 @@ import time import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from contextlib import contextmanager from datetime import datetime from unittest import mock @@ -177,6 +178,18 @@ def outcomes(self, job_dir): j = parse.parse_job(job_dir) return {t.trial_name.split("__")[0]: t for t in j.trials}, j + def windows(self, job_dir, starts): + """The phases one trial recorded as (start, end) seconds, its ``result.json`` whole, and what the + live feed was told of it: the phases in order and its end's durations.""" + trial = [d for d in os.listdir(job_dir) if d.startswith(starts)][0] + with open(os.path.join(job_dir, trial, "result.json")) as f: + res = json.load(f) + at = lambda key: tuple(datetime.fromisoformat(res[key][k]).timestamp() for k in ("started_at", "finished_at")) if res.get(key) else None + with open(os.path.join(job_dir, "events.jsonl")) as f: + mine = [e for e in map(json.loads, f) if e.get("trial") == trial] + return ({k: at(k) for k in ("environment_setup", "agent_setup", "agent_execution", "verifier")}, res, + [e["phase"] for e in mine if e["event"] == "trial.phase"], next(e for e in mine if e["event"] == "trial.end")["durations"]) + def assert_no_key(self, job_dir): for root, _, files in os.walk(job_dir): for n in files: @@ -573,6 +586,47 @@ def test_an_agents_trial_writes_a_trace_with_its_turn_and_tool_call(self): self.assertEqual(tool["attributes"]["tool_name"], "write answer.txt") self.assertEqual(tool["timing_quality"], "measured") + def test_an_agents_setup_is_timed_apart_from_its_run_and_the_board_is_told_the_same(self): + """Over ACP, the agent's start and its session are its setup: the + environment, the setup, the run and the check follow one another on + shared stamps, and the live feed carries the same phases and the same + seconds as the trial's result.""" + w, res, phases, told = self.windows(self.go("fakebench", "fake", limit=1), "good__") + self.assertEqual(w["environment_setup"][1], w["agent_setup"][0]) + self.assertEqual(w["agent_setup"][1], w["agent_execution"][0]) + self.assertLess(w["agent_setup"][0], w["agent_setup"][1]) + self.assertLessEqual(w["agent_execution"][1], w["verifier"][0]) + self.assertEqual(phases, ["env_setup", "agent_setup", "agent_run", "verify"]) + self.assertEqual(sorted(told), ["agent_run", "agent_setup", "env_setup", "verify"]) + for phase, key in (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier")): + self.assertAlmostEqual(told[phase], w[key][1] - w[key][0], places=3, msg=phase) + + def test_a_forced_agent_limit_is_the_runs_never_the_setups(self): + """``--timeout agent`` bounds the agent's work from its open session, where its clock starts: + the board is told that limit for the run, and ACP's two start limits for the setup.""" + job_dir = self.go("fakebench", "fake", limit=1, timeouts={"agent": 45}) + with open(os.path.join(job_dir, "events.jsonl")) as f: + limits = [(e["phase"], e["timeout_sec"]) for e in map(json.loads, f) if e["event"] == "trial.phase"] + self.assertEqual(limits[1:3], [("agent_setup", acp.INITIALIZE_SECONDS + acp.SESSION_SECONDS), ("agent_run", 45)]) + + def test_the_tasks_door_is_the_environments_not_the_agents(self): + """A task's internet door opens before the agent begins and closes + after it ended: neither is the agent's setup nor its run.""" + @contextmanager + def slow_door(self, setup, box, rl, trial): + threading.Event().wait(0.4) + try: + yield + finally: + threading.Event().wait(0.4) + t0 = datetime.now().timestamp() + with mock.patch.object(run.Run, "agent_internet", slow_door): + w, res, phases, told = self.windows(self.go("fakebench", "fake", limit=1), "good__") + self.assertGreaterEqual(w["agent_setup"][0] - w["environment_setup"][0], 0.4) # the door opening is the environment's + self.assertGreaterEqual(w["verifier"][0] - w["agent_execution"][1], 0.4) # its closing is nobody's phase + self.assertEqual(w["environment_setup"][1], w["agent_setup"][0]) + self.assertGreater(w["environment_setup"][0], t0) + def test_each_workspace_is_a_repository_of_its_own(self): # an agent looking up for its instructions (AGENTS.md) stops there, not at the checkout holding the jobs job_dir = self.go("fakebench", "fake", limit=1) @@ -606,6 +660,12 @@ def slow_start(c, method, params, timeout=600): got, j = self.outcomes(self.go("fakebench", "fake")) self.assertEqual(len(got), 4) # every unit written, the run ended self.assertTrue(all(t.outcome == classify.SETUP_ERROR for t in got.values()), {k: t.outcome for k, t in got.items()}) + # a start with no session is all setup: no run was timed, and the board ended each unit in its agent setup + self.assertTrue(all(t.dur_agent_setup is not None and t.dur_agent is None for t in got.values()), + {k: (t.dur_agent_setup, t.dur_agent) for k, t in got.items()}) + w, res, phases, told = self.windows(j.dir, "good__") + self.assertEqual(phases, ["env_setup", "agent_setup"]) + self.assertEqual(sorted(told), ["agent_setup", "env_setup"]) def test_a_rate_limit_is_infrastructure_not_a_miss(self): Echo.status = 429 @@ -899,6 +959,9 @@ def test_a_feature_run_drives_the_feature_through_its_port(self): got, j = self.outcomes(job_dir) self.assertEqual(set(got), {"q1", "q2"}) self.assertTrue(all(t.outcome == classify.SCORED and t.score == 1.0 for t in got.values())) # a score, no pass + # a capability has no setup of its own: its process answers nothing until its first request + self.assertTrue(all(t.dur_agent_setup is None and t.dur_agent is not None for t in got.values())) + self.assertEqual(self.windows(job_dir, "q1__")[2], ["env_setup", "agent_run", "verify"]) trial = [d for d in os.listdir(job_dir) if d.startswith("q1__")][0] with open(os.path.join(job_dir, trial, "agent", "trajectory.json")) as f: t = json.load(f) @@ -1148,6 +1211,12 @@ def test_a_conversation_the_upstream_starts_again_is_a_new_session(self): detail = json.load(f)["detail"] self.assertEqual(detail["said"], ["INTRO\n\nYOU SAID: OPEN", "INTRO\n\nYOU SAID: AGAIN"]) self.assertEqual(opened.call_count, 2) + # its setup ended with its first session: the second is part of its run + w, res, phases, told = self.windows(job_dir, os.path.basename(trial)) + self.assertEqual(phases.count("agent_setup"), 1) + self.assertEqual(w["agent_setup"][1], w["agent_execution"][0]) + with open(os.path.join(trial, "agent", "acp.jsonl")) as f: + self.assertGreaterEqual(json.loads(f.readline())["t"], w["agent_execution"][0]) def test_a_simulator_is_only_for_a_conversation(self): b = definitions.get("benchmark", "fakebench", ROOT) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267..ec8588f 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -4,6 +4,7 @@ import json import os +import shutil import tempfile import unittest @@ -136,6 +137,64 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): exception=("UnitSpendingCapError", "its own spending cap"), calls=[call(1, 2), refused, failed], trajectory=None) self.assertEqual((res["agent_result"]["n_model_calls"], res["agent_result"]["n_input_tokens"]), (2, 10)) + def scratch(self): + d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + self.addCleanup(shutil.rmtree, d, True) + return d + + def trial(self, events, calls, **kw): + d = self.scratch() + traj = job.trajectory_from_events(list(events), list(calls), AGENT, "m", "j/u") if events is not None else None + res = job.write_trial(d, job="j", bench=BENCH, other=AGENT, model="m", unit="u", t0=T, t1=T + 40, produce=(T + 1, T + 30), + score=kw.pop("score", {"reward": 1.0}), exception=kw.pop("exception", None), calls=list(calls), + trajectory=traj, setup_end=T + 1, **kw) + at = os.path.join(d, "agent", "trace.json") + if not os.path.isfile(at): + return res, None + with open(at) as f: + return res, json.load(f) + + #: an agent first asked at T+4, once it is in its session + LATE = [ev(4, "user_message_chunk", content={"type": "text", "text": "first"}), + ev(6, "agent_message_chunk", content={"type": "text", "text": "done"})] + + def test_the_agent_s_setup_is_its_own_phase_and_its_run_and_its_trace_start_after_it(self): + """In its session at T+3: its setup is T+1 to T+3, its run and its + trace's window T+3 to T+30, as Harbor's agent_setup and agent_execution are.""" + res, doc = self.trial(self.LATE, [call(5, 6)], agent_ready=T + 3) + self.assertEqual(res["agent_setup"], {"started_at": job.iso(T + 1), "finished_at": job.iso(T + 3)}) + self.assertEqual(res["agent_execution"], {"started_at": job.iso(T + 3), "finished_at": job.iso(T + 30)}) + session = doc["spans"][0] + self.assertEqual((session["start_ns"], session["end_ns"]), (int((T + 3) * 10**9), int((T + 30) * 10**9))) + self.assertEqual(doc["totals"]["wall_ns"], 27 * 10**9) + unset, same = self.trial(self.LATE, [call(5, 6)]) # no clock given: the window is the whole of produce + self.assertEqual(same["spans"][0]["start_ns"], int((T + 1) * 10**9)) + self.assertEqual(same["totals"]["wall_ns"], 29 * 10**9) + + def test_a_call_made_while_the_agent_set_up_widens_its_trace_and_is_never_clamped(self): + """A model call at T+2, before the session opened at T+3: it is in the + setup, and the trace's session starts with it.""" + res, doc = self.trial(self.LATE, [call(2, 2.5), call(5, 6)], agent_ready=T + 3) + self.assertEqual(res["agent_execution"]["started_at"], job.iso(T + 3)) + self.assertEqual(doc["spans"][0]["start_ns"], int((T + 2) * 10**9)) + self.assertEqual(res["agent_result"]["n_model_calls"], 2) # counted all the same + + def test_a_start_that_ended_with_no_session_is_all_setup_and_no_run(self): + """It never answered its start, or exited: Harbor writes a failed agent + setup with its window and no agent_execution, and so does a standard run.""" + res, doc = self.trial(None, [], agent_unready=T + 5, score=None, exception=("AgentSetupTimeoutError", "the agent did not answer initialize in time")) + self.assertEqual(res["agent_setup"], {"started_at": job.iso(T + 1), "finished_at": job.iso(T + 5)}) + self.assertIsNone(res["agent_execution"]) + self.assertIsNone(doc) + self.assertEqual(job.feed_durations(res), {"env_setup": 1.0, "agent_setup": 4.0}) + + def test_a_trial_given_no_agent_clock_has_no_setup_phase(self): + res, _ = self.trial(None, []) + self.assertIsNone(res["agent_setup"]) + self.assertEqual(res["agent_execution"]["started_at"], job.iso(T + 1)) + self.assertEqual(job.feed_durations(res), {"env_setup": 1.0, "agent_run": 29.0}) + self.assertIsNone(job.feed_durations({})) + def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) diff --git a/docs/field-guide.md b/docs/field-guide.md index 480433a..6d4f8f6 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -165,8 +165,8 @@ How the agent worked, not whether it won. Computed over layer 3. | `task_broken` | On the feasibility page: `nop`, which changes nothing, failed passing tests: the task is broken as shipped, or its verifier unstable. | derived | swe-bench | | `patch_exists / patch_applied` | Whether a patch was produced, and whether it applied. | measured | swe-bench | | `dur_total` | Trial wall clock. | measured | any | -| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. An imported trial charges its whole window to `dur_agent`. | measured | any | -| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, the unit's setup, the benchmark's driver and its scorer. | derived | any | +| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases a trial records, drawn in order. An imported trial charges its whole window to `dur_agent`. | measured | any | +| `dur_other` | `dur_total` minus the phases: waits and teardown. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | Prompt tokens read from cache, excluding cache writes: a subset, never added to input tokens. Older standard records may include writes; stored results are not rewritten. | measured | any | | `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any |