From b189804c262e9bf332f36fe41d58b40973957c38 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Mon, 16 Nov 2026 22:25:23 +0100 Subject: [PATCH 1/6] fix(standard): time the agent's setup apart from its run A unit's agent started and opened its session inside agent_execution, so the trial page had no Setup phase and Agent time carried the handshake and the benchmark's MCP servers starting. The run now notes when the session opened (an ACP agent) or the process started (a capability) and writes it as Harbor's agent_setup; agent_execution and the trace's window start there. A command agent and an upstream setup start and run in one call, so they have none. --- app/nullius/standard/job.py | 10 ++++++++-- app/nullius/standard/run.py | 18 ++++++++++++------ app/tests/test_standard_trace.py | 23 +++++++++++++++++++++++ docs/field-guide.md | 2 +- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 683d4e14..4779f0e5 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -403,15 +403,20 @@ def backfill_trace(trial_dir: str, write: bool = False) -> str: def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float, produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", - image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict: + image: str = "", setup_end: Optional[float] = None, agent_ready: Optional[float] = None, + checking: tuple = (None, None)) -> dict: """Write one unit's trial and return its result. ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began - (its setup, relay and container from ``t0``); ``checking``: the check's + (its setup, relay and container from ``t0``); ``agent_ready``: when the + agent, started and in its session, was first asked - its setup from + ``setup_end``, its run from there; ``checking``: the check's start and end (its own step in the container, then the driver's score).""" os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True) os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True) + if produce and produce[0] and agent_ready: # its run starts once it is set up: the trace's window too + produce = (agent_ready, produce[1]) tot = _usage_total(calls) # the pages read a task's name after its last slash: a unit's own slashes # (a unit id like "set/12") would leave only its last part @@ -420,6 +425,7 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""), "model_info": {"name": model}}, "environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None, + "agent_setup": {"started_at": iso(setup_end), "finished_at": iso(agent_ready)} if setup_end and agent_ready else None, "agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None, "verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None, "agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"], diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 2ea72822..7866ed71 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -993,6 +993,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS) produce, score, exc, traj, logs = (None, None), None, None, None, {} p0, checking = None, (None, None) # when the agent began; the check's start and end + ready: List[float] = [] # when the agent, started and in its session, was first asked setup: dict = {} transcript: List[dict] = [] rl, environment, box, changes = None, None, None, None @@ -1042,9 +1043,9 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: elif self.upstream: answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) elif self.other["_kind"] == "agent": - answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name) + answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name, ready=ready) else: - answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl, ready=ready) finally: produce = (p0, time.time()) if box is not None: # what the agent (or the control) left in the container, before the check @@ -1118,7 +1119,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")]) res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit, t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, - setup_end=p0 or time.time(), checking=checking, + setup_end=p0 or time.time(), agent_ready=ready[0] if ready else None, checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), environment=environment or "", image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") @@ -1739,13 +1740,14 @@ def turn_limit(self, setup: dict) -> float: return float(own) return self.turn_seconds - def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name=""): + def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name="", ready=None): """The agent over ACP, its session's ``cwd`` the unit's workspace - in an own image (``box``), the container's working directory, which this machine does not see. Files are served (``acp.Client(fs=)``) only to an agent running here: one in a container has its workspace mounted, and serving it from this machine would let it swap a folder for a link - between the check and the open and reach any file of the user's.""" + between the check and the open and reach any file of the user's. + ``ready`` gets the time its session opened: its setup ends there.""" a = self.other env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options) log = os.path.join(os.path.dirname(ws), "agent", "agent.log") @@ -1763,6 +1765,8 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name try: c.start() sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or []) + if ready is not None: + ready.append(time.time()) except TimeoutError as e: # the agent never answered its start: this unit's setup failed, not the run raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})") @@ -1833,7 +1837,7 @@ def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl return r.stdout.strip(), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {} # -- a capability, through its port ------------------------------------------- - def drive_feature(self, unit, ws, home, tag, setup, transcript, rl): + def drive_feature(self, unit, ws, home, tag, setup, transcript, rl, ready=None): f = self.other cmd = install.words(f["start"]["command"], {"here": f["_here"], "install": self.inst}) env = agent_env(f, self.inst, home, rl.unit_url, self.model, self.options) @@ -1854,6 +1858,8 @@ def send(op, **kw): try: try: proc.start() + if ready is not None: + ready.append(time.time()) plan = setup.get("memory") or {} for s in plan.get("add") or []: send("add", session=s) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index f4ea267d..622cde73 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -136,6 +136,29 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): exception=("UnitSpendingCapError", "its own spending cap"), calls=[call(1, 2), refused, failed], trajectory=None) self.assertEqual((res["agent_result"]["n_model_calls"], res["agent_result"]["n_input_tokens"]), (2, 10)) + def test_the_agent_s_setup_is_its_own_phase_and_its_run_starts_after_it(self): + """Started and in its session at T+3: its setup is T+1 to T+3, its run + and its trace's window from T+3, as Harbor's agent_setup and + agent_execution are.""" + d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + traj = job.trajectory_from_events(list(EVENTS), [call(5, 6)], AGENT, "m", "j/u") + res = job.write_trial(d, job="j", bench=BENCH, other=AGENT, model="m", unit="u", t0=T, t1=T + 40, produce=(T + 1, T + 30), + score={"reward": 1.0}, exception=None, calls=[call(5, 6)], trajectory=traj, + setup_end=T + 1, agent_ready=T + 3) + self.assertEqual(res["agent_setup"], {"started_at": job.iso(T + 1), "finished_at": job.iso(T + 3)}) + self.assertEqual(res["agent_execution"], {"started_at": job.iso(T + 3), "finished_at": job.iso(T + 30)}) + with open(os.path.join(d, "agent", "trace.json")) as f: + session = json.load(f)["spans"][0] + self.assertLessEqual(session["start_ns"], int((T + 3) * 10**9)) + self.assertEqual(session["end_ns"], int((T + 30) * 10**9)) + + def test_without_a_session_there_is_no_setup_phase(self): + d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + res = job.write_trial(d, job="j", bench=BENCH, other=AGENT, model="m", unit="u", t0=T, t1=T + 40, produce=(T + 1, T + 30), + score={"reward": 1.0}, exception=None, calls=[], trajectory=None, setup_end=T + 1) + self.assertIsNone(res["agent_setup"]) + self.assertEqual(res["agent_execution"]["started_at"], job.iso(T + 1)) + def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) diff --git a/docs/field-guide.md b/docs/field-guide.md index 18a9b1f2..b12dac46 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -160,7 +160,7 @@ How the agent worked, not whether it won. Computed over layer 3. | `task_broken` | On the feasibility page: `nop`, which changes nothing, failed passing tests: the task is broken as shipped, or its verifier unstable. | derived | swe-bench | | `patch_exists / patch_applied` | Whether a patch was produced, and whether it applied. | measured | swe-bench | | `dur_total` | Trial wall clock. | measured | any | -| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. An imported trial charges its whole window to `dur_agent`. | measured | any | +| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. A standard run records the same four: setup is the agent's start and session. An imported trial charges its whole window to `dur_agent`. | measured | any | | `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, the unit's setup, the benchmark's driver and its scorer. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | Prompt tokens read from cache, excluding cache writes: a subset, never added to input tokens. Older standard records may include writes; stored results are not rewritten. | measured | any | From 8a359c334e48ce864d546d701e6f33d8354e92ee Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Tue, 17 Nov 2026 01:48:49 +0100 Subject: [PATCH 2/6] fix(standard): count a failed start as setup, give a capability none, and tell the board the same Review of the first commit found the trial's result and the live board disagreeing, and three windows that were not what they said: - a start that ended with no session (never answered, exited, stopped) was written as the agent's run; it is now its setup, with no run, as Harbor writes a failed agent setup; - a capability got a setup of a millisecond, the time to spawn its process, which answers nothing until its first request; it has none now; - the task's internet door opened inside the agent's setup and closed inside its run; the agent now begins once it is open and ends before it closes; - the live feed still sent one duration, timed from before the setup, and no agent_setup phase; it now sends the phase, starts the turn's limit when the session opens, and takes every duration from the result. The trace-window test could not fail (its events began before the setup); it now pins the window, and the run's side has tests of its own: the four windows on shared stamps, a failed start, a capability, a conversation started again, the door. The field guide says which runs have a setup and what Other is on a standard run, and the trial page's hover no longer calls a standard agent's setup an installation. --- app/nullius/standard/acp.py | 10 +++-- app/nullius/standard/job.py | 26 ++++++++++-- app/nullius/standard/run.py | 62 ++++++++++++++++++----------- app/templates/traces/trial.html | 2 +- app/tests/test_standard_run.py | 61 ++++++++++++++++++++++++++++ app/tests/test_standard_trace.py | 68 ++++++++++++++++++++++++-------- docs/field-guide.md | 4 +- 7 files changed, 185 insertions(+), 48 deletions(-) diff --git a/app/nullius/standard/acp.py b/app/nullius/standard/acp.py index 464d7c00..534b3001 100644 --- a/app/nullius/standard/acp.py +++ b/app/nullius/standard/acp.py @@ -43,6 +43,10 @@ from typing import Dict, List, Optional PROTOCOL_VERSION = 1 +# How long an agent has to answer each of the two requests that set it up: together, the limit of +# a unit's agent setup (``run``). +INITIALIZE_SECONDS = 120 +SESSION_SECONDS = 180 class AgentFailed(Exception): @@ -135,7 +139,7 @@ def start(self) -> "Client": self.info = self.request("initialize", {"protocolVersion": PROTOCOL_VERSION, "clientCapabilities": {"fs": {"readTextFile": self.fs, "writeTextFile": self.fs}, "terminal": False, "session": {"notices": {}}}, - "clientInfo": {"name": "nullius", "version": "1"}}, timeout=120) + "clientInfo": {"name": "nullius", "version": "1"}}, timeout=INITIALIZE_SECONDS) return self def _send(self, msg: dict): @@ -242,11 +246,11 @@ def new_session(self, cwd: str, mcp: Optional[List[dict]] = None) -> str: "env": [{"name": k, "value": v} for k, v in (s.get("env") or {}).items()]} for s in mcp or []] params = {"cwd": os.path.abspath(cwd), "mcpServers": servers} try: - res = self.request("session/new", params, timeout=180) + res = self.request("session/new", params, timeout=SESSION_SECONDS) except AgentFailed as e: if "auth" not in str(e).lower() or not self.authenticate(): raise - res = self.request("session/new", params, timeout=180) + res = self.request("session/new", params, timeout=SESSION_SECONDS) sid = res["sessionId"] self.roots[sid] = os.path.abspath(cwd) return sid diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 4779f0e5..81708021 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -404,19 +404,23 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict], trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "", image: str = "", setup_end: Optional[float] = None, agent_ready: Optional[float] = None, - checking: tuple = (None, None)) -> dict: + agent_unready: Optional[float] = None, checking: tuple = (None, None)) -> dict: """Write one unit's trial and return its result. ``environment``: the benchmark's environment the unit asked for (none: the base image), and ``image``, the one it ran in on the container backend - names only. ``setup_end``: when the unit's environment was ready and the agent began (its setup, relay and container from ``t0``); ``agent_ready``: when the agent, started and in its session, was first asked - its setup from - ``setup_end``, its run from there; ``checking``: the check's + ``setup_end``, its run from there; ``agent_unready``: when its start ended + with no session (it never answered, it exited) - its setup ends there and + it has no run, as Harbor writes a failed agent setup; ``checking``: the check's start and end (its own step in the container, then the driver's score).""" os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True) os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True) if produce and produce[0] and agent_ready: # its run starts once it is set up: the trace's window too produce = (agent_ready, produce[1]) + elif agent_unready: # it never got a session: no run to time + produce = (None, None) tot = _usage_total(calls) # the pages read a task's name after its last slash: a unit's own slashes # (a unit id like "set/12") would leave only its last part @@ -425,7 +429,8 @@ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: st "agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""), "model_info": {"name": model}}, "environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None, - "agent_setup": {"started_at": iso(setup_end), "finished_at": iso(agent_ready)} if setup_end and agent_ready else None, + "agent_setup": {"started_at": iso(setup_end), "finished_at": iso(agent_ready or agent_unready)} + if setup_end and (agent_ready or agent_unready) else None, "agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None, "verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None, "agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"], @@ -506,6 +511,21 @@ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str "ModelNotFound") +FEED_PHASES = (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier")) + + +def feed_durations(res: dict) -> Optional[dict]: + """The phases a trial recorded, in the live feed's words and in seconds, read from its result so + the board and the trial's page say the same (Harbor's writer sends its four the same way). A + phase it has none of is left out; None when it recorded none.""" + out = {} + for phase, key in FEED_PHASES: + window = res.get(key) or {} + if window.get("started_at") and window.get("finished_at"): + out[phase] = round((datetime.fromisoformat(window["finished_at"]) - datetime.fromisoformat(window["started_at"])).total_seconds(), 3) + return out or None + + def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict: """A unit's end in the live feed's words (``nullius.runs.live``): its outcome, with the reward (a pass benchmark) or the score, and the error's diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 7866ed71..549f1447 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -971,6 +971,12 @@ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = Non if self.feed is not None: self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout) + def over_acp(self) -> bool: + """Whether the unit's other is an agent driven over ACP: the one kind with a setup of its own, + its start and its session. A command agent starts and runs in one call, an upstream setup and a + control are the driver's, and a capability's process answers nothing until its first request.""" + return self.other["_kind"] == "agent" and "command" not in self.other["start"] + def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()): return None # the endpoint refused the run: nothing more is begun @@ -992,8 +998,8 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: self.progress.emit("unit", "start", unit=unit) self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS) produce, score, exc, traj, logs = (None, None), None, None, None, {} - p0, checking = None, (None, None) # when the agent began; the check's start and end - ready: List[float] = [] # when the agent, started and in its session, was first asked + p0, p1, checking = None, None, (None, None) # when the agent began and ended; the check's start and end + clock: dict = {} # an agent's start: "ready" when its session opened, "failed" when it ended with none setup: dict = {} transcript: List[dict] = [] rl, environment, box, changes = None, None, None, None @@ -1033,21 +1039,27 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: box = self.own_box(tag, setup, rl, home) if setup.get("prepare") is not None: self.prepare(setup["prepare"], box, trial) - p0 = time.time() - self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" - else DRIVER_SECONDS if self.other["key"] == "oracle" else None) try: with self.agent_internet(setup, box, rl, trial): - if self.other["_kind"] == "control": - answer = self.control_answer(unit, ws, box, name) - elif self.upstream: - answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) - elif self.other["_kind"] == "agent": - answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name, ready=ready) - else: - answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl, ready=ready) + p0 = time.time() # the task's door is open: the environment is ready, the agent begins + try: + if self.over_acp(): # its start and session are its setup: the turn's limit begins once it is in one + self.phase(name, unit, "agent_setup", acp.INITIALIZE_SECONDS + acp.SESSION_SECONDS) + else: + self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control" + else DRIVER_SECONDS if self.other["key"] == "oracle" else None) + if self.other["_kind"] == "control": + answer = self.control_answer(unit, ws, box, name) + elif self.upstream: + answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl) + elif self.other["_kind"] == "agent": + answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name, clock=clock) + else: + answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl) + finally: + p1 = time.time() # before the door closes finally: - produce = (p0, time.time()) + produce = (p0, p1 if p1 is not None else time.time()) if box is not None: # what the agent (or the control) left in the container, before the check changes = self.box_changes(box) cut = self.cap_reached(rl, tag) @@ -1119,7 +1131,8 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")]) res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit, t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls, - setup_end=p0 or time.time(), agent_ready=ready[0] if ready else None, checking=checking, + setup_end=p0 or time.time(), agent_ready=clock.get("ready"), agent_unready=clock.get("failed"), + checking=checking, trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})), environment=environment or "", image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "") @@ -1131,7 +1144,7 @@ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]: self.feed.emit("trial.end", name, task=unit, agent=self.other["key"], **job.feed_outcome(self.bench["score"]["kind"], score, exc), **{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS}, - durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None) + durations=job.feed_durations(res)) self.note_refusal(exc) if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None: with self._said: @@ -1740,14 +1753,16 @@ def turn_limit(self, setup: dict) -> float: return float(own) return self.turn_seconds - def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name="", ready=None): + def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name="", clock=None): """The agent over ACP, its session's ``cwd`` the unit's workspace - in an own image (``box``), the container's working directory, which this machine does not see. Files are served (``acp.Client(fs=)``) only to an agent running here: one in a container has its workspace mounted, and serving it from this machine would let it swap a folder for a link between the check and the open and reach any file of the user's. - ``ready`` gets the time its session opened: its setup ends there.""" + ``clock`` gets when its setup ended: ``ready``, its session open, or + ``failed``, its start over with none (it never answered, it exited, the + run was stopped).""" a = self.other env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options) log = os.path.join(os.path.dirname(ws), "agent", "agent.log") @@ -1765,11 +1780,12 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name try: c.start() sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or []) - if ready is not None: - ready.append(time.time()) except TimeoutError as e: # the agent never answered its start: this unit's setup failed, not the run raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})") + if clock is not None: + clock["ready"] = time.time() # started and in its session: its setup ends, its run begins + self.phase(name, unit, "agent_run", self.turn_limit(setup)) prompt = setup["prompt"] answer = "" # a conversation's turns carry its simulated user's route and its judge's (each when it has one) @@ -1814,6 +1830,8 @@ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name except jsonl.Failed as e: raise _Ended("VerifierError", "the benchmark's next turn: " + str(e)) finally: + if clock is not None and "ready" not in clock: + clock["failed"] = time.time() # its start ended with no session: all of it was its setup c.close() raw = "".join(json.dumps(e) + "\n" for e in events) return answer, job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {"acp.jsonl": raw} @@ -1837,7 +1855,7 @@ def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl return r.stdout.strip(), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {} # -- a capability, through its port ------------------------------------------- - def drive_feature(self, unit, ws, home, tag, setup, transcript, rl, ready=None): + def drive_feature(self, unit, ws, home, tag, setup, transcript, rl): f = self.other cmd = install.words(f["start"]["command"], {"here": f["_here"], "install": self.inst}) env = agent_env(f, self.inst, home, rl.unit_url, self.model, self.options) @@ -1858,8 +1876,6 @@ def send(op, **kw): try: try: proc.start() - if ready is not None: - ready.append(time.time()) plan = setup.get("memory") or {} for s in plan.get("add") or []: send("add", session=s) diff --git a/app/templates/traces/trial.html b/app/templates/traces/trial.html index 0967b4cf..e20ab9ec 100644 --- a/app/templates/traces/trial.html +++ b/app/templates/traces/trial.html @@ -142,7 +142,7 @@

Where the time went

{# A 0s phase emits no segment at all, so a control's empty setup leaves no stray sliver. #} {% if trial.dur_env|phase %}{% endif %} - {% if trial.dur_agent_setup|phase %}{% endif %} + {% if trial.dur_agent_setup|phase %}{% endif %} {% if trial.dur_agent|phase %}{% endif %} {% if trial.dur_verifier|phase %}{% endif %} {% set items = job.runner_items[1] if job.runner_items else 'items' %} diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 2a7ec268..6ad17427 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -10,6 +10,7 @@ import threading import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from contextlib import contextmanager from datetime import datetime from unittest import mock @@ -139,6 +140,18 @@ def outcomes(self, job_dir): j = parse.parse_job(job_dir) return {t.trial_name.split("__")[0]: t for t in j.trials}, j + def windows(self, job_dir, starts): + """The phases one trial recorded as (start, end) seconds, its ``result.json`` whole, and what the + live feed was told of it: the phases in order and its end's durations.""" + trial = [d for d in os.listdir(job_dir) if d.startswith(starts)][0] + with open(os.path.join(job_dir, trial, "result.json")) as f: + res = json.load(f) + at = lambda key: tuple(datetime.fromisoformat(res[key][k]).timestamp() for k in ("started_at", "finished_at")) if res.get(key) else None + with open(os.path.join(job_dir, "events.jsonl")) as f: + mine = [e for e in map(json.loads, f) if e.get("trial") == trial] + return ({k: at(k) for k in ("environment_setup", "agent_setup", "agent_execution", "verifier")}, res, + [e["phase"] for e in mine if e["event"] == "trial.phase"], next(e for e in mine if e["event"] == "trial.end")["durations"]) + def assert_no_key(self, job_dir): for root, _, files in os.walk(job_dir): for n in files: @@ -528,6 +541,39 @@ def test_an_agents_trial_writes_a_trace_with_its_turn_and_tool_call(self): self.assertEqual(tool["attributes"]["tool_name"], "write answer.txt") self.assertEqual(tool["timing_quality"], "measured") + def test_an_agents_setup_is_timed_apart_from_its_run_and_the_board_is_told_the_same(self): + """Over ACP, the agent's start and its session are its setup: the + environment, the setup, the run and the check follow one another on + shared stamps, and the live feed carries the same phases and the same + seconds as the trial's result.""" + w, res, phases, told = self.windows(self.go("fakebench", "fake", limit=1), "good__") + self.assertEqual(w["environment_setup"][1], w["agent_setup"][0]) + self.assertEqual(w["agent_setup"][1], w["agent_execution"][0]) + self.assertLess(w["agent_setup"][0], w["agent_setup"][1]) + self.assertLessEqual(w["agent_execution"][1], w["verifier"][0]) + self.assertEqual(phases, ["env_setup", "agent_setup", "agent_run", "verify"]) + self.assertEqual(sorted(told), ["agent_run", "agent_setup", "env_setup", "verify"]) + for phase, key in (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier")): + self.assertAlmostEqual(told[phase], w[key][1] - w[key][0], places=3, msg=phase) + + def test_the_tasks_door_is_the_environments_not_the_agents(self): + """A task's internet door opens before the agent begins and closes + after it ended: neither is the agent's setup nor its run.""" + @contextmanager + def slow_door(self, setup, box, rl, trial): + threading.Event().wait(0.4) + try: + yield + finally: + threading.Event().wait(0.4) + t0 = datetime.now().timestamp() + with mock.patch.object(run.Run, "agent_internet", slow_door): + w, res, phases, told = self.windows(self.go("fakebench", "fake", limit=1), "good__") + self.assertGreaterEqual(w["agent_setup"][0] - w["environment_setup"][0], 0.4) # the door opening is the environment's + self.assertGreaterEqual(w["verifier"][0] - w["agent_execution"][1], 0.4) # its closing is nobody's phase + self.assertEqual(w["environment_setup"][1], w["agent_setup"][0]) + self.assertGreater(w["environment_setup"][0], t0) + def test_each_workspace_is_a_repository_of_its_own(self): # an agent looking up for its instructions (AGENTS.md) stops there, not at the checkout holding the jobs job_dir = self.go("fakebench", "fake", limit=1) @@ -561,6 +607,12 @@ def slow_start(c, method, params, timeout=600): got, j = self.outcomes(self.go("fakebench", "fake")) self.assertEqual(len(got), 4) # every unit written, the run ended self.assertTrue(all(t.outcome == classify.SETUP_ERROR for t in got.values()), {k: t.outcome for k, t in got.items()}) + # a start with no session is all setup: no run was timed, and the board ended each unit in its agent setup + self.assertTrue(all(t.dur_agent_setup is not None and t.dur_agent is None for t in got.values()), + {k: (t.dur_agent_setup, t.dur_agent) for k, t in got.items()}) + w, res, phases, told = self.windows(j.dir, "good__") + self.assertEqual(phases, ["env_setup", "agent_setup"]) + self.assertEqual(sorted(told), ["agent_setup", "env_setup"]) def test_a_rate_limit_is_infrastructure_not_a_miss(self): Echo.status = 429 @@ -629,6 +681,9 @@ def test_a_feature_run_drives_the_feature_through_its_port(self): got, j = self.outcomes(job_dir) self.assertEqual(set(got), {"q1", "q2"}) self.assertTrue(all(t.outcome == classify.SCORED and t.score == 1.0 for t in got.values())) # a score, no pass + # a capability has no setup of its own: its process answers nothing until its first request + self.assertTrue(all(t.dur_agent_setup is None and t.dur_agent is not None for t in got.values())) + self.assertEqual(self.windows(job_dir, "q1__")[2], ["env_setup", "agent_run", "verify"]) trial = [d for d in os.listdir(job_dir) if d.startswith("q1__")][0] with open(os.path.join(job_dir, trial, "agent", "trajectory.json")) as f: t = json.load(f) @@ -808,6 +863,12 @@ def test_a_conversation_the_upstream_starts_again_is_a_new_session(self): detail = json.load(f)["detail"] self.assertEqual(detail["said"], ["INTRO\n\nYOU SAID: OPEN", "INTRO\n\nYOU SAID: AGAIN"]) self.assertEqual(opened.call_count, 2) + # its setup ended with its first session: the second is part of its run + w, res, phases, told = self.windows(job_dir, os.path.basename(trial)) + self.assertEqual(phases.count("agent_setup"), 1) + self.assertEqual(w["agent_setup"][1], w["agent_execution"][0]) + with open(os.path.join(trial, "agent", "acp.jsonl")) as f: + self.assertGreaterEqual(json.loads(f.readline())["t"], w["agent_execution"][0]) def test_a_simulator_is_only_for_a_conversation(self): b = definitions.get("benchmark", "fakebench", ROOT) diff --git a/app/tests/test_standard_trace.py b/app/tests/test_standard_trace.py index 622cde73..ec8588fc 100644 --- a/app/tests/test_standard_trace.py +++ b/app/tests/test_standard_trace.py @@ -4,6 +4,7 @@ import json import os +import shutil import tempfile import unittest @@ -136,28 +137,63 @@ def test_a_call_its_cap_refused_was_never_sent_and_is_no_model_call(self): exception=("UnitSpendingCapError", "its own spending cap"), calls=[call(1, 2), refused, failed], trajectory=None) self.assertEqual((res["agent_result"]["n_model_calls"], res["agent_result"]["n_input_tokens"]), (2, 10)) - def test_the_agent_s_setup_is_its_own_phase_and_its_run_starts_after_it(self): - """Started and in its session at T+3: its setup is T+1 to T+3, its run - and its trace's window from T+3, as Harbor's agent_setup and - agent_execution are.""" + def scratch(self): d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) - traj = job.trajectory_from_events(list(EVENTS), [call(5, 6)], AGENT, "m", "j/u") + self.addCleanup(shutil.rmtree, d, True) + return d + + def trial(self, events, calls, **kw): + d = self.scratch() + traj = job.trajectory_from_events(list(events), list(calls), AGENT, "m", "j/u") if events is not None else None res = job.write_trial(d, job="j", bench=BENCH, other=AGENT, model="m", unit="u", t0=T, t1=T + 40, produce=(T + 1, T + 30), - score={"reward": 1.0}, exception=None, calls=[call(5, 6)], trajectory=traj, - setup_end=T + 1, agent_ready=T + 3) + score=kw.pop("score", {"reward": 1.0}), exception=kw.pop("exception", None), calls=list(calls), + trajectory=traj, setup_end=T + 1, **kw) + at = os.path.join(d, "agent", "trace.json") + if not os.path.isfile(at): + return res, None + with open(at) as f: + return res, json.load(f) + + #: an agent first asked at T+4, once it is in its session + LATE = [ev(4, "user_message_chunk", content={"type": "text", "text": "first"}), + ev(6, "agent_message_chunk", content={"type": "text", "text": "done"})] + + def test_the_agent_s_setup_is_its_own_phase_and_its_run_and_its_trace_start_after_it(self): + """In its session at T+3: its setup is T+1 to T+3, its run and its + trace's window T+3 to T+30, as Harbor's agent_setup and agent_execution are.""" + res, doc = self.trial(self.LATE, [call(5, 6)], agent_ready=T + 3) self.assertEqual(res["agent_setup"], {"started_at": job.iso(T + 1), "finished_at": job.iso(T + 3)}) self.assertEqual(res["agent_execution"], {"started_at": job.iso(T + 3), "finished_at": job.iso(T + 30)}) - with open(os.path.join(d, "agent", "trace.json")) as f: - session = json.load(f)["spans"][0] - self.assertLessEqual(session["start_ns"], int((T + 3) * 10**9)) - self.assertEqual(session["end_ns"], int((T + 30) * 10**9)) - - def test_without_a_session_there_is_no_setup_phase(self): - d = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) - res = job.write_trial(d, job="j", bench=BENCH, other=AGENT, model="m", unit="u", t0=T, t1=T + 40, produce=(T + 1, T + 30), - score={"reward": 1.0}, exception=None, calls=[], trajectory=None, setup_end=T + 1) + session = doc["spans"][0] + self.assertEqual((session["start_ns"], session["end_ns"]), (int((T + 3) * 10**9), int((T + 30) * 10**9))) + self.assertEqual(doc["totals"]["wall_ns"], 27 * 10**9) + unset, same = self.trial(self.LATE, [call(5, 6)]) # no clock given: the window is the whole of produce + self.assertEqual(same["spans"][0]["start_ns"], int((T + 1) * 10**9)) + self.assertEqual(same["totals"]["wall_ns"], 29 * 10**9) + + def test_a_call_made_while_the_agent_set_up_widens_its_trace_and_is_never_clamped(self): + """A model call at T+2, before the session opened at T+3: it is in the + setup, and the trace's session starts with it.""" + res, doc = self.trial(self.LATE, [call(2, 2.5), call(5, 6)], agent_ready=T + 3) + self.assertEqual(res["agent_execution"]["started_at"], job.iso(T + 3)) + self.assertEqual(doc["spans"][0]["start_ns"], int((T + 2) * 10**9)) + self.assertEqual(res["agent_result"]["n_model_calls"], 2) # counted all the same + + def test_a_start_that_ended_with_no_session_is_all_setup_and_no_run(self): + """It never answered its start, or exited: Harbor writes a failed agent + setup with its window and no agent_execution, and so does a standard run.""" + res, doc = self.trial(None, [], agent_unready=T + 5, score=None, exception=("AgentSetupTimeoutError", "the agent did not answer initialize in time")) + self.assertEqual(res["agent_setup"], {"started_at": job.iso(T + 1), "finished_at": job.iso(T + 5)}) + self.assertIsNone(res["agent_execution"]) + self.assertIsNone(doc) + self.assertEqual(job.feed_durations(res), {"env_setup": 1.0, "agent_setup": 4.0}) + + def test_a_trial_given_no_agent_clock_has_no_setup_phase(self): + res, _ = self.trial(None, []) self.assertIsNone(res["agent_setup"]) self.assertEqual(res["agent_execution"]["started_at"], job.iso(T + 1)) + self.assertEqual(job.feed_durations(res), {"env_setup": 1.0, "agent_run": 29.0}) + self.assertIsNone(job.feed_durations({})) def test_a_control_writes_none(self): self.assertFalse(os.path.exists(self.write({"key": "nop", "_kind": "control"}, None))) diff --git a/docs/field-guide.md b/docs/field-guide.md index b12dac46..bd9885ad 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -160,8 +160,8 @@ How the agent worked, not whether it won. Computed over layer 3. | `task_broken` | On the feasibility page: `nop`, which changes nothing, failed passing tests: the task is broken as shipped, or its verifier unstable. | derived | swe-bench | | `patch_exists / patch_applied` | Whether a patch was produced, and whether it applied. | measured | swe-bench | | `dur_total` | Trial wall clock. | measured | any | -| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. A standard run records the same four: setup is the agent's start and session. An imported trial charges its whole window to `dur_agent`. | measured | any | -| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, the unit's setup, the benchmark's driver and its scorer. | derived | any | +| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. On a standard run, setup is an ACP agent's start and session, or all of a start that ends with no session; a command agent, a capability, an Original setup and a control have none. Older standard records carry the start in `dur_agent`; stored results are not rewritten. An imported trial charges its whole window to `dur_agent`. | measured | any | +| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, what follows the agent and the check: the task's door closed and the container's changes read, then the container and the relay stopped. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | Prompt tokens read from cache, excluding cache writes: a subset, never added to input tokens. Older standard records may include writes; stored results are not rewritten. | measured | any | | `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | From d7385d44948f2dccc933b97b1ac61739307ad17b Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Wed, 18 Nov 2026 00:01:00 +0100 Subject: [PATCH 3/6] test(standard): pin a forced agent limit to the run phase, and say where it starts main lets a run force the agent's limit (--timeout agent). The board is told that limit for the agent's run, from its open session, and ACP's two start limits for its setup; the run command's page says an ACP agent starts before its limit does. --- app/tests/test_standard_run.py | 8 ++++++++ docs/cli/run.md | 2 ++ 2 files changed, 10 insertions(+) diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 07b7748b..f23ef69e 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -601,6 +601,14 @@ def test_an_agents_setup_is_timed_apart_from_its_run_and_the_board_is_told_the_s for phase, key in (("env_setup", "environment_setup"), ("agent_setup", "agent_setup"), ("agent_run", "agent_execution"), ("verify", "verifier")): self.assertAlmostEqual(told[phase], w[key][1] - w[key][0], places=3, msg=phase) + def test_a_forced_agent_limit_is_the_runs_never_the_setups(self): + """``--timeout agent`` bounds the agent's work from its open session, where its clock starts: + the board is told that limit for the run, and ACP's two start limits for the setup.""" + job_dir = self.go("fakebench", "fake", limit=1, timeouts={"agent": 45}) + with open(os.path.join(job_dir, "events.jsonl")) as f: + limits = [(e["phase"], e["timeout_sec"]) for e in map(json.loads, f) if e["event"] == "trial.phase"] + self.assertEqual(limits[1:3], [("agent_setup", acp.INITIALIZE_SECONDS + acp.SESSION_SECONDS), ("agent_run", 45)]) + def test_the_tasks_door_is_the_environments_not_the_agents(self): """A task's internet door opens before the agent begins and closes after it ended: neither is the agent's setup nor its run.""" diff --git a/docs/cli/run.md b/docs/cli/run.md index 6ce9f6be..f4bff995 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -71,6 +71,8 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | `verify` | The verify step's `seconds`, else the agent's default | Verifier error | | `score` | 600 s: the driver's `score` | Verifier error | +An agent over ACP starts before its `agent` limit does: 120 s to answer its start, 180 s to open its session, `AgentSetupTimeoutError` past either. The limit runs from the open session. + `--max-turns N` caps a conversation's agent turns over `limits.turns` (default 20); the conversation is scored as it stands. `config.json` records what was forced under `standard.timeouts` and `standard.max_turns`. From the builder, *Limits* takes them in minutes. ## A cap on model calls From 414469be83675e48db416127b97da3d92067f6de Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Wed, 18 Nov 2026 00:23:55 +0100 Subject: [PATCH 4/6] refactor(traces): leave the trial page's Setup hover to #173 #173 merges first and words each phase's hover by the kind of run, a standard run's setup included; this branch's own wording of the same line would only conflict with it. --- app/templates/traces/trial.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/traces/trial.html b/app/templates/traces/trial.html index 60b2ef7f..13e19f9e 100644 --- a/app/templates/traces/trial.html +++ b/app/templates/traces/trial.html @@ -152,7 +152,7 @@

Where the time went

{# A 0s phase emits no segment at all, so a control's empty setup leaves no stray sliver. #} {% if trial.dur_env|phase %}{% endif %} - {% if trial.dur_agent_setup|phase %}{% endif %} + {% if trial.dur_agent_setup|phase %}{% endif %} {% if trial.dur_agent|phase %}{% endif %} {% if trial.dur_verifier|phase %}{% endif %} {% set items = job.runner_items[1] if job.runner_items else 'items' %} From 61547d22794fe7f8bf2e34cd700d9afae5f9b579 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Wed, 18 Nov 2026 00:28:52 +0100 Subject: [PATCH 5/6] docs: cut the setup phase's doc lines to what a reader needs --- docs/cli/run.md | 2 +- docs/field-guide.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index f4bff995..41704e18 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -71,7 +71,7 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | `verify` | The verify step's `seconds`, else the agent's default | Verifier error | | `score` | 600 s: the driver's `score` | Verifier error | -An agent over ACP starts before its `agent` limit does: 120 s to answer its start, 180 s to open its session, `AgentSetupTimeoutError` past either. The limit runs from the open session. +An ACP agent's start (120 s) and session (180 s) come before its `agent` limit. `--max-turns N` caps a conversation's agent turns over `limits.turns` (default 20); the conversation is scored as it stands. `config.json` records what was forced under `standard.timeouts` and `standard.max_turns`. From the builder, *Limits* takes them in minutes. diff --git a/docs/field-guide.md b/docs/field-guide.md index 2364e400..1d7a93f8 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -165,8 +165,8 @@ How the agent worked, not whether it won. Computed over layer 3. | `task_broken` | On the feasibility page: `nop`, which changes nothing, failed passing tests: the task is broken as shipped, or its verifier unstable. | derived | swe-bench | | `patch_exists / patch_applied` | Whether a patch was produced, and whether it applied. | measured | swe-bench | | `dur_total` | Trial wall clock. | measured | any | -| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. On a standard run, setup is an ACP agent's start and session, or all of a start that ends with no session; a command agent, a capability, an Original setup and a control have none. Older standard records carry the start in `dur_agent`; stored results are not rewritten. An imported trial charges its whole window to `dur_agent`. | measured | any | -| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, what follows the agent and the check: the task's door closed and the container's changes read, then the container and the relay stopped. | derived | any | +| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. On a standard run, setup is an ACP agent's start and session. An imported trial charges its whole window to `dur_agent`. | measured | any | +| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, gaps and teardown. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | Prompt tokens read from cache, excluding cache writes: a subset, never added to input tokens. Older standard records may include writes; stored results are not rewritten. | measured | any | | `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any | From 360127b64304bb6d3a6e416b0aca14faf0f84c64 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Wed, 18 Nov 2026 00:41:35 +0100 Subject: [PATCH 6/6] docs: shorten the phase rows of the field guide, drop the start limits line --- docs/cli/run.md | 2 -- docs/field-guide.md | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/cli/run.md b/docs/cli/run.md index 41704e18..6ce9f6be 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -71,8 +71,6 @@ Each phase of a unit has a time limit. `--timeout PHASE=SECONDS` (repeatable) fo | `verify` | The verify step's `seconds`, else the agent's default | Verifier error | | `score` | 600 s: the driver's `score` | Verifier error | -An ACP agent's start (120 s) and session (180 s) come before its `agent` limit. - `--max-turns N` caps a conversation's agent turns over `limits.turns` (default 20); the conversation is scored as it stands. `config.json` records what was forced under `standard.timeouts` and `standard.max_turns`. From the builder, *Limits* takes them in minutes. ## A cap on model calls diff --git a/docs/field-guide.md b/docs/field-guide.md index 1d7a93f8..89f93bf8 100644 --- a/docs/field-guide.md +++ b/docs/field-guide.md @@ -165,8 +165,8 @@ How the agent worked, not whether it won. Computed over layer 3. | `task_broken` | On the feasibility page: `nop`, which changes nothing, failed passing tests: the task is broken as shipped, or its verifier unstable. | derived | swe-bench | | `patch_exists / patch_applied` | Whether a patch was produced, and whether it applied. | measured | swe-bench | | `dur_total` | Trial wall clock. | measured | any | -| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases Harbor records, drawn in order. On a standard run, setup is an ACP agent's start and session. An imported trial charges its whole window to `dur_agent`. | measured | any | -| `dur_other` | `dur_total` minus the phases: on Harbor, image pull, container start and teardown; on an imported script run, the wait for the judge; on a standard run, gaps and teardown. | derived | any | +| `dur_env / dur_agent_setup / dur_agent / dur_verifier` | The four phases a trial records, drawn in order. An imported trial charges its whole window to `dur_agent`. | measured | any | +| `dur_other` | `dur_total` minus the phases: waits and teardown. | derived | any | | `n_input_tokens` | Prompt tokens for the trial. | measured | any | | `n_cache_tokens` | Prompt tokens read from cache, excluding cache writes: a subset, never added to input tokens. Older standard records may include writes; stored results are not rewritten. | measured | any | | `cost_usd` | The cost `result.json` records: on Harbor, priced by the agent from its own table; on a standard run, the relay's record: the provider's own figure where its reply carried one, else an estimate from the user's price list (Settings, Model prices) for exactly that model name. A benchmark option's embeddings (a search index its driver builds) are the run's, in `.work/relay-benchmark`, never a trial's. None recorded reads —, never `$0.00`. | derived | any |