Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 20 additions & 12 deletions app/nullius/judge/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
OBSERVATION = 600 # characters kept of a tool observation: the head, and the tail below
OBSERVATION_TAIL = 200 # of those, the characters kept from the end - where a traceback names its exception and a command its exit
MARKER = "\n[… {n} characters left out here …]\n"
FINAL_MARKER = "[agent] [the final message; its text is under {heading} below]"

# Observation relevance. Accuracy falls as state grows with material unrelated
# to the decision, so a tool observation is sent in full (up to OBSERVATION)
Expand Down Expand Up @@ -82,13 +83,15 @@ def _relevant(call: dict, obs: str, instruction: str, terms: set, next_message:
return len(_terms(obs) & terms) >= MIN_OVERLAP


def transcript_of(traj, relevance: str = RELEVANCE) -> str:
def transcript_of(traj, relevance: str = RELEVANCE, final_heading: str = "") -> str:
"""The agent's steps in order: each message and its tool calls with a
bounded observation - in full when relevant to the question
(`_relevant`), a stub with a marker otherwise; `relevance="all"` keeps
every observation. A stub carries no observation text at all, so a
secret in an omitted observation cannot leak through it."""
return _transcript(traj, relevance)[0]
secret in an omitted observation cannot leak through it. With
`final_heading`, the step the final message came from is a marker
naming that heading."""
return _transcript(traj, relevance, final_heading)[0]


def cap_observation(obs: str, limit: int = OBSERVATION, tail: int = OBSERVATION_TAIL) -> str:
Expand All @@ -99,14 +102,18 @@ def cap_observation(obs: str, limit: int = OBSERVATION, tail: int = OBSERVATION_
return obs[:head] + f" … [{len(obs) - limit} characters left out here] … " + obs[-tail:]


def _transcript(traj, relevance: str) -> tuple:
def _transcript(traj, relevance: str, final_heading: str) -> tuple:
instruction = instruction_of(traj) if relevance != "all" else ""
terms = _terms(instruction)
steps = [s for s in traj.steps if s.source == "agent"]
# the step `last_agent_message` is read from, by position: an earlier step with the same text stays
final = max((i for i, s in enumerate(steps) if s.message and s.message != "(tool use)"), default=-1) if final_heading else -1
out = []
kept = omitted = omitted_chars = 0
for i, s in enumerate(steps):
if s.message:
if i == final:
out.append(FINAL_MARKER.format(heading=final_heading))
elif s.message:
out.append(f"[agent] {s.message}")
following = next((t.message for t in steps[i + 1:] if t.message), "") or ""
for c in s.tool_calls or []:
Expand Down Expand Up @@ -146,26 +153,27 @@ def build(kind: str, traj, trial=None, trial_dir: Optional[str] = None, other=No
`relevance` is the observation policy `transcript_of` applies to the
built-in kinds ("question" by default, "all" keeps every observation).

`trial`: instruction + steps + final message. `failure`: the same plus
the recorded error and the verifier text. `final`: the final message.
`trial`: instruction + steps + final message, the step of the final
message a one-line marker among the steps. `failure`: the same plus the
recorded error and the verifier text. `final`: the final message.
`pair`: instruction + transcript A + transcript B (`other` is B's
trajectory). Any other kind is a registered builder's (`register`),
given the same keywords."""
trajectory), each with the same marker. Any other kind is a registered
builder's (`register`), given the same keywords."""
if kind in BUILDERS:
text = BUILDERS[kind](traj, trial=trial, trial_dir=trial_dir, other=other, **extra)
elif kind == "final":
text = "[final message]\n" + (traj.last_agent_message or "")
elif kind == "pair":
# each transcript truncated on its own, so a long A cannot push B out of the state
half = max(2_000, (limit - 4_000) // 2)
a, ta = truncate(transcript_of(traj, relevance), half)
b, tb = truncate(transcript_of(other, relevance), half)
a, ta = truncate(transcript_of(traj, relevance, "[final message A]"), half)
b, tb = truncate(transcript_of(other, relevance, "[final message B]"), half)
text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript A]\n" + a + "\n[final message A]\n" + (traj.last_agent_message or "") \
+ "\n\n[transcript B]\n" + b + "\n[final message B]\n" + (other.last_agent_message or "")
text, t = truncate(redact(text), limit)
return text, t or ta or tb
else:
text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript]\n" + transcript_of(traj, relevance) + "\n\n[final message]\n" + (traj.last_agent_message or "")
text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript]\n" + transcript_of(traj, relevance, "[final message]") + "\n\n[final message]\n" + (traj.last_agent_message or "")
if kind == "failure":
err = ""
if trial is not None and (trial.exception_type or trial.exception_message):
Expand Down
89 changes: 88 additions & 1 deletion app/tests/test_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import unittest
from pathlib import Path

from nullius import workspace
from nullius import redact, workspace
from nullius.judge import batch, client, questions, state, store
from nullius.traces import trajectory

Expand Down Expand Up @@ -444,6 +444,93 @@ def test_a_failed_call_is_kept_whatever_it_says_and_nothing_but_observations_is_
self.assertIn(line, state.transcript_of(trajectory.load(str(CLEAN))), line)


class FinalMessageOnce(unittest.TestCase):
"""The step the final message came from is a one-line marker at its
place in the transcript, and the text is written once, under its heading."""
MARKER = "[agent] [the final message; its text is under [final message] below]"
MARKER_A = "[agent] [the final message; its text is under [final message A] below]"
MARKER_B = "[agent] [the final message; its text is under [final message B] below]"

def made_up(self, steps):
"""A trajectory read from the clean fixture with its steps replaced."""
raw = json.loads(CLEAN.read_text())
raw["steps"] = steps(raw["steps"])
tmp = Path(tempfile.mkdtemp(prefix="nullius-judge-final-")); self.addCleanup(shutil.rmtree, tmp, True)
(tmp / "t.json").write_text(json.dumps(raw))
traj = trajectory.load(str(tmp / "t.json")); self.assertFalse(traj.error)
return traj

def transcript_lines(self, text):
return text.split("\n\n[transcript]\n", 1)[1].split("\n\n[final message]\n", 1)[0].splitlines()

def test_each_kind_holds_the_final_message_once_per_trajectory(self):
for fixture in (CLEAN, ADVERSARIAL):
traj = trajectory.load(str(fixture)); final = traj.last_agent_message; self.assertTrue(final)
with self.subTest(fixture=fixture.name):
for kind in ("trial", "failure", "final"):
self.assertEqual(state.build(kind, traj)[0].count(final), 1, kind)
self.assertEqual(state.build("pair", traj, other=traj)[0].count(final), 2)

def test_the_marker_stands_where_the_final_message_stood(self):
for fixture in (CLEAN, ADVERSARIAL):
traj = trajectory.load(str(fixture)); final = traj.last_agent_message
line = "[agent] " + final
head = "[instruction]\n" + state.instruction_of(traj) + "\n\n[transcript]\n" + state.transcript_of(traj)
with self.subTest(fixture=fixture.name):
trial = state.build("trial", traj)[0]
self.assertEqual(trial.count(self.MARKER), 1)
self.assertEqual(trial.replace(self.MARKER, line), redact.redact(head + "\n\n[final message]\n" + final))
failure = state.build("failure", traj)[0]
self.assertEqual(failure.replace(self.MARKER, line).partition("\n\n[recorded error]\n")[0],
redact.redact(head + "\n\n[final message]\n" + final))
pair = state.build("pair", traj, other=traj)[0]
self.assertEqual((pair.count(self.MARKER_A), pair.count(self.MARKER_B)), (1, 1))
transcript = state.transcript_of(traj)
self.assertEqual(pair.replace(self.MARKER_A, line).replace(self.MARKER_B, line), redact.redact(
"[instruction]\n" + state.instruction_of(traj) + "\n\n[transcript A]\n" + transcript + "\n[final message A]\n" + final
+ "\n\n[transcript B]\n" + transcript + "\n[final message B]\n" + final))

def test_an_earlier_step_with_the_same_text_stays_and_the_last_is_the_marker(self):
def same_text_earlier(steps):
steps[2]["message"] = steps[-1]["message"]
return steps
traj = self.made_up(same_text_earlier); final = traj.last_agent_message
lines = self.transcript_lines(state.build("trial", traj)[0])
self.assertEqual([l for l in lines if l.startswith("[agent] ")], ["[agent] " + final, self.MARKER])
self.assertEqual(lines[-1], self.MARKER)

def test_an_agent_step_after_the_final_message_stays_after_the_marker(self):
def tool_step_after(steps):
steps.append({"step_id": 6, "timestamp": "2026-09-18T10:00:06.000Z", "source": "agent", "message": "(tool use)",
"tool_calls": [{"tool_call_id": "c6", "function_name": "Bash", "arguments": {"command": "git status"}}],
"observation": {"results": [{"source_call_id": "c6", "content": "nothing to commit\n"}]}})
return steps
traj = self.made_up(tool_step_after); final = traj.last_agent_message
self.assertEqual(final, json.loads(CLEAN.read_text())["steps"][-1]["message"])
lines = self.transcript_lines(state.build("trial", traj)[0])
self.assertEqual([l for l in lines if l.startswith("[agent] ")], [self.MARKER, "[agent] (tool use)"])
self.assertEqual(lines[lines.index(self.MARKER) + 1:][:2], ["[agent] (tool use)", '[tool Bash] {"command": "git status"}'])
self.assertNotIn("[agent] " + final, lines)

def test_a_trajectory_with_no_agent_message_gets_no_marker(self):
for name, steps in (("no message", lambda s: s[:-1]),
("tool use only", lambda s: s[:-1] + [dict(s[-1], message="(tool use)")])):
with self.subTest(name):
traj = self.made_up(steps); self.assertEqual(traj.last_agent_message, "")
for kind in ("trial", "failure"):
self.assertNotIn("[the final message;", state.build(kind, traj)[0])
self.assertNotIn("[the final message;", state.build("pair", traj, other=traj)[0])

def test_the_control_state_keeps_the_last_message_in_its_transcript(self):
for fixture in (CLEAN, ADVERSARIAL):
traj = trajectory.load(str(fixture))
with self.subTest(fixture=fixture.name):
text = state.build("control", traj, control="nop")[0]
self.assertIn("[agent] " + traj.last_agent_message, text)
self.assertNotIn("[the final message;", text)
self.assertTrue(text.endswith(redact.redact("[transcript]\n" + state.transcript_of(traj))))


class PrJudgeWorkflow(unittest.TestCase):
"""The PR judge's workflow, read by line, since the standard library has
no YAML reader and the file is small. The verdict is a new PR comment
Expand Down
Loading