diff --git a/app/nullius/judge/state.py b/app/nullius/judge/state.py index 923a439e..8bc0bd3b 100644 --- a/app/nullius/judge/state.py +++ b/app/nullius/judge/state.py @@ -14,6 +14,7 @@ OBSERVATION = 600 # characters kept of a tool observation: the head, and the tail below OBSERVATION_TAIL = 200 # of those, the characters kept from the end - where a traceback names its exception and a command its exit MARKER = "\n[… {n} characters left out here …]\n" +FINAL_MARKER = "[agent] [the final message; its text is under {heading} below]" # Observation relevance. Accuracy falls as state grows with material unrelated # to the decision, so a tool observation is sent in full (up to OBSERVATION) @@ -82,13 +83,15 @@ def _relevant(call: dict, obs: str, instruction: str, terms: set, next_message: return len(_terms(obs) & terms) >= MIN_OVERLAP -def transcript_of(traj, relevance: str = RELEVANCE) -> str: +def transcript_of(traj, relevance: str = RELEVANCE, final_heading: str = "") -> str: """The agent's steps in order: each message and its tool calls with a bounded observation - in full when relevant to the question (`_relevant`), a stub with a marker otherwise; `relevance="all"` keeps every observation. A stub carries no observation text at all, so a - secret in an omitted observation cannot leak through it.""" - return _transcript(traj, relevance)[0] + secret in an omitted observation cannot leak through it. With + `final_heading`, the step the final message came from is a marker + naming that heading.""" + return _transcript(traj, relevance, final_heading)[0] def cap_observation(obs: str, limit: int = OBSERVATION, tail: int = OBSERVATION_TAIL) -> str: @@ -99,14 +102,18 @@ def cap_observation(obs: str, limit: int = OBSERVATION, tail: int = OBSERVATION_ return obs[:head] + f" … [{len(obs) - limit} characters left out here] … " + obs[-tail:] -def _transcript(traj, relevance: str) -> tuple: +def _transcript(traj, relevance: str, final_heading: str) -> tuple: instruction = instruction_of(traj) if relevance != "all" else "" terms = _terms(instruction) steps = [s for s in traj.steps if s.source == "agent"] + # the step `last_agent_message` is read from, by position: an earlier step with the same text stays + final = max((i for i, s in enumerate(steps) if s.message and s.message != "(tool use)"), default=-1) if final_heading else -1 out = [] kept = omitted = omitted_chars = 0 for i, s in enumerate(steps): - if s.message: + if i == final: + out.append(FINAL_MARKER.format(heading=final_heading)) + elif s.message: out.append(f"[agent] {s.message}") following = next((t.message for t in steps[i + 1:] if t.message), "") or "" for c in s.tool_calls or []: @@ -146,11 +153,12 @@ def build(kind: str, traj, trial=None, trial_dir: Optional[str] = None, other=No `relevance` is the observation policy `transcript_of` applies to the built-in kinds ("question" by default, "all" keeps every observation). - `trial`: instruction + steps + final message. `failure`: the same plus - the recorded error and the verifier text. `final`: the final message. + `trial`: instruction + steps + final message, the step of the final + message a one-line marker among the steps. `failure`: the same plus the + recorded error and the verifier text. `final`: the final message. `pair`: instruction + transcript A + transcript B (`other` is B's - trajectory). Any other kind is a registered builder's (`register`), - given the same keywords.""" + trajectory), each with the same marker. Any other kind is a registered + builder's (`register`), given the same keywords.""" if kind in BUILDERS: text = BUILDERS[kind](traj, trial=trial, trial_dir=trial_dir, other=other, **extra) elif kind == "final": @@ -158,14 +166,14 @@ def build(kind: str, traj, trial=None, trial_dir: Optional[str] = None, other=No elif kind == "pair": # each transcript truncated on its own, so a long A cannot push B out of the state half = max(2_000, (limit - 4_000) // 2) - a, ta = truncate(transcript_of(traj, relevance), half) - b, tb = truncate(transcript_of(other, relevance), half) + a, ta = truncate(transcript_of(traj, relevance, "[final message A]"), half) + b, tb = truncate(transcript_of(other, relevance, "[final message B]"), half) text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript A]\n" + a + "\n[final message A]\n" + (traj.last_agent_message or "") \ + "\n\n[transcript B]\n" + b + "\n[final message B]\n" + (other.last_agent_message or "") text, t = truncate(redact(text), limit) return text, t or ta or tb else: - text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript]\n" + transcript_of(traj, relevance) + "\n\n[final message]\n" + (traj.last_agent_message or "") + text = "[instruction]\n" + instruction_of(traj) + "\n\n[transcript]\n" + transcript_of(traj, relevance, "[final message]") + "\n\n[final message]\n" + (traj.last_agent_message or "") if kind == "failure": err = "" if trial is not None and (trial.exception_type or trial.exception_message): diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 85f34ed5..7f58350d 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -11,7 +11,7 @@ import unittest from pathlib import Path -from nullius import workspace +from nullius import redact, workspace from nullius.judge import batch, client, questions, state, store from nullius.traces import trajectory @@ -444,6 +444,93 @@ def test_a_failed_call_is_kept_whatever_it_says_and_nothing_but_observations_is_ self.assertIn(line, state.transcript_of(trajectory.load(str(CLEAN))), line) +class FinalMessageOnce(unittest.TestCase): + """The step the final message came from is a one-line marker at its + place in the transcript, and the text is written once, under its heading.""" + MARKER = "[agent] [the final message; its text is under [final message] below]" + MARKER_A = "[agent] [the final message; its text is under [final message A] below]" + MARKER_B = "[agent] [the final message; its text is under [final message B] below]" + + def made_up(self, steps): + """A trajectory read from the clean fixture with its steps replaced.""" + raw = json.loads(CLEAN.read_text()) + raw["steps"] = steps(raw["steps"]) + tmp = Path(tempfile.mkdtemp(prefix="nullius-judge-final-")); self.addCleanup(shutil.rmtree, tmp, True) + (tmp / "t.json").write_text(json.dumps(raw)) + traj = trajectory.load(str(tmp / "t.json")); self.assertFalse(traj.error) + return traj + + def transcript_lines(self, text): + return text.split("\n\n[transcript]\n", 1)[1].split("\n\n[final message]\n", 1)[0].splitlines() + + def test_each_kind_holds_the_final_message_once_per_trajectory(self): + for fixture in (CLEAN, ADVERSARIAL): + traj = trajectory.load(str(fixture)); final = traj.last_agent_message; self.assertTrue(final) + with self.subTest(fixture=fixture.name): + for kind in ("trial", "failure", "final"): + self.assertEqual(state.build(kind, traj)[0].count(final), 1, kind) + self.assertEqual(state.build("pair", traj, other=traj)[0].count(final), 2) + + def test_the_marker_stands_where_the_final_message_stood(self): + for fixture in (CLEAN, ADVERSARIAL): + traj = trajectory.load(str(fixture)); final = traj.last_agent_message + line = "[agent] " + final + head = "[instruction]\n" + state.instruction_of(traj) + "\n\n[transcript]\n" + state.transcript_of(traj) + with self.subTest(fixture=fixture.name): + trial = state.build("trial", traj)[0] + self.assertEqual(trial.count(self.MARKER), 1) + self.assertEqual(trial.replace(self.MARKER, line), redact.redact(head + "\n\n[final message]\n" + final)) + failure = state.build("failure", traj)[0] + self.assertEqual(failure.replace(self.MARKER, line).partition("\n\n[recorded error]\n")[0], + redact.redact(head + "\n\n[final message]\n" + final)) + pair = state.build("pair", traj, other=traj)[0] + self.assertEqual((pair.count(self.MARKER_A), pair.count(self.MARKER_B)), (1, 1)) + transcript = state.transcript_of(traj) + self.assertEqual(pair.replace(self.MARKER_A, line).replace(self.MARKER_B, line), redact.redact( + "[instruction]\n" + state.instruction_of(traj) + "\n\n[transcript A]\n" + transcript + "\n[final message A]\n" + final + + "\n\n[transcript B]\n" + transcript + "\n[final message B]\n" + final)) + + def test_an_earlier_step_with_the_same_text_stays_and_the_last_is_the_marker(self): + def same_text_earlier(steps): + steps[2]["message"] = steps[-1]["message"] + return steps + traj = self.made_up(same_text_earlier); final = traj.last_agent_message + lines = self.transcript_lines(state.build("trial", traj)[0]) + self.assertEqual([l for l in lines if l.startswith("[agent] ")], ["[agent] " + final, self.MARKER]) + self.assertEqual(lines[-1], self.MARKER) + + def test_an_agent_step_after_the_final_message_stays_after_the_marker(self): + def tool_step_after(steps): + steps.append({"step_id": 6, "timestamp": "2026-09-18T10:00:06.000Z", "source": "agent", "message": "(tool use)", + "tool_calls": [{"tool_call_id": "c6", "function_name": "Bash", "arguments": {"command": "git status"}}], + "observation": {"results": [{"source_call_id": "c6", "content": "nothing to commit\n"}]}}) + return steps + traj = self.made_up(tool_step_after); final = traj.last_agent_message + self.assertEqual(final, json.loads(CLEAN.read_text())["steps"][-1]["message"]) + lines = self.transcript_lines(state.build("trial", traj)[0]) + self.assertEqual([l for l in lines if l.startswith("[agent] ")], [self.MARKER, "[agent] (tool use)"]) + self.assertEqual(lines[lines.index(self.MARKER) + 1:][:2], ["[agent] (tool use)", '[tool Bash] {"command": "git status"}']) + self.assertNotIn("[agent] " + final, lines) + + def test_a_trajectory_with_no_agent_message_gets_no_marker(self): + for name, steps in (("no message", lambda s: s[:-1]), + ("tool use only", lambda s: s[:-1] + [dict(s[-1], message="(tool use)")])): + with self.subTest(name): + traj = self.made_up(steps); self.assertEqual(traj.last_agent_message, "") + for kind in ("trial", "failure"): + self.assertNotIn("[the final message;", state.build(kind, traj)[0]) + self.assertNotIn("[the final message;", state.build("pair", traj, other=traj)[0]) + + def test_the_control_state_keeps_the_last_message_in_its_transcript(self): + for fixture in (CLEAN, ADVERSARIAL): + traj = trajectory.load(str(fixture)) + with self.subTest(fixture=fixture.name): + text = state.build("control", traj, control="nop")[0] + self.assertIn("[agent] " + traj.last_agent_message, text) + self.assertNotIn("[the final message;", text) + self.assertTrue(text.endswith(redact.redact("[transcript]\n" + state.transcript_of(traj)))) + + class PrJudgeWorkflow(unittest.TestCase): """The PR judge's workflow, read by line, since the standard library has no YAML reader and the file is small. The verdict is a new PR comment