From d05683d2aaa37f02881bd10b2086e019e9fc3d27 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:41:44 +0100 Subject: [PATCH 1/2] test(judge): expect text in the failure state's verifier section With no verifier file the failure state ends with an empty [verifier] heading. These tests cover a missing file with no trial, with a trial that records no reward and no verifier phase, and with a trial that records either, and a present file, whose section stays as it is. --- app/tests/test_judge.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/app/tests/test_judge.py b/app/tests/test_judge.py index 85f34ed5..c882d03f 100644 --- a/app/tests/test_judge.py +++ b/app/tests/test_judge.py @@ -13,6 +13,7 @@ from nullius import workspace from nullius.judge import batch, client, questions, state, store +from nullius.runs import parse from nullius.traces import trajectory HERE = Path(__file__).resolve().parent @@ -393,6 +394,51 @@ def flagging(url, body, key): self.assertNotIn("[transcript]", json.dumps(rec)); self.assertNotIn("AKIAIOSFODNN7EXAMPLE", json.dumps(rec)) # the record carries readings, never the state +class VerifierSection(unittest.TestCase): + """The failure state's [verifier] section always has text: the + verifier's files when there are any, otherwise a sentence that says + whether the verifier ran.""" + DID_NOT_RUN = "[verifier]\ndid not run: the trial records no reward and no verifier phase" + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="nullius-judge-verifier-")); self.addCleanup(shutil.rmtree, self.tmp, True) + self.traj = trajectory.load(str(CLEAN)) + + def trial(self, reward, dur_verifier): + return parse.Trial(job_id="j", dir=str(self.tmp), task_name="x/y", trial_name="t", task_base="y", source="", + model_name="", outcome="", reward=reward, exception_type="", exception_message="", + patch_exists=None, patch_applied=None, f2p_pass=0, f2p_fail=0, p2p_pass=0, p2p_fail=0, + n_input_tokens=0, n_output_tokens=0, n_cache_tokens=0, cost_usd=None, dur_total=None, + dur_env=None, dur_agent_setup=None, dur_agent=None, dur_verifier=dur_verifier) + + def section(self, text): + """The failure state's last section, from its heading to the end.""" + return "[verifier]\n" + text.rpartition("\n\n[verifier]\n")[2] + + def failure(self, trial): + return self.section(state.build("failure", self.traj, trial=trial, trial_dir=str(self.tmp))[0]) + + def test_no_verifier_file_and_no_trial_is_nothing_recorded(self): + self.assertEqual(self.failure(None), "[verifier]\nnothing recorded") + self.assertEqual(self.section(state.build("failure", self.traj)[0]), "[verifier]\nnothing recorded") + + def test_a_trial_with_no_reward_and_no_verifier_phase_says_the_verifier_did_not_run(self): + self.assertEqual(self.failure(self.trial(None, None)), self.DID_NOT_RUN) + + def test_a_trial_with_a_reward_or_a_verifier_phase_is_nothing_recorded(self): + for reward, dur_verifier in ((0.0, None), (None, 4.5), (1.0, 4.5)): + with self.subTest(reward=reward, dur_verifier=dur_verifier): + self.assertEqual(self.failure(self.trial(reward, dur_verifier)), "[verifier]\nnothing recorded") + + def test_a_verifier_file_is_the_section_whatever_the_trial_records(self): + (self.tmp / "verifier").mkdir() + (self.tmp / "verifier" / "reward.txt").write_text("0\n") + self.assertEqual(state.verifier_text(str(self.tmp)), "[verifier/reward.txt]\n0\n") + for trial in (None, self.trial(None, None), self.trial(0.0, 4.5)): + with self.subTest(trial=trial and (trial.reward, trial.dur_verifier)): + self.assertEqual(self.failure(trial), "[verifier]\n[verifier/reward.txt]\n0\n") + + class Instruction(unittest.TestCase): """mini-swe-agent, swe-agent, OpenHands and Codex trajectories open with the agent's own system prompt; the task is the first user step.""" From f4658723c0431edcc9a57ea84a15cdff5911ad1c Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:41:44 +0100 Subject: [PATCH 2/2] fix(judge): say why the failure state's verifier section is empty When none of the verifier's files exists, the [verifier] section was left empty, so the judge could not tell whether the verifier ran and wrote nothing or never ran. The section now says "nothing recorded", the wording of the control state, and says the verifier did not run when the trial records neither a reward nor a verifier phase. A present verifier file is written as before. --- app/nullius/judge/state.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/nullius/judge/state.py b/app/nullius/judge/state.py index 923a439e..13456111 100644 --- a/app/nullius/judge/state.py +++ b/app/nullius/judge/state.py @@ -171,7 +171,12 @@ def build(kind: str, traj, trial=None, trial_dir: Optional[str] = None, other=No if trial is not None and (trial.exception_type or trial.exception_message): err = f"{trial.exception_type}: {trial.exception_message}".strip(": ") text += "\n\n[recorded error]\n" + (err or "none recorded") - text += "\n\n[verifier]\n" + (verifier_text(trial_dir) if trial_dir else "") + verifier = verifier_text(trial_dir) if trial_dir else "" + if not verifier: + # a trial stopped before its verifier phase records neither a reward nor that phase + did_not_run = trial is not None and trial.reward is None and trial.dur_verifier is None + verifier = "did not run: the trial records no reward and no verifier phase" if did_not_run else "nothing recorded" + text += "\n\n[verifier]\n" + verifier return truncate(redact(text), limit)