From f6d8a4bbd02d35ef9327a3ec15e3e645f6eb198d Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Mon, 16 Nov 2026 13:21:51 +0100 Subject: [PATCH 01/15] feat(runs): read a task's repeated attempts together, by pass rate, with pass@k and pass^k Trials of one task in one job are its attempts, numbered by when they started (nullius.runs.attempts). The trial page gets a tab per attempt and an All attempts table; the job page shows one row per task with its pass count, the instruction score once and the mean tokens and times; the run page and Compare show pass@k and pass^k. Compare judges a task by its pass rate over every attempt, ranks by how far it moved (read with one pass and one fail added, so a single flipped attempt ranks below a steady gap), and its side-by-side page gives each run its own attempt tabs. Attempts are read as the adjusted rate reads trials: an infra attempt is counted apart and left out of the pass count, the means and pass@k; an agent's timeout or crash counts as a failure. The standard's pass_hat now calls the same estimators; what it writes is unchanged. --- AGENTS.md | 3 +- app/nullius/compare/comparison.py | 73 ++++++-- app/nullius/runs/attempts.py | 228 ++++++++++++++++++++++++ app/nullius/standard/job.py | 17 +- app/static/compare/comparison.css | 1 + app/static/runs/board.css | 18 +- app/static/runs/board.js | 43 ++++- app/static/style.css | 29 ++- app/static/traces/conversation.js | 2 + app/templates/compare/compare.html | 23 ++- app/templates/compare/compare_task.html | 17 +- app/templates/runs/job.html | 44 ++++- app/templates/traces/_attempts.html | 25 +++ app/templates/traces/attempts.html | 40 +++++ app/templates/traces/trial.html | 7 + app/tests/field_contract.json | 22 ++- app/tests/field_contract.py | 1 + app/tests/test_attempts.py | 120 +++++++++++++ app/tests/test_comparison.py | 108 +++++++++++ app/ui_tests/test_attempt_routes.py | 183 +++++++++++++++++++ app/ui_tests/test_comparison_routes.py | 33 +++- app/web/compare.py | 24 ++- app/web/runs.py | 5 +- app/web/traces.py | 13 +- docs/field-guide.md | 8 +- 25 files changed, 1025 insertions(+), 62 deletions(-) create mode 100644 app/nullius/runs/attempts.py create mode 100644 app/templates/traces/_attempts.html create mode 100644 app/templates/traces/attempts.html create mode 100644 app/tests/test_attempts.py create mode 100644 app/ui_tests/test_attempt_routes.py diff --git a/AGENTS.md b/AGENTS.md index 8a336c0c..edff2b34 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,8 @@ and its resources. The source layout stays under `app/`. - `app/nullius/` — its backend, one package per purpose: `runs/` (reading one Harbor job tree: `parse`, `classify`, `live`, `artifacts`, `feasibility`, `launches`, `verifier` - a verifier's report read by its - benchmark's report block - and the `cli`), `traces/` (`trace`, `trajectory`, + benchmark's report block - `attempts`, a task's attempts in one job with + their pass count and pass@k, and the `cli`), `traces/` (`trace`, `trajectory`, `conversation`, `charts`, `behavior`, `otel`), `compare/` (`comparison` and its `cli`), `catalogue/` (`catalog`, which loads one folder per benchmark under `catalogue/benchmarks/`, named by its key: its `benchmark.json` with its diff --git a/app/nullius/compare/comparison.py b/app/nullius/compare/comparison.py index cdf7984b..0a0b69f8 100644 --- a/app/nullius/compare/comparison.py +++ b/app/nullius/compare/comparison.py @@ -7,10 +7,10 @@ from __future__ import annotations -from collections import Counter from dataclasses import dataclass, replace from typing import Mapping, Sequence +from nullius.runs import attempts as task_attempts from nullius.runs.parse import Job, Trial @@ -77,16 +77,47 @@ def resolve(values: Sequence[str], jobs_by_workspace: Mapping[str, Sequence[Job] @dataclass(frozen=True) class TaskRow: + """One task across the runs. ``trials`` holds each run's last indexed + attempt, the one its cell links to; ``attempts`` counts each run's + attempts and ``tried`` holds them, read as ``attempts.Attempts`` reads + them (infra apart).""" dataset: str task: str trials: tuple[Trial | None, ...] attempts: tuple[int, ...] + tried: tuple[task_attempts.Attempts | None, ...] = () + + def _compared(self) -> list: + """The runs whose rates are set against each other, one kind of number + only: those that passed or failed the task when any did, else those + with a mean score. A pass rate is never set against a mean score.""" + tried = [a for a in self.tried if a] + return [a for a in tried if a.has_verdicts] or [a for a in tried if a.mean_score is not None] + + @property + def rate_moved(self) -> bool: + """The runs' pass rates (mean scores) differ.""" + return len({round(a.rate, 3) for a in self._compared()}) > 1 @property def differs(self) -> bool: - # a scored task differs by its score too: two readers that both scored a - # conversation are told apart by how much - return len({(trial.outcome, None if trial.score is None else round(trial.score, 3)) for trial in self.trials if trial}) > 1 + """The pass rate moved, or the attempts that ran ended differently: + one failed verification, another timed out. A run none of whose + attempts ran differs from one that ran.""" + return self.rate_moved or len({a.endings for a in self.tried if a}) > 1 + + @property + def movement(self) -> float: + """How far apart the runs' pass rates are, each read as if the run had + one more pass and one more fail, so a gap over many attempts outranks + the same gap over one: 1/1 against 0/1 reads 67% against 33%, 2/3 + against 0/3 reads 60% against 20%. Mean scores are taken as they are. + 0 when the rates are the same, and it can be 0 when they are not (1/1 + against 3/4 reads 67% against 67%): ``rate_moved`` says which.""" + if not self.rate_moved: + return 0.0 + values = [(a.n_passed + 1) / (a.n_ran + 2) if a.has_verdicts else a.mean_score for a in self._compared()] + return max(values) - min(values) @property def shared(self) -> bool: @@ -104,6 +135,7 @@ class Comparison: repeated_trials: tuple[int, ...] datasets_match: bool datasets_known: bool + pass_k: tuple[dict | None, ...] = () @property def references(self) -> list[str]: @@ -118,8 +150,11 @@ def build(selected: Sequence[SelectedRun], cohort: str = "whole") -> Comparison: """Build a whole-run or exact shared-task view without changing any Job. Matching uses the dataset string as recorded, without alias/path guessing. - Every attempt on a shared task remains in the rates, costs and token sums; - task rows retain the last indexed attempt, as the existing table does. + Every attempt on a shared task remains in the rates, costs and token sums. + A task row holds every attempt and keeps the last indexed one to link to; + rows sort by how far the pass rate moved (``TaskRow.movement``), the + largest first, then the other rows whose rate moved, then the rest that + differ. Unknown outcomes remain in the adjusted-rate denominator because Job owns the arithmetic. A cohort cannot manufacture a new wall-clock run interval. """ @@ -137,6 +172,7 @@ def key(selection, trial): sets = [{key(selection, trial) for trial in selection.job.trials} for selection in selected] common = set.intersection(*sets) if sets else set() + # each task's attempts, one list per run, the tasks in the order they appear jobs, excluded, repeated, by_task = [], [], [], {} for column, selection in enumerate(selected): original = selection.job @@ -148,19 +184,22 @@ def key(selection, trial): **({"started_at": None, "finished_at": None} if cohort == "matched" else {})) jobs.append(view) excluded.append(len(original.trials) - len(trials)) - counts = Counter(key(selection, trial) for trial in trials) - repeated.append(sum(count - 1 for count in counts.values())) + repeated.append(len(trials) - len({key(selection, trial) for trial in trials})) for trial in trials: - task_key = key(selection, trial) - row, attempts = by_task.setdefault(task_key, ([None] * len(selected), [0] * len(selected))) - row[column] = trial - attempts[column] += 1 - - rows = [TaskRow(dataset, task, tuple(trials), tuple(attempts)) - for (dataset, task), (trials, attempts) in by_task.items()] - rows.sort(key=lambda row: (not row.differs, row.dataset, row.task)) + by_task.setdefault(key(selection, trial), [[] for _ in selected])[column].append(trial) + + def task_row(task_key, columns): + # the cell links to the last indexed attempt + return TaskRow(*task_key, tuple(ts[-1] if ts else None for ts in columns), tuple(len(ts) for ts in columns), + tuple(task_attempts.Attempts(tuple(task_attempts.ordered(ts))) if ts else None for ts in columns)) + + rows = [task_row(task_key, columns) for task_key, columns in by_task.items()] + rows.sort(key=lambda row: (-round(row.movement, 3), not row.rate_moved, not row.differs, row.dataset, row.task)) + # pass@k from the rows' own tasks, so the summary and the table count alike + pass_k = tuple(task_attempts.for_tasks(row.tried[column].trials for row in rows if row.tried[column]) + for column in range(len(selected))) return Comparison(selected, cohort, tuple(jobs), tuple(rows), len(common), - tuple(excluded), tuple(repeated), datasets_match, known) + tuple(excluded), tuple(repeated), datasets_match, known, pass_k) diff --git a/app/nullius/runs/attempts.py b/app/nullius/runs/attempts.py new file mode 100644 index 00000000..272af6cf --- /dev/null +++ b/app/nullius/runs/attempts.py @@ -0,0 +1,228 @@ +"""A task's attempts in one job. + +A job that ran each task more than once (Harbor's ``n_attempts``, the standard's +``--repeat``) records one trial per attempt, and nothing in a trial says which +attempt it was. Two trials are attempts at the same task when they carry the +same ``task_name``; they are numbered by when they started, a trial with no +recorded start after every one that has, ties by trial name. The number is +this module's, not the producer's: it is stable for one job tree and says +nothing about the order Harbor indexed them in. + +The attempts at a task are read as the run's adjusted rate reads its trials: +an infra attempt (a rate limit, a provider or setup error, the spending cap, a +cancellation) never got a fair try, so it is left out of the pass count, the +rate and the means and counted apart; an unknown outcome stays in once anything +at the task was passed or failed. ``has_verdict`` is that rule, for every page. + +pass@k and pass^k are here too: ``estimate`` is the estimators, which the +standard's ``pass_hat_k`` uses as well, and ``for_tasks`` the pages' reading of +a run's tasks. + +Nothing here reads a file: it takes the parsed job. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import cached_property +from math import comb +from typing import Iterable, Sequence + +from . import classify + +_LAST = datetime.max.replace(tzinfo=timezone.utc) + + +def _start(trial) -> datetime: + at = trial.started_at + if at is None: + return _LAST + return at if at.tzinfo else at.replace(tzinfo=timezone.utc) + + +def ordered(trials) -> list: + """Attempts in the order they are numbered.""" + return sorted(trials, key=lambda t: (_start(t), t.trial_name)) + + +# Outcomes that are neither a pass nor a fail: nothing scored the attempt, or +# the benchmark gives a score and no pass or fail. +NO_VERDICT = frozenset({classify.UNKNOWN, classify.SCORED}) + +# The numbers a task's row shows as a mean over its attempts. +MEANS = ("tokens", "dur_total", "dur_env", "dur_agent", "dur_verifier") + + +def ran(trials) -> tuple: + """The attempts that got a fair try: every one but infra.""" + return tuple(t for t in trials if t.outcome not in classify.INFRA_OUTCOMES) + + +def has_verdict(trials) -> bool: + """Whether an attempt that ran was passed or failed, so a pass count over + them means something.""" + return any(t.outcome not in NO_VERDICT for t in ran(trials)) + + +@dataclass(frozen=True) +class Attempts: + """Every attempt at one task in one job, numbered from 1.""" + trials: tuple + + @property + def n(self) -> int: + return len(self.trials) + + @cached_property + def ran(self) -> tuple: + return ran(self.trials) + + @property + def n_ran(self) -> int: + return len(self.ran) + + @property + def n_infra(self) -> int: + return self.n - self.n_ran + + @property + def n_passed(self) -> int: + return sum(1 for t in self.trials if t.outcome == classify.RESOLVED) + + @property + def has_verdicts(self) -> bool: + """False when no attempt that ran was passed or failed (all unknown, + scored or infra), so a pass count would say nothing.""" + return has_verdict(self.trials) + + @property + def scores(self) -> list: + return [t.score for t in self.ran if t.score is not None] + + @property + def n_scored(self) -> int: + return len(self.scores) + + @property + def mean_score(self) -> float | None: + return sum(self.scores) / len(self.scores) if self.scores else None + + @property + def rate(self) -> float | None: + """The share of the attempts that ran which passed, when one was passed + or failed; else their mean score; else None (nothing ran, or nothing + scored it).""" + if self.has_verdicts: + return self.n_passed / self.n_ran + return self.mean_score + + @property + def endings(self) -> frozenset: + """How the attempts that ran ended, each way once: an outcome and its + score. Empty when none ran.""" + return frozenset((t.outcome, None if t.score is None else round(t.score, 3)) for t in self.ran) + + def spread(self, field: str, kind: str = "agent") -> dict | None: + """The mean of one number over the attempts that ran and recorded it, + with its lowest and highest: ``field`` a trial attribute, or ``tokens`` + for input and output together. An agent's 0 tokens is a count that was + not captured, as the token column reads it; a control's is real + (``kind`` is the job's). None when no attempt that ran recorded it.""" + if field == "tokens": + values = [(t.n_input_tokens or 0) + (t.n_output_tokens or 0) for t in self.ran + if t.n_input_tokens is not None or t.n_output_tokens is not None] + if kind != "control": + values = [v for v in values if v] + else: + values = [v for v in (getattr(t, field) for t in self.ran) if v is not None] + if not values: + return None + return {"mean": sum(values) / len(values), "lo": min(values), "hi": max(values), "n": len(values)} + + def means(self, kind: str = "agent") -> dict: + """``spread`` of each number a task's row shows, by field.""" + return {field: self.spread(field, kind) for field in MEANS} + + def number(self, trial) -> int | None: + return next((i for i, t in enumerate(self.trials, 1) if t.trial_name == trial.trial_name), None) + + +def _by_task_name(job) -> dict[str, list]: + """The job's trials by task name, the tasks in the order they first appear: + what makes two trials attempts at one task, for every page of one run.""" + groups: dict[str, list] = {} + for t in job.trials: + groups.setdefault(t.task_name, []).append(t) + return groups + + +def by_task(job) -> dict[str, Attempts]: + """Every task of the job that was attempted more than once, by task name.""" + return {task: Attempts(tuple(ordered(ts))) for task, ts in _by_task_name(job).items() if len(ts) > 1} + + +def of(job, trial) -> Attempts | None: + """The attempts at this trial's task, or None when it was attempted once.""" + same = [t for t in job.trials if t.task_name == trial.task_name] + return Attempts(tuple(ordered(same))) if len(same) > 1 else None + + +def grouped(job) -> list[tuple]: + """``(trial, attempts)`` for every trial, each task's attempts together and in + number order, the tasks in the order they first appear. ``attempts`` is None + for a task attempted once.""" + groups = by_task(job) + out, seen = [], set() + for t in job.trials: + group = groups.get(t.task_name) + if group is None: + out.append((t, None)) + elif t.task_name not in seen: + seen.add(t.task_name) + out.extend((a, group) for a in group.trials) + return out + + +def pass_at(n: int, c: int, k: int) -> float: + """pass@k, as HumanEval's estimator reads n attempts with c passed: the + chance that at least one of k drawn from them passed.""" + return 1 - comb(n - c, k) / comb(n, k) + + +def pass_hat(n: int, c: int, k: int) -> float: + """pass^k, as τ-bench defines it: the chance that all k drawn passed.""" + return comb(c, k) / comb(n, k) + + +def estimate(per_task: Iterable[Sequence[bool]]) -> dict | None: + """pass^k (``hat``) and pass@k (``at``) for k from 1 to n, averaged over the + tasks, n the smallest attempt count among them and each task read on its + first n attempts. None below two attempts.""" + tasks = [list(v) for v in per_task] + n = min((len(v) for v in tasks), default=0) + if n < 2: + return None + passed = [sum(v[:n]) for v in tasks] + return {"units": len(tasks), "attempts": n, + "hat": {k: sum(pass_hat(n, c, k) for c in passed) / len(tasks) for k in range(1, n + 1)}, + "at": {k: sum(pass_at(n, c, k) for c in passed) / len(tasks) for k in range(1, n + 1)}} + + +def for_tasks(tasks: Iterable[Sequence]) -> dict | None: + """``estimate`` over a run's tasks, each the trials of its attempts. Every + attempt that ran counts: a pass when resolved, else a fail - an agent's + timeout or crash is its own result, and an unknown outcome stays in, as in + the adjusted rate. A task with an infra attempt is left out, as τ-bench + leaves out an infrastructure error, and so is one with a scored attempt, + which has no pass or fail. None when nothing left was passed or failed, or + below two attempts.""" + whole = [ordered(ts) for ts in tasks + if not any(t.outcome in classify.INFRA_OUTCOMES or t.outcome == classify.SCORED for t in ts)] + if not any(t.outcome != classify.UNKNOWN for ts in whole for t in ts): + return None + return estimate([t.outcome == classify.RESOLVED for t in ts] for ts in whole) + + +def for_job(job) -> dict | None: + """``for_tasks`` over a parsed job, its tasks by task name.""" + return for_tasks(_by_task_name(job).values()) diff --git a/app/nullius/standard/job.py b/app/nullius/standard/job.py index 683d4e14..17e584d7 100644 --- a/app/nullius/standard/job.py +++ b/app/nullius/standard/job.py @@ -521,8 +521,9 @@ def pass_hat(results: List[dict]) -> Optional[dict]: """pass^k of a run that tried each unit several times (``repeat``), as τ-bench defines it: over the units, the chance that k attempts drawn from its n all passed, C(c, k) / C(n, k), averaged - pass^1 is the pass rate; and pass@k (``at``), the chance that one of them did, as HumanEval reads it. A unit with an attempt that ended in an error, with no verdict, is left - out, as the upstreams leave an infrastructure error out. None for a run of one attempt each.""" - from math import comb + out, as the upstreams leave an infrastructure error out. None for a run of one attempt each. + The estimators are ``nullius.runs.attempts``', the ones the pages use.""" + from nullius.runs import attempts per: Dict[str, List[bool]] = {} broken = set() for r in results: @@ -534,14 +535,12 @@ def pass_hat(results: List[dict]) -> Optional[dict]: broken.add(u) continue per.setdefault(u, []).append(rewards["reward"] == 1.0) - whole = {u: v for u, v in per.items() if u not in broken} - n = min((len(v) for v in whole.values()), default=0) - if n < 2: + got = attempts.estimate(v for u, v in per.items() if u not in broken) + if got is None: return None - return {"units": len(whole), "attempts": n, - **{str(k): round(sum(comb(sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}, - # pass@k, as HumanEval's estimate_pass_at_k reads n attempts with c passed: 1 - C(n-c, k) / C(n, k), averaged - "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}} + return {"units": got["units"], "attempts": got["attempts"], + **{str(k): round(v, 4) for k, v in got["hat"].items()}, + "at": {str(k): round(v, 4) for k, v in got["at"].items()}} def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None): diff --git a/app/static/compare/comparison.css b/app/static/compare/comparison.css index 4c71d2ac..a9e8a52c 100644 --- a/app/static/compare/comparison.css +++ b/app/static/compare/comparison.css @@ -115,3 +115,4 @@ white-space: normal; overflow-wrap: anywhere; } +.compare-table-wrap .grid.compare tr.pass-k th.rowhead { text-transform: none; letter-spacing: 0; font-family: var(--mono); } diff --git a/app/static/runs/board.css b/app/static/runs/board.css index 15c3128b..90d346a7 100644 --- a/app/static/runs/board.css +++ b/app/static/runs/board.css @@ -40,7 +40,7 @@ .rp .oc-no_patch { --oc: var(--no_patch); } .rp .oc-agent_error { --oc: var(--agent_error); } .rp .oc-rate_limited { --oc: var(--rate_limited); } .rp .oc-api_error { --oc: var(--api_error); } .rp .oc-setup_error { --oc: var(--setup_error); } .rp .oc-unknown { --oc: var(--unknown-mark); } -.rp .oc-spending_cap { --oc: var(--spending_cap); } +.rp .oc-spending_cap { --oc: var(--spending_cap); } .rp .oc-scored { --oc: var(--scored); } .rp .oc-cancelled { --oc: var(--cancelled); } .rp .sw { display: inline-block; width: 10px; height: 10px; border-radius: 2px; background: var(--oc); flex: none; } .rp .odot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: var(--oc); flex: none; } .rp .adv { display: inline-block; font: 500 10.5px/16px var(--sans); letter-spacing: .06em; text-transform: uppercase; color: var(--text-faint); @@ -111,6 +111,11 @@ .rp .hk .em { color: var(--text); font-weight: 500; } .rp .hk .warn { color: var(--warn); font-weight: 500; } .rp .hk .bad { color: var(--bad); font-weight: 500; } +.rp .hk .pass-k { border-collapse: collapse; margin-top: 6px; font: 12px/20px var(--mono); } +.rp .hk .pass-k th { color: var(--text-faint); font-weight: 400; text-align: right; padding: 0 0 0 10px; } +.rp .hk .pass-k th[scope=col] { font-family: var(--sans); } +.rp .hk .pass-k th:first-child { text-align: left; padding: 0; } +.rp .hk .pass-k td { color: var(--text-dim); text-align: right; padding: 0 0 0 10px; } /* outcomes: the bar, and its legend rows, which are the filter and each the start of its lanes */ .rp .mix { flex: 1; display: flex; gap: 2px; height: 12px; } @@ -291,6 +296,17 @@ .rp .tt tbody.pv tr[data-outcome]:not(.lit) td { opacity: .34; } .rp .tt td.num { text-align: right; font: 12.5px/20px var(--mono); font-variant-numeric: tabular-nums; } .rp .task { font: 12.5px/20px var(--mono); color: var(--text); overflow-wrap: break-word; } +.rp .tt tr.task-group { cursor: pointer; } +.rp .tt tr.task-group:hover td, .rp .tt tr.task-group:focus-within td, .rp .tt tr.task-group.lit td { background: var(--surface-2); } +.rp .tt tbody.pv tr.task-group:not(.lit) td { opacity: .34; } +.rp .group-toggle { display: inline-flex; align-items: center; padding: 0; border: 0; background: none; color: inherit; font: inherit; cursor: pointer; text-align: left; } +.rp .group-toggle::before { color: var(--text-dim); } +.rp .group-toggle:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; } +.rp .group-pass { display: inline-flex; align-items: center; gap: 8px; white-space: nowrap; } +.rp .group-pass b { font: 600 12.5px/20px var(--mono); } +.rp .sqs { display: inline-flex; gap: 3px; } +.rp .tt tr.in-group td:first-child { padding-left: 39px; } +.rp .attempt-link { color: var(--text-dim); } .rp .tt td:first-child { min-width: 150px; } .rp tr:hover .task { text-decoration: underline; text-decoration-color: var(--hush); text-underline-offset: 4px; } .rp .tt .badge { margin-left: 6px; } diff --git a/app/static/runs/board.js b/app/static/runs/board.js index 4eca8ee8..b222d5cb 100644 --- a/app/static/runs/board.js +++ b/app/static/runs/board.js @@ -1,6 +1,8 @@ // The run page's board and trials. Every number and row is rendered by the server; this adds the filters (outcome, judge // cause and status, flag - ANDed, carried in the query string so a link reproduces the view), the linked hover between the // board and the rows, Tokens/Total sorting, Esc, the live count, and one tooltip for hover and keyboard focus alike. +// A task attempted more than once is one row that opens onto its attempts; a filter opens the groups it matches, and +// clearing it brings back the groups the reader had opened. (() => { const $ = (s, r = document) => r.querySelector(s), $$ = (s, r = document) => [...r.querySelectorAll(s)]; const page = $('#run-page'); if (!page) return; @@ -15,6 +17,21 @@ const hits = (d, r) => { const x = F.get(r); return (!('o' in d) || x.o === d.o) && (!('c' in d) || x.c === d.c) && (!('s' in d) || x.s === d.s) && (!('f' in d) || x.f.includes(d.f)) && (!('t' in d) || x.t === d.t); }; const fxs = $$('.fx', page), hls = board ? $$('.hl', board) : []; + // groups: the task's row, its attempts, whether it is open, and whether the reader opened it + const members = new Map(); rows.forEach(r => { const k = r.dataset.group; if (k) { if (!members.has(k)) members.set(k, []); members.get(k).push(r); } }); + const groups = new Map($$('#trial-rows tr.task-group').map(h => [h.dataset.group, + { head: h, rows: members.get(h.dataset.group) || [], open: false, chosen: false, btn: $('.group-toggle', h) }])); + const groupOf = r => groups.get(r.dataset.group); + function reveal() { + rows.forEach(r => { const g = groupOf(r); r.hidden = !r.matched || (!!g && !g.open); }); + groups.forEach(g => { g.head.hidden = !g.rows.some(r => r.matched); g.btn.setAttribute('aria-expanded', String(g.open)); }); + } + groups.forEach(g => g.head.addEventListener('click', e => { + if (e.target.closest('a')) return; + g.open = g.chosen = !g.open; reveal(); + if (g.open && !matchMedia('(prefers-reduced-motion: reduce)').matches) + g.rows.forEach(r => { if (!r.hidden) r.animate([{ opacity: 0, transform: 'translateY(-4px)' }, { opacity: 1, transform: 'none' }], { duration: 240, easing: 'cubic-bezier(.2, 0, 0, 1)' }); }); + })); const q = new URLSearchParams(location.search); const st = { o: q.get('outcome') || '', c: q.get('cause') || '', s: q.get('status') || '', f: q.get('flag') || '' }; @@ -38,8 +55,10 @@ function apply() { const want = {}; KEYS.forEach(k => { if (st[k]) want[k] = st[k]; }); let n = 0, solved = 0; - rows.forEach(r => { const on = hits(want, r); r.hidden = !on; if (on) { n++; if (F.get(r).o === 'resolved') solved++; } }); + rows.forEach(r => { const on = hits(want, r); r.matched = on; if (on) { n++; if (F.get(r).o === 'resolved') solved++; } }); const any = Object.keys(want).length > 0; + groups.forEach(g => { g.open = any ? g.rows.some(r => r.matched) : g.chosen; }); + reveal(); empty.hidden = n > 0; clear.hidden = !any; count.textContent = any ? `${n} of ${N} · ${solved} solved` : String(N); liveEl.textContent = any ? `${n} of ${N} trials shown, ${solved} solved` : `All ${N} trials shown`; @@ -67,17 +86,20 @@ let cur = null; const pips = $$('#trial-rows .pip'); const light = src => { if (src === cur) return; cur = src; - const isPip = !!src && src.matches('.pip'), isRow = !!src && !isPip && src.matches('tr[data-outcome]'); - const S = src && !isRow && !isPip ? rows.filter(r => !r.hidden && hits(src.dataset, r)) : []; + const isPip = !!src && src.matches('.pip'), isHead = !!src && src.matches('tr.task-group'); + const isRow = !!src && !isPip && (isHead || src.matches('tr[data-outcome]')); + const own = isHead ? groups.get(src.dataset.group).rows.filter(r => r.matched) : isRow ? [src] : []; + const S = src && !isRow && !isPip ? rows.filter(r => r.matched && hits(src.dataset, r)) : []; tbody.classList.toggle('pv', !!src && !isRow && !isPip); rows.forEach(r => r.classList.toggle('lit', S.includes(r))); + groups.forEach(g => g.head.classList.toggle('lit', g.rows.some(r => S.includes(r)))); if (board) board.classList.toggle('pv', isRow || isPip); - hls.forEach(e => { e.classList.toggle('lit', isPip ? e.dataset.f === src.dataset.f : isRow && hits(e.dataset, src)); + hls.forEach(e => { e.classList.toggle('lit', isPip ? e.dataset.f === src.dataset.f : isRow && own.some(r => hits(e.dataset, r))); e.classList.toggle('hot', !!src && !isRow && !isPip && inside(e, src)); }); const fl = src && !isRow && !isPip && 'f' in src.dataset ? src.dataset.f : null; pips.forEach(p => p.classList.toggle('hot', p.dataset.f === fl)); }; - const target = t => { const el = t && t.closest && t.closest('#trial-rows .pip, .fx, #trial-rows tr[data-outcome]'); return el && page.contains(el) ? el : null; }; + const target = t => { const el = t && t.closest && t.closest('#trial-rows .pip, .fx, #trial-rows tr[data-outcome], #trial-rows tr.task-group'); return el && page.contains(el) ? el : null; }; document.addEventListener('pointerover', e => light(target(e.target))); document.addEventListener('focusin', e => light(target(e.target))); document.addEventListener('focusout', e => { if (!target(e.relatedTarget)) light(null); }); @@ -90,8 +112,15 @@ sort = { k: dir ? k : '', dir }; sorts.forEach(b => b.closest('th').removeAttribute('aria-sort')); if (dir) a.closest('th').setAttribute('aria-sort', dir); - const before = new Map(rows.map(r => [r, r.getBoundingClientRect().top])), v = r => +r.dataset[k]; - const order = [...rows].sort((x, y) => (!dir ? 0 : dir === 'ascending' ? v(x) - v(y) : v(y) - v(x)) || x.dataset.i - y.dataset.i); + const all = [...groups.values()].map(g => g.head).concat(rows); + const before = new Map(all.map(r => [r, r.getBoundingClientRect().top])), v = r => +r.dataset[k]; + const cmp = (x, y) => (!dir ? 0 : dir === 'ascending' ? v(x) - v(y) : v(y) - v(x)) || x.dataset.i - y.dataset.i; + // a group sorts by its mean over the attempts, and its attempts sort among themselves + const units = [], placed = new Set(); + rows.forEach(r => { const g = groupOf(r); if (!g) units.push({ lead: r, list: [r] }); + else if (!placed.has(g)) { placed.add(g); const list = [...g.rows].sort(cmp); units.push({ lead: g.head, list: [g.head, ...list] }); } }); + units.sort((a, b) => cmp(a.lead, b.lead)); + const order = units.flatMap(u => u.list); order.forEach(r => tbody.insertBefore(r, empty)); if (matchMedia('(prefers-reduced-motion: reduce)').matches) return; order.forEach(r => { const dy = before.get(r) - r.getBoundingClientRect().top; if (!dy) return; diff --git a/app/static/style.css b/app/static/style.css index b14f6a4e..af254f3c 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1006,6 +1006,25 @@ tr.control-row td { color: var(--text-dim); } .oc-cancelled::before, .seg.oc-cancelled, .tag.oc-cancelled::before { background: var(--cancelled); } .seg.oc-unknown { background: var(--unknown); } +/* a task's attempts: one tab each (traces/_attempts.html), on the trial page, + its All attempts view and each side of a compared task */ +.attempt-head { display: flex; flex-wrap: wrap; align-items: center; gap: .5rem .9rem; margin: .9rem 0 0; } +.attempt-count { color: var(--text-dim); font-size: .85rem; } +.attempt-count b { color: var(--text); font-weight: 600; } +.attempt-tabs { display: flex; flex-wrap: wrap; gap: .35rem; } +.attempt-tab { + display: inline-flex; align-items: center; gap: .45rem; padding: .3rem .7rem; + border: 1px solid var(--border); border-radius: 6px; color: var(--text-dim); + font-size: .8rem; text-decoration: none; white-space: nowrap; +} +.attempt-tab:not(.all)::before { content: ""; width: 8px; height: 8px; border-radius: 2px; flex: none; } +.attempt-tab.oc-unknown::before { background: var(--unknown); } +.attempt-tab:hover { color: var(--text); border-color: var(--text-faint); } +.attempt-tab[aria-current="page"] { color: var(--text); background: var(--accent-soft); border-color: var(--accent); font-weight: 600; } +.attempt-tab:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.attempt-table .num { text-align: right; font-variant-numeric: tabular-nums; } +.attempt-table tr[aria-current="true"] td { background: var(--accent-soft); } + /* outcome tag inside a table row */ .tag { font-size: .74rem; padding: .2rem .55rem; border-radius: 6px; @@ -1777,15 +1796,17 @@ select.ui-select-native { display: none !important; } .nav::-webkit-scrollbar-track { background: transparent; } .nav::-webkit-scrollbar-thumb { background: color-mix(in srgb, var(--text-faint) 28%, var(--bg)); border-radius: 8px; box-shadow: inset 0 0 2px #0005; } -/* Shared disclosure controls; popovers and filter menus keep their own behavior. */ +/* Shared disclosure controls; popovers and filter menus keep their own behavior. + A control that cannot be a
(a table row's) takes the same chevron as + .disclosure-chevron, turned by its aria-expanded. */ :root .slide-disclosure { box-sizing: border-box; } :root .slide-disclosure > summary { cursor: pointer; list-style: none; color: var(--text-dim); } :root .slide-disclosure > summary::-webkit-details-marker { display: none; } -:root .slide-disclosure > summary::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; } -:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before { transform: rotate(45deg); } +:root .slide-disclosure > summary::before, :root .disclosure-chevron::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; } +:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before, :root .disclosure-chevron[aria-expanded="true"]::before { transform: rotate(45deg); } :root .slide-disclosure > summary > .tw { display: none; } :root .slide-disclosure > summary:hover { color: var(--text); } :root .slide-disclosure > summary:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; } .tree-kids { box-sizing: border-box; } @media (prefers-reduced-motion: no-preference) { .shell { transition: grid-template-columns .24s ease; } .tree-caret .ic { transition: transform .24s ease; } } -@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .tree-caret .ic, .side { transition: none; } } +@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, :root .disclosure-chevron::before, .tree-caret .ic, .side { transition: none; } } diff --git a/app/static/traces/conversation.js b/app/static/traces/conversation.js index 0ba17730..5e590258 100644 --- a/app/static/traces/conversation.js +++ b/app/static/traces/conversation.js @@ -165,6 +165,8 @@ tabs.forEach(tab => { const active = tab.dataset.tab === name; tab.setAttribute('aria-selected', String(active)); tab.tabIndex = active ? 0 : -1; }); document.querySelectorAll('.trial-tab').forEach(panel => { panel.hidden = panel.id !== `trial-${name}`; }); if (update) { const url = new URL(location.href); url.searchParams.set('tab', name); if (name !== 'trace') url.hash = ''; history.replaceState(null, '', url); } + // another attempt opens on the same evidence tab + document.querySelectorAll('a[data-keep-tab]').forEach(link => { const url = new URL(link.href, location.href); if (name === 'overview') url.searchParams.delete('tab'); else url.searchParams.set('tab', name); link.href = url; }); if (name === 'conversation' && !conversationStarted) loadConversation(); } tabs.forEach((tab, index) => { diff --git a/app/templates/compare/compare.html b/app/templates/compare/compare.html index 7b9b8f4c..ee3fe3fb 100644 --- a/app/templates/compare/compare.html +++ b/app/templates/compare/compare.html @@ -1,4 +1,5 @@ {% from 'runs/_runtime.html' import what_ran %} +{% from 'traces/_attempts.html' import attempt_count %} {% extends "base.html" %} {% block title %}Compare {{ jobs|length }} runs — Nullius{% endblock %} {% block head %}{% endblock %} @@ -31,7 +32,7 @@ {% if not comparison.datasets_known %}

A run has no recorded dataset identity. Its task names are kept separate because their identity cannot be verified.

{% elif not comparison.datasets_match %}

The recorded datasets differ. Matching requires exact dataset names and versions; identical task names from different datasets are kept separate.

{% endif %} {% if cohort == 'matched' and not n_shared %}

No shared cohort is available. Choose runs with overlapping coverage or return to whole runs.

{% endif %} - {% if comparison.repeated_trials|sum %}

Some tasks have repeated attempts. All attempts count in the summaries; each task row and its detail show the last indexed attempt, not necessarily the latest by timestamp.

{% endif %} + {% if comparison.repeated_trials|sum %}

Some tasks have repeated attempts. A task row counts every attempt, infra apart, and disagrees when its pass rate or how its attempts ended differs; its outcome is the last indexed attempt.

{% endif %}

Adjusted rate excludes only rate limits, API errors, setup errors and spending caps. Unknown outcomes stay in its denominator. Controls are never ranked.

@@ -108,6 +109,18 @@

Side by side

{% for v in r.cells %}{{ v }}{% endfor %} {% endfor %} + {% set pk = comparison.pass_k %} + {% if pk and pk|select|list|length == pk|length %} + {% set top = pk|map(attribute='attempts')|min %} + {% for name, key, hint in [('pass@', 'at', 'The chance that at least one of k attempts passed, averaged over the tasks'), ('pass^', 'hat', 'The chance that all k attempts passed, averaged over the tasks')] %} + {% for k in range(1, top + 1) %} + + {{ name }}{{ k }} + {% for p in pk %}{{ '%.0f'|format(p[key][k] * 100) }}%{% endfor %} + + {% endfor %} + {% endfor %} + {% endif %} Rerun {% for j in jobs %} @@ -120,7 +133,7 @@

Side by side

Task by task

-

Tasks where the runs disagree are listed first. A blank cell means that run +

Tasks whose pass rate moved most are listed first. A blank cell means that run didn't include the task.

{% if judgement %}
@@ -153,8 +166,7 @@

Task by task

{% for taskrow in tasks %} {% set task_index = loop.index0 %} {% set row = taskrow.trials %} - {% set outcomes = row|select|map(attribute='outcome')|list %} - +
{# One glyph, rotated by CSS on aria-expanded. #} @@ -185,7 +197,8 @@

Task by task

Tokens{{ used_tokens|tokens(jobs[loop.index0].kind) }}
- {% if taskrow.attempts[loop.index0] > 1 %}{{ taskrow.attempts[loop.index0] }} attempts · last indexed shown{% endif %} + {%- set tried = taskrow.tried[loop.index0] %} + {% if tried.n > 1 %}{{ attempt_count(tried) }}{% endif %} {% else %} — {% endif %} diff --git a/app/templates/compare/compare_task.html b/app/templates/compare/compare_task.html index 4b6ccb12..d1a0bf7c 100644 --- a/app/templates/compare/compare_task.html +++ b/app/templates/compare/compare_task.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% from 'traces/_attempts.html' import attempt_count %} {% block title %}{{ task }} side by side — Nullius{% endblock %} {% block head %}