diff --git a/app/nullius/runs/artifacts.py b/app/nullius/runs/artifacts.py index 388ede17..5adf4e70 100644 --- a/app/nullius/runs/artifacts.py +++ b/app/nullius/runs/artifacts.py @@ -48,12 +48,18 @@ def listing(trial_dir): return records -def read(trial_dir, key, page=1): +def whole(trial_dir, key): + """The record redacted, up to ``MAX_READ``, and whether it was cut there: + what a download or a copy carries, never more than the reader shows.""" path = path_for(trial_dir, key) with open(path, encoding="utf-8", errors="replace") as handle: raw = handle.read(MAX_READ + 1) + return redact(raw[:MAX_READ]), len(raw) > MAX_READ + + +def read(trial_dir, key, page=1): # Redact before slicing so credentials cannot straddle visible chunks. - text = redact(raw[:MAX_READ]) + text, truncated = whole(trial_dir, key) pages = max(1, (len(text) + CHUNK_SIZE - 1) // CHUNK_SIZE) try: page = int(page) @@ -63,5 +69,5 @@ def read(trial_dir, key, page=1): start = (page - 1) * CHUNK_SIZE return dict(key=key, label=FILES[key][0], filename=FILES[key][1], page=page, pages=pages, text=text[start:start + CHUNK_SIZE], empty=not text, - truncated=len(raw) > MAX_READ, characters=len(text), start=start, + truncated=truncated, characters=len(text), start=start, end=min(start + CHUNK_SIZE, len(text))) diff --git a/app/nullius/runs/cli.py b/app/nullius/runs/cli.py index ee3b396f..30e3929c 100644 --- a/app/nullius/runs/cli.py +++ b/app/nullius/runs/cli.py @@ -154,7 +154,7 @@ def _first_line(text: str, width: int = 96) -> str: def _secs(value) -> str: - return "—" if value is None else f"{value:.1f}s" + return "—" if value is None else f"{value:.2f}s" def _print_trial(job: parse.Job, trial: parse.Trial) -> None: @@ -202,7 +202,7 @@ def _print_trial(job: parse.Job, trial: parse.Trial) -> None: for c in s["tool_calls"]) text = _first_line(s["message"]) if s["message"] and s["message"] != "(tool use)" else "" body = " · ".join(x for x in (text, calls) if x) - dur = "" if s["dur"] is None else f" {s['dur']:.1f}s" + dur = "" if s["dur"] is None else f" {s['dur']:.2f}s" print(f" {s['step_id']:>{width}} {s['source']:<6} {body}{dur}") diff --git a/app/static/catalogue/agents/README.md b/app/static/catalogue/agents/README.md index 445f8a35..039fcda5 100644 --- a/app/static/catalogue/agents/README.md +++ b/app/static/catalogue/agents/README.md @@ -1,6 +1,6 @@ # Agent marks -The catalogue, entry pages, sidebar and Nullius run builder share `marks.json`. +The catalogue, entry pages, sidebar, Nullius run builder and a trial's head share `marks.json`. Keys are agent catalogue keys; versions use the same mark. Unknown agents and capabilities retain their initial as a fallback. Files are bundled locally; rendering a logo makes no request to its upstream website. diff --git a/app/static/style.css b/app/static/style.css index 87529940..75de2ae6 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -261,6 +261,7 @@ h1 { font-size: clamp(1.75rem, 2.5vw, 2.15rem); font-weight: 500; line-height: 1 h2 { font-size: 1.05rem; margin: 2rem 0 .75rem; } h3 { font-size: .72rem; margin: 1.25rem 0 .6rem; color: var(--text-dim); text-transform: uppercase; letter-spacing: .09em; font-weight: 500; } +h3 .total { margin-left: .15rem; color: var(--text); font-weight: 600; text-transform: none; letter-spacing: normal; } .subtitle { color: var(--text-dim); font-size: .9rem; line-height: 1.75; margin: .75rem 0 1.75rem; max-width: 105ch; } .subtitle .mono { color: var(--text); } .muted { color: var(--text-dim); } @@ -299,7 +300,7 @@ h3 { font-size: .72rem; margin: 1.25rem 0 .6rem; color: var(--text-dim); .jflags th { text-align: left; font-weight: 500; color: var(--text-dim); font-size: .8rem; padding: .7rem 1rem; border-bottom: 1px solid var(--border-soft); } .jflags td { padding: .9rem 1rem; border-bottom: 1px solid var(--border-soft); vertical-align: middle; } .jflags tr:last-child td, .jflags tr.jflag-detail td { border-bottom: none; } -.jflags tr.jflag-detail td { padding-top: 0; border-bottom: 1px solid var(--border-soft); } +.jflags tr.jflag-detail td { padding-top: 0; padding-left: calc(1rem + 7px + .5rem); border-bottom: 1px solid var(--border-soft); } .jflags tr.tone-good > td { background: rgba(46, 160, 67, 0.08); } .jflags tr.tone-warn > td { background: rgba(212, 167, 44, 0.10); } .jflags tr.tone-bad > td { background: rgba(220, 53, 69, 0.09); } @@ -312,14 +313,12 @@ tr.tone-good .jbar .fill { background: #4cc26a; } tr.tone-warn .jbar .fill { bac .jmore { width: 2rem; text-align: right; } .jdisclose { border: 0; background: none; color: var(--text-dim); font-size: 1.2rem; cursor: pointer; padding: 0 .3rem; } .jdisclose[aria-expanded="true"] { transform: rotate(90deg); } -.judgement-flags-summary { white-space: nowrap; } .flag { display: inline-block; padding: 1px 8px; border-radius: 999px; font-size: 0.8em; font-weight: 500; border: 1px solid var(--border-soft); } .flag.raised { color: var(--bad); border-color: currentColor; } .flag.clear { color: var(--muted, #888); } .flag.undecided { color: var(--warn, #b8860b); border-color: currentColor; border-style: dashed; } .hygiene { display: inline-block; padding: 1px 8px; border-radius: 4px; font-size: 0.8em; font-weight: 500; border: 1px solid var(--border-soft); } .hygiene.warn { color: var(--warn, #b8860b); border-color: currentColor; } -.hygiene.clear { color: var(--muted, #888); } .badge.task-quality { margin-left: 6px; } .badge.task-quality[data-task-quality="0"] { color: var(--bad); border-color: currentColor; } .badge.task-quality[data-task-quality="1"] { color: var(--warn, #b8860b); border-color: currentColor; } @@ -355,6 +354,8 @@ button.seg:hover { filter: brightness(1.25); } /* table toolbar + column menu */ .table-tools { display: flex; align-items: center; gap: .75rem; margin: 0 0 .6rem; } +.artifact-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: .35rem; } +.artifact-tools { flex-wrap: wrap; } .menu-wrap { position: relative; } .btn.sm { font-size: .8rem; padding: .42rem .75rem; } .menu { @@ -626,6 +627,13 @@ button.seg:hover { filter: brightness(1.25); } color-mix(in srgb, var(--text-faint) 85%, transparent) 0 3px, color-mix(in srgb, var(--text-faint) 35%, transparent) 3px 6px); } +/* the trial's overview: small tiles side by side, the time one among them */ +.overview-tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 22rem), 1fr)); gap: 12px; margin-bottom: 1rem; } +.overview-tiles > .panel { margin: 0; padding: .85rem 1rem; } +.overview-tiles h3 { margin: 0 0 .6rem; } +.overview-tiles .bar.phases { height: 8px; } +.overview-tiles .legend { gap: .3rem; margin-top: .6rem; } +.overview-tiles .legend .chip { font-size: .72rem; padding: .12rem .5rem; } .chip.ph-other { color: var(--text-dim); } .chip.ph-other::before { background: var(--text-faint); } .chip.ph-env::before { background: var(--rate_limited); } @@ -956,14 +964,11 @@ button.seg:hover { filter: brightness(1.25); } .control-verdict { color: var(--text-dim); white-space: nowrap; } a.control-verdict { text-decoration: none; border: 1px solid var(--border); border-radius: 999px; padding: .1rem .6rem; background: var(--surface-2); } a.control-verdict:hover { border-color: var(--text-dim); } -.control-verdict.ok b, td.control-verdict.ok, .card.control-verdict.ok .v { color: var(--text); } -.control-verdict.warn b, td.control-verdict.warn, .card.control-verdict.warn .v { color: var(--bad); } -.card.control-verdict.warn { border-color: color-mix(in srgb, var(--agent_error) 55%, var(--border)); } +.control-verdict.ok b, td.control-verdict.ok { color: var(--text); } +.control-verdict.warn b, td.control-verdict.warn { color: var(--bad); } .seg.ctl-ok { background: var(--text-dim); } .seg.ctl-warn { background: var(--bad); } tr.control-row td { color: var(--text-dim); } -.card.danger { border-color: color-mix(in srgb, var(--agent_error) 55%, var(--border)); } -.card.danger .v { color: var(--bad); } .chip.regression-chip { border-color: color-mix(in srgb, var(--agent_error) 45%, var(--border)); } .chip.regression-chip::before { background: var(--bad); } .panel.danger-panel { border-color: color-mix(in srgb, var(--agent_error) 45%, var(--border)); @@ -1085,13 +1090,6 @@ tr.control-row td { color: var(--text-dim); } .tag.oc-unknown { color: var(--text-dim); } .tag.oc-unknown::before { background: var(--text-dim); } -.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: .75rem; margin: 1.5rem 0; } -.card { background: var(--surface); border: 1px solid var(--border-soft); border-radius: var(--radius); padding: 1.2rem 1.25rem; } -.card.warn { border-color: color-mix(in srgb, var(--patch_failed_tests) 45%, var(--border)); } -.card .k { color: var(--text-dim); font-size: .75rem; font-weight: 500; } -.card .v { font-size: 1.9rem; font-weight: 500; margin-top: .45rem; letter-spacing: -0.04em; font-variant-numeric: tabular-nums; } -.card .v small { font-size: .95rem; color: var(--text-dim); font-weight: 500; } - table.grid { width: 100%; border-collapse: collapse; font-variant-numeric: tabular-nums; font-size: .95rem; } .grid th, .grid td { text-align: left; padding: 1rem .9rem; border-bottom: 1px solid var(--border-soft); } .grid thead th { @@ -1567,8 +1565,6 @@ a.chip { text-decoration: none; } main { padding: 1.6rem 1rem 3rem; } .topbar { padding: 0 1rem; } .panel { padding: 1rem; } - .cards { gap: .6rem; } - .card { padding: 1rem; } } /* ---- workspaces ---- */ diff --git a/app/static/traces/conversation.css b/app/static/traces/conversation.css index f4b24040..784975a1 100644 --- a/app/static/traces/conversation.css +++ b/app/static/traces/conversation.css @@ -4,10 +4,8 @@ .trial-tabs button{border:0;border-bottom:2px solid transparent;background:none;color:var(--text-dim);font:inherit;padding:12px 0;cursor:pointer;white-space:nowrap} .trial-tabs button[aria-selected=true]{color:var(--text);border-bottom-color:var(--accent)} .trial-tab[hidden]{display:none} -.evidence-strip{display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin:16px 0;color:var(--text-dim);font-size:12px} -.evidence-strip span{padding:5px 9px;border:1px solid var(--border);border-radius:6px} .reader-heading{display:flex;justify-content:space-between;gap:20px;align-items:center;margin:0 0 20px} -.reader-heading h2{margin:0 0 6px}.reader-heading p{margin:0}.reader-total{font-size:13px;color:var(--text-dim);white-space:nowrap} +.reader-total{font-size:13px;color:var(--text-dim);white-space:nowrap} .reader-search{display:flex;gap:10px;margin-bottom:16px}.reader-search input{flex:1;min-width:0} .reader-search input,.reader-jump input{background:var(--surface);color:var(--text);border:1px solid var(--border);border-radius:8px;font:inherit;padding:9px 12px} .reader-search input::placeholder,.reader-jump input::placeholder{color:var(--text-dim)} @@ -46,4 +44,22 @@ .reader-layout button:focus-visible,.reader-toolbar button:focus-visible,.trial-tabs button:focus-visible{outline:2px solid var(--accent);outline-offset:2px} .wf-detail .reader-content{margin-top:18px}.wf-detail .reader-sections{margin-top:20px} @media(max-width:850px){.reader-selected{padding:18px}.reader-layout{grid-template-columns:minmax(175px,30%) minmax(0,1fr)}.reader-step-nav{align-items:flex-start;flex-direction:column;gap:12px}} -@media(max-width:600px){.reader-layout{grid-template-columns:minmax(0,1fr);min-height:0}.reader-index{border-right:0;border-bottom:1px solid var(--border)}.reader-rows{max-height:230px;overflow:auto}.reader-row{padding:10px 12px}.reader-row-preview{-webkit-line-clamp:1}.reader-selected{padding:16px}.reader-content pre{max-height:360px;padding:12px}.reader-heading{align-items:flex-start}.reader-total{padding-top:5px}.reader-search{gap:6px}.reader-toolbar{gap:10px}.reader-jump{width:100%}.reader-jump input{flex:1;min-width:0}.reader-content-nav{gap:6px;font-size:10px}.trial-tabs{gap:22px}.reader-filters{width:100%;justify-content:space-between}.reader-filters button{padding:7px 8px}} +@media(max-width:600px){.reader-layout{grid-template-columns:minmax(0,1fr);min-height:0}.reader-index{border-right:0;border-bottom:1px solid var(--border)}.reader-rows{max-height:230px;overflow:auto}.reader-row{padding:10px 12px}.reader-row-preview{-webkit-line-clamp:1}.reader-selected{padding:16px}.reader-content pre{max-height:360px;padding:12px}.reader-search{gap:6px}.reader-toolbar{gap:10px}.reader-jump{width:100%}.reader-jump input{flex:1;min-width:0}.reader-content-nav{gap:6px;font-size:10px}.trial-tabs{gap:22px}.reader-filters{width:100%;justify-content:space-between}.reader-filters button{padding:7px 8px}} + +/* the trial's head: what ran, then how it went, one row that wraps */ +.trial-title h1{font-family:var(--mono);font-size:1.25rem;line-height:1.3;overflow-wrap:anywhere} +.trial-title{min-height:1.875rem} +.trial-head{display:flex;flex-wrap:wrap;gap:10px;margin:14px 0 6px;align-items:stretch} +.th-card{display:flex;align-items:center;gap:12px;border:1px solid var(--border);border-radius:var(--radius);background:var(--surface);padding:9px 14px;min-width:0} +.th-card .k,.th-strip .k{display:block;color:var(--text-dim);font-size:.75rem} +.th-card .v{display:block;font-weight:500} +.th-sep{width:1px;align-self:stretch;background:var(--border)} +.th-mark{width:32px;height:32px;border-radius:8px;background:var(--accent-soft);color:var(--accent);display:grid;place-items:center;font:500 15px/1 var(--mono);flex:none;padding:6px} +.th-mark:has(.agent-logo){background:color-mix(in srgb,var(--text) 8%,transparent);color:var(--text)} +.th-model{display:block;font-size:.8rem;color:var(--text-dim);overflow-wrap:anywhere} +/* The cells keep their own width: the strip sits beside the cards while all of them fit on one + line, moves under the cards when they do not, and wraps its cells when the page itself is narrower. + The 1px gaps over the border colour are the rules between cells, on every row. */ +.th-strip{display:flex;flex-wrap:wrap;flex:1 0 auto;max-width:100%;gap:1px;background:var(--border);border:1px solid var(--border);border-radius:var(--radius);overflow:hidden} +.th-strip>div{flex:1 0 auto;padding:9px 14px;background:var(--surface);white-space:nowrap} +.th-strip .n{display:block;font-size:1.1rem;font-weight:500;font-variant-numeric:tabular-nums} diff --git a/app/templates/base.html b/app/templates/base.html index be69205e..5ec09fc9 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -394,25 +394,42 @@ } }); - // One handler for every [data-copy] button: put the text on the - // clipboard and confirm inline. The execCommand fallback covers - // non-secure origins, where navigator.clipboard is undefined. + // One handler for every [data-copy] button: put the text on the clipboard and say so, or say + // that it failed. The execCommand fallback covers non-secure origins, where navigator.clipboard + // is undefined. [data-copy-url] is text too long for an attribute (a whole record), fetched on + // the click: the clipboard is handed the pending text at once, inside the click, because a + // browser that drops the gesture across an await (Safari) refuses a write made after the fetch. document.addEventListener('click', async (e) => { - const btn = e.target.closest('[data-copy]'); + const btn = e.target.closest('[data-copy], [data-copy-url]'); if (!btn) return; - const text = btn.dataset.copy; - try { - await navigator.clipboard.writeText(text); - } catch { + const legacy = (text) => { const ta = document.createElement('textarea'); ta.value = text; document.body.appendChild(ta); ta.select(); - document.execCommand('copy'); + let done = false; + try { done = document.execCommand('copy'); } catch {} ta.remove(); + return done; + }; + const write = async (text) => { + try { await navigator.clipboard.writeText(text); return true; } catch { return legacy(text); } + }; + let copied = false; + if (btn.dataset.copy !== undefined) { + copied = await write(btn.dataset.copy); + } else { + const text = nulliusFetch(btn.dataset.copyUrl, {cache: 'no-store'}).then((r) => { if (!r.ok) throw new Error(r.status); return r.text(); }); + text.catch(() => {}); // read again below: never an unhandled rejection + try { + await navigator.clipboard.write([new ClipboardItem({'text/plain': text.then((t) => new Blob([t], {type: 'text/plain'}))})]); + copied = true; + } catch { + try { copied = await write(await text); } catch {} + } } - const label = btn.textContent; - btn.textContent = 'copied ✓'; + const label = btn.dataset.copyLabel || (btn.dataset.copyLabel = btn.textContent); // the button's own word, however fast it is clicked again + btn.textContent = copied ? 'copied ✓' : 'copy failed'; setTimeout(() => { btn.textContent = label; }, 1200); }); })(); diff --git a/app/templates/runs/_execution.html b/app/templates/runs/_execution.html index f3f22009..96f29457 100644 --- a/app/templates/runs/_execution.html +++ b/app/templates/runs/_execution.html @@ -45,7 +45,7 @@ let since = 0, live = box.dataset.live === 'yes'; // the stages, drawn as the template draws them: the same marks, times and units line. A step keeps // its element from one answer to the next, so its mark changes in place and the running one's beat goes on. - const secs = v => v < 9.95 ? v.toFixed(1) + 's' : v < 60 ? Math.round(v) + 's' : (v / 60).toFixed(1) + 'm'; + const secs = v => v < 59.995 ? v.toFixed(2) + 's' : (v / 60).toFixed(2) + 'm'; const clock = s => Math.floor(s / 60) + ':' + String(Math.floor(s % 60)).padStart(2, '0'); const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; const stages = sp => { diff --git a/app/templates/traces/_conversation.html b/app/templates/traces/_conversation.html index 47e2652e..c2c902c6 100644 --- a/app/templates/traces/_conversation.html +++ b/app/templates/traces/_conversation.html @@ -1,10 +1,11 @@
+ {% if traj and not traj.error %}
-

Conversation

One source step at a time, with its tool calls and results. Search reads the full recorded content.

- {{ traj.n_steps if traj and not traj.error else '—' }} steps + {{ traj.n_steps }} steps
+ {% endif %} {% if not traj or traj.error %}

{% if traj and traj.error %}Trajectory unreadable: {{ traj.error }}{% elif trial.agent_type == 'control' %}This control has no recorded conversation.{% else %}No trajectory recorded for this trial.{% endif %}

{% else %} diff --git a/app/templates/traces/_head.html b/app/templates/traces/_head.html new file mode 100644 index 00000000..b3931452 --- /dev/null +++ b/app/templates/traces/_head.html @@ -0,0 +1,30 @@ +{% from 'runs/_runtime.html' import what_ran %} +{# What ran, said the same on a trial's page and on its task's All attempts: where the task is from + and what ran it, then the agent, its mark (`mark`: base.html's agent_mark) and the model that + answered. An imported trial says its importer: no benchmark gave it, and no runner ran it here. + A block with nothing to say is not drawn. #} +{% macro ran_cards(job, trial, imported_from, mark) -%} + {# The trial records a bare model name ("gpt-5.6-luna"); the job keeps the + provider-qualified one ("openai/gpt-5.6-luna"). Prefer the fuller id. #} + {% set model = job.model_name if job.model_name and job.model_name.endswith(trial.model_name or '~') + else (trial.model_name or job.model_name) %} + {% set agent = trial.agent_info_name or job.agent_label %} + {% if imported_from %} +
Imported from{{ imported_from }}
+ {% else %} +
+ {% if trial.source %}
Benchmark{{ trial.source|dsname }}
+ {% endif %} +
Runner{% if job.runner %}{{ what_ran(job) }}{% else %}Harbor{% endif %}
+
+ {% endif %} + {% if agent or model %} +
+ {% if agent %}{% endif %} +
+ {% if agent %}{{ agent }}{% if trial.agent_version %} {{ trial.agent_version }}{% endif %}{% endif %} + {% if model %}{{ model }}{% endif %} +
+
+ {% endif %} +{%- endmacro %} diff --git a/app/templates/traces/_judgement.html b/app/templates/traces/_judgement.html index 1ad7e4d4..9f06c5e3 100644 --- a/app/templates/traces/_judgement.html +++ b/app/templates/traces/_judgement.html @@ -11,17 +11,19 @@
-

{{ 'LLM judge' if llm else 'System One judge' }} experimental

-

{% if llm %}The same questions, asked of {{ j.judge.label if j.keyed else 'the LLM judge Settings names' }}. A choice or a score carries no confidence.{% else %}Task outcome, answer quality and behaviour, as {{ j.record.judge.model if j.record else (j.judge.label if j.keyed else 'the System One model Settings names') }} reads the transcript.{% endif %} Experimental: the recorded outcome and reward stand.

+ experimental
{% if j.keyed %}
{% else %}{{ 'LLM judge' if llm else 'System One' }} not configured{% if j.judge is not none and j.judge.missing %}: needs {{ j.judge.missing }}{% endif %} — set it in Settings{% endif %}
- {% if j.record %}

{{ j.record.judge.model }}{% if j.record.judge.provider not in ('typesafe.ai', 'typesafe') %} via {{ j.record.judge.provider|provider }}{% endif %}{% if j.record.refusals %} · {{ j.record.refusals|length }} refused{% endif %} · {{ j.record.questions }}{% if j.earlier %} earlier vocabulary{% endif %} · asked {{ j.record.asked_at[:16]|replace('T', ' ') }}Z{% if j.record.truncated %} · state truncated{% if j.record.states %} ({% for kind, k in j.record.states.items() if k.truncated %}{{ kind }}{{ ', ' if not loop.last }}{% endfor %}){% endif %}{% endif %}{% if j.record.usage and j.record.usage.input_tokens %} · {{ j.record.usage.input_tokens }} input tokens{% endif %}

{% endif %} - {% if j.hygiene and j.hygiene.kinds %}

- {% if j.hygiene.flagged %}⚠ state hygiene {% for kind, raised in j.hygiene.flagged.items() %}`{{ kind }}` state: {% for id, label, p, decided in raised %}{{ label }} {{ '%.0f'|format(p * 100) }}%{% if not decided %} (undecided){% endif %}{{ '; ' if not loop.last }}{% endfor %}{{ ' · ' if not loop.last }}{% endfor %} — the answers from that state are marked ⚠; none is changed - {% else %}state hygiene nothing read as addressing the judge or showing a credential on {{ j.hygiene.kinds|join(', ') }}{% endif %}

{% endif %} + {% if j.record and (j.record.refusals or j.earlier or j.record.truncated) %}{% set note = namespace(sep='') %}

+ {%- if j.record.refusals %}{{ j.record.refusals|length }} refused{% set note.sep = ' · ' %}{% endif -%} + {%- if j.earlier %}{{ note.sep }}earlier vocabulary{% set note.sep = ' · ' %}{% endif -%} + {%- if j.record.truncated %}{{ note.sep }}state truncated{% if j.record.states %} ({% for kind, k in j.record.states.items() if k.truncated %}{{ kind }}{{ ', ' if not loop.last }}{% endfor %}){% endif %}{% endif -%} +

{% endif %} + {% if j.hygiene and j.hygiene.flagged %}

+ ⚠ state hygiene {% for kind, raised in j.hygiene.flagged.items() %}`{{ kind }}` state: {% for id, label, p, decided in raised %}{{ label }} {{ '%.0f'|format(p * 100) }}%{% if not decided %} (undecided){% endif %}{{ '; ' if not loop.last }}{% endfor %}{{ ' · ' if not loop.last }}{% endfor %} — the answers from that state are marked ⚠; none is changed

{% endif %}
{% for r in j.cards %} @@ -41,15 +43,14 @@

{{ 'LLM judge' if llm else 'System One judge' }} -

Behaviour flags — yes/no, each one thing

- {% if j.flags_judged %}{{ j.raised|length }} raised · {{ j.undecided|length }} undecided · {{ j.cleared|length }} clear{% endif %} +

Behaviour flags

{% for r in j.flags %} - + diff --git a/app/templates/traces/artifact.html b/app/templates/traces/artifact.html index aa9bbb37..5a7deadd 100644 --- a/app/templates/traces/artifact.html +++ b/app/templates/traces/artifact.html @@ -2,14 +2,19 @@ {% block title %}{{ artifact.label }} — Nullius{% endblock %} {% block crumb %}/ runs / {{ job.job_id }} / {{ trial.task_base }} / artifacts{% endblock %} {% block body %} -

← Back to trial artifacts

+{% set download = url_for('artifact_download', job_id=job.job_id, trial_name=trial.trial_name, key=artifact.key, workspace=request.args.get('workspace')) %} +

← Artifacts

{{ artifact.label }}

{{ artifact.filename }} · source record · credential-shaped values redacted

{% if artifact.truncated %}

This record exceeds the viewer’s 16 MiB limit. The first portion is available here; the original file is intact.

{% endif %} -
+
Page {{ artifact.page }} of {{ artifact.pages }} · characters {{ artifact.start + 1 if artifact.characters else 0 }}–{{ artifact.end }} of {{ artifact.characters }} {% if artifact.page > 1 %}Previous{% endif %} {% if artifact.page < artifact.pages %}Next{% endif %} + {% if not artifact.empty %} + + Download + {% endif %}
{% if artifact.empty %}

This record is empty.

{% else %}
{{ artifact.text }}
{% endif %} {% endblock %} diff --git a/app/templates/traces/attempts.html b/app/templates/traces/attempts.html index 2eff8602..aacde6cb 100644 --- a/app/templates/traces/attempts.html +++ b/app/templates/traces/attempts.html @@ -1,14 +1,15 @@ {% extends "base.html" %} {% from 'traces/_attempts.html' import attempt_tabs, attempt_count %} +{% from 'traces/_head.html' import ran_cards %} {% block title %}{{ trial.task_base }}, all attempts — Nullius{% endblock %} {% block crumb %}/ {{ job.job_id }} / {{ trial.task_base }}{% endblock %} {% block body %} {% set workspace = request.args.get('workspace') %} -

{{ trial.task_base }}

-

- {{ job.model_name or trial.model_name }}· - {{ trial.source|dsname }} -

+{# the head of an attempt's page (traces/trial.html): the name, what ran, then the tabs, so a tab + changes what is under it and nothing above #} + +

{{ trial.task_base }}

+
{{ ran_cards(job, trial, imported_from, agent_mark) }}
{{ attempt_count(attempts) }} {{ attempt_tabs(job, attempts, none, workspace) }} diff --git a/app/templates/traces/trial.html b/app/templates/traces/trial.html index 13e19f9e..394eaeaa 100644 --- a/app/templates/traces/trial.html +++ b/app/templates/traces/trial.html @@ -1,4 +1,3 @@ -{% from 'runs/_runtime.html' import what_ran %} {% extends "base.html" %} {% block title %}{{ trial.task_base }} — Nullius{% endblock %} {% block crumb %}/ {{ job.job_id }} / {{ trial.task_base }}{% endblock %} @@ -24,11 +23,12 @@ {% from 'traces/_judgement.html' import judgement_panel %} {% from 'traces/_attempts.html' import attempt_tabs, attempt_count, try_tabs %} +{% from 'traces/_head.html' import ran_cards %} {% block body %} -
+

{{ trial.task_base }}

{{ trial.outcome_label }} {# The recorded exception type, beside the label that came from it. For an @@ -38,17 +38,25 @@

{{ trial.task_base }}

{% if trial.exception_type %}{{ trial.exception_type }}{% endif %}
-{# The trial records a bare model name ("gpt-5.6-luna"); the job keeps the - provider-qualified one ("openai/gpt-5.6-luna"). Prefer the fuller id. #} -{% set model = job.model_name if job.model_name and job.model_name.endswith(trial.model_name or '~') - else (trial.model_name or job.model_name) %} -

- {{ model }}· - {{ trial.source|dsname }} - {%- if trial.agent_version %}·v{{ trial.agent_version }}{% endif %} - {%- if job.runner %}·{{ what_ran(job) }}{% endif %} -

+{# an earlier try of an attempt: evidence only, never judged #} +{% set replaced = trial.trial_name != attempt.trial_name %} +{# What ran (traces/_head.html, the same on All attempts), then how it went: the dataset's own + checks, the reward or the score, the agent's time and its steps. A value the trial does not + have has no cell, and a block with no cell is not drawn. A task run several times, or an attempt + tried again, has its tabs between the two: what ran is every attempt's and stays above them, + the numbers are this one's and change under them. #} +{% set vocab = trial.vocab %} +{% set cells %} + {%- if vocab.gained and (trial.f2p_pass or trial.f2p_fail) %}
{{ vocab.gained }}{{ trial.f2p_pass }}/{{ trial.f2p_fail }}
{% endif -%} + {%- if vocab.held and (trial.p2p_pass or trial.p2p_fail) %}
{{ vocab.held }}{{ trial.p2p_pass }}/{{ trial.p2p_fail }}
{% endif -%} + {%- if vocab.has_patch and trial.patch_exists is not none %}
Patch applied{{ 'Yes' if trial.patch_applied else 'No' }}
{% endif -%} + {%- if trial.reward is not none %}
Reward{{ trial.reward }}
{% endif -%} + {%- if trial.score is not none %}
Score{{ '%.2f'|format(trial.score) }}
{% endif -%} + {%- if trial.dur_agent is not none %}
Agent time{{ trial.dur_agent|secs }}
{% endif -%} + {%- if traj and not traj.error %}
Steps{{ traj.n_steps }}
{% endif -%} +{% endset %} +{% if attempts or tries %} +
{{ ran_cards(job, trial, imported_from, agent_mark) }}
{% if attempts %}
{{ attempt_count(attempts) }} @@ -56,54 +64,14 @@

{{ trial.task_base }}

{% endif %} {% if tries %}{{ try_tabs(job, tries, trial, request.args.get('workspace')) }}{% endif %} -{# an earlier try of an attempt: evidence only, never judged #} -{% set replaced = trial.trial_name != attempt.trial_name %} - -
- {# The check cards speak the dataset's language: labels come from its - Vocabulary, and a category it doesn't have (a verifier with no - regression guard and no patches; a session-derived trial has neither) - is not drawn at all rather than dashed under a foreign name. The dash - inside a drawn card still means "this trial reported nothing". #} - {% set vocab = trial.vocab %} - {% if vocab.gained %} -
{{ vocab.gained }}
{% if trial.f2p_pass or trial.f2p_fail %}{{ trial.f2p_pass }}/{{ trial.f2p_fail }}{% else %}—{% endif %}
- {% endif %} - {% if vocab.held %} -
{{ vocab.held }}
{% if trial.p2p_pass or trial.p2p_fail %}{{ trial.p2p_pass }}/{{ trial.p2p_fail }}{% else %}—{% endif %}
- {% endif %} - {% if vocab.has_patch %} -
Patch applied
{% if trial.patch_exists is none %}—{% else %}{{ 'Yes' if trial.patch_applied else 'No' }}{% endif %}
- {% endif %} - {# The verifier's numeric score. Mostly redundant with the outcome tag on - a pass/fail verifier (0 or 1), but it's the number Harbor's mean is built from, and - partial-credit verifiers make it a real scalar. #} -
-
Reward
-
{{ trial.reward if trial.reward is not none else '—' }}
-
-
Agent time
{{ trial.dur_agent|secs }}
- {% if traj and not traj.error %} -
Steps
{{ traj.n_steps }}
-
Tool calls
{{ traj.n_tool_calls }}
- {% endif %} +{% if cells|trim %}
{{ cells }}
{% endif %} +{% else %} +
+ {{ ran_cards(job, trial, imported_from, agent_mark) }} + {% if cells|trim %}
{{ cells }}
{% endif %}
+{% endif %} -
- Trajectory · {{ 'recorded' if trial.has_trajectory else 'absent' }} - Trace · {{ 'recorded' if trial.has_trace else 'absent' }} - Reward · {{ 'recorded' if trial.reward is not none else 'absent' }} - {# Rate limits the agent's SDK retried and the run survived. They are in the - gateway log and nowhere else — result.json records only the 429 that - killed a run — so a trial without that log reads as unmeasured, not clean. #} - {% if trial.n_rate_limited_retries is none %} - Rate limits · not recorded - {% elif trial.n_rate_limited_retries %} - {{ trial.n_rate_limited_retries }} rate-limited call{{ '' if trial.n_rate_limited_retries == 1 else 's' }} retried - {% else %} - Rate limits · none logged - {% endif %} -
{% endif %} -

Shape of the run

- {# The phase bar reads only trial.*, so it renders whether or not a trajectory was recorded: a trial that failed before writing one still shows its timing. #} {% set total = trial.dur_phases + trial.dur_other %} {% if total %} +
-

Where the time went

- {# All four phases Harbor records. Leaving agent_setup out understated the - bar by a third on some agents, and it is the most actionable number in - the set — installing the agent can cost more than solving the task. #} +

Where the time went{% if trial.dur_total is not none %} {{ trial.dur_total|secs }}{% endif %}

+ {# The phases the trial recorded, Harbor's four names for them. On a runner's trial Env is the + unit's environment, Setup (when it has one) the agent's start and session, Verify its check and + score, a model judge included. A recorded phase has its chip however short it was: a check that + took no time was still made. One never recorded has none, and Other, which is what the phases + leave of the trial, only when there is some. A segment needs enough time to draw, and its + weight is in milliseconds: weights that sum under 1 (a trial shorter than a second) would + leave the bar part empty. #} + {%- if job.runner %} + {%- set says = {'env': "The unit's environment: the driver's setup, the relay, its image", 'setup': "The agent's start and its session, its MCP servers included", 'verify': "The unit's check and its score, a model judge included", 'other': 'What the phases leave of the trial: the waits between them and the work after the check'} %} + {%- else %} + {%- set says = {'env': 'Environment build', 'setup': 'Installing the agent', 'verify': 'Verifier', 'other': 'Time Harbor does not attribute to a phase: image pull, container start and teardown, artifact collection'} %} + {%- endif %} + {%- set phases = [('env', 'Env', trial.dur_env, says.env), ('setup', 'Setup', trial.dur_agent_setup, says.setup), ('agent', 'Agent', trial.dur_agent, 'The agent working'), ('verify', 'Verify', trial.dur_verifier, says.verify), ('other', 'Other', trial.dur_other, says.other)] %}
- {# A 0s phase emits no segment at all, so a control's empty setup leaves - no stray sliver. #} - {% if trial.dur_env|phase %}{% endif %} - {% if trial.dur_agent_setup|phase %}{% endif %} - {% if trial.dur_agent|phase %}{% endif %} - {% if trial.dur_verifier|phase %}{% endif %} - {% set items = job.runner_items[1] if job.runner_items else 'items' %} - {% set other_is = 'the wait between the answer and its judgement: the judge reads the answers once its share of the ' ~ items ~ ' is answered' if job.runner else 'time Harbor does not attribute to a phase: image pull, container start and teardown, artifact collection' %} - {% if trial.dur_other|phase %}{% endif %} + {% for key, label, dur, title in phases if dur|phase %}{% endfor %}
- Env {{ trial.dur_env|secs }} - Setup {{ trial.dur_agent_setup|secs }} - Agent {{ trial.dur_agent|secs }} - Verify {{ trial.dur_verifier|secs }} - Other {{ trial.dur_other|secs }} + {% for key, label, dur, title in phases if (dur is not none and key != 'other') or dur|phase %}{{ label }} {{ dur|secs }} + {% endfor %}
-

The bar spans the trial's full {{ trial.dur_total|secs }}. - {% if job.runner %}“Other” is the wait for the judge, which reads the answers once its share of the {{ items }} is answered.{% else %}“Other” is harness overhead Harbor doesn't break out — the same tax for every agent.{% endif %} - {% if trace and trace.has_spans %}The Trace tab shows recorded span timing and its - measured or inferred provenance.{% elif traj and traj.has_step_clock %}Conversation - reports time between source steps; these are not per-tool durations.{% else %}No - per-step clock is available. Conversation preserves the recorded order.{% endif %}

+
{% endif %} {% if traj and traj.error %}

Trajectory unreadable

{{ traj.error }}
{% elif traj %} + {# usage and tool calls; a trajectory with neither (one answer, no per-step usage) has no panel #} + {% if traj.token_series or traj.total_reasoning_tokens is not none or traj.calls %}
{% set has_growth = traj.token_series|length > 1 %} {% if has_growth %} @@ -273,8 +235,7 @@

Usage

{% endif %} {% endif %}
- - + {% endif %} {% if loc %}
@@ -319,19 +280,6 @@

{{ job.runner }}'s score gives the {% endfor %}

CheckResultLikelihood of flag
{{ r.label }}{% if r.warnings %} ⚠{% endif %}{{ r.label }}{% if r.warnings %} ⚠{% endif %} {% if r.answer %}{{ pill(r) }}{% else %}not judged{% endif %} {% if r.answer %}{{ r.headline }}% {% if r.floor is not none %}{% endif %}{% if r.floor is not none %} ±{{ '%.0f'|format(r.floor * 100) }}{% else %}±?{% endif %}{% endif %}
-

Below, Nullius's own trajectory judge: experimental, asked on a click.

-
-{% elif job.runner %} -
-

{{ job.runner }}'s judge gives the outcome

- {% if bench_judge %} -

{{ bench_judge.judge or job.runner_judge }}{% if bench_judge.model %} · sent as {{ bench_judge.model }}{% endif %}

-

Said {{ (bench_judge.said or '—')[:200] }}: {{ 'resolved' if trial.outcome == 'resolved' else trial.outcome|replace('_', ' ') }}.

-
What it was asked
{{ bench_judge.asked }}
- {% else %} -

{% if job.runner_judge %}{{ job.runner_judge }}. {% endif %}No record of its call for this one{% if trial.outcome not in ('resolved', 'failed_verification') %}: it never got to judge it{% endif %}.

- {% endif %} -

Below, Nullius's own trajectory judge: experimental, asked on a click.

{% endif %} {{ judgement_panel(judgement, job, trial, csrf) }} @@ -496,7 +444,7 @@

Time breakdown {% endfor %}

Categories, not a sequence. Reported by the trace, never summed from the bars below: spans nest.{% if trace.totals.unaccounted_ns %} Unaccounted is wall clock outside the instrumented spans.{% endif %}

- {% else %} + {%- else %}

The trace reports no totals.

{% endif %} @@ -529,7 +477,7 @@

Without the {{ trace.hidden_count }} span{{ '' if trace.hidden_count == 1 el the call it ran inside), because summing spans that nest or overlap attributes more than the run. So hiding a span moves its time into “Unaccounted” rather than out of the total: the run took just as long.

- {% else %} + {%- else %}

Nothing is left to break down.

{% endif %} {% endif %} @@ -656,7 +604,7 @@

Waterfall

{% for q in trace.qualities_present %}{{ q }}{% endfor %}
- {% else %} + {%- else %}

Waterfall

Every span is filtered out — nothing left to draw.

{% endif %} @@ -678,14 +626,16 @@

Waterfall

{% include 'traces/_conversation.html' %}
-

Artifacts

Read the recorded source files. Content is redacted and paged; source files remain intact.

+ {% if artifacts %}
- +
RecordAvailabilitySize
{% for artifact in artifacts %} - + + {% endfor %}
RecordSize
{{ artifact.label }}{{ 'Recorded' if artifact.exists else 'Not recorded' }}{{ ((artifact.size / 1024)|round(1)) ~ ' KB' if artifact.exists else '—' }}{% if artifact.exists %}Read →{% endif %}
{{ artifact.label }}{{ '%.2f'|format(artifact.size / 1024) }} KBReadDownload
+ {% endif %}