Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions app/nullius/runs/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,18 @@ def listing(trial_dir):
return records


def read(trial_dir, key, page=1):
def whole(trial_dir, key):
"""The record redacted, up to ``MAX_READ``, and whether it was cut there:
what a download or a copy carries, never more than the reader shows."""
path = path_for(trial_dir, key)
with open(path, encoding="utf-8", errors="replace") as handle:
raw = handle.read(MAX_READ + 1)
return redact(raw[:MAX_READ]), len(raw) > MAX_READ


def read(trial_dir, key, page=1):
# Redact before slicing so credentials cannot straddle visible chunks.
text = redact(raw[:MAX_READ])
text, truncated = whole(trial_dir, key)
pages = max(1, (len(text) + CHUNK_SIZE - 1) // CHUNK_SIZE)
try:
page = int(page)
Expand All @@ -63,5 +69,5 @@ def read(trial_dir, key, page=1):
start = (page - 1) * CHUNK_SIZE
return dict(key=key, label=FILES[key][0], filename=FILES[key][1], page=page,
pages=pages, text=text[start:start + CHUNK_SIZE], empty=not text,
truncated=len(raw) > MAX_READ, characters=len(text), start=start,
truncated=truncated, characters=len(text), start=start,
end=min(start + CHUNK_SIZE, len(text)))
4 changes: 2 additions & 2 deletions app/nullius/runs/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def _first_line(text: str, width: int = 96) -> str:


def _secs(value) -> str:
return "—" if value is None else f"{value:.1f}s"
return "—" if value is None else f"{value:.2f}s"


def _print_trial(job: parse.Job, trial: parse.Trial) -> None:
Expand Down Expand Up @@ -202,7 +202,7 @@ def _print_trial(job: parse.Job, trial: parse.Trial) -> None:
for c in s["tool_calls"])
text = _first_line(s["message"]) if s["message"] and s["message"] != "(tool use)" else ""
body = " · ".join(x for x in (text, calls) if x)
dur = "" if s["dur"] is None else f" {s['dur']:.1f}s"
dur = "" if s["dur"] is None else f" {s['dur']:.2f}s"
print(f" {s['step_id']:>{width}} {s['source']:<6} {body}{dur}")


Expand Down
2 changes: 1 addition & 1 deletion app/static/catalogue/agents/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Agent marks

The catalogue, entry pages, sidebar and Nullius run builder share `marks.json`.
The catalogue, entry pages, sidebar, Nullius run builder and a trial's head share `marks.json`.
Keys are agent catalogue keys; versions use the same mark. Unknown agents and
capabilities retain their initial as a fallback. Files are bundled locally;
rendering a logo makes no request to its upstream website.
Expand Down
30 changes: 13 additions & 17 deletions app/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,7 @@ h1 { font-size: clamp(1.75rem, 2.5vw, 2.15rem); font-weight: 500; line-height: 1
h2 { font-size: 1.05rem; margin: 2rem 0 .75rem; }
h3 { font-size: .72rem; margin: 1.25rem 0 .6rem; color: var(--text-dim);
text-transform: uppercase; letter-spacing: .09em; font-weight: 500; }
h3 .total { margin-left: .15rem; color: var(--text); font-weight: 600; text-transform: none; letter-spacing: normal; }
.subtitle { color: var(--text-dim); font-size: .9rem; line-height: 1.75; margin: .75rem 0 1.75rem; max-width: 105ch; }
.subtitle .mono { color: var(--text); }
.muted { color: var(--text-dim); }
Expand Down Expand Up @@ -299,7 +300,7 @@ h3 { font-size: .72rem; margin: 1.25rem 0 .6rem; color: var(--text-dim);
.jflags th { text-align: left; font-weight: 500; color: var(--text-dim); font-size: .8rem; padding: .7rem 1rem; border-bottom: 1px solid var(--border-soft); }
.jflags td { padding: .9rem 1rem; border-bottom: 1px solid var(--border-soft); vertical-align: middle; }
.jflags tr:last-child td, .jflags tr.jflag-detail td { border-bottom: none; }
.jflags tr.jflag-detail td { padding-top: 0; border-bottom: 1px solid var(--border-soft); }
.jflags tr.jflag-detail td { padding-top: 0; padding-left: calc(1rem + 7px + .5rem); border-bottom: 1px solid var(--border-soft); }
.jflags tr.tone-good > td { background: rgba(46, 160, 67, 0.08); }
.jflags tr.tone-warn > td { background: rgba(212, 167, 44, 0.10); }
.jflags tr.tone-bad > td { background: rgba(220, 53, 69, 0.09); }
Expand All @@ -312,14 +313,12 @@ tr.tone-good .jbar .fill { background: #4cc26a; } tr.tone-warn .jbar .fill { bac
.jmore { width: 2rem; text-align: right; }
.jdisclose { border: 0; background: none; color: var(--text-dim); font-size: 1.2rem; cursor: pointer; padding: 0 .3rem; }
.jdisclose[aria-expanded="true"] { transform: rotate(90deg); }
.judgement-flags-summary { white-space: nowrap; }
.flag { display: inline-block; padding: 1px 8px; border-radius: 999px; font-size: 0.8em; font-weight: 500; border: 1px solid var(--border-soft); }
.flag.raised { color: var(--bad); border-color: currentColor; }
.flag.clear { color: var(--muted, #888); }
.flag.undecided { color: var(--warn, #b8860b); border-color: currentColor; border-style: dashed; }
.hygiene { display: inline-block; padding: 1px 8px; border-radius: 4px; font-size: 0.8em; font-weight: 500; border: 1px solid var(--border-soft); }
.hygiene.warn { color: var(--warn, #b8860b); border-color: currentColor; }
.hygiene.clear { color: var(--muted, #888); }
.badge.task-quality { margin-left: 6px; }
.badge.task-quality[data-task-quality="0"] { color: var(--bad); border-color: currentColor; }
.badge.task-quality[data-task-quality="1"] { color: var(--warn, #b8860b); border-color: currentColor; }
Expand Down Expand Up @@ -355,6 +354,8 @@ button.seg:hover { filter: brightness(1.25); }

/* table toolbar + column menu */
.table-tools { display: flex; align-items: center; gap: .75rem; margin: 0 0 .6rem; }
.artifact-actions { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: .35rem; }
.artifact-tools { flex-wrap: wrap; }
.menu-wrap { position: relative; }
.btn.sm { font-size: .8rem; padding: .42rem .75rem; }
.menu {
Expand Down Expand Up @@ -626,6 +627,13 @@ button.seg:hover { filter: brightness(1.25); }
color-mix(in srgb, var(--text-faint) 85%, transparent) 0 3px,
color-mix(in srgb, var(--text-faint) 35%, transparent) 3px 6px);
}
/* the trial's overview: small tiles side by side, the time one among them */
.overview-tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 22rem), 1fr)); gap: 12px; margin-bottom: 1rem; }
.overview-tiles > .panel { margin: 0; padding: .85rem 1rem; }
.overview-tiles h3 { margin: 0 0 .6rem; }
.overview-tiles .bar.phases { height: 8px; }
.overview-tiles .legend { gap: .3rem; margin-top: .6rem; }
.overview-tiles .legend .chip { font-size: .72rem; padding: .12rem .5rem; }
.chip.ph-other { color: var(--text-dim); }
.chip.ph-other::before { background: var(--text-faint); }
.chip.ph-env::before { background: var(--rate_limited); }
Expand Down Expand Up @@ -956,14 +964,11 @@ button.seg:hover { filter: brightness(1.25); }
.control-verdict { color: var(--text-dim); white-space: nowrap; }
a.control-verdict { text-decoration: none; border: 1px solid var(--border); border-radius: 999px; padding: .1rem .6rem; background: var(--surface-2); }
a.control-verdict:hover { border-color: var(--text-dim); }
.control-verdict.ok b, td.control-verdict.ok, .card.control-verdict.ok .v { color: var(--text); }
.control-verdict.warn b, td.control-verdict.warn, .card.control-verdict.warn .v { color: var(--bad); }
.card.control-verdict.warn { border-color: color-mix(in srgb, var(--agent_error) 55%, var(--border)); }
.control-verdict.ok b, td.control-verdict.ok { color: var(--text); }
.control-verdict.warn b, td.control-verdict.warn { color: var(--bad); }
.seg.ctl-ok { background: var(--text-dim); }
.seg.ctl-warn { background: var(--bad); }
tr.control-row td { color: var(--text-dim); }
.card.danger { border-color: color-mix(in srgb, var(--agent_error) 55%, var(--border)); }
.card.danger .v { color: var(--bad); }
.chip.regression-chip { border-color: color-mix(in srgb, var(--agent_error) 45%, var(--border)); }
.chip.regression-chip::before { background: var(--bad); }
.panel.danger-panel { border-color: color-mix(in srgb, var(--agent_error) 45%, var(--border));
Expand Down Expand Up @@ -1085,13 +1090,6 @@ tr.control-row td { color: var(--text-dim); }
.tag.oc-unknown { color: var(--text-dim); }
.tag.oc-unknown::before { background: var(--text-dim); }

.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: .75rem; margin: 1.5rem 0; }
.card { background: var(--surface); border: 1px solid var(--border-soft); border-radius: var(--radius); padding: 1.2rem 1.25rem; }
.card.warn { border-color: color-mix(in srgb, var(--patch_failed_tests) 45%, var(--border)); }
.card .k { color: var(--text-dim); font-size: .75rem; font-weight: 500; }
.card .v { font-size: 1.9rem; font-weight: 500; margin-top: .45rem; letter-spacing: -0.04em; font-variant-numeric: tabular-nums; }
.card .v small { font-size: .95rem; color: var(--text-dim); font-weight: 500; }

table.grid { width: 100%; border-collapse: collapse; font-variant-numeric: tabular-nums; font-size: .95rem; }
.grid th, .grid td { text-align: left; padding: 1rem .9rem; border-bottom: 1px solid var(--border-soft); }
.grid thead th {
Expand Down Expand Up @@ -1567,8 +1565,6 @@ a.chip { text-decoration: none; }
main { padding: 1.6rem 1rem 3rem; }
.topbar { padding: 0 1rem; }
.panel { padding: 1rem; }
.cards { gap: .6rem; }
.card { padding: 1rem; }
}

/* ---- workspaces ---- */
Expand Down
24 changes: 20 additions & 4 deletions app/static/traces/conversation.css

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 28 additions & 11 deletions app/templates/base.html
Original file line number Diff line number Diff line change
Expand Up @@ -394,25 +394,42 @@
}
});

// One handler for every [data-copy] button: put the text on the
// clipboard and confirm inline. The execCommand fallback covers
// non-secure origins, where navigator.clipboard is undefined.
// One handler for every [data-copy] button: put the text on the clipboard and say so, or say
// that it failed. The execCommand fallback covers non-secure origins, where navigator.clipboard
// is undefined. [data-copy-url] is text too long for an attribute (a whole record), fetched on
// the click: the clipboard is handed the pending text at once, inside the click, because a
// browser that drops the gesture across an await (Safari) refuses a write made after the fetch.
document.addEventListener('click', async (e) => {
const btn = e.target.closest('[data-copy]');
const btn = e.target.closest('[data-copy], [data-copy-url]');
if (!btn) return;
const text = btn.dataset.copy;
try {
await navigator.clipboard.writeText(text);
} catch {
const legacy = (text) => {
const ta = document.createElement('textarea');
ta.value = text;
document.body.appendChild(ta);
ta.select();
document.execCommand('copy');
let done = false;
try { done = document.execCommand('copy'); } catch {}
ta.remove();
return done;
};
const write = async (text) => {
try { await navigator.clipboard.writeText(text); return true; } catch { return legacy(text); }
};
let copied = false;
if (btn.dataset.copy !== undefined) {
copied = await write(btn.dataset.copy);
} else {
const text = nulliusFetch(btn.dataset.copyUrl, {cache: 'no-store'}).then((r) => { if (!r.ok) throw new Error(r.status); return r.text(); });
text.catch(() => {}); // read again below: never an unhandled rejection
try {
await navigator.clipboard.write([new ClipboardItem({'text/plain': text.then((t) => new Blob([t], {type: 'text/plain'}))})]);
copied = true;
} catch {
try { copied = await write(await text); } catch {}
}
}
const label = btn.textContent;
btn.textContent = 'copied ✓';
const label = btn.dataset.copyLabel || (btn.dataset.copyLabel = btn.textContent); // the button's own word, however fast it is clicked again
btn.textContent = copied ? 'copied ✓' : 'copy failed';
setTimeout(() => { btn.textContent = label; }, 1200);
});
})();
Expand Down
2 changes: 1 addition & 1 deletion app/templates/runs/_execution.html
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
let since = 0, live = box.dataset.live === 'yes';
// the stages, drawn as the template draws them: the same marks, times and units line. A step keeps
// its element from one answer to the next, so its mark changes in place and the running one's beat goes on.
const secs = v => v < 9.95 ? v.toFixed(1) + 's' : v < 60 ? Math.round(v) + 's' : (v / 60).toFixed(1) + 'm';
const secs = v => v < 59.995 ? v.toFixed(2) + 's' : (v / 60).toFixed(2) + 'm';
const clock = s => Math.floor(s / 60) + ':' + String(Math.floor(s % 60)).padStart(2, '0');
const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; };
const stages = sp => {
Expand Down
5 changes: 3 additions & 2 deletions app/templates/traces/_conversation.html
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<section class="conversation-reader" id="conversation-reader"
data-endpoint="{{ url_for('conversation_data', job_id=job.job_id, trial_name=trial.trial_name) }}"
aria-label="Conversation reader">
{% if traj and not traj.error %}
<div class="reader-heading">
<div><h2>Conversation</h2><p class="muted small">One source step at a time, with its tool calls and results. Search reads the full recorded content.</p></div>
<span class="reader-total mono">{{ traj.n_steps if traj and not traj.error else '—' }} steps</span>
<span class="reader-total mono">{{ traj.n_steps }} steps</span>
</div>
{% endif %}
{% if not traj or traj.error %}
<div class="panel"><p class="muted" style="margin:0">{% if traj and traj.error %}Trajectory unreadable: {{ traj.error }}{% elif trial.agent_type == 'control' %}This control has no recorded conversation.{% else %}No trajectory recorded for this trial.{% endif %}</p></div>
{% else %}
Expand Down
30 changes: 30 additions & 0 deletions app/templates/traces/_head.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
{% from 'runs/_runtime.html' import what_ran %}
{# What ran, said the same on a trial's page and on its task's All attempts: where the task is from
and what ran it, then the agent, its mark (`mark`: base.html's agent_mark) and the model that
answered. An imported trial says its importer: no benchmark gave it, and no runner ran it here.
A block with nothing to say is not drawn. #}
{% macro ran_cards(job, trial, imported_from, mark) -%}
{# The trial records a bare model name ("gpt-5.6-luna"); the job keeps the
provider-qualified one ("openai/gpt-5.6-luna"). Prefer the fuller id. #}
{% set model = job.model_name if job.model_name and job.model_name.endswith(trial.model_name or '~')
else (trial.model_name or job.model_name) %}
{% set agent = trial.agent_info_name or job.agent_label %}
{% if imported_from %}
<div class="th-card"><div><span class="k">Imported from</span><span class="v">{{ imported_from }}</span></div></div>
{% else %}
<div class="th-card">
{% if trial.source %}<div><span class="k">Benchmark</span><span class="v" title="{{ trial.source }}">{{ trial.source|dsname }}</span></div>
<span class="th-sep" aria-hidden="true"></span>{% endif %}
<div><span class="k">Runner</span><span class="v">{% if job.runner %}{{ what_ran(job) }}{% else %}<span{% if job.harbor_label %} title="harbor {{ job.harbor_label }}"{% endif %}>Harbor</span>{% endif %}</span></div>
</div>
{% endif %}
{% if agent or model %}
<div class="th-card">
{% if agent %}<span class="th-mark" aria-hidden="true">{{ mark('agent', agent, agent) }}</span>{% endif %}
<div>
{% if agent %}<span class="v">{{ agent }}{% if trial.agent_version %} <span class="mono faint" title="The agent's version as the trial's agent_info reports it">{{ trial.agent_version }}</span>{% endif %}</span>{% endif %}
{% if model %}<span class="th-model mono">{{ model }}</span>{% endif %}
</div>
</div>
{% endif %}
{%- endmacro %}
Loading
Loading