diff --git a/app/nullius/runs/launches.py b/app/nullius/runs/launches.py index ae567f37..1295ea02 100644 --- a/app/nullius/runs/launches.py +++ b/app/nullius/runs/launches.py @@ -20,15 +20,21 @@ launch: a shard joins the launch that holds another shard of its id before any other rule is asked. -Nothing here reads a file: it takes the parsed jobs and the records' job lists, -and groups. Newest launch first. +``group`` reads no file: it takes the parsed jobs and the records' job lists, +and groups. Newest launch first. ``fold`` reads one participant's shards as +``cli merge`` would put them together, again only when the workspace parsed one of +them again, and writes nothing. """ from __future__ import annotations import os -from dataclasses import dataclass, field +import threading +from dataclasses import dataclass, field, replace from datetime import datetime, timezone +from nullius.runs.parse import _budget, _iso +from nullius.standard import merge + WINDOW_SEC = 30 * 60 @@ -76,6 +82,51 @@ def merged_sources(jobs) -> set[str]: for source in getattr(j, "merged_from", []) if source != j.job_id} +# Folds already read in this process, by their shards' dirs: (the shard jobs, (check, record)). The workspace +# hands out the same Job until its signature moves (``workspace.load_job``), so the same jobs read the same: +# a page renders again without reading every trial record, nor stating every file, while no shard moved. +# A dir is in one entry at most: a new set of shards replaces each fold it shares a dir with. +_FOLDED: dict = {} +_FOLDED_LOCK = threading.Lock() + + +def _merged(dirs): + """``merge.check`` of ``dirs`` without the trial records, and the job's record; (None, None) when merge refuses them.""" + try: + got = merge.check(list(dirs)) + except merge.Refused: + return None, None + res = merge.result_of(got["shards"], [r for s in got["shards"] for r in s["results"]]) + return {**got, "shards": [{k: v for k, v in s.items() if k != "results"} for s in got["shards"]]}, res + + +def fold(shards): + """One participant's shard jobs as the one job ``cli merge`` would make of + them, in memory: their trials together, the job's record ``merge.result_of``, + and what ``merge.check`` says no shard ran. ``(job, check)``; None when + merge refuses them, so they stay apart.""" + dirs = tuple(j.dir for j in shards) + with _FOLDED_LOCK: + known = _FOLDED.get(dirs) + if known is not None and all(a is b for a, b in zip(known[0], shards)): + got, res = known[1] + else: + got, res = _merged(dirs) + with _FOLDED_LOCK: + for stale in [k for k in _FOLDED if k != dirs and not set(k).isdisjoint(dirs)]: + del _FOLDED[stale] + _FOLDED[dirs] = (tuple(shards), (got, res)) + if got is None: + return None + return replace(shards[0], trials=[t for j in shards for t in j.trials], n_tasks=res["n_total_trials"], + n_total_trials=res["n_total_trials"], started_at=_iso(res["started_at"]), finished_at=_iso(res["finished_at"]), + n_errored_trials=res["stats"]["n_errored_trials"], budget=_budget(res.get("budget")), + # still going: what a merge, made of ended shards, never sees + n_running_trials=sum(j.n_running_trials for j in shards), + n_pending_trials=sum(j.n_pending_trials for j in shards), + preparing=all(j.preparing for j in shards)), got + + def group(jobs, records: dict | None = None, window: float = WINDOW_SEC) -> list[Launch]: """``jobs``: the parsed jobs (``job_id``, ``dataset``, ``started_at``). ``records``: ``{execution id: (name, [job ids], launched at)}``, the builder's diff --git a/app/nullius/standard/merge.py b/app/nullius/standard/merge.py index c0d6636f..97cb517f 100644 --- a/app/nullius/standard/merge.py +++ b/app/nullius/standard/merge.py @@ -133,8 +133,9 @@ def check(job_dirs: List[str], force: bool = False) -> dict: names: Dict[str, List[str]] = {} for s in shards: s["trials"] = _trials(s["dir"]) - for t in s["trials"]: - unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t + s["results"] = [_read(os.path.join(s["dir"], t, "result.json")) for t in s["trials"]] + for t, res in zip(s["trials"], s["results"]): + unit = _at(res, ("standard", "unit")) or t by_unit.setdefault(str(unit), []).append((s["name"], t)) names.setdefault(t, []).append(s["name"]) twice = {u: w for u, w in by_unit.items() if len(w) > 1} @@ -203,6 +204,26 @@ def _budget(budgets: list) -> Optional[dict]: **({"unit_max_usd": each, "over_unit": [u for x in budgets for u in x.get("over_unit") or []]} if number(each) else {})} +def result_of(shards: list, results: List[dict]) -> dict: + """The merged job's ``result.json``, but its ``id``: ``shards`` as ``check`` reads them, ``results`` every + trial's record. The runs page folds a launch's shards with it (``runs.launches.fold``).""" + planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards) + starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")] + ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")] + errored = sum(1 for r in results if r.get("exception_info")) + rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results] + scores = [w["score"] for w in rewards if "score" in w] + budget = _budget([s["result"].get("budget") for s in shards]) + # ISO stamps in UTC sort as their instants + return {"started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None, + "n_total_trials": planned, + "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, + "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, + "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), + "scored": len(scores), "mean_score": job.mean_score(results)}, + **({"budget": budget} if budget else {})} + + def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: """Merge the shard jobs ``job_dirs`` into the new job ``out``. {out, trials, merged_from, notes, missing}.""" @@ -226,22 +247,10 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict: with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f: json.dump(res, f, indent=1) results.append(res) - planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards) - starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")] - ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")] - errored = sum(1 for r in results if r.get("exception_info")) - rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results] - scores = [w["score"] for w in rewards if "score" in w] - budget = _budget([s["result"].get("budget") for s in shards]) + summary = result_of(shards, results) + planned = summary["n_total_trials"] with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f: - # ISO stamps in UTC sort as their instants - json.dump({"id": name, "started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None, - "n_total_trials": planned, - "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored, - "n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0}, - "standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0), - "scored": len(scores), "mean_score": job.mean_score(results)}, - **({"budget": budget} if budget else {})}, f, indent=1) + json.dump({"id": name, **summary}, f, indent=1) cfg = json.loads(json.dumps(shards[0]["config"])) cfg["job_name"] = name std = cfg.setdefault("standard", {}) diff --git a/app/static/runs/launches.css b/app/static/runs/launches.css index f5c225cf..e32156fc 100644 --- a/app/static/runs/launches.css +++ b/app/static/runs/launches.css @@ -29,8 +29,8 @@ .launch-agents { width: 100%; border-collapse: collapse; } .launch-agents th { text-align: left; font: 400 11.5px var(--sans); color: var(--text-dim); padding: 8px 12px 6px; border-top: 1px solid var(--border-soft); border-bottom: 1px solid var(--border-soft); white-space: nowrap; } -.launch-agents td { padding: 10px 12px; border-bottom: 1px solid var(--border-soft); vertical-align: middle; font-size: 13px; } -.launch-agents tbody tr:last-child td { border-bottom: 0; } +/* rules above each row, not below: a table of several bodies, some hidden, ends without one */ +.launch-agents td { padding: 10px 12px; border-top: 1px solid var(--border-soft); vertical-align: middle; font-size: 13px; } .launch-agents th:first-child, .launch-agents td:first-child { padding-left: 18px; } .launch-agents th:last-child, .launch-agents td:last-child { padding-right: 18px; } .launch-agents .num { text-align: right; font-family: var(--mono); font-variant-numeric: tabular-nums; white-space: nowrap; } @@ -79,10 +79,27 @@ /* a task check's row: above the agents, set apart by a rule, its verdict where an agent has its rate */ .launch-agents tr.la-control td { background: color-mix(in srgb, var(--surface-2) 45%, transparent); } -.launch-agents tr.la-control + tr:not(.la-control) td { border-top: 2px solid var(--border); } +.launch-agents tr.la-first td { border-top: 2px solid var(--border); } .la-kind { font-size: 11px; font-weight: 500; letter-spacing: .04em; text-transform: uppercase; color: var(--text-faint); margin-left: 6px; } .la-verdict { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; } .la-verdict b { font-weight: 500; } .la-verdict.ok .lc-mark { color: var(--ok); } .la-verdict.warn .lc-mark { color: var(--bad); } .la-verdict.pending .lc-mark { color: var(--text-faint); } + +/* a participant run as several shards: one row, its chevron the shared one, unfolding into a row per shard */ +.launch-agents tr.la-fold { cursor: pointer; } +/* the chevron is a column of its own, so the name and its "N shards" line, and the shard rows below, start at one edge */ +.launch-agents { --la-fold-w: 14px; --la-fold-gap: 6px; } +.la-fold-who { display: flex; align-items: flex-start; gap: var(--la-fold-gap); min-width: 0; } +.la-fold-btn { flex: none; display: inline-flex; align-items: center; justify-content: center; width: var(--la-fold-w); padding: 2px 0; border: 0; background: none; color: var(--text-dim); cursor: pointer; } +.la-fold-btn:hover { color: var(--text); } +.la-fold-btn:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 3px; } +.la-fold-btn::before { margin-left: 0; } +.launch-agents tr.la-fold .la-name { font-weight: 500; } +.launch-agents tr.la-shard td:first-child .la-who { margin-left: calc(var(--la-fold-w) + var(--la-fold-gap)); } +.launch-agents tr.la-shard td:has(> .la-cell) { padding-top: 0; padding-bottom: 0; } +.la-cell::before, .la-cell::after { content: ''; display: block; height: 10px; } /* the row's padding, inside what slides */ +.launch-agents tr.la-shard .la-who a { font-weight: 400; color: var(--text-dim); } +.la-note { color: var(--patch_failed_tests); } +.la-gap { display: block; margin-top: 2px; font: 11.5px var(--sans); color: var(--patch_failed_tests); } diff --git a/app/static/style.css b/app/static/style.css index b14f6a4e..2ac1c745 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1781,11 +1781,11 @@ select.ui-select-native { display: none !important; } :root .slide-disclosure { box-sizing: border-box; } :root .slide-disclosure > summary { cursor: pointer; list-style: none; color: var(--text-dim); } :root .slide-disclosure > summary::-webkit-details-marker { display: none; } -:root .slide-disclosure > summary::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; } -:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before { transform: rotate(45deg); } +:root .slide-disclosure > summary::before, .la-fold-btn::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; } +:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before, .la-fold-btn[aria-expanded=true]::before { transform: rotate(45deg); } :root .slide-disclosure > summary > .tw { display: none; } :root .slide-disclosure > summary:hover { color: var(--text); } :root .slide-disclosure > summary:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; } .tree-kids { box-sizing: border-box; } @media (prefers-reduced-motion: no-preference) { .shell { transition: grid-template-columns .24s ease; } .tree-caret .ic { transition: transform .24s ease; } } -@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .tree-caret .ic, .side { transition: none; } } +@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .la-fold-btn::before, .tree-caret .ic, .side { transition: none; } } diff --git a/app/templates/runs/index.html b/app/templates/runs/index.html index 9cb01006..b019f3e0 100644 --- a/app/templates/runs/index.html +++ b/app/templates/runs/index.html @@ -21,6 +21,64 @@ {% endmacro %} +{#- One row of a launch's table: a task check (`kind` 'control') or an agent; `part` 'fold' for a participant's + shards as one row, 'shard' for one of them, '' for a run of one job. #} +{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %} + + {%- if part == 'shard' %} +
shard {{ r.shard }}{% if r.host %} · {{ r.host }}{% endif %}
+ {%- else %} + {% if part == 'fold' %}
{% endif %}
+ {%- if kind == 'control' %} + {%- if part == 'fold' %}{{ r.control }}{% else %}{{ r.control }}{% endif %} task check + {%- else %} + {%- set name = j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id %} + {%- if part == 'fold' %}{{ name }}{% else %}{{ name }}{% endif %} + {%- if j and j.runner %} {{ what_ran(j) }}{% elif j and j.agent_version %} {{ j.agent_version }}{% endif %} + {%- endif %} + {%- if part == 'fold' %} + {{ r.shards|length }} shards{% if r.note %} · ⚠ {{ r.note }}{% endif %} + {%- else %} + {{ r.job_id }}{% endif %}
{% if part == 'fold' %}
{% endif %} + {%- endif %} + {%- if kind == 'control' %} + {{ r.state }} +
+ {%- if v %}{{ v.label }}{% else %}{{ 'must reject every task' if r.control == 'nop' else 'must pass every task' }}{% endif %}
+ {%- else %} + {{ r.state }} + {% if j and j.scored_only %} +
{{ '%.2f'|format(j.mean_score) }}
+ {%- elif j and j.adjusted_solve_rate is not none %} +
{{ j.adjusted_solve_rate|pct }}
+ {%- else %}
—
{% endif %} + {%- if j and j.trials and not j.scored_only and j.counts.unknown and j.counts.unknown * 4 >= j.trials|length %}⚠ {{ j.counts.unknown }} of {{ j.trials|length }} have no verdict{% endif %} + {%- endif %} + {% if j and j.trials %}{% for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor %}{% else %}{% endif %} + {%- if kind == 'control' %} + — + {%- else %} + {% if j and j.scored_only %}—{% elif j and part == 'fold' %}{{ j.n_resolved }}/{{ j.trials|length }}{% if r.not_run %}{{ r.not_run }} of {{ r.total }} not run{% endif %}{% elif j %}{{ j.n_resolved }}/{{ j.n_planned or j.trials|length }}{% else %}0{% if g.n_tasks %}/{{ g.n_tasks }}{% endif %}{% endif %} + {%- endif %} + {% if j and (j.total_input_tokens or j.total_output_tokens) %}{{ (j.total_input_tokens + j.total_output_tokens)|tokens(j.kind) }}{% else %}—{% endif %} + {% if j and j.total_cost_usd is not none %}{{ j.total_cost_usd|usd(j.kind) }}{% else %}—{% endif %} + {% if j and j.duration_min %}{{ j.duration_min|runtime }}{% else %}—{% endif %} + {% if part != 'fold' and r.state != 'not in this workspace' %}{{ 'Open to resume' if r.state == 'paused' else 'Open →' }}{% endif %} + +{%- endmacro %} + +{#- A participant run as several shards is one row that unfolds into its shards' own; a run of one job is its row. #} +{% macro launch_rows(r, g, kind, first=False) %}{% set L = g.launch %} +{%- if r.shards %} + {{ launch_row(r, g, kind, 'fold', first) }} + + {%- for s in r.shards %}{{ launch_row(s, g, kind, 'shard') }}{% endfor %} + +{%- else %}{{ launch_row(r, g, kind, '', first) }}{% endif %} +{%- endmacro %} + {% block body %}

Runs

@@ -91,41 +149,8 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n AgentStateAdjusted solve rateOutcomesSolvedTokensCostTime {#- the task checks first: their verdict where an agent has its rate, never ranked, kept once they end #} - {% for r in g.controls %}{% set j = r.job %}{% set v = r.verdict %} - -
{{ r.control }} task check - {{ r.job_id }}
- {{ r.state }} -
- {%- if v %}{{ v.label }}{% else %}{{ 'must reject every task' if r.control == 'nop' else 'must pass every task' }}{% endif %}
- {% if j and j.trials %}{% for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor %}{% else %}{% endif %} - — - {% if j and (j.total_input_tokens or j.total_output_tokens) %}{{ (j.total_input_tokens + j.total_output_tokens)|tokens(j.kind) }}{% else %}—{% endif %} - {% if j and j.total_cost_usd is not none %}{{ j.total_cost_usd|usd(j.kind) }}{% else %}—{% endif %} - {% if j and j.duration_min %}{{ j.duration_min|runtime }}{% else %}—{% endif %} - Open → - - {% endfor %} - {% for r in g.agents %}{% set j = r.job %} - -
{{ j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id }} - {%- if j and j.runner %} {{ what_ran(j) }}{% elif j and j.agent_version %} {{ j.agent_version }}{% endif %} - {{ r.job_id }}
- {{ r.state }} - {% if j and j.scored_only %} -
{{ '%.2f'|format(j.mean_score) }}
- {%- elif j and j.adjusted_solve_rate is not none %} -
{{ j.adjusted_solve_rate|pct }}
- {%- else %}
—
{% endif %} - {%- if j and j.trials and not j.scored_only and j.counts.unknown and j.counts.unknown * 4 >= j.trials|length %}⚠ {{ j.counts.unknown }} of {{ j.trials|length }} have no verdict{% endif %} - {% if j and j.trials %}{% for k in ORDER %}{% if j.counts[k] %}{% endif %}{% endfor %}{% else %}{% endif %} - {% if j and j.scored_only %}—{% elif j %}{{ j.n_resolved }}/{{ j.n_planned or j.trials|length }}{% else %}0{% if g.n_tasks %}/{{ g.n_tasks }}{% endif %}{% endif %} - {% if j and (j.total_input_tokens or j.total_output_tokens) %}{{ (j.total_input_tokens + j.total_output_tokens)|tokens(j.kind) }}{% else %}—{% endif %} - {% if j and j.total_cost_usd is not none %}{{ j.total_cost_usd|usd(j.kind) }}{% else %}—{% endif %} - {% if j and j.duration_min %}{{ j.duration_min|runtime }}{% else %}—{% endif %} - {% if r.state != 'not in this workspace' %}{{ 'Open to resume' if r.state == 'paused' else 'Open →' }}{% endif %} - - {% endfor %} + {% for r in g.controls %}{{ launch_rows(r, g, 'control') }}{% endfor %} + {% for r in g.agents %}{{ launch_rows(r, g, 'agent', loop.first and g.controls) }}{% endfor %}

@@ -537,6 +562,33 @@

{{ d.info.title }}

a feed growing, a result landing), reads itself again and puts in only what moved. No reload: the scroll, the view, the filters, the sort and the ticks stay. A hidden tab does not ask, and a selection is left to be copied. */ + /* a participant's shards fold into one row: the row, or its chevron, unfolds them */ + const still = matchMedia('(prefers-reduced-motion: reduce)'); + function setFold(btn, open, animate = true) { + const body = document.getElementById(btn.getAttribute('aria-controls')); + btn.setAttribute('aria-expanded', String(open)); + body.inert = !open; + if (!animate || still.matches) { body.hidden = !open; return; } + // a table row is never shorter than its content: what slides is each cell's content, the shared way + const cells = [...body.querySelectorAll(':scope > tr > td')].map(td => { + if (!td.firstElementChild?.matches('.la-cell')) { + const cell = document.createElement('div'); + cell.className = 'la-cell'; + cell.append(...td.childNodes); + td.append(cell); + } + return td.firstElementChild; + }); + if (open) { cells.forEach(c => { c.hidden = true; }); body.hidden = false; } + cells.forEach(c => NulliusDisclosure.setHidden(c, open)); + if (!open) setTimeout(() => { if (btn.getAttribute('aria-expanded') !== 'true') body.hidden = true; }, 260); + } + document.addEventListener('click', e => { + const row = e.target.closest('.launch-agents tr.la-fold'); + if (!row || e.target.closest('a')) return; + const btn = row.querySelector('.la-fold-btn'); + setFold(btn, btn.getAttribute('aria-expanded') !== 'true'); + }); function refreshLaunches(fresh) { const wrap = document.getElementById('view-launch'); const coming = [...fresh.querySelectorAll('#view-launch > .launch')], keys = new Set(coming.map(l => l.dataset.launch)); @@ -545,7 +597,12 @@

{{ d.info.title }}

coming.forEach(l => { const name = l.dataset.launch, html = l.outerHTML; let el = wrap.querySelector(`:scope > .launch[data-launch="${CSS.escape(name)}"]`); - if (el && sent.get(name) !== html) { el.replaceWith(l); el = l; } + if (el && sent.get(name) !== html) { + // what the reader unfolded stays unfolded + const open = [...el.querySelectorAll('.la-fold-btn[aria-expanded="true"]')].map(b => b.getAttribute('aria-controls')); + el.replaceWith(l); el = l; + open.forEach(id => { const b = l.querySelector(`.la-fold-btn[aria-controls="${CSS.escape(id)}"]`); if (b) setFold(b, true, false); }); + } el = el || l; sent.set(name, html); const next = before ? before.nextElementSibling : wrap.firstElementChild; diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index 7d84a045..5eba633e 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -13,7 +13,9 @@ import unittest from datetime import datetime, timezone from types import SimpleNamespace +from unittest.mock import patch +from nullius import workspace from nullius.runs import launches, parse from nullius.standard import job, merge, shard @@ -61,7 +63,7 @@ def test_shards_started_apart_agree_on_their_launch(self): def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="gpt-5-mini", bench=BENCH, other=AGENT, - planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local"): + planned=None, t0=1_760_000_000.0, failed=(), budget=None, backend="local", reward=lambda place: float(place % 2)): """A shard job as a run writes it: its config, a trial per unit (tagged by the unit's place in the whole dataset), its result.""" d = os.path.join(root, name) @@ -78,7 +80,7 @@ def write_shard(root, name, units, index, of, *, total=10, launch="L1", model="g f.write("print(1)\n") exc = ("AgentTimeoutError", "slow") if u in failed else None results.append(job.write_trial(trial, job=name, bench=bench, other=other, model=model, unit=u, t0=t0 + k, t1=t0 + k + 5, - produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": 1.0 if place % 2 else 0.0}, + produce=(t0 + k, t0 + k + 4), score=None if exc else {"reward": reward(place)}, exception=exc, calls=[], trajectory=None)) job.write_result(d, name, t0, t0 + 10 * (index + 1), results, len(planned), budget=budget) return d @@ -212,6 +214,54 @@ def test_a_job_that_is_no_shard_or_an_out_that_is_taken_is_refused(self): self.assertIn("not a shard", str(e.exception)) +class Fold(unittest.TestCase): + """The runs page folds one participant's shards into one row: what it shows is the job ``cli merge`` makes.""" + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="nullius-fold-"); self.addCleanup(shutil.rmtree, self.tmp, True) + units = [f"t-{i}" for i in range(10)] + self.split = {i: [u for _, u in shard.select(units, i, 3)] for i in (1, 2, 3)} + + def test_a_fold_reads_as_the_merged_job(self): + planned = self.split[2] + dirs = [write_shard(self.tmp, "s1", self.split[1], 1, 3, failed=(self.split[1][0],), reward=lambda place: 1.0), + write_shard(self.tmp, "s2", planned[:-1], 2, 3, planned=planned)] # shard 3 never ran; nor did a unit of shard 2 + folded, check = launches.fold([parse.parse_job(d) for d in dirs]) + made = merge.merge(dirs, os.path.join(self.tmp, "out", "run")) + merged = parse.parse_job(made["out"]) + for name in ("n_resolved", "adjusted_solve_rate", "raw_solve_rate", "counts", "n_planned", "n_errored_trials", "started_at", + "finished_at", "duration_min", "total_input_tokens", "total_output_tokens", "total_cost_usd", "mean_score", "budget"): + self.assertEqual(getattr(folded, name), getattr(merged, name), name) + self.assertEqual(sorted(t.trial_name for t in folded.trials), sorted(t.trial_name for t in merged.trials)) + # the rate is over every trial with a verdict across the shards, never the mean of the shards' rates + rates = [parse.parse_job(d).adjusted_solve_rate for d in dirs] + self.assertNotAlmostEqual(folded.adjusted_solve_rate, sum(rates) / len(rates)) + # the units no shard ran are merge's own: counted apart, never as misses + self.assertEqual(check["notes"], made["notes"]) + self.assertEqual(check["units_total"] - len(check["by_unit"]), 10 - len(folded.trials)) + self.assertEqual(sum(folded.counts.values()), len(check["by_unit"])) + + def test_a_fold_is_read_again_only_when_the_workspace_parses_a_shard_again(self): + dirs = [write_shard(self.tmp, f"c{i}", self.split[i], i, 3) for i in (1, 2)] + ws = workspace.Workspace("w", "W", [self.tmp], "2026-01-01", "2026-01-01", os.path.join(self.tmp, "ws"), [self.tmp]) + with patch.object(merge, "check", wraps=merge.check) as check: + first = launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) + self.assertEqual(launches.fold([workspace.load_job(ws, d)[0] for d in dirs]), first) + self.assertEqual(check.call_count, 1) + trial = os.path.join(dirs[1], sorted(n for n in os.listdir(dirs[1]) if n.startswith("t-"))[0], "result.json") + with open(trial, "a") as f: + f.write("\n") + launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) + self.assertEqual(check.call_count, 2) + # a shard more replaces the fold without it: one entry per dir, however long the app runs + dirs.append(write_shard(self.tmp, "c3", self.split[3], 3, 3)) + launches.fold([workspace.load_job(ws, d)[0] for d in dirs]) + self.assertEqual([k for k in launches._FOLDED if dirs[0] in k], [tuple(dirs)]) + + def test_shards_merge_refuses_are_not_folded(self): + dirs = [write_shard(self.tmp, "a", self.split[1], 1, 3, launch="L1"), write_shard(self.tmp, "b", self.split[2], 2, 3, launch="L2")] + self.assertIsNone(launches.fold([parse.parse_job(d) for d in dirs])) + + T0 = datetime(2026, 10, 12, 9, 0, tzinfo=timezone.utc) diff --git a/app/ui_tests/test_launch_shards.py b/app/ui_tests/test_launch_shards.py new file mode 100644 index 00000000..1738feca --- /dev/null +++ b/app/ui_tests/test_launch_shards.py @@ -0,0 +1,172 @@ +"""A launch run over several machines, on the Runs page's "by launch" view. + +Each participant's shards fold into one row whose numbers are the job +``cli merge`` would make of them (``runs.launches.fold``); the row unfolds into +a row per shard, each with its own Open. The header counts every shard, and +every shard has its row. Jobs are written as a standard run writes them +(``tests.test_standard_shard.write_shard``); nothing is run. +""" +from __future__ import annotations + +import os +import re +import shutil +import tempfile +import unittest +from contextlib import ExitStack +from pathlib import Path +from unittest.mock import patch + +from nullius import execution, hosts, workspace +from nullius.runs import parse +from nullius.standard import merge +from tests.test_standard_shard import write_shard +from .test_controls_apart import text_of +from .test_routes import web + +NAME = "fold" +UNITS = [f"t-{i}" for i in range(8)] +SPLIT = {1: UNITS[0::2], 2: UNITS[1::2]} +AGENTS = ("terminus-2", "jiuwenswarm") +CONTROLS = ("nop", "oracle") +DONE = {"rc": 0, "at": 1_760_000_100.0, "label": "done"} + + +def write_launch(root, runs, nop=None, stopped="terminus-2", written=("nop", "jiuwenswarm", "terminus-2")): + """One launch over two machines, 8 units: every participant's shard 1 on + this machine, done; shard 2 on ``host-2``, stopped while ``stopped`` ran + (2 of its 4 units, no job record), after the ``written`` ones before it. + ``nop``: shard 2's nop rewards by place. The jobs and the two run records.""" + jobs, executions = [], {} + for i, host in ((1, None), (2, "host-2")): + order = list(CONTROLS + AGENTS) if i == 1 else list(written) + [p for p in CONTROLS + AGENTS if p not in written] + for p in order: + if i == 2 and p not in written: + continue + other = {"key": p, "_kind": "control" if p in CONTROLS else "agent", "_ref": p, "source": "nullius"} + units = SPLIT[i][:2] if i == 2 and p == stopped else SPLIT[i] + reward = {"nop": lambda place: 0.0, "oracle": lambda place: 1.0}.get(p, lambda place: float(place % 3 == 0)) + if i == 2 and p == "nop" and nop: + reward = nop + d = write_shard(root, f"{NAME}-{p}-shard-{i}of2", units, i, 2, total=len(UNITS), launch=f"{NAME}-{p}", + other=other, planned=SPLIT[i], reward=reward, t0=1_760_000_000.0 + 3600 * (i - 1)) + if i == 2 and p == stopped: + os.remove(os.path.join(d, "result.json")) + jobs.append(parse.parse_job(d)) + plan = execution.Plan(name=f"{NAME}-shard-{i}of2", script="", runner="standard", host=hosts.parse(host) if host else None, + jobs=[{"name": f"{NAME}-{p}-shard-{i}of2", "agent": p} for p in order], + parts=[{"id": p, "label": p, "script": "true"} for p in order]) + record = ({"id": "s1", "state": "done", "launched": order[-1], "part": len(order) - 1, "ends": {p: DONE for p in order}} if i == 1 else + {"id": "s2", "state": "stopped", "launched": stopped, "part": order.index(stopped), "exit_code": 130, + "ends": {p: DONE for p in order[:order.index(stopped)]}}) + ex = execution.Execution(plan, root=runs, record={"started": 1_760_000_000.0 + i, "finished": 1_760_010_000.0, **record}) + ex.jobs = root + executions[ex.id] = ex + return jobs, executions + + +def rows_of(block): + return re.findall(r'', response.get_data(as_text=True), re.S) + self.assertEqual(len(blocks), 1) + return blocks[0] + + def fold_row(self, block, who): + row = re.search(r']*>(?:(?!).)*?aria-label="Shards of ' + re.escape(who) + r'".*?', block, re.S) + self.assertIsNotNone(row, who) + return row.group(0) + + def test_each_participant_is_one_row_of_its_shards_and_every_shard_keeps_its_row(self): + block = self.block() + # the header counts every shard; each has its row, under a folded row that says how many it holds + self.assertIn("· 8 runs", block) + classes = rows_of(block) + self.assertEqual(sum("la-shard" in c for c in classes), 8) + self.assertEqual(sum("la-fold" in c for c in classes), 4) + self.assertEqual(len(classes), 12) + self.assertEqual(sum(int(n) for n in re.findall(r"(\d+) shards", block)), 8) + # folded by default; the row has no Open of its own, each shard has one, named by shard and machine + self.assertEqual(block.count('aria-expanded="false"'), 4) + self.assertEqual(len(re.findall(r'', block)), 4) + self.assertEqual(block.count("Open →"), 8) + self.assertNotIn("Open", self.fold_row(block, "terminus-2")) + self.assertIn('title="fold-terminus-2-shard-2of2">shard 2 · host-2', block) + self.assertIn('title="fold-terminus-2-shard-1of2">shard 1 · this machine', block) + + def test_a_folded_row_shows_the_merged_job_and_says_what_did_not_run(self): + block = self.block() + row = self.fold_row(block, "terminus-2") + merged = parse.parse_job(merge.merge([os.path.join(self.root, f"fold-terminus-2-shard-{i}of2") for i in (1, 2)], + os.path.join(self.runs, "merged"))["out"]) + pct = web.app.jinja_env.filters["pct"] + self.assertIn(f"{pct(merged.adjusted_solve_rate)}", row) + # 6 of the 8 units ran: the count is over them, and the 2 not run stand beside it, never as misses + self.assertIn(f'{merged.n_resolved}/6{web.app.jinja_env.filters["runtime"](merged.duration_min)}', row) + done = self.fold_row(block, "jiuwenswarm") + self.assertIn('la-state finished', done) + self.assertNotIn("not run", done); self.assertNotIn("⚠", done) + + def test_controls_fold_alike_and_are_judged_over_every_shard(self): + block = self.block() + self.assertIn("Task checks incomplete: oracle not run on shard 2.", text_of(block)) + for control in CONTROLS: + row = self.fold_row(block, control) + self.assertIn("la-control", row) + self.assertIn("task check", row) + self.assertIn("1 of 2 shards not run", self.fold_row(block, "oracle")) + self.assertIn("8/8 rejected ✓", self.fold_row(block, "nop")) + # never ranked nor offered to Compare: Compare lists the agents' finished shards only + compare = re.search(r'href="/compare\?([^"]+)"', block).group(1) + self.assertNotIn("nop", compare); self.assertNotIn("oracle", compare) + self.assertLess(block.index('aria-label="Shards of oracle"'), block.index('aria-label="Shards of terminus-2"')) + + def test_a_nop_that_passes_a_task_on_any_shard_is_flagged(self): + block = self.block(nop=lambda place: float(place == 3)) + self.assertIn("A control did not behave", text_of(block)) + + def test_a_control_stopped_on_one_shard_is_named_with_its_shard(self): + block = self.block(stopped="nop", written=("nop",)) + self.assertIn("Task checks incomplete: nop stopped on shard 2, oracle not run on shard 2.", text_of(block)) + + def test_a_participant_of_one_shard_is_its_plain_row(self): + self.jobs, _ = write_launch(self.root, self.runs, written=()) + self.jobs = [j for j in self.jobs if j.job_id.endswith("1of2")] + response = self.client.get("/runs") + self.addCleanup(response.close) + block = re.search(r'
', response.get_data(as_text=True), re.S).group(0) + self.assertNotIn("la-fold", block); self.assertNotIn("la-shard", block) + self.assertIn("· 4 runs", block) + self.assertEqual(len(rows_of(block)), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/web/runs.py b/app/web/runs.py index d4266fda..9ae2e702 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -4,6 +4,7 @@ import hashlib import json import os +import re import tempfile from dataclasses import replace from flask import ( @@ -176,6 +177,63 @@ def _cut_short(r) -> str: return "" +# A shard's job as a recipe names it (standard/recipe.py): ``-shard-of``. +_SHARD = re.compile(r"-shard-(\d+)of\d+$") +# A participant's state is its shards' most telling: one still working, else the one that ended worst. +_TELLING = ("running", "preparing", "stopping", "paused", "queued", "failed", "refused", "stopped", execution.NOT_RUN, + "not in this workspace", "finished") + + +def _host_of(job_id) -> str: + ex = execution.for_job(job_id) + return "" if ex is None else ex.plan.host.label if ex.plan.host else "this machine" + + +def _fold(rows, ended, control=False) -> list: + """A participant's shards as one row, ``shards`` its shards' own rows, when + it has two or more and ``cli merge`` would put them together: their numbers + are the merged job's (``launches.fold``). Missing evidence is not a zero: + ``not_run`` is the units no shard ran, beside the count, never in it.""" + def key(r): + # a written shard by the launch id its config records; one not written yet by the name its record gives it + if r["job"] is not None: + return (r.get("control"), r["job"].launch_id) if r["job"].launch_id else id(r) + return (r.get("control"), _SHARD.sub("", r["job_id"])) if _SHARD.search(r["job_id"]) else id(r) + by = {} + for r in rows: + by.setdefault(key(r), []).append(r) + out = [] + for k, group in by.items(): + written = [r["job"] for r in group if r["job"] is not None] + got = launches.fold(written) if len(group) > 1 and written else None + if len(group) < 2 or (written and got is None): + out += group + continue + job, check = got or (None, None) + index = {s["name"]: s["shard"].get("index") for s in check["shards"]} if check else {} + for r in group: + m = _SHARD.search(r["job_id"]) + r["shard"] = index.get(r["job_id"]) or (int(m.group(1)) if m else None) + r["host"] = _host_of(r["job_id"]) + group.sort(key=lambda r: (r["shard"] is None, r["shard"] or 0, r["job_id"])) + states = [r["state"] for r in group] + state = min(states, key=lambda s: _TELLING.index(s) if s in _TELLING else len(_TELLING)) + bad = [s for s in dict.fromkeys(states) if s in _ENDED_SHORT] + fold = {"job": job, "job_id": k[1], "state": state, "shards": group, "control": k[0], + "note": ", ".join(f"{states.count(s)} of {len(group)} shards {s}" for s in bad), + "notes": check["notes"] if check else []} + if job is not None: + total = check["units_total"] or job.n_planned + fold["total"] = total + # while a shard still works, its units are to come, not missing + fold["not_run"] = max(0, total - len(check["by_unit"])) if state in _ENDED_SHORT + ("finished",) else 0 + if control: + v = feasibility.summary_of(job) if job is not None else None + fold["verdict"] = v and replace(v, running=any(r["verdict"] and r["verdict"].running for r in group)) + out.append(fold) + return out + + def _launch_view(g, by_id, verdicts, ended) -> dict: """One launch as its block draws it: the task checks, the agents ranked on one scale, the runs its record named that Harbor has not written yet.""" @@ -187,15 +245,20 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: for r in rows: if r["state"] == "refused": r["why"] = execution.for_job(r["job_id"]).refused(r["job_id"]) + rows = _fold(rows, ended) rate = lambda r: r["job"].adjusted_solve_rate if r["job"] is not None and r["job"].adjusted_solve_rate is not None else -1.0 rows.sort(key=lambda r: (_STATE_ORDER.get(r["state"], 9), -rate(r), r["job_id"])) - ok = {v.control for _, v in checks if v.state == "ok"} # The controls are rows of their own at the top of the table, written or still to come, nop before oracle: # a check on the tasks, never ranked with the agents nor offered to Compare, and kept once they end. controls = [{"job": j, "job_id": j.job_id, "state": _state_of(j.job_id, j, ended), "control": v.control, "verdict": v} for j, v in checks] controls += [{"job": None, "job_id": i, "state": _state_of(i, None, ended), "control": c, "verdict": None} for i, c in waiting] + controls = _fold(controls, ended, control=True) controls.sort(key=lambda r: (feasibility.CONTROLS.index(r["control"]) if r["control"] in feasibility.CONTROLS else 9, r["job_id"])) - short = list(dict.fromkeys(filter(None, map(_cut_short, controls)))) + # judged over every shard: a folded control's verdict is over all its trials, and what one shard lacks is said of that shard + checks = [(r["job"], r["verdict"]) for r in controls if r["verdict"] is not None] + ok = {v.control for _, v in checks if v.state == "ok"} + lacks = lambda r: [f"{c} on shard {s['shard']}" for s in r["shards"] for c in [_cut_short(s)] if c] if "shards" in r else [_cut_short(r)] + short = list(dict.fromkeys(filter(None, (x for r in controls for x in lacks(r))))) if any(r["state"] != "finished" and r["state"] not in _ENDED_SHORT for r in controls): verdict, checked = "Not checked yet: a control has not finished.", False elif not checks and not short: @@ -212,10 +275,12 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: verdict, checked = "An empty answer fails every task; no oracle ran to show each can pass.", True else: verdict, checked = "The reference passes every task; no nop ran to show an empty answer fails.", True - n_tasks = max([j.n_planned or len(j.trials) for j in jobs] + [0]) or None - finished = [r["job_id"] for r in rows if r["state"] == "finished" and r["job"] is not None and r["job"].has_verdict] + n_tasks = max([r.get("total") or r["job"].n_planned or len(r["job"].trials) for r in controls + rows if r["job"] is not None] + [0]) or None + # each shard is a run of its own: it is counted, offered to Compare and opened as one + leaves = lambda rs: [s for r in rs for s in r.get("shards", [r])] + finished = [r["job_id"] for r in leaves(rows) if r["state"] == "finished" and r["job"] is not None and r["job"].has_verdict] counts = {} - for r in controls + rows: + for r in leaves(controls + rows): counts[r["state"]] = counts.get(r["state"], 0) + 1 ex = execution.for_job((g.job_ids + g.pending)[0]) return {"launch": g, "checks": checks, "verdict": verdict, diff --git a/docs/cli/shard-and-merge.md b/docs/cli/shard-and-merge.md index 89b15456..e9e9fc9e 100644 --- a/docs/cli/shard-and-merge.md +++ b/docs/cli/shard-and-merge.md @@ -13,7 +13,7 @@ nullius standard merge jobs_standard/*__shard-* -o jobs_standard/humaneval-oracl | No central relay | Each unit's relay runs on its machine with its own key; a provider's limit is per key | | Shards by the unit's id | `sha256(unit id) mod N` (`app/nullius/standard/shard.py`): every unit in exactly one shard, new units leave the rest in place; changing N moves most. Order: `units`, the units `--units FILE` lists, the shard, then `--limit` (`--shard 1/2 --limit 6`: the first six of shard 1) | | A unit keeps its place | `u0042` is the 43rd unit of the dataset in every shard; shard jobs never share a trial name | -| One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch`: `--launch`, or the day (UTC), benchmark, agent or capability and model. The runs page groups them until merged, then shows and counts the merged job in place of the source jobs named by its `merged_from` record. Unmerged jobs still count; source job files are kept | +| One launch | Each shard's `config.json` records `standard.shard` (`index`, `of`, `units_total`, `partition`, `planned`, `limit`) and `standard.launch`: `--launch`, or the day (UTC), benchmark, agent or capability and model. The runs page groups them until merged, each agent's or control's shards as one row with the numbers `merge` would give and the units not run beside the count; the row unfolds into one row per shard. Once merged, it shows and counts the merged job in place of the source jobs named by its `merged_from` record. Unmerged jobs still count; source job files are kept | | `merge JOB_DIR... -o OUT` | One job (`app/nullius/standard/merge.py`): trials hard-linked, each `result.json` naming the merged job and shard; the job's `result.json` recomputed; `config.json` with `merged_from` for `shard`. Different launches, versions, datasets, agents, models, backends or N refused unless `--force`; lists of different units and two trials of one unit always. Units no shard ran are named | | A list of units | `--units FILE` on every machine: each shard runs the listed units of its own; its `units_total` counts the listed units ([Run a benchmark](run.md#a-list-of-units)). `merge` compares the units each list holds, in any order. A list's file name, comments and sha256 may differ from shard to shard; the merged job records the first shard's list | | A spending cap | Each shard's own; the builder shares a recipe's cap over its shards. `merge` adds them into one `budget` when every shard had one |