Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 54 additions & 3 deletions app/nullius/runs/launches.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,21 @@
launch: a shard joins the launch that holds another shard of its id before
any other rule is asked.

Nothing here reads a file: it takes the parsed jobs and the records' job lists,
and groups. Newest launch first.
``group`` reads no file: it takes the parsed jobs and the records' job lists,
and groups. Newest launch first. ``fold`` reads one participant's shards as
``cli merge`` would put them together, again only when the workspace parsed one of
them again, and writes nothing.
"""
from __future__ import annotations

import os
from dataclasses import dataclass, field
import threading
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone

from nullius.runs.parse import _budget, _iso
from nullius.standard import merge

WINDOW_SEC = 30 * 60


Expand Down Expand Up @@ -76,6 +82,51 @@ def merged_sources(jobs) -> set[str]:
for source in getattr(j, "merged_from", []) if source != j.job_id}


# Folds already read in this process, by their shards' dirs: (the shard jobs, (check, record)). The workspace
# hands out the same Job until its signature moves (``workspace.load_job``), so the same jobs read the same:
# a page renders again without reading every trial record, nor stating every file, while no shard moved.
# A dir is in one entry at most: a new set of shards replaces each fold it shares a dir with.
_FOLDED: dict = {}
_FOLDED_LOCK = threading.Lock()


def _merged(dirs):
"""``merge.check`` of ``dirs`` without the trial records, and the job's record; (None, None) when merge refuses them."""
try:
got = merge.check(list(dirs))
except merge.Refused:
return None, None
res = merge.result_of(got["shards"], [r for s in got["shards"] for r in s["results"]])
return {**got, "shards": [{k: v for k, v in s.items() if k != "results"} for s in got["shards"]]}, res


def fold(shards):
"""One participant's shard jobs as the one job ``cli merge`` would make of
them, in memory: their trials together, the job's record ``merge.result_of``,
and what ``merge.check`` says no shard ran. ``(job, check)``; None when
merge refuses them, so they stay apart."""
dirs = tuple(j.dir for j in shards)
with _FOLDED_LOCK:
known = _FOLDED.get(dirs)
if known is not None and all(a is b for a, b in zip(known[0], shards)):
got, res = known[1]
else:
got, res = _merged(dirs)
with _FOLDED_LOCK:
for stale in [k for k in _FOLDED if k != dirs and not set(k).isdisjoint(dirs)]:
del _FOLDED[stale]
_FOLDED[dirs] = (tuple(shards), (got, res))
if got is None:
return None
return replace(shards[0], trials=[t for j in shards for t in j.trials], n_tasks=res["n_total_trials"],
n_total_trials=res["n_total_trials"], started_at=_iso(res["started_at"]), finished_at=_iso(res["finished_at"]),
n_errored_trials=res["stats"]["n_errored_trials"], budget=_budget(res.get("budget")),
# still going: what a merge, made of ended shards, never sees
n_running_trials=sum(j.n_running_trials for j in shards),
n_pending_trials=sum(j.n_pending_trials for j in shards),
preparing=all(j.preparing for j in shards)), got


def group(jobs, records: dict | None = None, window: float = WINDOW_SEC) -> list[Launch]:
"""``jobs``: the parsed jobs (``job_id``, ``dataset``, ``started_at``).
``records``: ``{execution id: (name, [job ids], launched at)}``, the builder's
Expand Down
43 changes: 26 additions & 17 deletions app/nullius/standard/merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,9 @@ def check(job_dirs: List[str], force: bool = False) -> dict:
names: Dict[str, List[str]] = {}
for s in shards:
s["trials"] = _trials(s["dir"])
for t in s["trials"]:
unit = _at(_read(os.path.join(s["dir"], t, "result.json")), ("standard", "unit")) or t
s["results"] = [_read(os.path.join(s["dir"], t, "result.json")) for t in s["trials"]]
for t, res in zip(s["trials"], s["results"]):
unit = _at(res, ("standard", "unit")) or t
by_unit.setdefault(str(unit), []).append((s["name"], t))
names.setdefault(t, []).append(s["name"])
twice = {u: w for u, w in by_unit.items() if len(w) > 1}
Expand Down Expand Up @@ -203,6 +204,26 @@ def _budget(budgets: list) -> Optional[dict]:
**({"unit_max_usd": each, "over_unit": [u for x in budgets for u in x.get("over_unit") or []]} if number(each) else {})}


def result_of(shards: list, results: List[dict]) -> dict:
"""The merged job's ``result.json``, but its ``id``: ``shards`` as ``check`` reads them, ``results`` every
trial's record. The runs page folds a launch's shards with it (``runs.launches.fold``)."""
planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards)
starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")]
ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")]
errored = sum(1 for r in results if r.get("exception_info"))
rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results]
scores = [w["score"] for w in rewards if "score" in w]
budget = _budget([s["result"].get("budget") for s in shards])
# ISO stamps in UTC sort as their instants
return {"started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None,
"n_total_trials": planned,
"stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
"n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0},
"standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0),
"scored": len(scores), "mean_score": job.mean_score(results)},
**({"budget": budget} if budget else {})}


def merge(job_dirs: List[str], out: str, force: bool = False) -> dict:
"""Merge the shard jobs ``job_dirs`` into the new job ``out``.
{out, trials, merged_from, notes, missing}."""
Expand All @@ -226,22 +247,10 @@ def merge(job_dirs: List[str], out: str, force: bool = False) -> dict:
with open(os.path.join(dest, "result.json"), "w", encoding="utf-8") as f:
json.dump(res, f, indent=1)
results.append(res)
planned = sum(int(s["result"].get("n_total_trials") or len(s["trials"])) for s in shards)
starts = [s["result"].get("started_at") for s in shards if s["result"].get("started_at")]
ends = [s["result"].get("finished_at") for s in shards if s["result"].get("finished_at")]
errored = sum(1 for r in results if r.get("exception_info"))
rewards = [(r.get("verifier_result") or {}).get("rewards") or {} for r in results]
scores = [w["score"] for w in rewards if "score" in w]
budget = _budget([s["result"].get("budget") for s in shards])
summary = result_of(shards, results)
planned = summary["n_total_trials"]
with open(os.path.join(out, "result.json"), "w", encoding="utf-8") as f:
# ISO stamps in UTC sort as their instants
json.dump({"id": name, "started_at": min(starts) if starts else None, "finished_at": max(ends) if ends else None,
"n_total_trials": planned,
"stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
"n_cancelled_trials": max(0, planned - len(results)), "n_running_trials": 0, "n_pending_trials": 0},
"standard": {"passed": sum(1 for w in rewards if w.get("reward") == 1.0),
"scored": len(scores), "mean_score": job.mean_score(results)},
**({"budget": budget} if budget else {})}, f, indent=1)
json.dump({"id": name, **summary}, f, indent=1)
cfg = json.loads(json.dumps(shards[0]["config"]))
cfg["job_name"] = name
std = cfg.setdefault("standard", {})
Expand Down
23 changes: 20 additions & 3 deletions app/static/runs/launches.css
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
.launch-agents { width: 100%; border-collapse: collapse; }
.launch-agents th { text-align: left; font: 400 11.5px var(--sans); color: var(--text-dim); padding: 8px 12px 6px; border-top: 1px solid var(--border-soft);
border-bottom: 1px solid var(--border-soft); white-space: nowrap; }
.launch-agents td { padding: 10px 12px; border-bottom: 1px solid var(--border-soft); vertical-align: middle; font-size: 13px; }
.launch-agents tbody tr:last-child td { border-bottom: 0; }
/* rules above each row, not below: a table of several bodies, some hidden, ends without one */
.launch-agents td { padding: 10px 12px; border-top: 1px solid var(--border-soft); vertical-align: middle; font-size: 13px; }
.launch-agents th:first-child, .launch-agents td:first-child { padding-left: 18px; }
.launch-agents th:last-child, .launch-agents td:last-child { padding-right: 18px; }
.launch-agents .num { text-align: right; font-family: var(--mono); font-variant-numeric: tabular-nums; white-space: nowrap; }
Expand Down Expand Up @@ -79,10 +79,27 @@

/* a task check's row: above the agents, set apart by a rule, its verdict where an agent has its rate */
.launch-agents tr.la-control td { background: color-mix(in srgb, var(--surface-2) 45%, transparent); }
.launch-agents tr.la-control + tr:not(.la-control) td { border-top: 2px solid var(--border); }
.launch-agents tr.la-first td { border-top: 2px solid var(--border); }
.la-kind { font-size: 11px; font-weight: 500; letter-spacing: .04em; text-transform: uppercase; color: var(--text-faint); margin-left: 6px; }
.la-verdict { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; }
.la-verdict b { font-weight: 500; }
.la-verdict.ok .lc-mark { color: var(--ok); }
.la-verdict.warn .lc-mark { color: var(--bad); }
.la-verdict.pending .lc-mark { color: var(--text-faint); }

/* a participant run as several shards: one row, its chevron the shared one, unfolding into a row per shard */
.launch-agents tr.la-fold { cursor: pointer; }
/* the chevron is a column of its own, so the name and its "N shards" line, and the shard rows below, start at one edge */
.launch-agents { --la-fold-w: 14px; --la-fold-gap: 6px; }
.la-fold-who { display: flex; align-items: flex-start; gap: var(--la-fold-gap); min-width: 0; }
.la-fold-btn { flex: none; display: inline-flex; align-items: center; justify-content: center; width: var(--la-fold-w); padding: 2px 0; border: 0; background: none; color: var(--text-dim); cursor: pointer; }
.la-fold-btn:hover { color: var(--text); }
.la-fold-btn:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 3px; }
.la-fold-btn::before { margin-left: 0; }
.launch-agents tr.la-fold .la-name { font-weight: 500; }
.launch-agents tr.la-shard td:first-child .la-who { margin-left: calc(var(--la-fold-w) + var(--la-fold-gap)); }
.launch-agents tr.la-shard td:has(> .la-cell) { padding-top: 0; padding-bottom: 0; }
.la-cell::before, .la-cell::after { content: ''; display: block; height: 10px; } /* the row's padding, inside what slides */
.launch-agents tr.la-shard .la-who a { font-weight: 400; color: var(--text-dim); }
.la-note { color: var(--patch_failed_tests); }
.la-gap { display: block; margin-top: 2px; font: 11.5px var(--sans); color: var(--patch_failed_tests); }
6 changes: 3 additions & 3 deletions app/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -1781,11 +1781,11 @@ select.ui-select-native { display: none !important; }
:root .slide-disclosure { box-sizing: border-box; }
:root .slide-disclosure > summary { cursor: pointer; list-style: none; color: var(--text-dim); }
:root .slide-disclosure > summary::-webkit-details-marker { display: none; }
:root .slide-disclosure > summary::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; }
:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before { transform: rotate(45deg); }
:root .slide-disclosure > summary::before, .la-fold-btn::before { content: ''; display: inline-block; width: 7px; height: 7px; flex: 0 0 7px; border-right: 1.5px solid currentColor; border-bottom: 1.5px solid currentColor; transform: rotate(-45deg); margin: 0 10px 0 3px; transition: transform .24s ease; }
:root .slide-disclosure[open]:not([data-expanded=false]) > summary::before, .la-fold-btn[aria-expanded=true]::before { transform: rotate(45deg); }
:root .slide-disclosure > summary > .tw { display: none; }
:root .slide-disclosure > summary:hover { color: var(--text); }
:root .slide-disclosure > summary:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; }
.tree-kids { box-sizing: border-box; }
@media (prefers-reduced-motion: no-preference) { .shell { transition: grid-template-columns .24s ease; } .tree-caret .ic { transition: transform .24s ease; } }
@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .tree-caret .ic, .side { transition: none; } }
@media (prefers-reduced-motion: reduce) { :root .slide-disclosure > summary, :root .slide-disclosure > summary::before, .la-fold-btn::before, .tree-caret .ic, .side { transition: none; } }
Loading
Loading