Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 89 additions & 1 deletion app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@
import os
import re
import shlex
import shutil
import subprocess
import sys
import threading
Expand All @@ -122,7 +123,9 @@
from datetime import datetime, timezone
from typing import Callable, Dict, List, Optional

from nullius import hosts, paths, prices as price_list, progress, providers, redact, secrets, settings
from nullius import hosts, paths, prices as price_list, progress, providers, redact, secrets, settings, workspace
from nullius.judge import store as judgements
from nullius.runs import parse
from nullius.harbor import source as harbor_source
from nullius.standard import container, payload as standard_payload

Expand Down Expand Up @@ -1659,3 +1662,88 @@ def restore(root: Optional[str] = None, **kw) -> List[Execution]:
if ex.state in ALIVE:
back.append(ex.resume())
return back


# --- deleting a launch ---------------------------------------------------------

def run_folder(name: str, root: Optional[str] = None) -> str:
"""The folder of the run *name* directly in the runs root, holding its
record; ValueError for any other path, a link included."""
root = os.path.realpath(root or runs_root())
path = os.path.join(root, name or "")
if not _NAME.fullmatch(name or "") or os.path.realpath(path) != path or not os.path.isfile(os.path.join(path, RECORD)):
raise ValueError(f"{name} is not a run Nullius made")
return path


def _run_deletion(name: str, home: str, root: Optional[str]) -> dict:
path = run_folder(name, root)
with open(os.path.join(path, RECORD), encoding="utf-8") as fh:
rec = json.load(fh)
ex = next((e for e in EXECUTIONS.values() if os.path.realpath(e.dir) == path), None)
state = ex.state if ex is not None else rec.get("state")
if state in ALIVE:
raise ValueError(f"{name} is {state}: stop it before you delete it")
plan = rec.get("plan") or {}
# a run refused at the name check never made the host folder: one of that name there is not its own
host = hosts.parse(plan["host"]) if plan.get("host") and (rec.get("launched") or rec.get("step") not in ("connect", "name")) else None
held = []
for ws, roots in workspace.holding(home, path):
dirs = parse.find_job_dirs(roots)
ids = [j.job_id for j in workspace.load_jobs(ws, roots)]
# a job of the same id from another root shares these judgements: they stay
others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs}
judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others])
held.append({"ws": ws, "roots": roots, "jobs": dirs,
"judged": judged, "judgements": sum(len(os.listdir(d)) for d in judged)})
size = sum(os.lstat(os.path.join(d, n)).st_size for d, _, names in os.walk(path) for n in names)
return {"name": name, "dir": path, "bytes": size, "host": host, "workspaces": held,
"judgements": sum(w["judgements"] for w in held)}


def deletion(names: List[str], home: str, root: Optional[str] = None) -> dict:
"""What deleting the runs *names* of one launch removes: per run (``runs``)
its folder and size, the host copy that is its own, and per workspace that
holds it its roots, parsed jobs and judgement folders; the totals, and the
number of copies per host. ValueError when any of them is alive or is not
a run: nothing goes then."""
runs = [_run_deletion(n, home, root) for n in dict.fromkeys(names)]
copies: Dict[str, int] = {}
for r in runs:
if r["host"]:
copies[r["host"].label] = copies.get(r["host"].label, 0) + 1
return {"runs": runs, "bytes": sum(r["bytes"] for r in runs), "judgements": sum(r["judgements"] for r in runs),
"workspaces": list(dict.fromkeys(w["ws"].label for r in runs for w in r["workspaces"])), "hosts": copies}


def delete(names: List[str], home: str, root: Optional[str] = None, copies=(), run: Optional[Callable] = None) -> dict:
"""Delete the runs *names* of one launch (``deletion`` says what goes):
per run its folder, its entry here, its roots, parsed jobs and judgements
in every workspace, and its copy on its host when that host's label is in
*copies*. Returns ``deletion``'s answer, with ``stays``: the runs whose
folder could not go (they keep everything else), and ``kept``: one line
per thing that stays."""
gone = deletion(names, home, root)
gone["stays"], gone["kept"] = [], []
for r in gone["runs"]:
try:
shutil.rmtree(r["dir"])
except OSError as e:
gone["stays"].append(r["name"])
gone["kept"].append(f"{r['name']} stays in {r['dir']}: {e.strerror or e}")
continue
for key in [k for k, e in EXECUTIONS.items() if os.path.realpath(e.dir) == r["dir"]]:
del EXECUTIONS[key]
for w in r["workspaces"]:
for d in w["judged"]:
shutil.rmtree(d, ignore_errors=True)
# read again: another run of the launch may have left this workspace already
ws = workspace.load(home, w["ws"].name)
if ws is not None:
workspace.forget(ws, w["roots"], w["jobs"])
host = r["host"]
if host and host.label in copies:
rc, out = (run or subprocess_run)(host.argv(f"rm -rf {_rq(REMOTE_HOME + '/runs/' + r['name'])}", timeout=15), 60)
if rc != 0:
gone["kept"].append(f"the copy of {r['name']} on {host.label} stays: " + redact.redact(out.strip()[-200:]))
return gone
5 changes: 5 additions & 0 deletions app/nullius/judge/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str =
return out


def job_dirs(ws_dir: str, job_ids) -> list:
"""The folders that hold these jobs' judgements, in every lane."""
return [d for lane in ROOTS for j in job_ids if os.path.isdir(d := os.path.join(_root(ws_dir, lane), _SAFE.sub("_", j)))]


def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "system-one"):
"""Every stored record, every version: yields (job_id, trial_name,
version, record). One job's when `job_id` is given."""
Expand Down
28 changes: 28 additions & 0 deletions app/nullius/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,34 @@ def delete(home_dir: str, name: str) -> bool:
return True


def holding(home_dir: str, folder: str) -> list:
"""Every workspace with a root in *folder*: [(workspace, those roots)]."""
inside = os.path.join(os.path.realpath(folder), "")
found = []
for ws in list_all(home_dir):
roots = [r for r in ws.roots if os.path.join(os.path.realpath(r), "").startswith(inside)]
if roots:
found.append((ws, roots))
return found


def forget(ws: Workspace, roots, job_dirs) -> None:
"""*ws* without *roots*, and without its parsed jobs of *job_dirs*."""
ws.roots = [r for r in ws.roots if r not in roots]
ws.job_roots = [r for r in ws.job_roots if r not in roots]
save(ws)
index = job_index(ws)
for d in job_dirs:
path = _cache_file(ws, d)
with _DECODED_LOCK:
_DECODED.pop(path, None)
try:
os.remove(path)
except OSError:
pass
index.pop(d, None)
_write_index(ws, index)


def signature(job_dir: str) -> str:
"""What the parse of *job_dir* depends on: the files it reads, by size
Expand Down
3 changes: 2 additions & 1 deletion app/static/notifications.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@
new MutationObserver(() => {
const tray = document.querySelector('.toasts');
if (tray && tray.parentElement !== host()) {
host().append(tray); if (tray.showPopover && !tray.matches(':popover-open')) tray.showPopover();
host().append(tray); tray.setAttribute('popover', 'manual'); // base.html's flashed tray has none
if (tray.showPopover && !tray.matches(':popover-open')) tray.showPopover();
}
}).observe(document.body, {subtree: true, attributes: true, attributeFilter: ['open']});
})();
5 changes: 5 additions & 0 deletions app/static/runs/launches.css
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@
.la-mix span + span { box-shadow: inset 1.5px 0 0 var(--surface); }
.la-go { text-align: right; white-space: nowrap; font-size: 12.5px; }
.launch-foot { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 10px 18px; border-top: 1px solid var(--border-soft); font-size: 12.5px; color: var(--text-dim); }
/* a launch's Delete asks in the Add dialog's frame (catalogue.css, dialog.std-how), narrower */
dialog.std-how.launch-delete { width: min(520px, calc(100vw - 32px)); }
.launch-delete ul { margin: 12px 0 0; padding-left: 1.2em; }
.launch-delete label { display: block; margin: 4px 0; overflow-wrap: anywhere; }
.launch-delete-acts { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 10px; margin-top: 20px; }

@media (max-width: 900px) { .launch-agents .lh-more { display: none; } }
@media (max-width: 640px) {
Expand Down
2 changes: 2 additions & 0 deletions app/static/style.css
Original file line number Diff line number Diff line change
Expand Up @@ -1614,6 +1614,8 @@ pre.instruction { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 32
/* what a run is doing: a stepper of its stages, then its units. A step is a mark, the
line to the next step, its name and its time; done reads filled and checked, the
running one ringed, the rest numbered. Stacked where a row would crush the names. */
.execution-line { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; }
.run-delete { margin-left: auto; color: var(--text-dim); }
.execution-stages { container: xs / inline-size; margin: 14px 0 8px; }
.xs-list { --mark: 1.6rem; list-style: none; margin: 0; padding: 0; display: flex; align-items: flex-start; }
.xs { position: relative; flex: 1 1 0; min-width: 0; display: flex; flex-direction: column; align-items: flex-start; gap: .1rem; padding-right: .9rem; font-size: .82rem; line-height: 1.3; color: var(--text-dim); }
Expand Down
5 changes: 4 additions & 1 deletion app/templates/runs/_execution.html
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
Stop and Continue answered 403. #}
{# `own`: this job's own (state, finished) when the run is of several agents - an arm that
answered everything is done though a later arm failed; the run's state is said beside it. #}
{% macro execution_strip(ex, csrf, waiting=false, own=none) %}
{# `deletable`: the key of this job's launch when the Runs page offers its Delete (web/runs._deletable). #}
{% macro execution_strip(ex, csrf, waiting=false, own=none, deletable=false) %}
{% set shown = own[0] if own else ex.state %}
<section class="execution" id="execution" data-run="{{ url_for('run_status', run_id=ex.id) }}" data-stop="{{ url_for('run_stop', run_id=ex.id) }}" data-continue="{{ url_for('run_continue', run_id=ex.id) }}"
data-live="{{ 'yes' if ex.state in ('preparing', 'running') else 'no' }}" data-waiting="{{ 'yes' if waiting else 'no' }}" data-csrf="{{ csrf }}" data-own="{{ own[0] if own else '' }}">
Expand All @@ -15,6 +16,8 @@
{% if ex.error and not own %}<span class="small bad" id="execution-error">{{ ex.error }}</span>{% endif %}
<span class="small muted" id="execution-run"{% if not own or ex.state == shown %} hidden{% endif %}>· the run: {{ ex.state }}{% if ex.error %} · {{ ex.error }}{% endif %}</span>
{% if ex.state in ('preparing', 'running', 'paused') %}<button type="button" class="btn sm ghost" id="execution-stop">Stop</button>{% endif %}
{#- a launch goes as one, from the Runs page: never this job alone #}
{% if deletable %}<a class="small run-delete" href="{{ url_for('index', view='launch', delete=deletable, _anchor=deletable|replace(':', '-')) }}">Delete its launch…</a>{% endif %}
</div>
{# an agent's run failed with more to come: the question, and the answer #}
<div class="execution-ask" id="execution-ask" {% if ex.state != 'paused' %}hidden{% endif %}>
Expand Down
46 changes: 43 additions & 3 deletions app/templates/runs/index.html
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
{% from 'runs/_runtime.html' import what_ran %}
{% extends "base.html" %}
{% block title %}Runs — Nullius{% endblock %}
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='runs/launches.css') }}">{% endblock %}
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='runs/launches.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='catalogue/catalogue.css') }}">{% endblock %}

{# Value filter for one column: a caret in the header opening a checkbox list.
Values start all-checked; unchecking hides the matching rows. #}
Expand All @@ -21,6 +22,9 @@
</span>
{% endmacro %}

{#- A row's name as the launch's table shows it: the control, the agent, else its job id past the launch's prefix. #}
{% macro row_name(r, L) %}{% set j = r.job %}{{ r.control or (j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id) }}{% endmacro %}

{#- One row of a launch's table: a task check (`kind` 'control') or an agent; `part` 'fold' for a participant's
shards as one row, 'shard' for one of them, '' for a run of one job. #}
{% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %}
Expand All @@ -34,7 +38,7 @@
{%- if kind == 'control' %}
{%- if part == 'fold' %}{{ r.control }}{% else %}<a href="{{ url_for('job', job_id=r.job_id) }}">{{ r.control }}</a>{% endif %} <span class="la-kind">task check</span>
{%- else %}
{%- set name = j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id %}
{%- set name = row_name(r, L) %}
{%- if part == 'fold' %}{{ name }}{% else %}<a href="{{ url_for('job', job_id=r.job_id) }}">{{ name }}</a>{% endif %}
{%- if j and j.runner %} {{ what_ran(j) }}{% elif j and j.agent_version %} <span class="la-ver mono">{{ j.agent_version }}</span>{% endif %}
{%- endif %}</span>
Expand Down Expand Up @@ -127,7 +131,7 @@
controls), then its agents on one 0-100% scale, then the runs its record named that Harbor has not written yet. -#}
<div id="view-launch" class="launches"{% if view != 'launch' %} hidden{% endif %}>
{% for g in launches %}{% set L = g.launch %}
<section class="launch" data-launch="{{ L.key }}" aria-label="Launch {{ L.title }}">
<section class="launch" data-launch="{{ L.key }}" id="{{ L.key|replace(':', '-') }}" aria-label="Launch {{ L.title }}">
{% set by_runner = g.agents|map(attribute='job')|select|map(attribute='runner')|select|first %}
{% set words = g.agents|map(attribute='job')|select|map(attribute='runner_items')|select|first %}
<header class="launch-head">
Expand Down Expand Up @@ -162,10 +166,46 @@ <h2>{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n
{% if g.recipe %}<a href="{{ url_for('new_run', run=g.recipe) }}">Open this launch's recipe</a>{% endif %}
{% if g.compare|length >= 2 %}<a class="btn sm" href="{{ url_for('compare', job=g.compare) }}">Compare these {{ g.compare|length }}</a>
{% elif g.agents %}<span class="faint" title="Two finished agents with a verdict are needed">Compare needs two finished agents</span>{% endif %}
{#- every row is of runs Nullius made: the launch goes as one, its shards with it, asked first in the dialog below #}
{% if g.gone %}{% set gone, rows = g.gone, g.controls + g.agents %}{% set n, wss, k = gone.judgements, gone.workspaces, gone.runs|length %}
<button type="button" class="btn sm ghost danger launch-delete-btn" aria-haspopup="dialog">Delete launch</button>
<template class="launch-ask">
<div class="std-how-head"><h2 id="launch-delete-h">Delete launch {{ L.title }}</h2></div>
<form method="post" action="{{ url_for('launch_delete', key=L.key) }}">
<input type="hidden" name="csrf" value="{{ csrf_token() }}">
<ul>{% for r in rows %}<li>{{ row_name(r, L) }}{% if r.shards %} ({{ r.shards|length }} shards){% endif %}</li>{% endfor %}</ul>
<p>{{ k }} folder{{ '' if k == 1 else 's' }} · {{ gone.bytes|filesizeformat }}{% if n %} · {{ n }} judgement{{ '' if n == 1 else 's' }}{% endif %}{% if wss %} · workspace{{ '' if wss|length == 1 else 's' }} {{ wss|join(', ') }}{% endif %}</p>
{% for h, c in gone.hosts.items() %}<label><input type="checkbox" name="host" value="{{ h }}" checked> Its cop{{ 'y' if c == 1 else 'ies' }} on <b>{{ h }}</b></label>{% endfor %}
<div class="launch-delete-acts"><button type="submit" formmethod="dialog" class="btn ghost" autofocus>Keep it</button><button type="submit" class="btn danger">Delete launch</button></div>
</form>
</template>{% endif %}
</footer>
</section>
{% endfor %}
{#- one dialog for every launch: its Delete fills it from the launch's template; ?delete=<key> opens it on load #}
<dialog class="std-how launch-delete" id="launch-delete" aria-labelledby="launch-delete-h"></dialog>
</div>
<script>
(() => {
const d = document.getElementById('launch-delete');
const btn = key => document.querySelector(`.launch[data-launch="${CSS.escape(key)}"] .launch-delete-btn`);
let key = '';
const open = b => {
const l = b.closest('.launch');
key = l.dataset.launch;
d.replaceChildren(l.querySelector('template.launch-ask').content.cloneNode(true));
d.showModal();
};
document.addEventListener('click', e => { const b = e.target.closest('.launch-delete-btn'); if (b) open(b); });
d.addEventListener('click', e => { // the backdrop, not the dialog's own padding
const r = d.getBoundingClientRect();
if (e.target === d && (e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom)) d.close();
});
d.addEventListener('close', () => btn(key)?.focus()); // the button again, though a refresh replaced its block
const b = btn(new URLSearchParams(location.search).get('delete') || '');
if (b) { b.focus(); open(b); }
})();
</script>

<div id="view-all"{% if view != 'all' %} hidden{% endif %}>
<div class="table-tools">
Expand Down
2 changes: 1 addition & 1 deletion app/templates/runs/job.html
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
</div>
</header>

{% if execution %}{{ execution_strip(execution, csrf, own=own if execution.plan.steps()|length > 1 else none) }}{% endif %}
{% if execution %}{{ execution_strip(execution, csrf, own=own if execution.plan.steps()|length > 1 else none, deletable=deletable) }}{% endif %}
{% if job.is_preparing and not live_board and not ended %}
{# Launched, not started: Harbor has written the configuration and nothing
else. There is no result record and no feed to fold yet, so this page
Expand Down
2 changes: 1 addition & 1 deletion app/templates/runs/preparing.html
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@
{% elif ex.plan.runner == 'standard' %}<p class="subtitle">Started from the builder with Nullius. This page becomes the live board as soon as the install is done and the run starts.</p>
{% else %}<p class="subtitle">Started from the builder with the {{ ex.plan.runner }} runner. This page becomes the live board as soon as this arm's questions are chosen, after the install.</p>{% endif %}
</header>
{{ execution_strip(ex, csrf, waiting=true, own=own) }}
{{ execution_strip(ex, csrf, waiting=true, own=own, deletable=deletable) }}
{% endblock %}
Loading
Loading