diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 283d7315..2eb16bc4 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -114,6 +114,7 @@ import os import re import shlex +import shutil import subprocess import sys import threading @@ -122,7 +123,9 @@ from datetime import datetime, timezone from typing import Callable, Dict, List, Optional -from nullius import hosts, paths, prices as price_list, progress, providers, redact, secrets, settings +from nullius import hosts, paths, prices as price_list, progress, providers, redact, secrets, settings, workspace +from nullius.judge import store as judgements +from nullius.runs import parse from nullius.harbor import source as harbor_source from nullius.standard import container, payload as standard_payload @@ -1659,3 +1662,88 @@ def restore(root: Optional[str] = None, **kw) -> List[Execution]: if ex.state in ALIVE: back.append(ex.resume()) return back + + +# --- deleting a launch --------------------------------------------------------- + +def run_folder(name: str, root: Optional[str] = None) -> str: + """The folder of the run *name* directly in the runs root, holding its + record; ValueError for any other path, a link included.""" + root = os.path.realpath(root or runs_root()) + path = os.path.join(root, name or "") + if not _NAME.fullmatch(name or "") or os.path.realpath(path) != path or not os.path.isfile(os.path.join(path, RECORD)): + raise ValueError(f"{name} is not a run Nullius made") + return path + + +def _run_deletion(name: str, home: str, root: Optional[str]) -> dict: + path = run_folder(name, root) + with open(os.path.join(path, RECORD), encoding="utf-8") as fh: + rec = json.load(fh) + ex = next((e for e in EXECUTIONS.values() if os.path.realpath(e.dir) == path), None) + state = ex.state if ex is not None else rec.get("state") + if state in ALIVE: + raise ValueError(f"{name} is {state}: stop it before you delete it") + plan = rec.get("plan") or {} + # a run refused at the name check never made the host folder: one of that name there is not its own + host = hosts.parse(plan["host"]) if plan.get("host") and (rec.get("launched") or rec.get("step") not in ("connect", "name")) else None + held = [] + for ws, roots in workspace.holding(home, path): + dirs = parse.find_job_dirs(roots) + ids = [j.job_id for j in workspace.load_jobs(ws, roots)] + # a job of the same id from another root shares these judgements: they stay + others = {e.get("job_id") for d, e in workspace.job_index(ws).items() if d not in dirs} + judged = judgements.job_dirs(ws.dir, [i for i in ids if i.split("~")[0] not in others]) + held.append({"ws": ws, "roots": roots, "jobs": dirs, + "judged": judged, "judgements": sum(len(os.listdir(d)) for d in judged)}) + size = sum(os.lstat(os.path.join(d, n)).st_size for d, _, names in os.walk(path) for n in names) + return {"name": name, "dir": path, "bytes": size, "host": host, "workspaces": held, + "judgements": sum(w["judgements"] for w in held)} + + +def deletion(names: List[str], home: str, root: Optional[str] = None) -> dict: + """What deleting the runs *names* of one launch removes: per run (``runs``) + its folder and size, the host copy that is its own, and per workspace that + holds it its roots, parsed jobs and judgement folders; the totals, and the + number of copies per host. ValueError when any of them is alive or is not + a run: nothing goes then.""" + runs = [_run_deletion(n, home, root) for n in dict.fromkeys(names)] + copies: Dict[str, int] = {} + for r in runs: + if r["host"]: + copies[r["host"].label] = copies.get(r["host"].label, 0) + 1 + return {"runs": runs, "bytes": sum(r["bytes"] for r in runs), "judgements": sum(r["judgements"] for r in runs), + "workspaces": list(dict.fromkeys(w["ws"].label for r in runs for w in r["workspaces"])), "hosts": copies} + + +def delete(names: List[str], home: str, root: Optional[str] = None, copies=(), run: Optional[Callable] = None) -> dict: + """Delete the runs *names* of one launch (``deletion`` says what goes): + per run its folder, its entry here, its roots, parsed jobs and judgements + in every workspace, and its copy on its host when that host's label is in + *copies*. Returns ``deletion``'s answer, with ``stays``: the runs whose + folder could not go (they keep everything else), and ``kept``: one line + per thing that stays.""" + gone = deletion(names, home, root) + gone["stays"], gone["kept"] = [], [] + for r in gone["runs"]: + try: + shutil.rmtree(r["dir"]) + except OSError as e: + gone["stays"].append(r["name"]) + gone["kept"].append(f"{r['name']} stays in {r['dir']}: {e.strerror or e}") + continue + for key in [k for k, e in EXECUTIONS.items() if os.path.realpath(e.dir) == r["dir"]]: + del EXECUTIONS[key] + for w in r["workspaces"]: + for d in w["judged"]: + shutil.rmtree(d, ignore_errors=True) + # read again: another run of the launch may have left this workspace already + ws = workspace.load(home, w["ws"].name) + if ws is not None: + workspace.forget(ws, w["roots"], w["jobs"]) + host = r["host"] + if host and host.label in copies: + rc, out = (run or subprocess_run)(host.argv(f"rm -rf {_rq(REMOTE_HOME + '/runs/' + r['name'])}", timeout=15), 60) + if rc != 0: + gone["kept"].append(f"the copy of {r['name']} on {host.label} stays: " + redact.redact(out.strip()[-200:])) + return gone diff --git a/app/nullius/judge/store.py b/app/nullius/judge/store.py index b24b873d..b484047c 100644 --- a/app/nullius/judge/store.py +++ b/app/nullius/judge/store.py @@ -103,6 +103,11 @@ def for_job(ws_dir: str, job_id: str, version: Optional[str] = None, lane: str = return out +def job_dirs(ws_dir: str, job_ids) -> list: + """The folders that hold these jobs' judgements, in every lane.""" + return [d for lane in ROOTS for j in job_ids if os.path.isdir(d := os.path.join(_root(ws_dir, lane), _SAFE.sub("_", j)))] + + def all_records(ws_dir: str, job_id: Optional[str] = None, lane: str = "system-one"): """Every stored record, every version: yields (job_id, trial_name, version, record). One job's when `job_id` is given.""" diff --git a/app/nullius/workspace.py b/app/nullius/workspace.py index f28ea682..190af5be 100644 --- a/app/nullius/workspace.py +++ b/app/nullius/workspace.py @@ -227,6 +227,34 @@ def delete(home_dir: str, name: str) -> bool: return True +def holding(home_dir: str, folder: str) -> list: + """Every workspace with a root in *folder*: [(workspace, those roots)].""" + inside = os.path.join(os.path.realpath(folder), "") + found = [] + for ws in list_all(home_dir): + roots = [r for r in ws.roots if os.path.join(os.path.realpath(r), "").startswith(inside)] + if roots: + found.append((ws, roots)) + return found + + +def forget(ws: Workspace, roots, job_dirs) -> None: + """*ws* without *roots*, and without its parsed jobs of *job_dirs*.""" + ws.roots = [r for r in ws.roots if r not in roots] + ws.job_roots = [r for r in ws.job_roots if r not in roots] + save(ws) + index = job_index(ws) + for d in job_dirs: + path = _cache_file(ws, d) + with _DECODED_LOCK: + _DECODED.pop(path, None) + try: + os.remove(path) + except OSError: + pass + index.pop(d, None) + _write_index(ws, index) + def signature(job_dir: str) -> str: """What the parse of *job_dir* depends on: the files it reads, by size diff --git a/app/static/notifications.js b/app/static/notifications.js index d9e79bef..d2c694f9 100644 --- a/app/static/notifications.js +++ b/app/static/notifications.js @@ -43,7 +43,8 @@ new MutationObserver(() => { const tray = document.querySelector('.toasts'); if (tray && tray.parentElement !== host()) { - host().append(tray); if (tray.showPopover && !tray.matches(':popover-open')) tray.showPopover(); + host().append(tray); tray.setAttribute('popover', 'manual'); // base.html's flashed tray has none + if (tray.showPopover && !tray.matches(':popover-open')) tray.showPopover(); } }).observe(document.body, {subtree: true, attributes: true, attributeFilter: ['open']}); })(); diff --git a/app/static/runs/launches.css b/app/static/runs/launches.css index e32156fc..89a5ac87 100644 --- a/app/static/runs/launches.css +++ b/app/static/runs/launches.css @@ -68,6 +68,11 @@ .la-mix span + span { box-shadow: inset 1.5px 0 0 var(--surface); } .la-go { text-align: right; white-space: nowrap; font-size: 12.5px; } .launch-foot { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 10px 18px; border-top: 1px solid var(--border-soft); font-size: 12.5px; color: var(--text-dim); } +/* a launch's Delete asks in the Add dialog's frame (catalogue.css, dialog.std-how), narrower */ +dialog.std-how.launch-delete { width: min(520px, calc(100vw - 32px)); } +.launch-delete ul { margin: 12px 0 0; padding-left: 1.2em; } +.launch-delete label { display: block; margin: 4px 0; overflow-wrap: anywhere; } +.launch-delete-acts { display: flex; flex-wrap: wrap; justify-content: flex-end; gap: 10px; margin-top: 20px; } @media (max-width: 900px) { .launch-agents .lh-more { display: none; } } @media (max-width: 640px) { diff --git a/app/static/style.css b/app/static/style.css index 2ac1c745..cc8ddce9 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -1614,6 +1614,8 @@ pre.instruction { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 32 /* what a run is doing: a stepper of its stages, then its units. A step is a mark, the line to the next step, its name and its time; done reads filled and checked, the running one ringed, the rest numbered. Stacked where a row would crush the names. */ +.execution-line { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; } +.run-delete { margin-left: auto; color: var(--text-dim); } .execution-stages { container: xs / inline-size; margin: 14px 0 8px; } .xs-list { --mark: 1.6rem; list-style: none; margin: 0; padding: 0; display: flex; align-items: flex-start; } .xs { position: relative; flex: 1 1 0; min-width: 0; display: flex; flex-direction: column; align-items: flex-start; gap: .1rem; padding-right: .9rem; font-size: .82rem; line-height: 1.3; color: var(--text-dim); } diff --git a/app/templates/runs/_execution.html b/app/templates/runs/_execution.html index 738589e4..155cf039 100644 --- a/app/templates/runs/_execution.html +++ b/app/templates/runs/_execution.html @@ -5,7 +5,8 @@ Stop and Continue answered 403. #} {# `own`: this job's own (state, finished) when the run is of several agents - an arm that answered everything is done though a later arm failed; the run's state is said beside it. #} -{% macro execution_strip(ex, csrf, waiting=false, own=none) %} +{# `deletable`: the key of this job's launch when the Runs page offers its Delete (web/runs._deletable). #} +{% macro execution_strip(ex, csrf, waiting=false, own=none, deletable=false) %} {% set shown = own[0] if own else ex.state %}
@@ -15,6 +16,8 @@ {% if ex.error and not own %}{{ ex.error }}{% endif %} · the run: {{ ex.state }}{% if ex.error %} · {{ ex.error }}{% endif %} {% if ex.state in ('preparing', 'running', 'paused') %}{% endif %} + {#- a launch goes as one, from the Runs page: never this job alone #} + {% if deletable %}Delete its launch…{% endif %} {# an agent's run failed with more to come: the question, and the answer #}
diff --git a/app/templates/runs/index.html b/app/templates/runs/index.html index b019f3e0..46c29380 100644 --- a/app/templates/runs/index.html +++ b/app/templates/runs/index.html @@ -1,7 +1,8 @@ {% from 'runs/_runtime.html' import what_ran %} {% extends "base.html" %} {% block title %}Runs — Nullius{% endblock %} -{% block head %}{% endblock %} +{% block head %} +{% endblock %} {# Value filter for one column: a caret in the header opening a checkbox list. Values start all-checked; unchecking hides the matching rows. #} @@ -21,6 +22,9 @@ {% endmacro %} +{#- A row's name as the launch's table shows it: the control, the agent, else its job id past the launch's prefix. #} +{% macro row_name(r, L) %}{% set j = r.job %}{{ r.control or (j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id) }}{% endmacro %} + {#- One row of a launch's table: a task check (`kind` 'control') or an agent; `part` 'fold' for a participant's shards as one row, 'shard' for one of them, '' for a run of one job. #} {% macro launch_row(r, g, kind, part='', first=False) %}{% set L = g.launch %}{% set j = r.job %}{% set v = r.verdict %} @@ -34,7 +38,7 @@ {%- if kind == 'control' %} {%- if part == 'fold' %}{{ r.control }}{% else %}{{ r.control }}{% endif %} task check {%- else %} - {%- set name = j.agent_label if j and j.agent_label else r.job_id[L.prefix|length - 1:] if L.prefix.endswith('*') and r.job_id.startswith(L.prefix[:-1]) else r.job_id %} + {%- set name = row_name(r, L) %} {%- if part == 'fold' %}{{ name }}{% else %}{{ name }}{% endif %} {%- if j and j.runner %} {{ what_ran(j) }}{% elif j and j.agent_version %} {{ j.agent_version }}{% endif %} {%- endif %} @@ -127,7 +131,7 @@ controls), then its agents on one 0-100% scale, then the runs its record named that Harbor has not written yet. -#}
{% for g in launches %}{% set L = g.launch %} -
+
{% set by_runner = g.agents|map(attribute='job')|select|map(attribute='runner')|select|first %} {% set words = g.agents|map(attribute='job')|select|map(attribute='runner_items')|select|first %}
@@ -162,10 +166,46 @@

{{ g.dataset_title if L.dataset else (L.name or g.dataset_title) }}{% if g.n {% if g.recipe %}Open this launch's recipe{% endif %} {% if g.compare|length >= 2 %}Compare these {{ g.compare|length }} {% elif g.agents %}Compare needs two finished agents{% endif %} + {#- every row is of runs Nullius made: the launch goes as one, its shards with it, asked first in the dialog below #} + {% if g.gone %}{% set gone, rows = g.gone, g.controls + g.agents %}{% set n, wss, k = gone.judgements, gone.workspaces, gone.runs|length %} + + {% endif %}

{% endfor %} + {#- one dialog for every launch: its Delete fills it from the launch's template; ?delete= opens it on load #} +
+
diff --git a/app/templates/runs/job.html b/app/templates/runs/job.html index 448b40d6..3100ee2f 100644 --- a/app/templates/runs/job.html +++ b/app/templates/runs/job.html @@ -32,7 +32,7 @@
-{% if execution %}{{ execution_strip(execution, csrf, own=own if execution.plan.steps()|length > 1 else none) }}{% endif %} +{% if execution %}{{ execution_strip(execution, csrf, own=own if execution.plan.steps()|length > 1 else none, deletable=deletable) }}{% endif %} {% if job.is_preparing and not live_board and not ended %} {# Launched, not started: Harbor has written the configuration and nothing else. There is no result record and no feed to fold yet, so this page diff --git a/app/templates/runs/preparing.html b/app/templates/runs/preparing.html index 8175a6bc..004d6b45 100644 --- a/app/templates/runs/preparing.html +++ b/app/templates/runs/preparing.html @@ -10,5 +10,5 @@ {% elif ex.plan.runner == 'standard' %}

Started from the builder with Nullius. This page becomes the live board as soon as the install is done and the run starts.

{% else %}

Started from the builder with the {{ ex.plan.runner }} runner. This page becomes the live board as soon as this arm's questions are chosen, after the install.

{% endif %} -{{ execution_strip(ex, csrf, waiting=true, own=own) }} +{{ execution_strip(ex, csrf, waiting=true, own=own, deletable=deletable) }} {% endblock %} diff --git a/app/tests/test_run_delete.py b/app/tests/test_run_delete.py new file mode 100644 index 00000000..b4bd4108 --- /dev/null +++ b/app/tests/test_run_delete.py @@ -0,0 +1,164 @@ +"""Deleting a launch Nullius made, one run or a shard per machine: per run its +folder, its roots and parsed jobs in every workspace, the judgements of its +trials there, and its host copy when asked; never a launch with a run still +going, nor a folder that is not a run's.""" +from __future__ import annotations + +import json +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from nullius import execution, hosts, workspace +from nullius.judge import store + +from tests.test_workspace import _job_dir, _write_json + + +class RunDelete(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="nullius-rundel-") + self.addCleanup(shutil.rmtree, self.tmp, True) + self.root, self.home = os.path.join(self.tmp, "runs"), os.path.join(self.tmp, "ws") + patcher = patch.dict(execution.EXECUTIONS, clear=True); patcher.start(); self.addCleanup(patcher.stop) + self.jobs = self.run_("r1") + other = os.path.join(self.tmp, "elsewhere"); _job_dir(other, "job-b") + self.a = workspace.create(self.home, [self.jobs, other], label="a") + self.b = workspace.create(self.home, [self.jobs], label="b") + self.c = workspace.create(self.home, [other], label="c") + for ws in (self.a, self.b, self.c): + workspace.load_jobs(ws, ws.roots) # parsed, as a page leaves them + for job in ("job-a", "job-b"): + store.save(ws.dir, job, "t0", {"questions": "v", "answers": {"q": 1}}) + store.save(ws.dir, job, "t0", {"questions": "v", "answers": {"q": 1}}, lane="llm") + + def run_(self, name, state="done", host=None, step="run", job="job-a"): + plan = execution.Plan(name=name, script="true", host=hosts.parse(host) if host else None) + _write_json(os.path.join(self.root, name, execution.RECORD), {"plan": plan.to_dict(), "state": state, "step": step}) + jobs = os.path.join(self.root, name, "jobs"); _job_dir(jobs, job) + return jobs + + def test_it_asks_what_goes(self): + gone = execution.deletion(["r1"], self.home, self.root) + self.assertEqual((gone["bytes"], gone["judgements"], gone["workspaces"], gone["hosts"]), (gone["runs"][0]["bytes"], 4, ["a", "b"], {})) + gone = gone["runs"][0] + self.assertEqual(gone["dir"], os.path.realpath(os.path.join(self.root, "r1"))) + self.assertGreater(gone["bytes"], 0) + self.assertEqual(sorted(w["ws"].name for w in gone["workspaces"]), ["a", "b"]) + self.assertEqual(gone["judgements"], 4) # job-a's, in both lanes, in a and b + self.assertIsNone(gone["host"]) + + def test_it_deletes_the_folder_its_roots_and_its_judgements_only(self): + execution.delete(["r1"], self.home, self.root) + self.assertFalse(os.path.exists(os.path.join(self.root, "r1"))) + a, b, c = (workspace.load(self.home, n) for n in "abc") + self.assertEqual(a.roots, [os.path.join(self.tmp, "elsewhere")]); self.assertEqual(b.roots, []) + self.assertEqual(c.roots, self.c.roots) + for ws in (a, b): + self.assertEqual(store.job_dirs(ws.dir, ["job-a"]), []) + self.assertNotIn(os.path.join(self.jobs, "job-a"), workspace.job_index(ws)) + self.assertEqual([j.job_id for j in workspace.load_jobs(ws, ws.roots)], ["job-b"] if ws is a else []) + for ws in (a, c): + self.assertEqual(len(store.job_dirs(ws.dir, ["job-b"])), 2) # another root's judgements stay + self.assertEqual(len(store.job_dirs(c.dir, ["job-a"])), 2) # c never held the run + + def test_a_job_of_the_same_id_in_another_root_keeps_its_judgements(self): + twin = os.path.join(self.tmp, "twin"); _job_dir(twin, "job-a") + self.a.roots.append(twin); workspace.save(self.a); workspace.load_jobs(self.a, self.a.roots) + execution.delete(["r1"], self.home, self.root) + self.assertEqual(len(store.job_dirs(self.a.dir, ["job-a"])), 2) + self.assertEqual(store.job_dirs(self.b.dir, ["job-a"]), []) + + def test_a_run_still_going_is_refused(self): + for state in execution.ALIVE: + with self.subTest(state=state): + self.run_("live", state=state) + with self.assertRaisesRegex(ValueError, f"live is {state}: stop it"): + execution.delete(["r1", "live"], self.home, self.root) + self.assertTrue(os.path.isdir(os.path.join(self.root, "live"))) + self.assertTrue(os.path.isdir(os.path.join(self.root, "r1"))) # nothing of the launch goes + + def test_the_registry_s_state_wins_over_the_record_s(self): + ex = execution.Execution(execution.Plan(name="r1", script="true"), root=self.root, record={"state": "done"}) + ex.state = "running"; execution.EXECUTIONS[ex.id] = ex + with self.assertRaisesRegex(ValueError, "is running"): + execution.delete(["r1"], self.home, self.root) + ex.state = "done" + execution.delete(["r1"], self.home, self.root) + self.assertNotIn(ex.id, execution.EXECUTIONS) + + def test_only_a_run_folder_in_the_runs_root_is_deleted(self): + outside = os.path.join(self.tmp, "outside"); _write_json(os.path.join(outside, execution.RECORD), {"plan": {}, "state": "done"}) + os.symlink(outside, os.path.join(self.root, "link")) + os.makedirs(os.path.join(self.root, "bare")) # no record: not a run + for name in ("../outside", "link", "bare", "", ".", "r1/jobs", os.path.join(self.tmp, "outside")): + with self.subTest(name=name), self.assertRaisesRegex(ValueError, "is not a run Nullius made"): + execution.delete([name], self.home, self.root) + self.assertTrue(os.path.isfile(os.path.join(outside, execution.RECORD))) + + def test_the_host_copy_goes_when_asked(self): + self.run_("far", host="ubuntu@10.0.0.4") + calls = [] + execution.delete(["far"], self.home, self.root, copies={"ubuntu@10.0.0.4"}, run=lambda argv, timeout: calls.append(argv) or (0, "")) + self.assertEqual(calls[0][-2:], ["ubuntu@10.0.0.4", 'rm -rf "$HOME/.nullius/runs/far"']) + self.run_("far", host="ubuntu@10.0.0.4"); calls.clear() + execution.delete(["far"], self.home, self.root, run=lambda argv, timeout: calls.append(argv) or (0, "")) + self.assertEqual(calls, []) + + def test_a_host_that_does_not_answer_keeps_its_copy_and_says_so(self): + self.run_("far", host="ubuntu@10.0.0.4") + gone = execution.delete(["far"], self.home, self.root, copies={"ubuntu@10.0.0.4"}, + run=lambda argv, timeout: (255, "ssh: connect to host 10.0.0.4: No route")) + self.assertEqual(gone["kept"], ["the copy of far on ubuntu@10.0.0.4 stays: ssh: connect to host 10.0.0.4: No route"]) + self.assertFalse(os.path.exists(os.path.join(self.root, "far"))) + + def test_a_run_refused_at_the_name_check_leaves_the_host_s_folder(self): + self.run_("far", state="failed", host="ubuntu@10.0.0.4", step="name") # that folder on the host is another run's + self.assertEqual(execution.deletion(["far"], self.home, self.root)["hosts"], {}) + + def test_a_sharded_launch_goes_whole_each_copy_from_its_own_host(self): + roots = [self.run_(f"s{i}", host=f"ubuntu@10.0.0.{3 + i}", job=f"L-shard-{i}of2") for i in (1, 2)] + ws = workspace.create(self.home, roots, label="d"); workspace.load_jobs(ws, ws.roots) + for i in (1, 2): + store.save(ws.dir, f"L-shard-{i}of2", "t0", {"questions": "v", "answers": {"q": 1}}) + asked = execution.deletion(["s1", "s2"], self.home, self.root) + self.assertEqual((asked["hosts"], asked["workspaces"]), ({"ubuntu@10.0.0.4": 1, "ubuntu@10.0.0.5": 1}, ["d"])) + calls = [] + gone = execution.delete(["s1", "s2"], self.home, self.root, copies={"ubuntu@10.0.0.4", "ubuntu@10.0.0.5"}, + run=lambda argv, timeout: calls.append(argv[-2:]) or (0, "")) + self.assertEqual(calls, [["ubuntu@10.0.0.4", 'rm -rf "$HOME/.nullius/runs/s1"'], ["ubuntu@10.0.0.5", 'rm -rf "$HOME/.nullius/runs/s2"']]) + self.assertEqual((gone["stays"], gone["kept"]), ([], [])) + for name in ("s1", "s2"): + self.assertFalse(os.path.exists(os.path.join(self.root, name))) + self.assertEqual(workspace.load(self.home, "d").roots, []) # both runs left it, neither wrote it back + self.assertEqual(store.job_dirs(ws.dir, ["L-shard-1of2", "L-shard-2of2"]), []) + + def test_a_launch_with_one_shard_still_going_is_refused_whole(self): + self.run_("s1", host="ubuntu@10.0.0.4"); self.run_("s2", state="running", host="ubuntu@10.0.0.5") + calls = [] + with self.assertRaisesRegex(ValueError, "^s2 is running: stop it before you delete it$"): + execution.delete(["s1", "s2"], self.home, self.root, copies={"ubuntu@10.0.0.4"}, run=lambda argv, timeout: calls.append(argv)) + self.assertEqual(calls, []) + self.assertTrue(all(os.path.isdir(os.path.join(self.root, n)) for n in ("s1", "s2"))) + + def test_a_folder_that_cannot_go_is_said_and_its_run_kept_whole(self): + self.run_("s1", host="ubuntu@10.0.0.4"); self.run_("s2", host="ubuntu@10.0.0.5") + real = shutil.rmtree + def rmtree(path, *a, **kw): + if os.path.basename(path) == "s2": + raise PermissionError(13, "Permission denied", path) + return real(path, *a, **kw) + calls = [] + with patch.object(execution.shutil, "rmtree", rmtree): + gone = execution.delete(["s1", "s2"], self.home, self.root, copies={"ubuntu@10.0.0.4", "ubuntu@10.0.0.5"}, + run=lambda argv, timeout: calls.append(argv[-2]) or (0, "")) + s2 = os.path.realpath(os.path.join(self.root, "s2")) + self.assertEqual((gone["stays"], gone["kept"]), (["s2"], [f"s2 stays in {s2}: Permission denied"])) + self.assertEqual(calls, ["ubuntu@10.0.0.4"]) # s2's host copy stays with it + self.assertFalse(os.path.exists(os.path.join(self.root, "s1"))); self.assertTrue(os.path.isdir(s2)) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/ui_tests/test_run_routes.py b/app/ui_tests/test_run_routes.py index d608816f..ac07f336 100644 --- a/app/ui_tests/test_run_routes.py +++ b/app/ui_tests/test_run_routes.py @@ -17,7 +17,7 @@ from pathlib import Path from unittest.mock import patch -from nullius import execution, workspace +from nullius import execution, hosts, workspace from nullius.runs import parse from .test_routes import web @@ -508,6 +508,149 @@ def test_delete(self): self.assertIn("No workspace called", again.get_data(as_text=True)) +class RunDelete(unittest.TestCase): + """Delete on a launch in Runs: asked in a dialog, with every run of the launch + and what else goes, its shards with it; the job page only links to it; a + form token and runs that have ended are needed; the launch leaves every page.""" + def setUp(self): + from nullius.judge import store + from tests.test_workspace import _job_dir, _write_json + self._job_dir, self._write_json = _job_dir, _write_json + self.stack = ExitStack(); self.addCleanup(self.stack.close) + self.tmp = Path(tempfile.mkdtemp(prefix="nullius-rundel-")); self.addCleanup(shutil.rmtree, self.tmp, True) + self.home = str(self.tmp / "ws") + self.stack.enter_context(patch.dict(os.environ, {"RUNS_DIR": str(self.tmp / "runs")})) + self.stack.enter_context(patch.dict(web.app.config, TESTING=True)) + self.stack.enter_context(patch.dict(execution.EXECUTIONS, clear=True)) + self.stack.enter_context(patch.object(web.shared, "WS_HOME", self.home)) + self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []})) + self.run_dir = self.tmp / "runs" / "r1" + _write_json(str(self.run_dir / "run.json"), {"plan": execution.Plan(name="r1", script="true", jobs=[{"name": "r1-a"}, {"name": "r1-b"}]).to_dict(), "state": "done", "step": "run"}) + _job_dir(str(self.run_dir / "jobs"), "r1-a"); _job_dir(str(self.run_dir / "jobs"), "r1-b") + execution.restore() + self.ex = next(iter(execution.EXECUTIONS.values())) + self.ws = workspace.create(self.home, [str(self.run_dir / "jobs")], label="witty pelican") + workspace.create(self.home, [str(self.run_dir / "jobs")], label="sharp lemur") + store.save(self.ws.dir, "r1-a", "t0", {"questions": "v", "answers": {}}) + web.shared.open_workspace(self.ws) + self.client = web.app.test_client() + r = self.client.get("/workspaces"); self.addCleanup(r.close) + with self.client.session_transaction() as s: + self.csrf = s["csrf"] + + def delete(self, key=None, csrf=None, hosts=()): + r = self.client.post(f"/launch/{key or 'run:' + self.ex.id}/delete", data={"csrf": self.csrf if csrf is None else csrf, "host": list(hosts)}, + headers={"Origin": "http://localhost"}) + self.addCleanup(r.close) + return r + + def get(self, path): + r = self.client.get(path); self.addCleanup(r.close) + return r + + def sharded(self, foreign=False): + """A launch of two shards, each a run of its own on a host of its own; with *foreign*, a third shard from elsewhere.""" + from tests.test_standard_shard import write_shard + roots = [] + for i in (1, 2): + d = self.tmp / "runs" / f"s{i}" + plan = execution.Plan(name=f"s{i}", script="true", host=hosts.parse(f"ubuntu@10.0.0.{3 + i}"), jobs=[{"name": f"L1-shard-{i}of3"}]) + self._write_json(str(d / "run.json"), {"plan": plan.to_dict(), "state": "done", "step": "run"}) + write_shard(str(d / "jobs"), f"L1-shard-{i}of3", [f"u-{i}"], i, 3, total=3) + roots.append(str(d / "jobs")) + if foreign: + write_shard(str(self.tmp / "elsewhere"), "L1-shard-3of3", ["u-3"], 3, 3, total=3) + roots.append(str(self.tmp / "elsewhere")) + execution.restore() + key = "run:" + next(e.id for e in execution.EXECUTIONS.values() if e.plan.name == "s1") + web.shared.open_workspace(workspace.create(self.home, roots, label="calm otter")) + return key + + def test_the_launch_asks_what_goes(self): + html = self.get("/runs?view=launch").get_data(as_text=True) + self.assertIn(f'action="/launch/run:{self.ex.id}/delete"', html) + self.assertIn(f'', html) + self.assertIn('', html) + self.assertEqual(html.count(''), 1) + said = re.sub(r"<[^>]+>", " ", re.search(r'', html, re.S).group(1)) + self.assertRegex(" ".join(said.split()), r"^Delete launch r1-\* \S+ \S+ 1 folder · \d+(\.\d)? (Bytes|kB) · 1 judgement · " + r"workspaces (witty pelican, sharp lemur|sharp lemur, witty pelican) Keep it Delete launch$") + self.assertIn('', html) + self.assertNotIn('name="host"', html) # ran here: no host copy + self.assertNotIn("Delete launch", self.get("/runs?view=all").get_data(as_text=True).split('id="view-all"')[1]) # rows there are jobs + + def test_the_job_page_links_to_its_launch_and_the_link_opens_the_question(self): + html = self.get("/job/r1-a").get_data(as_text=True) + link = f"/runs?view=launch&delete=run:{self.ex.id}#run-{self.ex.id}" + self.assertIn(f'href="{link}">Delete its launch…', html) + self.assertNotIn("/delete\"", html) # never this job alone + runs = self.get(link.replace("&", "&").split("#")[0]).get_data(as_text=True) + self.assertIn(f'id="run-{self.ex.id}"', runs) + self.assertIn("launch-delete-btn", runs.split(f'id="run-{self.ex.id}"')[1].split("
")[0]) # the button ?delete= opens the dialog from + + def test_a_sharded_launch_goes_whole_each_copy_from_its_own_host(self): + key = self.sharded() + html = self.get("/runs?view=launch").get_data(as_text=True) + self.assertEqual(html.count('class="launch"'), 1) + said = re.sub(r"<[^>]+>", " ", re.search(r'', html, re.S).group(1)) + self.assertRegex(" ".join(said.split()), r"^Delete launch L1 \S+ \(2 shards\) 2 folders · [^·]+ · workspace calm otter Its copy on") + for h in ("ubuntu@10.0.0.4", "ubuntu@10.0.0.5"): + self.assertIn(f' Its copy on {h}', html) + self.assertIn(f"delete={key}", self.get("/job/L1-shard-2of3").get_data(as_text=True)) + calls = [] + with patch.object(execution, "subprocess_run", lambda argv, timeout: calls.append(argv[-2]) or (0, "")): + r = self.delete(key, hosts=["ubuntu@10.0.0.4", "ubuntu@10.0.0.5"]) + self.assertEqual(sorted(calls), ["ubuntu@10.0.0.4", "ubuntu@10.0.0.5"]) + self.assertFalse((self.tmp / "runs" / "s1").exists()); self.assertFalse((self.tmp / "runs" / "s2").exists()) + self.assertIn('Deleted launch L1.', self.get(r.headers["Location"]).get_data(as_text=True)) + + def test_a_host_that_does_not_answer_is_said_in_the_notice(self): + key = self.sharded() + with patch.object(execution, "subprocess_run", lambda argv, timeout: (255, "ssh: connect to host: No route") if argv[-2].endswith(".5") else (0, "")): + r = self.delete(key, hosts=["ubuntu@10.0.0.4", "ubuntu@10.0.0.5"]) + self.assertIn('Deleted launch L1; the copy of s2 on ubuntu@10.0.0.5 stays: ssh: connect to host: No route.', + self.get(r.headers["Location"]).get_data(as_text=True)) + + def test_a_launch_with_a_job_from_elsewhere_has_no_delete(self): + key = self.sharded(foreign=True) + html = self.get("/runs?view=launch").get_data(as_text=True) + self.assertEqual(html.count('class="launch"'), 1) + self.assertNotIn("Delete launch", html) + self.assertNotIn("Delete its launch", self.get("/job/L1-shard-1of3").get_data(as_text=True)) + r = self.delete(key) + self.assertIn("L1 has jobs Nullius did not run: it cannot be deleted here.", self.get(r.headers["Location"]).get_data(as_text=True)) + self.assertTrue((self.tmp / "runs" / "s1").is_dir()) + + def test_a_launch_with_a_run_still_going_has_no_delete_and_is_refused(self): + key = self.sharded() + next(e for e in execution.EXECUTIONS.values() if e.plan.name == "s2").state = "running" + self.assertNotIn("Delete launch", self.get("/runs?view=launch").get_data(as_text=True)) + self.assertNotIn("Delete its launch", self.get("/job/L1-shard-1of3").get_data(as_text=True)) + r = self.delete(key) + self.assertEqual(r.status_code, 302) + self.assertIn("s2 is running: stop it before you delete it", self.get(r.headers["Location"]).get_data(as_text=True)) + self.assertTrue((self.tmp / "runs" / "s1").is_dir()) + + def test_a_form_token_is_needed(self): + self.assertEqual(self.delete(csrf="").status_code, 403) + self.assertEqual(self.delete(csrf="stale").status_code, 403) + self.assertTrue(self.run_dir.is_dir()) + + def test_delete_removes_the_launch_from_every_page(self): + r = self.delete() + self.assertEqual((r.status_code, r.headers["Location"]), (302, "/runs?view=launch")) + notice = self.client.get(r.headers["Location"], follow_redirects=True); self.addCleanup(notice.close) + self.assertIn('Deleted launch r1-*.', notice.get_data(as_text=True)) + self.assertFalse(self.run_dir.exists()) + self.assertEqual(web.shared.STATE["workspace"].roots, []); self.assertEqual(web.shared.STATE["job_roots"], []) + self.assertTrue(all(w.roots == [] for w in workspace.list_all(self.home))) + self.assertFalse((Path(self.ws.dir) / "judgements" / "r1-a").exists()) + self.assertEqual(self.get("/job/r1-a").status_code, 404) + self.assertEqual(self.delete().status_code, 404) # gone from every launch too + runs = self.client.get("/runs?view=launch", follow_redirects=True); self.addCleanup(runs.close) + self.assertEqual(runs.status_code, 200); self.assertNotIn("r1-a", runs.get_data(as_text=True)) + + if __name__ == "__main__": unittest.main() diff --git a/app/ui_tests/test_runs_by_launch.py b/app/ui_tests/test_runs_by_launch.py index 967818f0..08320799 100644 --- a/app/ui_tests/test_runs_by_launch.py +++ b/app/ui_tests/test_runs_by_launch.py @@ -145,8 +145,8 @@ def test_a_part_held_back_by_a_pause_or_an_ended_run_reads_not_run(self): def test_a_later_paused_shard_has_a_resume_link(self): from web.runs import _launch_view group = SimpleNamespace(job_ids=[], pending=["first", "second"], source="record", dataset="") - first = SimpleNamespace(state="done", plan=SimpleNamespace(recipe={}), refused=lambda _: None, never_ran=lambda _: False) - second = SimpleNamespace(state="paused", plan=SimpleNamespace(recipe={}), live_state=lambda _: "paused") + first = SimpleNamespace(state="done", plan=SimpleNamespace(recipe={}, jobs=[]), refused=lambda _: None, never_ran=lambda _: False) + second = SimpleNamespace(state="paused", plan=SimpleNamespace(recipe={}, jobs=[]), live_state=lambda _: "paused") with patch.object(execution, "for_job", side_effect=lambda key: {"first": first, "second": second}[key]), \ patch("web.runs._control_planned", return_value=""): self.assertEqual(_launch_view(group, {}, {}, {})["paused"], "second") diff --git a/app/web/runs.py b/app/web/runs.py index 9ae2e702..e1220e3d 100644 --- a/app/web/runs.py +++ b/app/web/runs.py @@ -25,6 +25,7 @@ from .core import app from . import shared +from . import settings as settings_web @@ -38,15 +39,9 @@ def home(): def index(): if shared.STATE["workspace"] is None: return render_template("runs/start.html", workspaces=workspace.list_all(shared.WS_HOME)) - jobs = shared._load_jobs() + jobs, have, mine, grouped = _grouped() # Runs started from the builder whose job Harbor has not written yet: # listed above the table, each a link to the page that follows it. - have = {j.job_id for j in jobs} - # The run records are the app's, not the workspace's: only those launched - # into this workspace's roots, or with a job already written here, belong. - mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have)] - merged_sources = launches.merged_sources(jobs) - jobs = [j for j in jobs if j.job_id not in merged_sources] preparing = [(jid, e) for e in mine if e.state in execution.ALIVE for jid in e.summary()["job_ids"] if jid not in have] # a launch whose runs wrote no job yet (or never will: refused, stopped) is still shown if not jobs and not mine: @@ -91,13 +86,8 @@ def index(): "model": sorted({(j.model_name, j.model_name) for j in jobs}), "agent": sorted({(j.agent_label, j.agent_label or "—") for j in jobs}), } - # The page's first view: the runs grouped by the launch that made them - # (nullius.runs.launches) - the builder's run record, else the same dataset - # within half an hour - each with its task checks and its agents side by side. - records = {e.id: (e.plan.name, [str(x.get("name")) for x in e.plan.jobs - if str(x.get("name")) not in merged_sources], e.started) for e in mine} by_id = {j.job_id: j for j in jobs} - groups = [_launch_view(g, by_id, verdicts, ended) for g in launches.group(jobs, records)] + groups = [_launch_view(g, by_id, verdicts, ended) for g in grouped] view = request.args.get("view") if request.args.get("view") in ("launch", "all") else "launch" states = {} for g in groups: @@ -112,6 +102,22 @@ def index(): ) +def _grouped(): + """The open workspace's jobs (less those a merge stands for), the ids of all + it parsed, the run records that belong to it, and the launches By launch + shows: the runs grouped by the launch that made them (nullius.runs.launches) + - the builder's run record, else the same dataset within half an hour.""" + jobs = shared._load_jobs() + have = {j.job_id for j in jobs} + # The run records are the app's, not the workspace's: only those launched + # into this workspace's roots, or with a job already written here, belong. + mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have)] + merged = launches.merged_sources(jobs) + jobs = [j for j in jobs if j.job_id not in merged] + records = {e.id: (e.plan.name, [str(x.get("name")) for x in e.plan.jobs if str(x.get("name")) not in merged], e.started) for e in mine} + return jobs, have, mine, launches.group(jobs, records) + + def _launched_here(ex, have) -> bool: """A run record belongs to the open workspace when its job root is one of the workspace's roots (a Run adopts it there) or one of its jobs is here.""" @@ -283,7 +289,8 @@ def _launch_view(g, by_id, verdicts, ended) -> dict: for r in leaves(controls + rows): counts[r["state"]] = counts.get(r["state"], 0) + 1 ex = execution.for_job((g.job_ids + g.pending)[0]) - return {"launch": g, "checks": checks, "verdict": verdict, + return {"gone": _deletion(_runs_of(g)), + "launch": g, "checks": checks, "verdict": verdict, "controls": controls, "checked": checked, "agents": rows, "n_tasks": n_tasks, "compare": finished, "paused": next((i for i in g.job_ids + g.pending @@ -466,14 +473,73 @@ def job(job_id): # Started from the builder and not written by Harbor yet: the run's # own page, which becomes this one when the job appears. if ex is not None and request.method == "GET": - return render_template("runs/preparing.html", job_id=job_id, ex=ex, csrf=shared._csrf_token(), own=own) + return render_template("runs/preparing.html", job_id=job_id, ex=ex, csrf=shared._csrf_token(), own=own, deletable=_deletable(job_id, ex)) abort(404) from . import judge board = _live_board_state(j) return render_template("runs/job.html", job=j, live_board=board, view=_view_of(board, j), execution=ex, csrf=shared._csrf_token(), judgement=judge.job_summary(j) if j.trials else None, - control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own) + control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own, deletable=_deletable(job_id, ex)) + + +def _runs_of(g) -> list: + """The runs Nullius made that hold every row of the launch *g*, and no job + outside it; [] when a row is not one of theirs.""" + ids = g.job_ids + g.pending + runs = list({id(e): e for e in map(execution.for_job, ids)}.values()) + if None in runs or any(not {str(x.get("name")) for x in e.plan.jobs} <= set(ids) for e in runs): + return [] + return runs + + +def _deletion(runs): + """What deleting these runs removes, for the question Delete asks; None while they cannot all be deleted.""" + if not runs or any(e.state in execution.ALIVE for e in runs): + return None + try: + return execution.deletion([e.plan.name for e in runs], shared.WS_HOME, runs[0].root) + except (OSError, ValueError): + return None + + +def _deletable(job_id, ex): + """The key of this job's launch when the Runs page offers its Delete, else None.""" + if ex is None or ex.state in execution.ALIVE: + return None + g = next((g for g in _grouped()[3] if job_id in g.job_ids + g.pending), None) + return g.key if g is not None and _deletion(_runs_of(g)) else None + + +@app.route("/launch//delete", methods=["POST"]) +def launch_delete(key): + """Delete a launch that has ended: every run of it (``execution.delete``), + each shard's copy on its own host when ticked. A launch with a run still + going, or with a job from elsewhere, is refused. The launch's block on + the Runs page asks first, in a dialog.""" + shared._require_csrf() + if not settings_web._local_client(): + abort(403) + if shared.STATE["workspace"] is None: + abort(404) + g = next((g for g in _grouped()[3] if g.key == key), None) + if g is None: + abort(404) + runs = _runs_of(g) + if not runs: + flash(f"{g.title} has jobs Nullius did not run: it cannot be deleted here.", "error") + return redirect(url_for("index", view="launch")) + try: + gone = execution.delete([e.plan.name for e in runs], shared.WS_HOME, runs[0].root, copies=set(request.form.getlist("host"))) + except (OSError, ValueError) as e: + flash(str(e), "error") + return redirect(url_for("index", view="launch")) + ws = shared.STATE["workspace"] + if any(w["ws"].name == ws.name for r in gone["runs"] for w in r["workspaces"]): + shared.open_workspace(workspace.load(shared.WS_HOME, ws.name)) # read without the runs' roots from now on + said = Markup('{} launch {}').format("Deleted part of" if gone["stays"] else "Deleted", g.title) + flash(said + "".join(f"; {k}" for k in gone["kept"]) + ".", "error" if gone["stays"] else "") + return redirect(url_for("index", view="launch")) def _view_of(board, j=None) -> str | None: diff --git a/docs/builder.md b/docs/builder.md index a11e9bee..4dba8b3b 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -106,6 +106,7 @@ a declared model list keeps those published choices. | Proofs | Local runs save to the user catalogue. A host uses a writable copy in its run's `catalogue/`, isolated from the host's own entries; that copy and its proofs mirror back alongside `jobs/`, without changing the active user catalogue | | Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts | | Stop | Ends the current part (exit 130), records `stopped`; the next part never starts | +| Delete launch | On the launch in Runs, *By launch*, once every run of it has ended; asks first. The job page links to it. Deletes each run's `//` (a shard per machine included), every job of the launch, their judgements in every workspace, and each run's copy on its host when ticked. A run still going is refused: stop it first. A launch with jobs from elsewhere has no Delete | | Restart of Nullius | `execution.restore` reattaches from `runs//run.json`; the job goes back into the workspace it started in, while that workspace exists | | Cannot run from the page | Generated tasks and custom agents are not supported by Run | | Refused | Reads *refused* on the runs page, with why; the job was never written | diff --git a/docs/cli/pages.md b/docs/cli/pages.md index 639ff2b0..79ea8ae4 100644 --- a/docs/cli/pages.md +++ b/docs/cli/pages.md @@ -38,6 +38,7 @@ Each page's command, or why there is none. | `/judge/…` | `judge.cli job --confirm` | | Rename or delete a workspace | `nullius workspace rename`, `nullius workspace delete --yes` | | `/new-run` | `recipes.cli`; today `POST /new-run/runner` | +| Delete a launch | `execution.delete`; today `POST /launch//delete` | ## None needed diff --git a/docs/judgement.md b/docs/judgement.md index c61f7bc9..8796dc21 100644 --- a/docs/judgement.md +++ b/docs/judgement.md @@ -69,6 +69,7 @@ Follow the evidence: a recorded rate limit is a provider's failure whatever the | What | Detail | | --- | --- | | Where | `workspaces//judgements//.json` below the user data directory (`WORKSPACES_DIR` can override it), never in the job tree | +| Deleted | With the launch they judge (*Delete launch* in Runs, [Run](builder.md#run)) | | Version | `judgement.questions.v2`; v1 records marked *earlier* | | Cost | A batch button names its calls; each is checked against the token budget (`batch.preflight`) | | Noise floors | `app/nullius/judge/floors.json`, each yes/no's run-to-run spread, measured once |