diff --git a/app/app.py b/app/app.py index e8ba24fe..66d72239 100644 --- a/app/app.py +++ b/app/app.py @@ -86,7 +86,7 @@ def main(argv=None) -> None: back = execution.restore() for ex in back: print(f"Run {ex.plan.name} picked up again on {ex.plan.host.label if ex.plan.host else 'this machine'} ({ex.state})") - shared.adopt_roots([ex.jobs]) # its job root back in a workspace, as when it started + shared.adopt_roots([ex.jobs], into=ex.workspace) # its job root back in the workspace it started in app.config["SERVED_HOST"] = args.host # a name the Host header may carry (shared._served_name) print(f"Serving on http://{args.host}:{args.port}") app.run(host=args.host, port=args.port, debug=args.debug) diff --git a/app/nullius/execution.py b/app/nullius/execution.py index 5f65c788..c38811fa 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -436,6 +436,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl self.tasks_on_target = rec.get("tasks_on_target", "") self.app_on_target = rec.get("app_on_target", "") # the standard's source tree or runner snapshot on the target self.catalogue_on_target = rec.get("catalogue_on_target", "") # writable definitions, separate from the immutable runner + self.workspace = rec.get("workspace", "") # the workspace its job was added to, the one a restart adds it back to # the trials the log has said already, (job, trial), so a restart says each once; none in a record from before self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()} if record: @@ -480,7 +481,7 @@ def _save(self) -> None: data = {"id": self.id, "plan": self.plan.to_dict(), "state": self.state, "step": self.step, "part": self.part, "pid": self.pid, "log_offset": self.log_offset, "exit_code": self.exit_code, "error": self.error, "question": self.question, "ends": self.ends, "started": self.started, "finished": self.finished, "harbor": self.harbor, "tasks_on_target": self.tasks_on_target, "app_on_target": self.app_on_target, - "catalogue_on_target": self.catalogue_on_target, + "catalogue_on_target": self.catalogue_on_target, "workspace": self.workspace, "launched": self.launched, "seen_trials": [list(k) for k in sorted(self._seen_trials, key=str)]} try: os.makedirs(self.dir, exist_ok=True) @@ -636,6 +637,11 @@ def start(self) -> "Execution": self.thread.start() return self + def file_under(self, workspace: str) -> None: + """Record the workspace the run's job went into.""" + self.workspace = workspace + self._save() + def resume(self) -> "Execution": """Pick a recorded run up where it was: the part that was running is followed again, a pause waits again, the rest runs on.""" diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index 356b1567..ae4d363e 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -48,7 +48,7 @@ def start(plan): self.started.append(plan) return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {}) self.stack.enter_context(patch.object(execution, "start", start)) - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None)) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None)) self.client = web.app.test_client() def test_examples_lead_catalogue_and_sidebar(self): @@ -338,7 +338,7 @@ def start(plan): self.started.append(plan) return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1"}) self.stack.enter_context(patch.object(execution, "start", start)) - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None)) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None)) self.client = web.app.test_client() r = self.client.get("/new-run"); self.addCleanup(r.close) self.schema = json.loads(re.search(r'', r.get_data(as_text=True), re.S).group(1)) @@ -428,7 +428,7 @@ def start(plan): self.started.append(plan) return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name}) self.stack.enter_context(patch.object(execution, "start", start)) - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None)) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None)) self.client = web.app.test_client() def get(self, url): diff --git a/app/ui_tests/test_harbor_routes.py b/app/ui_tests/test_harbor_routes.py index a4c436a7..1324c9bc 100644 --- a/app/ui_tests/test_harbor_routes.py +++ b/app/ui_tests/test_harbor_routes.py @@ -40,7 +40,7 @@ def start(plan): self.started.append(plan) return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1", "state": "preparing"}) self.stack.enter_context(patch.object(execution, "start", start)) - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None)) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None)) self.client = web.app.test_client() r = self.client.get("/new-run"); self.addCleanup(r.close) with self.client.session_transaction() as s: diff --git a/app/ui_tests/test_prerequisites_route.py b/app/ui_tests/test_prerequisites_route.py index 3eb90cc5..717f9361 100644 --- a/app/ui_tests/test_prerequisites_route.py +++ b/app/ui_tests/test_prerequisites_route.py @@ -29,7 +29,7 @@ def setUp(self): self.started = [] self.stack.enter_context(patch.object(execution, "start", lambda plan: self.started.append(plan) or SimpleNamespace( jobs=str(self.tmp), summary=lambda: {"id": "x", "state": "preparing"}))) - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None)) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None)) self.client = web.app.test_client() r = self.client.get("/new-run"); self.addCleanup(r.close) with self.client.session_transaction() as s: diff --git a/app/ui_tests/test_run_routes.py b/app/ui_tests/test_run_routes.py index d0ff0fa1..d608816f 100644 --- a/app/ui_tests/test_run_routes.py +++ b/app/ui_tests/test_run_routes.py @@ -78,7 +78,7 @@ def run(argv, timeout=600, stdin=None, stop=None): return 0, "" self.stack.enter_context(patch.object(execution, "subprocess_run", run)) self.stack.enter_context(patch.object(execution, "docker_here", lambda sh, needs="": None)) # docker answers here, as the probe says it does there - self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: self.adopted.extend(roots))) + self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: self.adopted.extend(ex.jobs for ex in runs))) self.adopted = [] self.client = web.app.test_client() r = self.client.get("/new-run"); self.addCleanup(r.close) @@ -347,6 +347,70 @@ def test_the_page_carries_the_token_the_stop_button_sends(self): self.assertEqual(self.client.post(f"/run/{r.json['id']}/stop", headers={"Origin": "http://localhost", "X-CSRF-Token": ""}).status_code, 403) +class ResumeIntoWorkspace(unittest.TestCase): + """A restart while a run started from the builder goes on: the run's job goes back into the workspace it started in.""" + def setUp(self): + self.stack = ExitStack(); self.addCleanup(self.stack.close) + self.tmp = Path(tempfile.mkdtemp(prefix="nullius-resume-")); self.addCleanup(shutil.rmtree, self.tmp, True) + self.home = str(self.tmp / "ws") + self.stack.enter_context(patch.dict(os.environ, {"RUNS_DIR": str(self.tmp / "runs"), "JOB_ROOTS": ""})) + self.stack.enter_context(patch.dict(web.app.config, TESTING=True)) + self.stack.enter_context(patch.dict(execution.EXECUTIONS, clear=True)) + self.stack.enter_context(patch.object(web.shared, "WS_HOME", self.home)) + self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []})) + self.stack.enter_context(patch.object(web.app, "run", lambda **kw: None)) # main() serves nothing + roots = [self.tmp / "a", self.tmp / "b"] + for r in roots: + r.mkdir() + self.held = [str(r) for r in roots] + self.ws = workspace.create(self.home, self.held, label="witty pelican") + web.shared.open_workspace(self.ws) + self.client = web.app.test_client() + r = self.client.get("/new-run"); self.addCleanup(r.close) + with self.client.session_transaction() as s: + self.csrf = s["csrf"] + # a run that pauses after its first part, so it is still alive when the app stops + parts = [{"id": "a", "label": "a", "script": "exit 2"}, {"id": "b", "label": "b", "script": "echo b"}] + r = self.client.post("/run", data=json.dumps({"name": "p", "script": "", "where": "local", "parts": parts}), content_type="application/json", + headers={"Origin": "http://localhost", "X-CSRF-Token": self.csrf}); self.addCleanup(r.close) + self.assertEqual(r.status_code, 200, r.get_data(as_text=True)) + self.jobs = str(self.tmp / "runs" / "p" / "jobs") + self.record = self.tmp / "runs" / "p" / "run.json" + for _ in range(250): + if json.loads(self.record.read_text())["state"] == "paused": + break + time.sleep(0.02) + ex = execution.EXECUTIONS[r.json["id"]] + ex._save = lambda: None # the app stops: the old thread writes no record again + execution.EXECUTIONS.clear() + web.shared.close_workspace() + + def restart(self): + web.main([]) + back = list(execution.EXECUTIONS.values()) + self.addCleanup(lambda: [e.stop() for e in back]) + return web.shared.STATE["workspace"] + + def test_a_restart_puts_the_run_back_into_its_workspace(self): + again = self.restart() + self.assertEqual(again.name, self.ws.name) + self.assertEqual([w.name for w in workspace.list_all(self.home)], [self.ws.name]) # no new workspace + self.assertIn(self.jobs, again.roots) + self.assertEqual(json.loads(self.record.read_text())["workspace"], self.ws.name) + + def test_a_run_whose_workspace_is_gone_gets_one_as_before(self): + workspace.delete(self.home, self.ws.name) + again = self.restart() + self.assertEqual(again.roots, [self.jobs]) + + def test_an_old_record_without_its_workspace_gets_one_as_before(self): + rec = json.loads(self.record.read_text()); rec.pop("workspace", None); self.record.write_text(json.dumps(rec)) + again = self.restart() + self.assertNotEqual(again.name, self.ws.name) + self.assertEqual(again.roots, [self.jobs]) + self.assertEqual(workspace.load(self.home, self.ws.name).roots, self.held + [self.jobs]) # added when it started, left as it is + + class Credentials(unittest.TestCase): def test_a_run_on_openai_is_given_openais_endpoint(self): """A Harbor part's script binds its clients' endpoint from API_URL and diff --git a/app/web/agents.py b/app/web/agents.py index a2e8efba..b791d254 100644 --- a/app/web/agents.py +++ b/app/web/agents.py @@ -158,7 +158,7 @@ def _check(kind: str, key: str): flash(problem, "error") return redirect(back) ex = execution.start(plan) - shared.adopt_roots([ex.jobs]) + shared.adopt_runs([ex]) n = len(got["jobs"]) flash(f"Checking {got['label']}'s ports: " + (f"{n} kits, one after the other." if n > 1 else "its one kit."), "ok") return redirect(url_for("job", job_id=got["jobs"][0]["name"])) diff --git a/app/web/catalogue.py b/app/web/catalogue.py index 2449c141..4c0fbde6 100644 --- a/app/web/catalogue.py +++ b/app/web/catalogue.py @@ -98,7 +98,7 @@ def benchmark_prove(key): flash(problem, "error") return redirect(back) ex = execution.start(plan) - shared.adopt_roots([ex.jobs]) + shared.adopt_runs([ex]) flash(f"Proving {got['label']}: its units and the leak check, then " + (f"{len(got['jobs'])} control runs." if got["jobs"] else "no control: it declares none."), "ok") return redirect(url_for("job", job_id=got["jobs"][0]["name"]) if got["jobs"] else back) diff --git a/app/web/execute.py b/app/web/execute.py index f07af346..6c61f26f 100644 --- a/app/web/execute.py +++ b/app/web/execute.py @@ -143,7 +143,7 @@ def _start_standard(recipe: dict): if os.path.exists(os.path.join(execution.runs_root(), plan.name)): return jsonify(ok=False, error=execution.name_taken(plan.name)), 409 started = [execution.start(plan) for plan in plans] - shared.adopt_roots([ex.jobs for ex in started]) + shared.adopt_runs(started) return jsonify(ok=True, open=url_for("index", view="launch"), runs=[ex.summary() for ex in started], **started[0].summary()) @@ -177,7 +177,7 @@ def _start(plan: execution.Plan): # The job lands (or is mirrored) under runs//jobs: the workspace # reads it from now on. The Runs page, by launch, shows it from the first # second: its runs queued, preparing, then running, each opening its job. - shared.adopt_roots([ex.jobs]) + shared.adopt_runs([ex]) return jsonify(ok=True, open=url_for("index", view="launch"), **ex.summary()) diff --git a/app/web/shared.py b/app/web/shared.py index 939d633c..3d72ffd2 100644 --- a/app/web/shared.py +++ b/app/web/shared.py @@ -317,17 +317,22 @@ def close_workspace(): STATE.update({"workspace": None, "roots": [], "label": "", "job_roots": []}) -def adopt_roots(roots, label="", new=False): +def adopt_roots(roots, label="", new=False, into=""): """Roots the user just pointed at: into the open workspace, or a new one. Without a label, roots that already have a workspace reopen it — so a restart with the same argv lands in the same place, cache and all. + ``into``: the workspace to add them to while it exists, opened only + when none is open. """ ws = STATE["workspace"] + if into and not (ws and ws.name == into): + ws = workspace.load(WS_HOME, into) or ws if ws and not new: ws.roots = list(ws.roots) + [r for r in roots if r not in ws.roots] workspace.save(ws) - open_workspace(ws) + if STATE["workspace"] in (None, ws): + open_workspace(ws) return ws ws = None if label else workspace.find_by_roots(WS_HOME, roots, besides=execution.runs_root()) ws = ws or workspace.create(WS_HOME, roots, label) @@ -335,6 +340,14 @@ def adopt_roots(roots, label="", new=False): return ws +def adopt_runs(runs): + """Runs just started: their jobs into a workspace as ``adopt_roots`` puts + them, each record naming it for a restart to put them back.""" + ws = adopt_roots([ex.jobs for ex in runs]) + for ex in runs: + ex.file_under(ws.name) + + def _load_jobs(): ws = STATE["workspace"] if ws is None: diff --git a/docs/builder.md b/docs/builder.md index 429985a7..127a5937 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -105,7 +105,7 @@ a declared model list keeps those published choices. | Proofs | Local runs save to the user catalogue. A host uses a writable copy in its run's `catalogue/`, isolated from the host's own entries; that copy and its proofs mirror back alongside `jobs/`, without changing the active user catalogue | | Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts | | Stop | Ends the current part (exit 130), records `stopped`; the next part never starts | -| Restart of Nullius | `execution.restore` reattaches from `runs//run.json` | +| Restart of Nullius | `execution.restore` reattaches from `runs//run.json`; the job goes back into the workspace it started in, while that workspace exists | | Cannot run from the page | Generated tasks and custom agents are not supported by Run | | Refused | Reads *refused* on the runs page, with why; the job was never written | | Files | `//`: script, log, `run.json`, `jobs/`; default `runs/` under the user data directory (`nullius paths`) |