diff --git a/app/app.py b/app/app.py
index e8ba24fe..66d72239 100644
--- a/app/app.py
+++ b/app/app.py
@@ -86,7 +86,7 @@ def main(argv=None) -> None:
back = execution.restore()
for ex in back:
print(f"Run {ex.plan.name} picked up again on {ex.plan.host.label if ex.plan.host else 'this machine'} ({ex.state})")
- shared.adopt_roots([ex.jobs]) # its job root back in a workspace, as when it started
+ shared.adopt_roots([ex.jobs], into=ex.workspace) # its job root back in the workspace it started in
app.config["SERVED_HOST"] = args.host # a name the Host header may carry (shared._served_name)
print(f"Serving on http://{args.host}:{args.port}")
app.run(host=args.host, port=args.port, debug=args.debug)
diff --git a/app/nullius/execution.py b/app/nullius/execution.py
index 5f65c788..c38811fa 100644
--- a/app/nullius/execution.py
+++ b/app/nullius/execution.py
@@ -436,6 +436,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl
self.tasks_on_target = rec.get("tasks_on_target", "")
self.app_on_target = rec.get("app_on_target", "") # the standard's source tree or runner snapshot on the target
self.catalogue_on_target = rec.get("catalogue_on_target", "") # writable definitions, separate from the immutable runner
+ self.workspace = rec.get("workspace", "") # the workspace its job was added to, the one a restart adds it back to
# the trials the log has said already, (job, trial), so a restart says each once; none in a record from before
self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()}
if record:
@@ -480,7 +481,7 @@ def _save(self) -> None:
data = {"id": self.id, "plan": self.plan.to_dict(), "state": self.state, "step": self.step, "part": self.part, "pid": self.pid,
"log_offset": self.log_offset, "exit_code": self.exit_code, "error": self.error, "question": self.question, "ends": self.ends,
"started": self.started, "finished": self.finished, "harbor": self.harbor, "tasks_on_target": self.tasks_on_target, "app_on_target": self.app_on_target,
- "catalogue_on_target": self.catalogue_on_target,
+ "catalogue_on_target": self.catalogue_on_target, "workspace": self.workspace,
"launched": self.launched, "seen_trials": [list(k) for k in sorted(self._seen_trials, key=str)]}
try:
os.makedirs(self.dir, exist_ok=True)
@@ -636,6 +637,11 @@ def start(self) -> "Execution":
self.thread.start()
return self
+ def file_under(self, workspace: str) -> None:
+ """Record the workspace the run's job went into."""
+ self.workspace = workspace
+ self._save()
+
def resume(self) -> "Execution":
"""Pick a recorded run up where it was: the part that was running is
followed again, a pause waits again, the rest runs on."""
diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py
index 356b1567..ae4d363e 100644
--- a/app/ui_tests/test_agents.py
+++ b/app/ui_tests/test_agents.py
@@ -48,7 +48,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {})
self.stack.enter_context(patch.object(execution, "start", start))
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
def test_examples_lead_catalogue_and_sidebar(self):
@@ -338,7 +338,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1"})
self.stack.enter_context(patch.object(execution, "start", start))
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
self.schema = json.loads(re.search(r'', r.get_data(as_text=True), re.S).group(1))
@@ -428,7 +428,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name})
self.stack.enter_context(patch.object(execution, "start", start))
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
def get(self, url):
diff --git a/app/ui_tests/test_harbor_routes.py b/app/ui_tests/test_harbor_routes.py
index a4c436a7..1324c9bc 100644
--- a/app/ui_tests/test_harbor_routes.py
+++ b/app/ui_tests/test_harbor_routes.py
@@ -40,7 +40,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1", "state": "preparing"})
self.stack.enter_context(patch.object(execution, "start", start))
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
with self.client.session_transaction() as s:
diff --git a/app/ui_tests/test_prerequisites_route.py b/app/ui_tests/test_prerequisites_route.py
index 3eb90cc5..717f9361 100644
--- a/app/ui_tests/test_prerequisites_route.py
+++ b/app/ui_tests/test_prerequisites_route.py
@@ -29,7 +29,7 @@ def setUp(self):
self.started = []
self.stack.enter_context(patch.object(execution, "start", lambda plan: self.started.append(plan) or SimpleNamespace(
jobs=str(self.tmp), summary=lambda: {"id": "x", "state": "preparing"})))
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
with self.client.session_transaction() as s:
diff --git a/app/ui_tests/test_run_routes.py b/app/ui_tests/test_run_routes.py
index d0ff0fa1..d608816f 100644
--- a/app/ui_tests/test_run_routes.py
+++ b/app/ui_tests/test_run_routes.py
@@ -78,7 +78,7 @@ def run(argv, timeout=600, stdin=None, stop=None):
return 0, ""
self.stack.enter_context(patch.object(execution, "subprocess_run", run))
self.stack.enter_context(patch.object(execution, "docker_here", lambda sh, needs="": None)) # docker answers here, as the probe says it does there
- self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: self.adopted.extend(roots)))
+ self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: self.adopted.extend(ex.jobs for ex in runs)))
self.adopted = []
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
@@ -347,6 +347,70 @@ def test_the_page_carries_the_token_the_stop_button_sends(self):
self.assertEqual(self.client.post(f"/run/{r.json['id']}/stop", headers={"Origin": "http://localhost", "X-CSRF-Token": ""}).status_code, 403)
+class ResumeIntoWorkspace(unittest.TestCase):
+ """A restart while a run started from the builder goes on: the run's job goes back into the workspace it started in."""
+ def setUp(self):
+ self.stack = ExitStack(); self.addCleanup(self.stack.close)
+ self.tmp = Path(tempfile.mkdtemp(prefix="nullius-resume-")); self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.home = str(self.tmp / "ws")
+ self.stack.enter_context(patch.dict(os.environ, {"RUNS_DIR": str(self.tmp / "runs"), "JOB_ROOTS": ""}))
+ self.stack.enter_context(patch.dict(web.app.config, TESTING=True))
+ self.stack.enter_context(patch.dict(execution.EXECUTIONS, clear=True))
+ self.stack.enter_context(patch.object(web.shared, "WS_HOME", self.home))
+ self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}))
+ self.stack.enter_context(patch.object(web.app, "run", lambda **kw: None)) # main() serves nothing
+ roots = [self.tmp / "a", self.tmp / "b"]
+ for r in roots:
+ r.mkdir()
+ self.held = [str(r) for r in roots]
+ self.ws = workspace.create(self.home, self.held, label="witty pelican")
+ web.shared.open_workspace(self.ws)
+ self.client = web.app.test_client()
+ r = self.client.get("/new-run"); self.addCleanup(r.close)
+ with self.client.session_transaction() as s:
+ self.csrf = s["csrf"]
+ # a run that pauses after its first part, so it is still alive when the app stops
+ parts = [{"id": "a", "label": "a", "script": "exit 2"}, {"id": "b", "label": "b", "script": "echo b"}]
+ r = self.client.post("/run", data=json.dumps({"name": "p", "script": "", "where": "local", "parts": parts}), content_type="application/json",
+ headers={"Origin": "http://localhost", "X-CSRF-Token": self.csrf}); self.addCleanup(r.close)
+ self.assertEqual(r.status_code, 200, r.get_data(as_text=True))
+ self.jobs = str(self.tmp / "runs" / "p" / "jobs")
+ self.record = self.tmp / "runs" / "p" / "run.json"
+ for _ in range(250):
+ if json.loads(self.record.read_text())["state"] == "paused":
+ break
+ time.sleep(0.02)
+ ex = execution.EXECUTIONS[r.json["id"]]
+ ex._save = lambda: None # the app stops: the old thread writes no record again
+ execution.EXECUTIONS.clear()
+ web.shared.close_workspace()
+
+ def restart(self):
+ web.main([])
+ back = list(execution.EXECUTIONS.values())
+ self.addCleanup(lambda: [e.stop() for e in back])
+ return web.shared.STATE["workspace"]
+
+ def test_a_restart_puts_the_run_back_into_its_workspace(self):
+ again = self.restart()
+ self.assertEqual(again.name, self.ws.name)
+ self.assertEqual([w.name for w in workspace.list_all(self.home)], [self.ws.name]) # no new workspace
+ self.assertIn(self.jobs, again.roots)
+ self.assertEqual(json.loads(self.record.read_text())["workspace"], self.ws.name)
+
+ def test_a_run_whose_workspace_is_gone_gets_one_as_before(self):
+ workspace.delete(self.home, self.ws.name)
+ again = self.restart()
+ self.assertEqual(again.roots, [self.jobs])
+
+ def test_an_old_record_without_its_workspace_gets_one_as_before(self):
+ rec = json.loads(self.record.read_text()); rec.pop("workspace", None); self.record.write_text(json.dumps(rec))
+ again = self.restart()
+ self.assertNotEqual(again.name, self.ws.name)
+ self.assertEqual(again.roots, [self.jobs])
+ self.assertEqual(workspace.load(self.home, self.ws.name).roots, self.held + [self.jobs]) # added when it started, left as it is
+
+
class Credentials(unittest.TestCase):
def test_a_run_on_openai_is_given_openais_endpoint(self):
"""A Harbor part's script binds its clients' endpoint from API_URL and
diff --git a/app/web/agents.py b/app/web/agents.py
index a2e8efba..b791d254 100644
--- a/app/web/agents.py
+++ b/app/web/agents.py
@@ -158,7 +158,7 @@ def _check(kind: str, key: str):
flash(problem, "error")
return redirect(back)
ex = execution.start(plan)
- shared.adopt_roots([ex.jobs])
+ shared.adopt_runs([ex])
n = len(got["jobs"])
flash(f"Checking {got['label']}'s ports: " + (f"{n} kits, one after the other." if n > 1 else "its one kit."), "ok")
return redirect(url_for("job", job_id=got["jobs"][0]["name"]))
diff --git a/app/web/catalogue.py b/app/web/catalogue.py
index 2449c141..4c0fbde6 100644
--- a/app/web/catalogue.py
+++ b/app/web/catalogue.py
@@ -98,7 +98,7 @@ def benchmark_prove(key):
flash(problem, "error")
return redirect(back)
ex = execution.start(plan)
- shared.adopt_roots([ex.jobs])
+ shared.adopt_runs([ex])
flash(f"Proving {got['label']}: its units and the leak check, then " + (f"{len(got['jobs'])} control runs." if got["jobs"] else "no control: it declares none."), "ok")
return redirect(url_for("job", job_id=got["jobs"][0]["name"]) if got["jobs"] else back)
diff --git a/app/web/execute.py b/app/web/execute.py
index f07af346..6c61f26f 100644
--- a/app/web/execute.py
+++ b/app/web/execute.py
@@ -143,7 +143,7 @@ def _start_standard(recipe: dict):
if os.path.exists(os.path.join(execution.runs_root(), plan.name)):
return jsonify(ok=False, error=execution.name_taken(plan.name)), 409
started = [execution.start(plan) for plan in plans]
- shared.adopt_roots([ex.jobs for ex in started])
+ shared.adopt_runs(started)
return jsonify(ok=True, open=url_for("index", view="launch"), runs=[ex.summary() for ex in started],
**started[0].summary())
@@ -177,7 +177,7 @@ def _start(plan: execution.Plan):
# The job lands (or is mirrored) under runs//jobs: the workspace
# reads it from now on. The Runs page, by launch, shows it from the first
# second: its runs queued, preparing, then running, each opening its job.
- shared.adopt_roots([ex.jobs])
+ shared.adopt_runs([ex])
return jsonify(ok=True, open=url_for("index", view="launch"), **ex.summary())
diff --git a/app/web/shared.py b/app/web/shared.py
index 939d633c..3d72ffd2 100644
--- a/app/web/shared.py
+++ b/app/web/shared.py
@@ -317,17 +317,22 @@ def close_workspace():
STATE.update({"workspace": None, "roots": [], "label": "", "job_roots": []})
-def adopt_roots(roots, label="", new=False):
+def adopt_roots(roots, label="", new=False, into=""):
"""Roots the user just pointed at: into the open workspace, or a new one.
Without a label, roots that already have a workspace reopen it — so a
restart with the same argv lands in the same place, cache and all.
+ ``into``: the workspace to add them to while it exists, opened only
+ when none is open.
"""
ws = STATE["workspace"]
+ if into and not (ws and ws.name == into):
+ ws = workspace.load(WS_HOME, into) or ws
if ws and not new:
ws.roots = list(ws.roots) + [r for r in roots if r not in ws.roots]
workspace.save(ws)
- open_workspace(ws)
+ if STATE["workspace"] in (None, ws):
+ open_workspace(ws)
return ws
ws = None if label else workspace.find_by_roots(WS_HOME, roots, besides=execution.runs_root())
ws = ws or workspace.create(WS_HOME, roots, label)
@@ -335,6 +340,14 @@ def adopt_roots(roots, label="", new=False):
return ws
+def adopt_runs(runs):
+ """Runs just started: their jobs into a workspace as ``adopt_roots`` puts
+ them, each record naming it for a restart to put them back."""
+ ws = adopt_roots([ex.jobs for ex in runs])
+ for ex in runs:
+ ex.file_under(ws.name)
+
+
def _load_jobs():
ws = STATE["workspace"]
if ws is None:
diff --git a/docs/builder.md b/docs/builder.md
index 429985a7..127a5937 100644
--- a/docs/builder.md
+++ b/docs/builder.md
@@ -105,7 +105,7 @@ a declared model list keeps those published choices.
| Proofs | Local runs save to the user catalogue. A host uses a writable copy in its run's `catalogue/`, isolated from the host's own entries; that copy and its proofs mirror back alongside `jobs/`, without changing the active user catalogue |
| Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts |
| Stop | Ends the current part (exit 130), records `stopped`; the next part never starts |
-| Restart of Nullius | `execution.restore` reattaches from `runs//run.json` |
+| Restart of Nullius | `execution.restore` reattaches from `runs//run.json`; the job goes back into the workspace it started in, while that workspace exists |
| Cannot run from the page | Generated tasks and custom agents are not supported by Run |
| Refused | Reads *refused* on the runs page, with why; the job was never written |
| Files | `//`: script, log, `run.json`, `jobs/`; default `runs/` under the user data directory (`nullius paths`) |