Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def main(argv=None) -> None:
back = execution.restore()
for ex in back:
print(f"Run {ex.plan.name} picked up again on {ex.plan.host.label if ex.plan.host else 'this machine'} ({ex.state})")
shared.adopt_roots([ex.jobs]) # its job root back in a workspace, as when it started
shared.adopt_roots([ex.jobs], into=ex.workspace) # its job root back in the workspace it started in
app.config["SERVED_HOST"] = args.host # a name the Host header may carry (shared._served_name)
print(f"Serving on http://{args.host}:{args.port}")
app.run(host=args.host, port=args.port, debug=args.debug)
Expand Down
8 changes: 7 additions & 1 deletion app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callabl
self.tasks_on_target = rec.get("tasks_on_target", "")
self.app_on_target = rec.get("app_on_target", "") # the standard's source tree or runner snapshot on the target
self.catalogue_on_target = rec.get("catalogue_on_target", "") # writable definitions, separate from the immutable runner
self.workspace = rec.get("workspace", "") # the workspace its job was added to, the one a restart adds it back to
# the trials the log has said already, (job, trial), so a restart says each once; none in a record from before
self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()}
if record:
Expand Down Expand Up @@ -480,7 +481,7 @@ def _save(self) -> None:
data = {"id": self.id, "plan": self.plan.to_dict(), "state": self.state, "step": self.step, "part": self.part, "pid": self.pid,
"log_offset": self.log_offset, "exit_code": self.exit_code, "error": self.error, "question": self.question, "ends": self.ends,
"started": self.started, "finished": self.finished, "harbor": self.harbor, "tasks_on_target": self.tasks_on_target, "app_on_target": self.app_on_target,
"catalogue_on_target": self.catalogue_on_target,
"catalogue_on_target": self.catalogue_on_target, "workspace": self.workspace,
"launched": self.launched, "seen_trials": [list(k) for k in sorted(self._seen_trials, key=str)]}
try:
os.makedirs(self.dir, exist_ok=True)
Expand Down Expand Up @@ -636,6 +637,11 @@ def start(self) -> "Execution":
self.thread.start()
return self

def file_under(self, workspace: str) -> None:
"""Record the workspace the run's job went into."""
self.workspace = workspace
self._save()

def resume(self) -> "Execution":
"""Pick a recorded run up where it was: the part that was running is
followed again, a pause waits again, the rest runs on."""
Expand Down
6 changes: 3 additions & 3 deletions app/ui_tests/test_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {})
self.stack.enter_context(patch.object(execution, "start", start))
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()

def test_examples_lead_catalogue_and_sidebar(self):
Expand Down Expand Up @@ -338,7 +338,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1"})
self.stack.enter_context(patch.object(execution, "start", start))
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
self.schema = json.loads(re.search(r'<script id="nb-schema" type="application/json">(.*?)</script>', r.get_data(as_text=True), re.S).group(1))
Expand Down Expand Up @@ -428,7 +428,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name})
self.stack.enter_context(patch.object(execution, "start", start))
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()

def get(self, url):
Expand Down
2 changes: 1 addition & 1 deletion app/ui_tests/test_harbor_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def start(plan):
self.started.append(plan)
return SimpleNamespace(jobs=str(self.tmp / "runs" / plan.name / "jobs"), summary=lambda: {"id": plan.name + "-1", "state": "preparing"})
self.stack.enter_context(patch.object(execution, "start", start))
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
with self.client.session_transaction() as s:
Expand Down
2 changes: 1 addition & 1 deletion app/ui_tests/test_prerequisites_route.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def setUp(self):
self.started = []
self.stack.enter_context(patch.object(execution, "start", lambda plan: self.started.append(plan) or SimpleNamespace(
jobs=str(self.tmp), summary=lambda: {"id": "x", "state": "preparing"})))
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: None))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: None))
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
with self.client.session_transaction() as s:
Expand Down
66 changes: 65 additions & 1 deletion app/ui_tests/test_run_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def run(argv, timeout=600, stdin=None, stop=None):
return 0, ""
self.stack.enter_context(patch.object(execution, "subprocess_run", run))
self.stack.enter_context(patch.object(execution, "docker_here", lambda sh, needs="": None)) # docker answers here, as the probe says it does there
self.stack.enter_context(patch.object(web.shared, "adopt_roots", lambda roots, **kw: self.adopted.extend(roots)))
self.stack.enter_context(patch.object(web.shared, "adopt_runs", lambda runs: self.adopted.extend(ex.jobs for ex in runs)))
self.adopted = []
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
Expand Down Expand Up @@ -347,6 +347,70 @@ def test_the_page_carries_the_token_the_stop_button_sends(self):
self.assertEqual(self.client.post(f"/run/{r.json['id']}/stop", headers={"Origin": "http://localhost", "X-CSRF-Token": ""}).status_code, 403)


class ResumeIntoWorkspace(unittest.TestCase):
"""A restart while a run started from the builder goes on: the run's job goes back into the workspace it started in."""
def setUp(self):
self.stack = ExitStack(); self.addCleanup(self.stack.close)
self.tmp = Path(tempfile.mkdtemp(prefix="nullius-resume-")); self.addCleanup(shutil.rmtree, self.tmp, True)
self.home = str(self.tmp / "ws")
self.stack.enter_context(patch.dict(os.environ, {"RUNS_DIR": str(self.tmp / "runs"), "JOB_ROOTS": ""}))
self.stack.enter_context(patch.dict(web.app.config, TESTING=True))
self.stack.enter_context(patch.dict(execution.EXECUTIONS, clear=True))
self.stack.enter_context(patch.object(web.shared, "WS_HOME", self.home))
self.stack.enter_context(patch.dict(web.shared.STATE, {"workspace": None, "roots": [], "label": "", "job_roots": []}))
self.stack.enter_context(patch.object(web.app, "run", lambda **kw: None)) # main() serves nothing
roots = [self.tmp / "a", self.tmp / "b"]
for r in roots:
r.mkdir()
self.held = [str(r) for r in roots]
self.ws = workspace.create(self.home, self.held, label="witty pelican")
web.shared.open_workspace(self.ws)
self.client = web.app.test_client()
r = self.client.get("/new-run"); self.addCleanup(r.close)
with self.client.session_transaction() as s:
self.csrf = s["csrf"]
# a run that pauses after its first part, so it is still alive when the app stops
parts = [{"id": "a", "label": "a", "script": "exit 2"}, {"id": "b", "label": "b", "script": "echo b"}]
r = self.client.post("/run", data=json.dumps({"name": "p", "script": "", "where": "local", "parts": parts}), content_type="application/json",
headers={"Origin": "http://localhost", "X-CSRF-Token": self.csrf}); self.addCleanup(r.close)
self.assertEqual(r.status_code, 200, r.get_data(as_text=True))
self.jobs = str(self.tmp / "runs" / "p" / "jobs")
self.record = self.tmp / "runs" / "p" / "run.json"
for _ in range(250):
if json.loads(self.record.read_text())["state"] == "paused":
break
time.sleep(0.02)
ex = execution.EXECUTIONS[r.json["id"]]
ex._save = lambda: None # the app stops: the old thread writes no record again
execution.EXECUTIONS.clear()
web.shared.close_workspace()

def restart(self):
web.main([])
back = list(execution.EXECUTIONS.values())
self.addCleanup(lambda: [e.stop() for e in back])
return web.shared.STATE["workspace"]

def test_a_restart_puts_the_run_back_into_its_workspace(self):
again = self.restart()
self.assertEqual(again.name, self.ws.name)
self.assertEqual([w.name for w in workspace.list_all(self.home)], [self.ws.name]) # no new workspace
self.assertIn(self.jobs, again.roots)
self.assertEqual(json.loads(self.record.read_text())["workspace"], self.ws.name)

def test_a_run_whose_workspace_is_gone_gets_one_as_before(self):
workspace.delete(self.home, self.ws.name)
again = self.restart()
self.assertEqual(again.roots, [self.jobs])

def test_an_old_record_without_its_workspace_gets_one_as_before(self):
rec = json.loads(self.record.read_text()); rec.pop("workspace", None); self.record.write_text(json.dumps(rec))
again = self.restart()
self.assertNotEqual(again.name, self.ws.name)
self.assertEqual(again.roots, [self.jobs])
self.assertEqual(workspace.load(self.home, self.ws.name).roots, self.held + [self.jobs]) # added when it started, left as it is


class Credentials(unittest.TestCase):
def test_a_run_on_openai_is_given_openais_endpoint(self):
"""A Harbor part's script binds its clients' endpoint from API_URL and
Expand Down
2 changes: 1 addition & 1 deletion app/web/agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def _check(kind: str, key: str):
flash(problem, "error")
return redirect(back)
ex = execution.start(plan)
shared.adopt_roots([ex.jobs])
shared.adopt_runs([ex])
n = len(got["jobs"])
flash(f"Checking {got['label']}'s ports: " + (f"{n} kits, one after the other." if n > 1 else "its one kit."), "ok")
return redirect(url_for("job", job_id=got["jobs"][0]["name"]))
Expand Down
2 changes: 1 addition & 1 deletion app/web/catalogue.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def benchmark_prove(key):
flash(problem, "error")
return redirect(back)
ex = execution.start(plan)
shared.adopt_roots([ex.jobs])
shared.adopt_runs([ex])
flash(f"Proving {got['label']}: its units and the leak check, then " + (f"{len(got['jobs'])} control runs." if got["jobs"] else "no control: it declares none."), "ok")
return redirect(url_for("job", job_id=got["jobs"][0]["name"]) if got["jobs"] else back)

Expand Down
4 changes: 2 additions & 2 deletions app/web/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def _start_standard(recipe: dict):
if os.path.exists(os.path.join(execution.runs_root(), plan.name)):
return jsonify(ok=False, error=execution.name_taken(plan.name)), 409
started = [execution.start(plan) for plan in plans]
shared.adopt_roots([ex.jobs for ex in started])
shared.adopt_runs(started)
return jsonify(ok=True, open=url_for("index", view="launch"), runs=[ex.summary() for ex in started],
**started[0].summary())

Expand Down Expand Up @@ -177,7 +177,7 @@ def _start(plan: execution.Plan):
# The job lands (or is mirrored) under runs/<name>/jobs: the workspace
# reads it from now on. The Runs page, by launch, shows it from the first
# second: its runs queued, preparing, then running, each opening its job.
shared.adopt_roots([ex.jobs])
shared.adopt_runs([ex])
return jsonify(ok=True, open=url_for("index", view="launch"), **ex.summary())


Expand Down
17 changes: 15 additions & 2 deletions app/web/shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,24 +317,37 @@ def close_workspace():
STATE.update({"workspace": None, "roots": [], "label": "", "job_roots": []})


def adopt_roots(roots, label="", new=False):
def adopt_roots(roots, label="", new=False, into=""):
"""Roots the user just pointed at: into the open workspace, or a new one.

Without a label, roots that already have a workspace reopen it — so a
restart with the same argv lands in the same place, cache and all.
``into``: the workspace to add them to while it exists, opened only
when none is open.
"""
ws = STATE["workspace"]
if into and not (ws and ws.name == into):
ws = workspace.load(WS_HOME, into) or ws
if ws and not new:
ws.roots = list(ws.roots) + [r for r in roots if r not in ws.roots]
workspace.save(ws)
open_workspace(ws)
if STATE["workspace"] in (None, ws):
open_workspace(ws)
return ws
ws = None if label else workspace.find_by_roots(WS_HOME, roots, besides=execution.runs_root())
ws = ws or workspace.create(WS_HOME, roots, label)
open_workspace(ws)
return ws


def adopt_runs(runs):
"""Runs just started: their jobs into a workspace as ``adopt_roots`` puts
them, each record naming it for a restart to put them back."""
ws = adopt_roots([ex.jobs for ex in runs])
for ex in runs:
ex.file_under(ws.name)


def _load_jobs():
ws = STATE["workspace"]
if ws is None:
Expand Down
2 changes: 1 addition & 1 deletion docs/builder.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ a declared model list keeps those published choices.
| Proofs | Local runs save to the user catalogue. A host uses a writable copy in its run's `catalogue/`, isolated from the host's own entries; that copy and its proofs mirror back alongside `jobs/`, without changing the active user catalogue |
| Docker or Podman | The engine must answer `ps` (`docker ps`, or `podman ps` for the standard's container backend), or the run stops before it starts |
| Stop | Ends the current part (exit 130), records `stopped`; the next part never starts |
| Restart of Nullius | `execution.restore` reattaches from `runs/<name>/run.json` |
| Restart of Nullius | `execution.restore` reattaches from `runs/<name>/run.json`; the job goes back into the workspace it started in, while that workspace exists |
| Cannot run from the page | Generated tasks and custom agents are not supported by Run |
| Refused | Reads *refused* on the runs page, with why; the job was never written |
| Files | `<RUNS_DIR>/<name>/`: script, log, `run.json`, `jobs/`; default `runs/` under the user data directory (`nullius paths`) |
Expand Down
Loading