diff --git a/app/nullius/execution.py b/app/nullius/execution.py index cbbade16..42132d34 100644 --- a/app/nullius/execution.py +++ b/app/nullius/execution.py @@ -570,10 +570,29 @@ def part_for(self, job_id: str) -> Optional[str]: def job_state(self, job_id: str) -> Optional[tuple]: """``(state, finished_at)`` for one job of this run once it has ended: + a complete, error-free standard result when jobs share a part, then its own part's result when that part ran to its end - an arm that answered every question is done though a later one failed - else the run's, once the run has ended. None while it is still to come.""" - end = self.ends.get(self.part_for(job_id) or "") + part = self.part_for(job_id) + # Conformance runs several kits in one process. Its exit code belongs + # to the whole step, not to each kit that already finished cleanly. + # Read only the small result record, including on a restored run or a + # host's mirrored jobs. Interrupted/errored jobs still use the runner. + if self.plan.runner == "standard" and part is not None and sum( + self.part_for(str(j.get("name"))) == part for j in self.plan.jobs) > 1: + try: + with open(os.path.join(self.jobs, job_id, "result.json"), encoding="utf-8") as f: + result = json.load(f) + stats = result["stats"] + if (result["id"] == job_id and result["n_total_trials"] > 0 + and stats["n_completed_trials"] == result["n_total_trials"] + and all(stats[k] == 0 for k in ("n_errored_trials", "n_cancelled_trials", "n_running_trials", "n_pending_trials"))): + at = datetime.fromisoformat(result["finished_at"].replace("Z", "+00:00")) + return "done", at + except (OSError, ValueError, KeyError, TypeError, AttributeError): + pass # absent or being written/synced: the runner still owns the state + end = self.ends.get(part or "") if end is not None: return ("done" if end["rc"] == 0 else "failed"), datetime.fromtimestamp(end["at"], tz=timezone.utc) if self.state in ENDED and self.finished: diff --git a/app/tests/test_execution_job_state.py b/app/tests/test_execution_job_state.py new file mode 100644 index 00000000..0ec5c256 --- /dev/null +++ b/app/tests/test_execution_job_state.py @@ -0,0 +1,69 @@ +"""One standard process can write several jobs, each with its own end.""" +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +from nullius import execution +from nullius.standard import job + + +class SharedStep(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.plan = execution.Plan(name="kits", script="conform", runner="standard", + jobs=[{"name": "kit-ok"}, {"name": "kit-next"}]) + self.ex = execution.Execution(self.plan, root=self.temp.name) + self.ex.state = "running" + self.ex.launched = "run" + self.folder = Path(self.ex.jobs) / "kit-ok" + self.folder.mkdir(parents=True) + job.write_result(str(self.folder), "kit-ok", 1, 2, [{}, {}], 2) + + def test_a_finished_kit_keeps_its_own_end_while_its_sibling_runs_or_fails(self): + for state in ("running", "failed", "stopped", "done"): + with self.subTest(state=state): + self.ex.state = state + self.ex.finished = 3 if state != "running" else None + self.ex.ends = {} if state == "running" else {"run": {"rc": 1, "at": 3}} + self.assertEqual(self.ex.job_state("kit-ok"), ("done", datetime.fromtimestamp(2, timezone.utc))) + self.assertEqual(self.ex.job_state("kit-next"), None if state == "running" else + ("failed", datetime.fromtimestamp(3, timezone.utc))) + self.assertEqual(self.ex.state, state) # the launch's state is untouched + + def test_a_restored_run_reads_each_job_end_from_disk(self): + ex = execution.Execution(self.plan, root=self.temp.name, + record={"state": "failed", "finished": 3, "ends": {"run": {"rc": 1, "at": 3}}}) + self.assertEqual(ex.job_state("kit-ok")[0], "done") + self.assertEqual(ex.job_state("kit-next")[0], "failed") + + def test_incomplete_errored_or_unreadable_results_keep_the_runner_state(self): + path = self.folder / "result.json" + original = path.read_text() + variants = ["", "{", "null", "[]", "{}"] + for field, value in (("n_completed_trials", 1), ("n_errored_trials", 1), + ("n_cancelled_trials", 1), ("n_running_trials", 1), ("n_pending_trials", 1)): + result = json.loads(original) + result["stats"][field] = value + variants.append(json.dumps(result)) + for field, value in (("finished_at", None), ("finished_at", "bad"), ("id", "another")): + result = json.loads(original) + result[field] = value + variants.append(json.dumps(result)) + for contents in variants: + with self.subTest(contents=contents): + path.write_text(contents) + self.ex.state, self.ex.finished = "running", None + self.assertIsNone(self.ex.job_state("kit-ok")) + self.ex.state, self.ex.finished = "stopped", 3 + self.assertEqual(self.ex.job_state("kit-ok")[0], "stopped") + + def test_single_job_and_harbor_steps_keep_their_exit_status(self): + self.ex.state, self.ex.finished = "failed", 3 + self.plan.runner = "harbor" + self.assertEqual(self.ex.job_state("kit-ok")[0], "failed") + self.plan.runner = "standard" + self.plan.jobs = [{"name": "kit-ok"}] + self.assertEqual(self.ex.job_state("kit-ok")[0], "failed") diff --git a/app/ui_tests/test_runs_by_launch.py b/app/ui_tests/test_runs_by_launch.py index 4a43e05f..002844d1 100644 --- a/app/ui_tests/test_runs_by_launch.py +++ b/app/ui_tests/test_runs_by_launch.py @@ -142,6 +142,33 @@ def test_a_part_held_back_by_a_pause_or_an_ended_run_reads_not_run(self): if says: self.assertIn(says, block) + def test_finished_kits_do_not_inherit_the_shared_process_state(self): + from nullius.standard import job + with tempfile.TemporaryDirectory() as root: + plan = execution.Plan(name="kits", script="conform", runner="standard", + jobs=[{"name": "kit-conversation"}, {"name": "kit-tools"}, {"name": "kit-computer"}]) + ex = execution.Execution(plan, root=root) + ex.launched = "run" + self.jobs = [] + for name in ("kit-conversation", "kit-tools", "kit-computer"): + ok = name != "kit-computer" + j = write_job(ex.jobs, name, "my-agent", {"a": 1 if ok else "RuntimeError"}, n_tasks=1) + job.write_result(j.dir, name, 1, 2, [{}] if ok else [{"exception_info": {"exception_type": "RuntimeError"}}], 1) + self.jobs.append(parse.parse_job(j.dir)) + execution.EXECUTIONS[ex.id] = ex + for state in ("running", "failed"): + with self.subTest(state=state): + ex.state = state + ex.finished = 3 if state == "failed" else None + ex.ends = {"run": {"rc": 1, "at": 3}} if state == "failed" else {} + html = self.get("/runs") + for name in ("kit-conversation", "kit-tools", "kit-computer"): + row = next(row for row in re.findall(r"