Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion app/nullius/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -570,10 +570,29 @@ def part_for(self, job_id: str) -> Optional[str]:

def job_state(self, job_id: str) -> Optional[tuple]:
"""``(state, finished_at)`` for one job of this run once it has ended:
a complete, error-free standard result when jobs share a part, then
its own part's result when that part ran to its end - an arm that
answered every question is done though a later one failed - else the
run's, once the run has ended. None while it is still to come."""
end = self.ends.get(self.part_for(job_id) or "")
part = self.part_for(job_id)
# Conformance runs several kits in one process. Its exit code belongs
# to the whole step, not to each kit that already finished cleanly.
# Read only the small result record, including on a restored run or a
# host's mirrored jobs. Interrupted/errored jobs still use the runner.
if self.plan.runner == "standard" and part is not None and sum(
self.part_for(str(j.get("name"))) == part for j in self.plan.jobs) > 1:
try:
with open(os.path.join(self.jobs, job_id, "result.json"), encoding="utf-8") as f:
result = json.load(f)
stats = result["stats"]
if (result["id"] == job_id and result["n_total_trials"] > 0
and stats["n_completed_trials"] == result["n_total_trials"]
and all(stats[k] == 0 for k in ("n_errored_trials", "n_cancelled_trials", "n_running_trials", "n_pending_trials"))):
at = datetime.fromisoformat(result["finished_at"].replace("Z", "+00:00"))
return "done", at
except (OSError, ValueError, KeyError, TypeError, AttributeError):
pass # absent or being written/synced: the runner still owns the state
end = self.ends.get(part or "")
if end is not None:
return ("done" if end["rc"] == 0 else "failed"), datetime.fromtimestamp(end["at"], tz=timezone.utc)
if self.state in ENDED and self.finished:
Expand Down
69 changes: 69 additions & 0 deletions app/tests/test_execution_job_state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""One standard process can write several jobs, each with its own end."""
import json
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path

from nullius import execution
from nullius.standard import job


class SharedStep(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.plan = execution.Plan(name="kits", script="conform", runner="standard",
jobs=[{"name": "kit-ok"}, {"name": "kit-next"}])
self.ex = execution.Execution(self.plan, root=self.temp.name)
self.ex.state = "running"
self.ex.launched = "run"
self.folder = Path(self.ex.jobs) / "kit-ok"
self.folder.mkdir(parents=True)
job.write_result(str(self.folder), "kit-ok", 1, 2, [{}, {}], 2)

def test_a_finished_kit_keeps_its_own_end_while_its_sibling_runs_or_fails(self):
for state in ("running", "failed", "stopped", "done"):
with self.subTest(state=state):
self.ex.state = state
self.ex.finished = 3 if state != "running" else None
self.ex.ends = {} if state == "running" else {"run": {"rc": 1, "at": 3}}
self.assertEqual(self.ex.job_state("kit-ok"), ("done", datetime.fromtimestamp(2, timezone.utc)))
self.assertEqual(self.ex.job_state("kit-next"), None if state == "running" else
("failed", datetime.fromtimestamp(3, timezone.utc)))
self.assertEqual(self.ex.state, state) # the launch's state is untouched

def test_a_restored_run_reads_each_job_end_from_disk(self):
ex = execution.Execution(self.plan, root=self.temp.name,
record={"state": "failed", "finished": 3, "ends": {"run": {"rc": 1, "at": 3}}})
self.assertEqual(ex.job_state("kit-ok")[0], "done")
self.assertEqual(ex.job_state("kit-next")[0], "failed")

def test_incomplete_errored_or_unreadable_results_keep_the_runner_state(self):
path = self.folder / "result.json"
original = path.read_text()
variants = ["", "{", "null", "[]", "{}"]
for field, value in (("n_completed_trials", 1), ("n_errored_trials", 1),
("n_cancelled_trials", 1), ("n_running_trials", 1), ("n_pending_trials", 1)):
result = json.loads(original)
result["stats"][field] = value
variants.append(json.dumps(result))
for field, value in (("finished_at", None), ("finished_at", "bad"), ("id", "another")):
result = json.loads(original)
result[field] = value
variants.append(json.dumps(result))
for contents in variants:
with self.subTest(contents=contents):
path.write_text(contents)
self.ex.state, self.ex.finished = "running", None
self.assertIsNone(self.ex.job_state("kit-ok"))
self.ex.state, self.ex.finished = "stopped", 3
self.assertEqual(self.ex.job_state("kit-ok")[0], "stopped")

def test_single_job_and_harbor_steps_keep_their_exit_status(self):
self.ex.state, self.ex.finished = "failed", 3
self.plan.runner = "harbor"
self.assertEqual(self.ex.job_state("kit-ok")[0], "failed")
self.plan.runner = "standard"
self.plan.jobs = [{"name": "kit-ok"}]
self.assertEqual(self.ex.job_state("kit-ok")[0], "failed")
27 changes: 27 additions & 0 deletions app/ui_tests/test_runs_by_launch.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,33 @@ def test_a_part_held_back_by_a_pause_or_an_ended_run_reads_not_run(self):
if says:
self.assertIn(says, block)

def test_finished_kits_do_not_inherit_the_shared_process_state(self):
from nullius.standard import job
with tempfile.TemporaryDirectory() as root:
plan = execution.Plan(name="kits", script="conform", runner="standard",
jobs=[{"name": "kit-conversation"}, {"name": "kit-tools"}, {"name": "kit-computer"}])
ex = execution.Execution(plan, root=root)
ex.launched = "run"
self.jobs = []
for name in ("kit-conversation", "kit-tools", "kit-computer"):
ok = name != "kit-computer"
j = write_job(ex.jobs, name, "my-agent", {"a": 1 if ok else "RuntimeError"}, n_tasks=1)
job.write_result(j.dir, name, 1, 2, [{}] if ok else [{"exception_info": {"exception_type": "RuntimeError"}}], 1)
self.jobs.append(parse.parse_job(j.dir))
execution.EXECUTIONS[ex.id] = ex
for state in ("running", "failed"):
with self.subTest(state=state):
ex.state = state
ex.finished = 3 if state == "failed" else None
ex.ends = {"run": {"rc": 1, "at": 3}} if state == "failed" else {}
html = self.get("/runs")
for name in ("kit-conversation", "kit-tools", "kit-computer"):
row = next(row for row in re.findall(r"<tr\b.*?</tr>", html, re.S)
if f"{name}</small>" in row)
expected = state if name == "kit-computer" else "finished"
self.assertIn(f'class="la-state {expected}"', row)
self.assertEqual(ex.state, state)

def test_a_later_paused_shard_has_a_resume_link(self):
from web.runs import _launch_view
group = SimpleNamespace(job_ids=[], pending=["first", "second"], source="record", dataset="")
Expand Down
Loading