Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions app/nullius/standard/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
import traceback
import tempfile
from collections import Counter
from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed
from concurrent.futures import FIRST_EXCEPTION, CancelledError, ThreadPoolExecutor, wait
from contextlib import contextmanager
from typing import Callable, Dict, List, Optional

Expand Down Expand Up @@ -738,10 +738,14 @@ def work_unit(iur):
self.loopback = container.Loopback(self.base, image=self.image, job=self.name).start()
for iur in work:
futures.append(pool.submit(work_unit, iur))
for future in as_completed(futures):
res = future.result()
if res is not None: # None: a unit not begun once the endpoint refused the run
results.append(res)
pending = set(futures)
while pending:
# timed: Python runs signal handlers only in this thread, and Stop's SIGTERM can reach another thread
done, pending = wait(pending, timeout=0.5, return_when=FIRST_EXCEPTION)
for future in done:
res = future.result()
if res is not None: # None: a unit not begun once the endpoint refused the run
results.append(res)
self.run_again(results, work, work_unit, again)
except BaseException:
self._stopped.set()
Expand Down
19 changes: 19 additions & 0 deletions app/tests/test_audit_regressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ def test_detached_child_holding_stdout_does_not_extend_timeout(self):
except (ProcessLookupError, ValueError): pass # ValueError: not written yet

def test_sigterm_stops_a_real_memory_process_and_writes_cancelled_results(self):
self._stop_a_memory_run(to_a_thread=False)

def test_sigterm_given_to_a_thread_other_than_main_stops_the_run(self):
# a process's SIGTERM can reach any of its threads, and Python runs the handler only in the main thread
self._stop_a_memory_run(to_a_thread=True)

def _stop_a_memory_run(self, to_a_thread):
"""Stop a run whose memory does not answer: SIGTERM from outside, or
from one of the run's own threads once the memory has started."""
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'memory.py').write_text("import os,sys,time\nopen('pid','w').write(str(os.getpid()))\nfor line in sys.stdin: time.sleep(60)\n")
Expand All @@ -84,7 +93,17 @@ def test_sigterm_stops_a_real_memory_process_and_writes_cancelled_results(self):
o=dict(o,start={{'command':[{sys.executable!r},{str(root / 'memory.py')!r}]}})
run.run(b,o,model='m',provider='custom',jobs_root={tmp!r},name='stopped',env={{'MODEL_API_URL':'http://127.0.0.1:9/v1'}},say=lambda s:None)
"""
if to_a_thread:
script = f"""
import glob,signal,threading,time
def stop():
while not glob.glob({tmp!r} + '/stopped/*/.work/capability/pid'): time.sleep(.05)
signal.pthread_kill(threading.get_ident(), signal.SIGTERM)
threading.Thread(target=stop, daemon=True).start()
""" + script
process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if to_a_thread:
process.terminate = lambda: None # the run's own thread sends SIGTERM
memory_pid = None
try:
deadline = time.monotonic() + 60
Expand Down
Loading