From f3ab8ad08332b410d9f936616edeb2aecde5ca0e Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Sat, 14 Nov 2026 17:36:52 +0100 Subject: [PATCH] fix(standard): stop a run when SIGTERM reaches a thread other than main Python runs a signal handler only in the main thread. During a run, that thread waited in as_completed with no timeout. POSIX lets the kernel give a process's SIGTERM to any thread. Linux gives it to the main thread when that thread can take it, so the defect is rare in practice, but a SIGTERM that reached another thread did not stop the run. The main thread now waits for the units in a loop with a 0.5 s timeout, so it wakes up and runs the handler. It returns at once when a unit raises. The signal mask stays as it is: child processes that the units start must still receive SIGTERM. --- app/nullius/standard/run.py | 14 +++++++++----- app/tests/test_audit_regressions.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 40251328..200ddac0 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -64,7 +64,7 @@ import traceback import tempfile from collections import Counter -from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed +from concurrent.futures import FIRST_EXCEPTION, CancelledError, ThreadPoolExecutor, wait from contextlib import contextmanager from typing import Callable, Dict, List, Optional @@ -738,10 +738,14 @@ def work_unit(iur): self.loopback = container.Loopback(self.base, image=self.image, job=self.name).start() for iur in work: futures.append(pool.submit(work_unit, iur)) - for future in as_completed(futures): - res = future.result() - if res is not None: # None: a unit not begun once the endpoint refused the run - results.append(res) + pending = set(futures) + while pending: + # timed: Python runs signal handlers only in this thread, and Stop's SIGTERM can reach another thread + done, pending = wait(pending, timeout=0.5, return_when=FIRST_EXCEPTION) + for future in done: + res = future.result() + if res is not None: # None: a unit not begun once the endpoint refused the run + results.append(res) self.run_again(results, work, work_unit, again) except BaseException: self._stopped.set() diff --git a/app/tests/test_audit_regressions.py b/app/tests/test_audit_regressions.py index 497f23e9..637d0f29 100644 --- a/app/tests/test_audit_regressions.py +++ b/app/tests/test_audit_regressions.py @@ -73,6 +73,15 @@ def test_detached_child_holding_stdout_does_not_extend_timeout(self): except (ProcessLookupError, ValueError): pass # ValueError: not written yet def test_sigterm_stops_a_real_memory_process_and_writes_cancelled_results(self): + self._stop_a_memory_run(to_a_thread=False) + + def test_sigterm_given_to_a_thread_other_than_main_stops_the_run(self): + # a process's SIGTERM can reach any of its threads, and Python runs the handler only in the main thread + self._stop_a_memory_run(to_a_thread=True) + + def _stop_a_memory_run(self, to_a_thread): + """Stop a run whose memory does not answer: SIGTERM from outside, or + from one of the run's own threads once the memory has started.""" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / 'memory.py').write_text("import os,sys,time\nopen('pid','w').write(str(os.getpid()))\nfor line in sys.stdin: time.sleep(60)\n") @@ -84,7 +93,17 @@ def test_sigterm_stops_a_real_memory_process_and_writes_cancelled_results(self): o=dict(o,start={{'command':[{sys.executable!r},{str(root / 'memory.py')!r}]}}) run.run(b,o,model='m',provider='custom',jobs_root={tmp!r},name='stopped',env={{'MODEL_API_URL':'http://127.0.0.1:9/v1'}},say=lambda s:None) """ + if to_a_thread: + script = f""" +import glob,signal,threading,time +def stop(): + while not glob.glob({tmp!r} + '/stopped/*/.work/capability/pid'): time.sleep(.05) + signal.pthread_kill(threading.get_ident(), signal.SIGTERM) +threading.Thread(target=stop, daemon=True).start() +""" + script process = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if to_a_thread: + process.terminate = lambda: None # the run's own thread sends SIGTERM memory_pid = None try: deadline = time.monotonic() + 60