From 5a200b7261a8ac2f2238fdd6098e0bc2b63b7e9c Mon Sep 17 00:00:00 2001 From: Raisal P Wardana Date: Sun, 15 Nov 2026 23:12:51 +0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(gateway):=20blue-green=20restart=20han?= =?UTF-8?q?dover=20=E2=80=94=20successor=20serves=20while=20blue=20drains?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restart no longer blocks new work on the drain. request_restart spawns a successor 'gateway run' child (same unit cgroup) before the drain: green boots with platform-lock takeover authority, binds the control socket name when ready (new sessions/relays land there), and flips systemd's MainPID via sd_notify so blue's exit never triggers Restart=always. Blue finishes its in-flight turns through the existing stop path; the PID record and the host role are claimed by green's deferred task only after blue exits. Gate: agent.blue_green_restart (default off) + a Type=notify unit (NOTIFY_SOCKET), because the MainPID switch is the piece that makes Restart=always safe. No notify socket -> classic drain-exit restart, unchanged. Green-not-ready -> same fallback, logged. Single-owner services need no extra deferral: green's adapter cold-connects carry the same takeover authority as --replace, fencing blue's pollers out of each platform lock. Failure convergence: green crashing after MainPID still respawns the unit normally; green wedged pre-notify recovers via the existing exit-75 retry path. --- .../workflows/gateway-blue-green-restart.yml | 70 ++++++ gateway/handover.py | 230 ++++++++++++++++++ gateway/run_bootstrap.py | 19 +- gateway/run_shutdown.py | 15 +- gateway/systemd_notify.py | 7 + hermes_cli/config_defaults.py | 5 + tests/gateway/test_handover.py | 196 +++++++++++++++ 7 files changed, 539 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/gateway-blue-green-restart.yml create mode 100644 gateway/handover.py create mode 100644 tests/gateway/test_handover.py diff --git a/.github/workflows/gateway-blue-green-restart.yml b/.github/workflows/gateway-blue-green-restart.yml new file mode 100644 index 0000000000000..fce14b41b6292 --- /dev/null +++ b/.github/workflows/gateway-blue-green-restart.yml @@ -0,0 +1,70 @@ +name: gateway-blue-green-restart + +# Runs ONLY on this branch: focused gate for the blue-green restart handover +# (successor-first restart with sd_notify MainPID switch; falls back to the +# classic drain-exit restart when green is not ready). The repo's own +# full-repo CI also fires on the PR. +on: + push: + branches: [feat/gateway-blue-green-restart] +permissions: + contents: read +concurrency: + group: gateway-blue-green-restart + cancel-in-progress: true + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Python 3.14 (.python-version) + run: uv python install 3.14 + + - name: "Private PM environment (canonical command; setup inputs modified: 4 UV_CONCURRENT caps = 1)" + env: + UV_CONCURRENT_DOWNLOADS: '1' + UV_CONCURRENT_BUILDS: '1' + UV_CONCURRENT_INSTALLS: '1' + UV_CONCURRENT_CACHE_READS: '1' + run: | + PY="$(uv python find 3.14)" + "$PY" -m pm.build_env --source . --out ci --group dev --group test --python "$PY" + + - name: Focused handover regressions (must PASS) + env: + TZ: UTC + LANG: C.UTF-8 + HERMES_HOME: ${{ runner.temp }}/hermes-home + run: | + mkdir -p "$HERMES_HOME" + HERMES_PYTHON="$PWD/ci/bin/python" bash scripts/run_tests.sh \ + tests/gateway/test_handover.py \ + --file-retries 0 --file-timeout 240 -j 2 -q 2>&1 | tee blue-green-handover.log + + - name: Gate on the focused result + run: | + grep -E "passed" blue-green-handover.log + ! grep -qE "[1-9][0-9]* failed" blue-green-handover.log + + - name: Byte-compile the touched modules + run: | + ci/bin/python -m py_compile gateway/handover.py gateway/systemd_notify.py \ + gateway/run_bootstrap.py gateway/run_shutdown.py hermes_cli/config_defaults.py + + - name: Upload evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: blue-green-handover-evidence + path: blue-green-handover.log + if-no-files-found: warn diff --git a/gateway/handover.py b/gateway/handover.py new file mode 100644 index 0000000000000..a15c54934cf53 --- /dev/null +++ b/gateway/handover.py @@ -0,0 +1,230 @@ +"""Blue-green gateway restart handover. + +Successor-first restart: blue (the running gateway) spawns green (``gateway run``) as a +child in the same supervisor cgroup. Green boots with platform-lock takeover authority, +binds the control socket name once it is ready (new sessions and relays land on green +while blue keeps serving its established connections), and points systemd's MainPID at +itself via ``sd_notify(MAINPID=...)`` so ``Restart=always`` stays quiet when blue exits. + +Ownership rules that keep the overlap safe: + +- The PID record (``gateway.pid``, O_EXCL) stays with blue until blue exits; green claims + it from a deferred task that polls for the predecessor's exit. +- Messaging/cron single-ownership needs no extra deferral: green boots with + ``_platform_lock_takeover_on_start`` (the same authority ``--replace`` uses), so its + adapter cold-connects fence blue's pollers out of each platform lock. +- If green never becomes ready, blue falls back to the classic drain-exit restart; a green + that crashes after taking MainPID still converges, because systemd then respawns the + unit normally. + +Requires a ``Type=notify`` unit (``NOTIFY_SOCKET`` present) so the MainPID switch works; +without it the handover is refused and restarts keep the classic drain-exit shape. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import os +import shlex +import subprocess +import sys +from dataclasses import dataclass +from typing import Any, Callable, Optional + +logger = logging.getLogger(__name__) + +HANDOVER_ENV = "HERMES_GATEWAY_HANDOVER" +PARENT_PID_ENV = "HERMES_GATEWAY_HANDOVER_PARENT_PID" +PARENT_START_ENV = "HERMES_GATEWAY_HANDOVER_PARENT_START_TICKS" + +READY_TIMEOUT_S = 20.0 +# Blue's drain budget already bounds its stop; this is only a safety net for arming the +# deferred PID-record claim, and arming late is safe (blue's own exit path also cleans up). +PREDECESSOR_EXIT_TIMEOUT_S = 1800.0 +_POLL_INTERVAL_S = 0.25 + + +def enabled(config: Any) -> bool: + """Blue-green restart gate: config opt-in + a notify socket (MainPID switch) + POSIX.""" + if sys.platform == "win32" or not os.environ.get("NOTIFY_SOCKET", "").strip(): + return False + agent = getattr(config, "agent", None) + return bool(getattr(agent, "blue_green_restart", False)) + + +def successor_env() -> dict[str, str]: + """Env additions that mark a child as blue's handover successor (PID-reuse guarded).""" + from gateway.status import get_process_start_time + return { + HANDOVER_ENV: "1", + PARENT_PID_ENV: str(os.getpid()), + PARENT_START_ENV: str(get_process_start_time(os.getpid())), + } + + +@dataclass(frozen=True) +class HandoverContext: + """Green-side view of its predecessor (``parent_start_ticks`` guards PID reuse).""" + + parent_pid: int + parent_start_ticks: Optional[int] + + +def handover_from_env() -> Optional[HandoverContext]: + """Parse the successor env; None when this boot is not a handover successor.""" + if os.environ.get(HANDOVER_ENV, "").strip() != "1": + return None + try: + parent_pid = int(os.environ.get(PARENT_PID_ENV, "")) + except ValueError: + logger.error("Handover env %s set but %s is missing/invalid; treating as a normal boot", + HANDOVER_ENV, PARENT_PID_ENV) + return None + if parent_pid <= 0 or parent_pid == os.getpid(): + logger.error("Handover parent pid %r is not a predecessor; treating as a normal boot", parent_pid) + return None + parent_start: Optional[int] + try: + parent_start = int(os.environ.get(PARENT_START_ENV, "")) + except ValueError: + parent_start = None + return HandoverContext(parent_pid=parent_pid, parent_start_ticks=parent_start) + + +def predecessor_alive(ctx: HandoverContext) -> bool: + """True while the predecessor process still exists (PID-reuse guarded when we have ticks).""" + try: + os.kill(ctx.parent_pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # exists, owned by someone else — still alive + if ctx.parent_start_ticks is not None: + from gateway.status import get_process_start_time + return get_process_start_time(ctx.parent_pid) == ctx.parent_start_ticks + return True + + +async def spawn_successor() -> Optional[int]: + """Blue side: spawn the ``gateway run`` successor child; its pid, or None on failure.""" + from gateway.run import _resolve_hermes_bin, logger as run_logger + argv = _resolve_hermes_bin() + if not argv: + run_logger.error("Blue-green handover: could not resolve the hermes argv; skipping") + return None + from hermes_cli.gateway import _profile_arg + full_argv = [*argv, *shlex.split(_profile_arg()), "gateway", "run"] + try: + proc = subprocess.Popen( # noqa: ASYNC220 -- spawn returns at once; the child outlives this task + full_argv, env={**os.environ, **successor_env()}, + ) + except OSError as exc: + run_logger.error("Blue-green handover: failed to spawn successor: %s", exc) + return None + run_logger.info("Blue-green handover: spawned successor gateway (PID %d)", proc.pid) + return proc.pid + + +def _identify_pid(socket_path) -> Optional[int]: + """One public ``identify`` round-trip on the control socket; the answering pid or None.""" + import socket as _socket + try: + with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock: + sock.settimeout(1.5) + sock.connect(str(socket_path)) + sock.sendall(json.dumps({"id": 1, "protocol": 1, "verb": "identify"}).encode() + b"\n") + with sock.makefile("rb") as reader: + line = reader.readline(4096) + payload = json.loads(line.decode("utf-8")) + except (OSError, ValueError, UnicodeDecodeError): + return None + if not isinstance(payload, dict) or payload.get("ok") is not True: + return None + result = payload.get("result") + pid = result.get("pid") if isinstance(result, dict) else None + return pid if isinstance(pid, int) and pid > 0 else None + + +async def wait_successor_ready(successor_pid: int, timeout: float = READY_TIMEOUT_S) -> bool: + """Blue side: poll the control socket until *successor_pid* is the gateway answering.""" + from gateway.control_socket import resolve_client_socket_path + from hermes_constants import get_hermes_home + deadline = asyncio.get_running_loop().time() + timeout + while asyncio.get_running_loop().time() < deadline: + path = resolve_client_socket_path(get_hermes_home()) + if path is not None: + # Blue may still answer identify until green binds; only green's pid counts. + if _identify_pid(path) == successor_pid: + return True + await asyncio.sleep(_POLL_INTERVAL_S) + return False + + +async def arm_after_predecessor_exit( + ctx: HandoverContext, + on_exit: Callable[[], Any], + timeout_s: float = PREDECESSOR_EXIT_TIMEOUT_S, +) -> None: + """Green side: run ``on_exit`` once the predecessor is gone (its stop path releases the + single-owner records). Arms anyway at the timeout — blue's exit path also cleans up.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + while loop.time() < deadline: + if not predecessor_alive(ctx): + break + await asyncio.sleep(_POLL_INTERVAL_S) + else: + logger.warning("Predecessor (PID %d) still alive after %.0fs; claiming deferred records anyway", + ctx.parent_pid, timeout_s) + outcome = on_exit() + if inspect.isawaitable(outcome): + await outcome + + +def _claim_pid_record(ctx: HandoverContext) -> None: + """Claim the PID record once the predecessor is gone (the body of the deferred claim).""" + import atexit + from gateway.status import get_process_start_time, get_running_pid, remove_pid_file, write_pid_file + recorded = get_running_pid() + if recorded is not None and recorded != os.getpid(): + try: + os.kill(recorded, 0) + alive = get_process_start_time(recorded) is not None + except OSError: + alive = False + if alive: + logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record", recorded) + return + logger.warning("Replacing stale predecessor PID record (PID %d)", recorded) + remove_pid_file() + try: + write_pid_file() + except FileExistsError as exc: + logger.error("Deferred PID claim lost the PID-file race: %s", exc) + return + atexit.register(remove_pid_file) + from gateway.run import _claim_host_gateway_role + try: + # The predecessor held the host role; with its exit the role is free, but claim it + # explicitly (force = the handover contract) so host-attach resolution points at us. + _claim_host_gateway_role(force=True) + except SystemExit as exc: # the role claim refuses via SystemExit(75) — degrade, not die + logger.error("Deferred host-role claim refused after handover: %s", exc) + except Exception as exc: # noqa: BLE001 -- role claim failure must not undo the PID claim + logger.error("Deferred host-role claim failed after handover: %s", exc) + from hermes_cli.gateway_multiplex_mode import publish_pending_multiplex_decision + publish_pending_multiplex_decision() + logger.info("Blue-green handover complete: PID record claimed (predecessor PID %d exited)", + ctx.parent_pid) + + +def deferred_pid_claim(ctx: HandoverContext) -> None: + """Green side: schedule ``_claim_pid_record`` to run once blue exits.""" + + async def _run() -> None: + await arm_after_predecessor_exit(ctx, lambda: _claim_pid_record(ctx)) + + asyncio.get_running_loop().create_task(_run()) diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py index b6f0ec9f62f43..feec34e74c34a 100644 --- a/gateway/run_bootstrap.py +++ b/gateway/run_bootstrap.py @@ -371,6 +371,13 @@ def _rescan_profiles_handler() -> dict: # Hosted-room transports install onto this listener; bind it here so every launcher # that starts the control socket (not only start_gateway) exposes it. runner.session_control_server = _control_server + from gateway import handover as _handover + if _handover.handover_from_env() is not None: + # Green is serving: point systemd's MainPID here so blue's exit (still draining + # its established connections) does not trigger Restart=always behind us. + from gateway import systemd_notify + if systemd_notify.notify_mainpid(os.getpid()): + logger.info("Blue-green handover: successor (PID %d) took over as MainPID", os.getpid()) except Exception as _cs_exc: logger.error("Control socket startup failed; the gateway cannot start without it: %s", _cs_exc) _control_server = None @@ -795,7 +802,14 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as # --force there disabled the one arbiter of the two-units-at-once race; a replace that took the # owner over already freed the lock with that process. - if not _start_gateway_claim_pid_file(force=force): + # A blue-green handover successor defers the claim instead: the predecessor owns the record + # until it exits, and the deferred task (armed after the control socket serves) takes it over. + from gateway import handover as _handover + _handover_ctx = _handover.handover_from_env() + if _handover_ctx is not None: + logger.info("Blue-green handover: this boot succeeds gateway PID %d", _handover_ctx.parent_pid) + _handover.deferred_pid_claim(_handover_ctx) + elif not _start_gateway_claim_pid_file(force=force): release_gateway_runtime_lock() return False @@ -817,7 +831,8 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = _enable_multiplex_log_routing(runner.config) # ``--replace`` is explicit startup authority, not a durable reconnect policy: GatewayRunner scopes # it to cold adapter connects and clears it before the background reconnect watcher starts. - runner._platform_lock_takeover_on_start = bool(replace) + # A blue-green handover successor carries the same authority over its predecessor. + runner._platform_lock_takeover_on_start = bool(replace) or _handover_ctx is not None # Unexpected signals exit non-zero so service managers revive us; planned stops write a marker first. _signal_initiated_shutdown = [False] diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 1606a343e27d4..b95ec2c9662b8 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1697,9 +1697,22 @@ def request_restart(self, *, detached: bool = False, via_service: bool = False) self._mark_api_runs_shutdown_requested() async def _run_restart() -> None: + # Blue-green handover: the successor boots FIRST, so new sessions land on it while + # blue keeps serving established connections through its drain. If green is not + # ready in time, fall through to the classic drain-exit restart unchanged. + handover_pid: int | None = None + from gateway import handover as _handover + if _handover.enabled(self.config): + handover_pid = await _handover.spawn_successor() + if handover_pid is None or not await _handover.wait_successor_ready(handover_pid): + logger.warning("Blue-green handover did not become ready (pid=%s); " + "falling back to the drain-exit restart", handover_pid) + handover_pid = None await self._await_active_work_before_restart() # Detached helper only AFTER the after-turn wait, or its drain_timeout+5 deadline fires mid-turn. - if detached: + # Never when the handover succeeded: systemd's MainPID is already the successor, and a + # post-exit `gateway restart` would bounce the new gateway for nothing. + if detached and handover_pid is None: with _log_suppressed(logging.ERROR, "Failed to launch detached gateway restart helper: %s"): await self._launch_detached_restart_command() await asyncio.sleep(0.05) diff --git a/gateway/systemd_notify.py b/gateway/systemd_notify.py index da66753533f02..1909c14f56a0f 100644 --- a/gateway/systemd_notify.py +++ b/gateway/systemd_notify.py @@ -28,6 +28,13 @@ def notify(message: str) -> bool: return False +def notify_mainpid(pid: int) -> bool: + """Point systemd's MainPID at ``pid`` (Type=notify units). The predecessor's exit then + looks like routine cgroup cleanup instead of a MainPID crash, so ``Restart=always`` + does not respawn the unit behind the successor's back.""" + return notify(f"MAINPID={pid}") + + def watchdog_interval_seconds() -> float | None: try: interval = float(os.environ.get("WATCHDOG_USEC", "") if _notify_socket() else "") / 1e6 diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 63ff490ee732e..a1716ed87bc83 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -109,6 +109,11 @@ def _aux(timeout, *, reasoning_effort=True, **extra): # TimeoutStopSec or risk SIGKILL mid-cleanup; for /restart prefer restart_after_turn_timeout # so turns finish BEFORE stop(). "restart_drain_timeout": 0, + # Blue-green restart: the successor gateway boots before the predecessor drains, new + # sessions land on it immediately, and sd_notify(MAINPID=...) keeps Restart=always + # quiet. Requires a Type=notify unit (NOTIFY_SOCKET); otherwise restarts keep the + # classic drain-exit shape. + "blue_green_restart": False, # Floor under the stop()/drain wait (seconds) for cron jobs and api_server runs. Interrupted # chat turns resume on the next message, but an interrupted cron run is recorded as a # permanent failure and an interrupted /v1 run fails its waiting caller, so neither may diff --git a/tests/gateway/test_handover.py b/tests/gateway/test_handover.py new file mode 100644 index 0000000000000..9a0830989ff3a --- /dev/null +++ b/tests/gateway/test_handover.py @@ -0,0 +1,196 @@ +"""Blue-green handover primitives: env contract, gates, readiness probe, deferred claim.""" + +from __future__ import annotations + +import asyncio +import json +import os +import socket +import subprocess +import sys +import types + +import pytest + +from gateway import handover, systemd_notify + + +class _AgentCfg: + blue_green_restart = True + + +class _Cfg: + agent = _AgentCfg() + + +class _CfgOff: + agent = types.SimpleNamespace(blue_green_restart=False) + + +def _clear_env(monkeypatch): + for key in (handover.HANDOVER_ENV, handover.PARENT_PID_ENV, handover.PARENT_START_ENV, + "NOTIFY_SOCKET"): + monkeypatch.delenv(key, raising=False) + + +def test_env_roundtrip(monkeypatch): + monkeypatch.setenv(handover.HANDOVER_ENV, "1") + monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid() + 12345)) + monkeypatch.delenv(handover.PARENT_START_ENV, raising=False) + ctx = handover.handover_from_env() + assert ctx is not None and ctx.parent_pid == os.getpid() + 12345 + assert ctx.parent_start_ticks is None + env = handover.successor_env() + assert env[handover.HANDOVER_ENV] == "1" + # successor_env() stamps the CALLER as the parent; from this process that is self -> rejected. + assert handover.handover_from_env() is None + + +def test_env_rejects_self_and_missing(monkeypatch): + _clear_env(monkeypatch) + assert handover.handover_from_env() is None + monkeypatch.setenv(handover.HANDOVER_ENV, "1") + monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid())) + assert handover.handover_from_env() is None # predecessor cannot be ourselves + monkeypatch.setenv(handover.PARENT_PID_ENV, "not-a-pid") + assert handover.handover_from_env() is None + + +def test_enabled_gate(monkeypatch): + _clear_env(monkeypatch) + assert handover.enabled(_Cfg()) is False # no NOTIFY_SOCKET -> cannot switch MainPID + monkeypatch.setenv("NOTIFY_SOCKET", "/run/user/1/notify") + assert handover.enabled(_Cfg()) is True + assert handover.enabled(_CfgOff()) is False + + +def test_notify_mainpid_payload(tmp_path): + sock_path = tmp_path / "notify.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + server.bind(str(sock_path)) + server.setblocking(False) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(systemd_notify, "_notify_socket", lambda: str(sock_path)) + assert systemd_notify.notify_mainpid(4242) is True + assert server.recv(4096) == b"MAINPID=4242" + server.close() + with pytest.MonkeyPatch.context() as mp: + mp.setattr(systemd_notify, "_notify_socket", lambda: "") + assert systemd_notify.notify_mainpid(1) is False + + +def test_wait_successor_ready_pid_match(tmp_path): + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock_path = tmp_path / "gateway.sock" + server.bind(str(sock_path)) + server.listen(1) + + successor_pid = os.getpid() + 777 + + async def serve_once(): + conn, _ = server.accept() + with conn: + conn.recv(4096) + conn.sendall(json.dumps({"ok": True, "result": {"pid": successor_pid}}).encode() + b"\n") + + loop = asyncio.new_event_loop() + try: + serve_task = loop.run_in_executor(None, serve_once) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(handover, "_POLL_INTERVAL_S", 0.05) + mp.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + ready = loop.run_until_complete( + asyncio.wait_for(handover.wait_successor_ready(successor_pid, timeout=5.0), 8)) + assert ready is True + serve_task.result() + finally: + loop.close() + server.close() + + +def test_wait_successor_ready_timeout(tmp_path, monkeypatch): + sock_path = tmp_path / "gateway.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(sock_path)) + server.listen(1) + monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) + loop = asyncio.new_event_loop() + try: + # Nobody answers with the successor's pid -> bounded False. + ready = loop.run_until_complete(handover.wait_successor_ready(os.getpid() + 999, timeout=0.2)) + assert ready is False + finally: + loop.close() + server.close() + + +def test_arm_after_predecessor_exit(monkeypatch): + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.2)"]) + ctx = handover.HandoverContext(parent_pid=child.pid, parent_start_ticks=None) + fired = [] + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) + loop = asyncio.new_event_loop() + try: + loop.run_until_complete( + asyncio.wait_for(handover.arm_after_predecessor_exit(ctx, lambda: fired.append(1)), 10)) + finally: + loop.close() + child.wait() + assert fired == [1] + + +def test_predecessor_alive_pid_reuse_guard(monkeypatch): + # A recycled pid with mismatched start ticks is not our predecessor. + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.5)"]) + try: + ctx = handover.HandoverContext(parent_pid=child.pid, parent_start_ticks=1) + monkeypatch.setattr("gateway.status.get_process_start_time", lambda pid: 999) + assert handover.predecessor_alive(ctx) is False + finally: + child.kill() + child.wait() + + +def _patch_claim_status(monkeypatch, recorded_pid, recorded_alive, writes): + import gateway.status as status + monkeypatch.setattr(status, "get_running_pid", lambda *a, **k: recorded_pid) + if recorded_alive: + monkeypatch.setattr(status, "get_process_start_time", lambda pid: 12345) + + def _kill(pid, sig): + return None + else: + def _kill(pid, sig): + raise ProcessLookupError() + monkeypatch.setattr(os, "kill", _kill) + monkeypatch.setattr(status, "write_pid_file", lambda: writes.append(1)) + monkeypatch.setattr(status, "remove_pid_file", lambda: writes.append("removed")) + import hermes_cli.gateway_multiplex_mode as gmm + monkeypatch.setattr(gmm, "publish_pending_multiplex_decision", lambda: None) + import gateway.run as run_mod + monkeypatch.setattr(run_mod, "_claim_host_gateway_role", lambda force=False: None) + + +def test_claim_skips_live_owner(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=True, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == [] # a live recorded gateway blocks the claim + + +def test_claim_replaces_stale_owner(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=False, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == ["removed", 1] + + +def test_claim_when_record_free(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=None, recorded_alive=False, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == [1] From 5be2fc8195872279d98550bdcd63635e5f1827f9 Mon Sep 17 00:00:00 2001 From: Raisal P Wardana Date: Mon, 16 Nov 2026 01:09:29 +0700 Subject: [PATCH 2/2] =?UTF-8?q?fix(gateway):=20blue-green=20handover=20rev?= =?UTF-8?q?iew=20pass=20=E2=80=94=20MAINPID=20sender,=20real=20config=20ga?= =?UTF-8?q?te,=20loop-safe=20probe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings (opus, systemd-probe-verified) addressed: - finding 1: the gate read config.agent which GatewayConfig lacks — the feature never enabled. Now loaded like restart_drain_timeout (_load_blue_green_restart -> runner._blue_green_restart) with an HERMES_BLUE_GREEN_RESTART env override. - finding 3: NotifyAccess=main drops a successor's own MAINPID notify; blue now sends MAINPID= after readiness, green never self-notifies. - finding 4: green carries no platform-lock takeover authority (takeover SIGTERMs the holder = blue mid-drain). Green's adapters fail the lock while blue holds it and reconnect when blue's stop releases it. - finding 5: the readiness probe ran blocking socket I/O on blue's loop (1.5s stalls, watchdog risk) — now via asyncio.to_thread, with an early exit when the successor process dies. - finding 6/8: readiness now requires identify state==ready (published after adapters/cron/housekeeping), not a bound socket; MAINPID only switches after that, so green's watchdog is already feeding. - finding 7/13: fallback terminates the unpromoted successor; stop() terminates any unpromoted child; the spawn is skipped once a signal stop was requested. - finding 11: a lost deferred PID-record claim is fatal for green (self-SIGTERM) instead of serving unrecorded. - finding 15: handover env vars are scrubbed and the parse cached. Open (design note to follow on the PR): profile-reservation/session-store handoff, control-socket affinity for parked approval turns, fixed-port listener handover (api_server EADDRINUSE is non-retryable), systemctl-stop during overlap. Flag stays default-off until those land. --- gateway/handover.py | 138 ++++++++++++++++++++++++--------- gateway/run_bootstrap.py | 16 ++-- gateway/run_config_loaders.py | 6 ++ gateway/run_runtime_init.py | 1 + gateway/run_shutdown.py | 40 +++++++--- tests/gateway/test_handover.py | 120 ++++++++++++++++------------ 6 files changed, 218 insertions(+), 103 deletions(-) diff --git a/gateway/handover.py b/gateway/handover.py index a15c54934cf53..7943193b0aa56 100644 --- a/gateway/handover.py +++ b/gateway/handover.py @@ -1,21 +1,27 @@ """Blue-green gateway restart handover. Successor-first restart: blue (the running gateway) spawns green (``gateway run``) as a -child in the same supervisor cgroup. Green boots with platform-lock takeover authority, -binds the control socket name once it is ready (new sessions and relays land on green -while blue keeps serving its established connections), and points systemd's MainPID at -itself via ``sd_notify(MAINPID=...)`` so ``Restart=always`` stays quiet when blue exits. +child in the same supervisor cgroup. Green serves sessions once its runtime reaches the +ready state; blue keeps serving its established connections until their turns finish, +then exits through the existing stop path. The MainPID switch is sent by BLUE (not green: +``NotifyAccess=main`` drops notify messages from any other pid) and only after green is +ready and feeding its watchdog, so ``Restart=always`` never fires behind green's back. Ownership rules that keep the overlap safe: - The PID record (``gateway.pid``, O_EXCL) stays with blue until blue exits; green claims - it from a deferred task that polls for the predecessor's exit. -- Messaging/cron single-ownership needs no extra deferral: green boots with - ``_platform_lock_takeover_on_start`` (the same authority ``--replace`` uses), so its - adapter cold-connects fence blue's pollers out of each platform lock. -- If green never becomes ready, blue falls back to the classic drain-exit restart; a green - that crashes after taking MainPID still converges, because systemd then respawns the - unit normally. + it from a deferred task that polls for the predecessor's exit. A lost claim is fatal for + green (SIGTERM self-stop): an unrecorded gateway must not keep serving. +- Platform locks: green carries NO takeover authority. Its adapters fail to lock while + blue holds them and pick up through the existing reconnect watcher once blue's stop + releases them; blue is never signalled by green. +- Blue sends ``MAINPID=`` only after green's control socket answers ``identify`` with the + successor pid AND descriptor state ``ready`` (adapters, cron, housekeeping running). + +Known protocol remainder (kept out of v1, flag stays default-off): the profile +reservation / session-store handoff, control-socket session affinity for turns parked on +approval/clarify, and fixed-port listeners (api_server) that cannot rebind until blue's +stop closes them. Requires a ``Type=notify`` unit (``NOTIFY_SOCKET`` present) so the MainPID switch works; without it the handover is refused and restarts keep the classic drain-exit shape. @@ -29,6 +35,7 @@ import logging import os import shlex +import signal import subprocess import sys from dataclasses import dataclass @@ -41,18 +48,20 @@ PARENT_START_ENV = "HERMES_GATEWAY_HANDOVER_PARENT_START_TICKS" READY_TIMEOUT_S = 20.0 +FALLBACK_TERMINATE_S = 10.0 # Blue's drain budget already bounds its stop; this is only a safety net for arming the # deferred PID-record claim, and arming late is safe (blue's own exit path also cleans up). PREDECESSOR_EXIT_TIMEOUT_S = 1800.0 _POLL_INTERVAL_S = 0.25 +_parsed_context: Optional["HandoverContext"] = None -def enabled(config: Any) -> bool: - """Blue-green restart gate: config opt-in + a notify socket (MainPID switch) + POSIX.""" + +def enabled(flag: bool) -> bool: + """Blue-green restart gate: pre-loaded config flag + notify socket (MainPID switch).""" if sys.platform == "win32" or not os.environ.get("NOTIFY_SOCKET", "").strip(): return False - agent = getattr(config, "agent", None) - return bool(getattr(agent, "blue_green_restart", False)) + return bool(flag) def successor_env() -> dict[str, str]: @@ -74,24 +83,38 @@ class HandoverContext: def handover_from_env() -> Optional[HandoverContext]: - """Parse the successor env; None when this boot is not a handover successor.""" + """Parse the successor env; None when this boot is not a handover successor. + + The variables are scrubbed from the environment on first parse and the result is + cached: a handover successor must not leak its authority to ``gateway run`` children + of its own, and bootstrap reads this twice. + """ + global _parsed_context + if _parsed_context is not None: + return _parsed_context or None if os.environ.get(HANDOVER_ENV, "").strip() != "1": + _parsed_context = False return None try: parent_pid = int(os.environ.get(PARENT_PID_ENV, "")) except ValueError: logger.error("Handover env %s set but %s is missing/invalid; treating as a normal boot", HANDOVER_ENV, PARENT_PID_ENV) - return None - if parent_pid <= 0 or parent_pid == os.getpid(): - logger.error("Handover parent pid %r is not a predecessor; treating as a normal boot", parent_pid) + _parsed_context = False return None parent_start: Optional[int] try: parent_start = int(os.environ.get(PARENT_START_ENV, "")) except ValueError: parent_start = None - return HandoverContext(parent_pid=parent_pid, parent_start_ticks=parent_start) + if parent_pid <= 0 or parent_pid == os.getpid(): + logger.error("Handover parent pid %r is not a predecessor; treating as a normal boot", parent_pid) + _parsed_context = False + return None + for key in (HANDOVER_ENV, PARENT_PID_ENV, PARENT_START_ENV): + os.environ.pop(key, None) + _parsed_context = HandoverContext(parent_pid=parent_pid, parent_start_ticks=parent_start) + return _parsed_context def predecessor_alive(ctx: HandoverContext) -> bool: @@ -108,8 +131,8 @@ def predecessor_alive(ctx: HandoverContext) -> bool: return True -async def spawn_successor() -> Optional[int]: - """Blue side: spawn the ``gateway run`` successor child; its pid, or None on failure.""" +def spawn_successor() -> Optional[subprocess.Popen]: + """Blue side: spawn the ``gateway run`` successor child; its handle, or None.""" from gateway.run import _resolve_hermes_bin, logger as run_logger argv = _resolve_hermes_bin() if not argv: @@ -125,11 +148,11 @@ async def spawn_successor() -> Optional[int]: run_logger.error("Blue-green handover: failed to spawn successor: %s", exc) return None run_logger.info("Blue-green handover: spawned successor gateway (PID %d)", proc.pid) - return proc.pid + return proc -def _identify_pid(socket_path) -> Optional[int]: - """One public ``identify`` round-trip on the control socket; the answering pid or None.""" +def _identify_pid_and_state(socket_path) -> Optional[tuple[int, str]]: + """One public ``identify`` round-trip; (answering pid, descriptor state) or None.""" import socket as _socket try: with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock: @@ -144,25 +167,56 @@ def _identify_pid(socket_path) -> Optional[int]: if not isinstance(payload, dict) or payload.get("ok") is not True: return None result = payload.get("result") - pid = result.get("pid") if isinstance(result, dict) else None - return pid if isinstance(pid, int) and pid > 0 else None + if not isinstance(result, dict): + return None + pid = result.get("pid") + state = result.get("state") + if not isinstance(pid, int) or pid <= 0 or not isinstance(state, str): + return None + return pid, state + +async def wait_successor_ready(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool: + """Blue side: poll until the successor ITSELF answers identify in the ready state. -async def wait_successor_ready(successor_pid: int, timeout: float = READY_TIMEOUT_S) -> bool: - """Blue side: poll the control socket until *successor_pid* is the gateway answering.""" + Blue may still answer until green binds the socket name, and a bound-but-starting + green answers ``state != 'ready'`` — neither counts. Blocking socket I/O runs in a + thread so the event loop (streaming, adapters, in-flight turns) never stalls. + """ from gateway.control_socket import resolve_client_socket_path from hermes_constants import get_hermes_home - deadline = asyncio.get_running_loop().time() + timeout - while asyncio.get_running_loop().time() < deadline: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if proc.poll() is not None: + logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid) + return False path = resolve_client_socket_path(get_hermes_home()) if path is not None: - # Blue may still answer identify until green binds; only green's pid counts. - if _identify_pid(path) == successor_pid: + answer = await asyncio.to_thread(_identify_pid_and_state, path) + if answer is not None and answer[0] == proc.pid and answer[1] == "ready": return True await asyncio.sleep(_POLL_INTERVAL_S) return False +def terminate_successor(proc: subprocess.Popen) -> None: + """Blue side: a successor that was never promoted must not outlive the fallback.""" + if proc.poll() is not None: + return + logger.warning("Blue-green handover: terminating unpromoted successor (PID %d)", proc.pid) + proc.terminate() + try: + proc.wait(timeout=FALLBACK_TERMINATE_S) + except subprocess.TimeoutExpired: + logger.error("Successor (PID %d) ignored SIGTERM; SIGKILL", proc.pid) + proc.kill() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + + async def arm_after_predecessor_exit( ctx: HandoverContext, on_exit: Callable[[], Any], @@ -184,8 +238,17 @@ async def arm_after_predecessor_exit( await outcome +def _self_stop() -> None: + """Stop this gateway through its own signal path (clean drain, supervisor revives).""" + os.kill(os.getpid(), signal.SIGTERM) + + def _claim_pid_record(ctx: HandoverContext) -> None: - """Claim the PID record once the predecessor is gone (the body of the deferred claim).""" + """Claim the PID record once the predecessor is gone (the body of the deferred claim). + + A lost claim leaves this process as an unrecorded gateway — fatal by self-stop so the + supervisor's next boot takes over cleanly instead of two gateways racing the record. + """ import atexit from gateway.status import get_process_start_time, get_running_pid, remove_pid_file, write_pid_file recorded = get_running_pid() @@ -196,14 +259,17 @@ def _claim_pid_record(ctx: HandoverContext) -> None: except OSError: alive = False if alive: - logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record", recorded) + logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record; " + "this successor stops itself", recorded) + _self_stop() return logger.warning("Replacing stale predecessor PID record (PID %d)", recorded) remove_pid_file() try: write_pid_file() except FileExistsError as exc: - logger.error("Deferred PID claim lost the PID-file race: %s", exc) + logger.error("Deferred PID claim lost the PID-file race (%s); this successor stops itself", exc) + _self_stop() return atexit.register(remove_pid_file) from gateway.run import _claim_host_gateway_role diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py index feec34e74c34a..dec71c0254d1f 100644 --- a/gateway/run_bootstrap.py +++ b/gateway/run_bootstrap.py @@ -371,13 +371,9 @@ def _rescan_profiles_handler() -> dict: # Hosted-room transports install onto this listener; bind it here so every launcher # that starts the control socket (not only start_gateway) exposes it. runner.session_control_server = _control_server - from gateway import handover as _handover - if _handover.handover_from_env() is not None: - # Green is serving: point systemd's MainPID here so blue's exit (still draining - # its established connections) does not trigger Restart=always behind us. - from gateway import systemd_notify - if systemd_notify.notify_mainpid(os.getpid()): - logger.info("Blue-green handover: successor (PID %d) took over as MainPID", os.getpid()) + # NOTE: a handover successor does NOT notify MAINPID itself — under NotifyAccess=main + # systemd drops notify messages from any pid but the current main. Blue sends the + # switch after green reports ready (gateway.run_shutdown.request_restart). except Exception as _cs_exc: logger.error("Control socket startup failed; the gateway cannot start without it: %s", _cs_exc) _control_server = None @@ -831,8 +827,10 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = _enable_multiplex_log_routing(runner.config) # ``--replace`` is explicit startup authority, not a durable reconnect policy: GatewayRunner scopes # it to cold adapter connects and clears it before the background reconnect watcher starts. - # A blue-green handover successor carries the same authority over its predecessor. - runner._platform_lock_takeover_on_start = bool(replace) or _handover_ctx is not None + # Deliberately NOT given to a blue-green handover successor: takeover SIGTERMs the holder, + # which would kill blue mid-drain. Green's adapters fail the lock while blue holds it and + # reconnect once blue's stop releases it. + runner._platform_lock_takeover_on_start = bool(replace) # Unexpected signals exit non-zero so service managers revive us; planned stops write a marker first. _signal_initiated_shutdown = [False] diff --git a/gateway/run_config_loaders.py b/gateway/run_config_loaders.py index 5230924b18411..87ae9cbf68281 100644 --- a/gateway/run_config_loaders.py +++ b/gateway/run_config_loaders.py @@ -380,6 +380,12 @@ def _warn_unparsable_timeout(cfg_key: str, raw: object, default: float) -> None: except (TypeError, ValueError): logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default) + @classmethod + def _load_blue_green_restart(cls) -> bool: + """Blue-green restart handover opt-in (successor-first restart; default off).""" + raw = cls._env_or_cfg_str("HERMES_BLUE_GREEN_RESTART", "agent", "blue_green_restart") + return raw.strip().lower() in {"1", "true", "yes", "on"} + @classmethod def _load_restart_drain_timeout(cls) -> float: """Graceful gateway restart/stop drain timeout in seconds.""" diff --git a/gateway/run_runtime_init.py b/gateway/run_runtime_init.py index 3e75e19c4b044..c800afe81cbff 100644 --- a/gateway/run_runtime_init.py +++ b/gateway/run_runtime_init.py @@ -23,6 +23,7 @@ def _init_runtime_settings(self) -> None: self._human_delay = self._human_delay_from_config(_load_gateway_config()) self._human_delay_by_profile: Dict[str, Optional[tuple[int, int]]] = {} self._restart_drain_timeout = self._load_restart_drain_timeout() + self._blue_green_restart = self._load_blue_green_restart() # Live launchd ``ExitTimeOut`` for this job (None when not launchd-owned). Read once at # boot — launchd fixes it at load — and applied only to signal-driven stops, which are the # only stops launchd times. See _load_launchd_exit_timeout(). diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index b95ec2c9662b8..c29658e923d41 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1698,21 +1698,34 @@ def request_restart(self, *, detached: bool = False, via_service: bool = False) async def _run_restart() -> None: # Blue-green handover: the successor boots FIRST, so new sessions land on it while - # blue keeps serving established connections through its drain. If green is not - # ready in time, fall through to the classic drain-exit restart unchanged. - handover_pid: int | None = None + # blue keeps serving established connections through its drain. Blue sends the + # MAINPID switch only after green reports ready (NotifyAccess=main drops green's + # own notify), and falls back to the classic drain-exit restart otherwise. + handover_child: subprocess.Popen | None = None from gateway import handover as _handover - if _handover.enabled(self.config): - handover_pid = await _handover.spawn_successor() - if handover_pid is None or not await _handover.wait_successor_ready(handover_pid): - logger.warning("Blue-green handover did not become ready (pid=%s); " - "falling back to the drain-exit restart", handover_pid) - handover_pid = None + if not self._stop_requested_by_signal and _handover.enabled(self._blue_green_restart): + handover_child = _handover.spawn_successor() + ready = handover_child is not None and await _handover.wait_successor_ready(handover_child) + if ready: + from gateway import systemd_notify + if not systemd_notify.notify_mainpid(handover_child.pid): + logger.warning("Blue-green handover: MAINPID notify failed (no systemd notify " + "socket); falling back to the drain-exit restart") + ready = False + if ready: + self._handover_promoted = True + else: + logger.warning("Blue-green handover did not become ready (pid=%s); falling back " + "to the drain-exit restart", getattr(handover_child, "pid", None)) + if handover_child is not None: + await asyncio.to_thread(_handover.terminate_successor, handover_child) + handover_child = None + self._handover_child = handover_child await self._await_active_work_before_restart() # Detached helper only AFTER the after-turn wait, or its drain_timeout+5 deadline fires mid-turn. # Never when the handover succeeded: systemd's MainPID is already the successor, and a # post-exit `gateway restart` would bounce the new gateway for nothing. - if detached and handover_pid is None: + if detached and handover_child is None: with _log_suppressed(logging.ERROR, "Failed to launch detached gateway restart helper: %s"): await self._launch_detached_restart_command() await asyncio.sleep(0.05) @@ -2257,6 +2270,13 @@ async def stop( ) -> None: """Stop the gateway and disconnect all adapters.""" from gateway.run import GatewayRunner + # A handover successor never promoted to MainPID (signal stop during the handover + # window, or a fallback race) must not outlive this stop as a duplicate gateway. + _handover_child = getattr(self, "_handover_child", None) + if _handover_child is not None and not getattr(self, "_handover_promoted", False): + from gateway import handover as _handover_mod + await asyncio.to_thread(_handover_mod.terminate_successor, _handover_child) + self._handover_child = None # getattr-guard: shutdown-path tests build bare runners via object.__new__ that lack the # liveness-guard machinery. _stop_guards = getattr(self, "_stop_loop_liveness_guards", None) diff --git a/tests/gateway/test_handover.py b/tests/gateway/test_handover.py index 9a0830989ff3a..b5cd821069f97 100644 --- a/tests/gateway/test_handover.py +++ b/tests/gateway/test_handover.py @@ -8,60 +8,54 @@ import socket import subprocess import sys -import types +import threading import pytest from gateway import handover, systemd_notify -class _AgentCfg: - blue_green_restart = True - - -class _Cfg: - agent = _AgentCfg() - - -class _CfgOff: - agent = types.SimpleNamespace(blue_green_restart=False) - - def _clear_env(monkeypatch): for key in (handover.HANDOVER_ENV, handover.PARENT_PID_ENV, handover.PARENT_START_ENV, "NOTIFY_SOCKET"): monkeypatch.delenv(key, raising=False) + monkeypatch.setattr(handover, "_parsed_context", None) -def test_env_roundtrip(monkeypatch): +def test_env_roundtrip_and_scrub(monkeypatch): + _clear_env(monkeypatch) monkeypatch.setenv(handover.HANDOVER_ENV, "1") monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid() + 12345)) monkeypatch.delenv(handover.PARENT_START_ENV, raising=False) ctx = handover.handover_from_env() assert ctx is not None and ctx.parent_pid == os.getpid() + 12345 assert ctx.parent_start_ticks is None + # Parsed-and-scrubbed: the authority must not leak to this process's own children, and + # bootstrap's second parse returns the cached context. + assert os.environ.get(handover.HANDOVER_ENV) is None + assert handover.handover_from_env() is ctx env = handover.successor_env() assert env[handover.HANDOVER_ENV] == "1" - # successor_env() stamps the CALLER as the parent; from this process that is self -> rejected. - assert handover.handover_from_env() is None def test_env_rejects_self_and_missing(monkeypatch): _clear_env(monkeypatch) assert handover.handover_from_env() is None + monkeypatch.setattr(handover, "_parsed_context", None) monkeypatch.setenv(handover.HANDOVER_ENV, "1") monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid())) assert handover.handover_from_env() is None # predecessor cannot be ourselves + monkeypatch.setattr(handover, "_parsed_context", None) monkeypatch.setenv(handover.PARENT_PID_ENV, "not-a-pid") assert handover.handover_from_env() is None def test_enabled_gate(monkeypatch): - _clear_env(monkeypatch) - assert handover.enabled(_Cfg()) is False # no NOTIFY_SOCKET -> cannot switch MainPID + monkeypatch.delenv("NOTIFY_SOCKET", raising=False) + assert handover.enabled(False) is False # no NOTIFY_SOCKET -> cannot switch MainPID monkeypatch.setenv("NOTIFY_SOCKET", "/run/user/1/notify") - assert handover.enabled(_Cfg()) is True - assert handover.enabled(_CfgOff()) is False + assert handover.enabled(True) is True + assert handover.enabled(False) is False def test_notify_mainpid_payload(tmp_path): @@ -79,50 +73,80 @@ def test_notify_mainpid_payload(tmp_path): assert systemd_notify.notify_mainpid(1) is False -def test_wait_successor_ready_pid_match(tmp_path): - server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock_path = tmp_path / "gateway.sock" - server.bind(str(sock_path)) - server.listen(1) +def _serve_identify(server, pid, state, replies=64): + server.setblocking(True) - successor_pid = os.getpid() + 777 + def serve(): + for _ in range(replies): + try: + conn, _ = server.accept() + except OSError: + return + with conn: + conn.recv(4096) + conn.sendall(json.dumps({"ok": True, "result": {"pid": pid, "state": state}}).encode() + b"\n") - async def serve_once(): - conn, _ = server.accept() - with conn: - conn.recv(4096) - conn.sendall(json.dumps({"ok": True, "result": {"pid": successor_pid}}).encode() + b"\n") + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return thread - loop = asyncio.new_event_loop() + +def test_wait_successor_ready_requires_ready_state(tmp_path, monkeypatch): + sock_path = tmp_path / "gateway.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(sock_path)) + server.listen(8) + successor_pid = os.getpid() + 777 + thread = _serve_identify(server, successor_pid, "ready") + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.05) + monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) try: - serve_task = loop.run_in_executor(None, serve_once) - with pytest.MonkeyPatch.context() as mp: - mp.setattr(handover, "_POLL_INTERVAL_S", 0.05) - mp.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + loop = asyncio.new_event_loop() + try: ready = loop.run_until_complete( - asyncio.wait_for(handover.wait_successor_ready(successor_pid, timeout=5.0), 8)) - assert ready is True - serve_task.result() + asyncio.wait_for(handover.wait_successor_ready(proc, timeout=5.0), 8)) + assert ready is True + finally: + loop.close() finally: - loop.close() + proc.kill() + proc.wait() server.close() + thread.join(timeout=2) -def test_wait_successor_ready_timeout(tmp_path, monkeypatch): +def test_wait_successor_ready_rejects_draining_and_wrong_pid(tmp_path, monkeypatch): sock_path = tmp_path / "gateway.sock" server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) server.bind(str(sock_path)) - server.listen(1) - monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + server.listen(8) + # A draining blue answering on its own socket must never satisfy the probe. + thread = _serve_identify(server, os.getpid(), "draining") monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) - loop = asyncio.new_event_loop() + monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) try: - # Nobody answers with the successor's pid -> bounded False. - ready = loop.run_until_complete(handover.wait_successor_ready(os.getpid() + 999, timeout=0.2)) - assert ready is False + loop = asyncio.new_event_loop() + try: + ready = loop.run_until_complete(handover.wait_successor_ready(os.getpid() + 999, timeout=0.3)) + assert ready is False + finally: + loop.close() finally: - loop.close() + proc.kill() + proc.wait() server.close() + thread.join(timeout=2) + + +def test_terminate_successor_fallback(): + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + handover.terminate_successor(child) + assert child.poll() is not None + already = subprocess.Popen([sys.executable, "-c", "pass"]) + already.wait() + handover.terminate_successor(already) # no-op on an exited child def test_arm_after_predecessor_exit(monkeypatch):