diff --git a/.github/workflows/gateway-blue-green-restart.yml b/.github/workflows/gateway-blue-green-restart.yml new file mode 100644 index 0000000000000..fce14b41b6292 --- /dev/null +++ b/.github/workflows/gateway-blue-green-restart.yml @@ -0,0 +1,70 @@ +name: gateway-blue-green-restart + +# Runs ONLY on this branch: focused gate for the blue-green restart handover +# (successor-first restart with sd_notify MainPID switch; falls back to the +# classic drain-exit restart when green is not ready). The repo's own +# full-repo CI also fires on the PR. +on: + push: + branches: [feat/gateway-blue-green-restart] +permissions: + contents: read +concurrency: + group: gateway-blue-green-restart + cancel-in-progress: true + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Python 3.14 (.python-version) + run: uv python install 3.14 + + - name: "Private PM environment (canonical command; setup inputs modified: 4 UV_CONCURRENT caps = 1)" + env: + UV_CONCURRENT_DOWNLOADS: '1' + UV_CONCURRENT_BUILDS: '1' + UV_CONCURRENT_INSTALLS: '1' + UV_CONCURRENT_CACHE_READS: '1' + run: | + PY="$(uv python find 3.14)" + "$PY" -m pm.build_env --source . --out ci --group dev --group test --python "$PY" + + - name: Focused handover regressions (must PASS) + env: + TZ: UTC + LANG: C.UTF-8 + HERMES_HOME: ${{ runner.temp }}/hermes-home + run: | + mkdir -p "$HERMES_HOME" + HERMES_PYTHON="$PWD/ci/bin/python" bash scripts/run_tests.sh \ + tests/gateway/test_handover.py \ + --file-retries 0 --file-timeout 240 -j 2 -q 2>&1 | tee blue-green-handover.log + + - name: Gate on the focused result + run: | + grep -E "passed" blue-green-handover.log + ! grep -qE "[1-9][0-9]* failed" blue-green-handover.log + + - name: Byte-compile the touched modules + run: | + ci/bin/python -m py_compile gateway/handover.py gateway/systemd_notify.py \ + gateway/run_bootstrap.py gateway/run_shutdown.py hermes_cli/config_defaults.py + + - name: Upload evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: blue-green-handover-evidence + path: blue-green-handover.log + if-no-files-found: warn diff --git a/gateway/handover.py b/gateway/handover.py new file mode 100644 index 0000000000000..7943193b0aa56 --- /dev/null +++ b/gateway/handover.py @@ -0,0 +1,296 @@ +"""Blue-green gateway restart handover. + +Successor-first restart: blue (the running gateway) spawns green (``gateway run``) as a +child in the same supervisor cgroup. Green serves sessions once its runtime reaches the +ready state; blue keeps serving its established connections until their turns finish, +then exits through the existing stop path. The MainPID switch is sent by BLUE (not green: +``NotifyAccess=main`` drops notify messages from any other pid) and only after green is +ready and feeding its watchdog, so ``Restart=always`` never fires behind green's back. + +Ownership rules that keep the overlap safe: + +- The PID record (``gateway.pid``, O_EXCL) stays with blue until blue exits; green claims + it from a deferred task that polls for the predecessor's exit. A lost claim is fatal for + green (SIGTERM self-stop): an unrecorded gateway must not keep serving. +- Platform locks: green carries NO takeover authority. Its adapters fail to lock while + blue holds them and pick up through the existing reconnect watcher once blue's stop + releases them; blue is never signalled by green. +- Blue sends ``MAINPID=`` only after green's control socket answers ``identify`` with the + successor pid AND descriptor state ``ready`` (adapters, cron, housekeeping running). + +Known protocol remainder (kept out of v1, flag stays default-off): the profile +reservation / session-store handoff, control-socket session affinity for turns parked on +approval/clarify, and fixed-port listeners (api_server) that cannot rebind until blue's +stop closes them. + +Requires a ``Type=notify`` unit (``NOTIFY_SOCKET`` present) so the MainPID switch works; +without it the handover is refused and restarts keep the classic drain-exit shape. +""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import os +import shlex +import signal +import subprocess +import sys +from dataclasses import dataclass +from typing import Any, Callable, Optional + +logger = logging.getLogger(__name__) + +HANDOVER_ENV = "HERMES_GATEWAY_HANDOVER" +PARENT_PID_ENV = "HERMES_GATEWAY_HANDOVER_PARENT_PID" +PARENT_START_ENV = "HERMES_GATEWAY_HANDOVER_PARENT_START_TICKS" + +READY_TIMEOUT_S = 20.0 +FALLBACK_TERMINATE_S = 10.0 +# Blue's drain budget already bounds its stop; this is only a safety net for arming the +# deferred PID-record claim, and arming late is safe (blue's own exit path also cleans up). +PREDECESSOR_EXIT_TIMEOUT_S = 1800.0 +_POLL_INTERVAL_S = 0.25 + +_parsed_context: Optional["HandoverContext"] = None + + +def enabled(flag: bool) -> bool: + """Blue-green restart gate: pre-loaded config flag + notify socket (MainPID switch).""" + if sys.platform == "win32" or not os.environ.get("NOTIFY_SOCKET", "").strip(): + return False + return bool(flag) + + +def successor_env() -> dict[str, str]: + """Env additions that mark a child as blue's handover successor (PID-reuse guarded).""" + from gateway.status import get_process_start_time + return { + HANDOVER_ENV: "1", + PARENT_PID_ENV: str(os.getpid()), + PARENT_START_ENV: str(get_process_start_time(os.getpid())), + } + + +@dataclass(frozen=True) +class HandoverContext: + """Green-side view of its predecessor (``parent_start_ticks`` guards PID reuse).""" + + parent_pid: int + parent_start_ticks: Optional[int] + + +def handover_from_env() -> Optional[HandoverContext]: + """Parse the successor env; None when this boot is not a handover successor. + + The variables are scrubbed from the environment on first parse and the result is + cached: a handover successor must not leak its authority to ``gateway run`` children + of its own, and bootstrap reads this twice. + """ + global _parsed_context + if _parsed_context is not None: + return _parsed_context or None + if os.environ.get(HANDOVER_ENV, "").strip() != "1": + _parsed_context = False + return None + try: + parent_pid = int(os.environ.get(PARENT_PID_ENV, "")) + except ValueError: + logger.error("Handover env %s set but %s is missing/invalid; treating as a normal boot", + HANDOVER_ENV, PARENT_PID_ENV) + _parsed_context = False + return None + parent_start: Optional[int] + try: + parent_start = int(os.environ.get(PARENT_START_ENV, "")) + except ValueError: + parent_start = None + if parent_pid <= 0 or parent_pid == os.getpid(): + logger.error("Handover parent pid %r is not a predecessor; treating as a normal boot", parent_pid) + _parsed_context = False + return None + for key in (HANDOVER_ENV, PARENT_PID_ENV, PARENT_START_ENV): + os.environ.pop(key, None) + _parsed_context = HandoverContext(parent_pid=parent_pid, parent_start_ticks=parent_start) + return _parsed_context + + +def predecessor_alive(ctx: HandoverContext) -> bool: + """True while the predecessor process still exists (PID-reuse guarded when we have ticks).""" + try: + os.kill(ctx.parent_pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # exists, owned by someone else — still alive + if ctx.parent_start_ticks is not None: + from gateway.status import get_process_start_time + return get_process_start_time(ctx.parent_pid) == ctx.parent_start_ticks + return True + + +def spawn_successor() -> Optional[subprocess.Popen]: + """Blue side: spawn the ``gateway run`` successor child; its handle, or None.""" + from gateway.run import _resolve_hermes_bin, logger as run_logger + argv = _resolve_hermes_bin() + if not argv: + run_logger.error("Blue-green handover: could not resolve the hermes argv; skipping") + return None + from hermes_cli.gateway import _profile_arg + full_argv = [*argv, *shlex.split(_profile_arg()), "gateway", "run"] + try: + proc = subprocess.Popen( # noqa: ASYNC220 -- spawn returns at once; the child outlives this task + full_argv, env={**os.environ, **successor_env()}, + ) + except OSError as exc: + run_logger.error("Blue-green handover: failed to spawn successor: %s", exc) + return None + run_logger.info("Blue-green handover: spawned successor gateway (PID %d)", proc.pid) + return proc + + +def _identify_pid_and_state(socket_path) -> Optional[tuple[int, str]]: + """One public ``identify`` round-trip; (answering pid, descriptor state) or None.""" + import socket as _socket + try: + with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock: + sock.settimeout(1.5) + sock.connect(str(socket_path)) + sock.sendall(json.dumps({"id": 1, "protocol": 1, "verb": "identify"}).encode() + b"\n") + with sock.makefile("rb") as reader: + line = reader.readline(4096) + payload = json.loads(line.decode("utf-8")) + except (OSError, ValueError, UnicodeDecodeError): + return None + if not isinstance(payload, dict) or payload.get("ok") is not True: + return None + result = payload.get("result") + if not isinstance(result, dict): + return None + pid = result.get("pid") + state = result.get("state") + if not isinstance(pid, int) or pid <= 0 or not isinstance(state, str): + return None + return pid, state + + +async def wait_successor_ready(proc: subprocess.Popen, timeout: float = READY_TIMEOUT_S) -> bool: + """Blue side: poll until the successor ITSELF answers identify in the ready state. + + Blue may still answer until green binds the socket name, and a bound-but-starting + green answers ``state != 'ready'`` — neither counts. Blocking socket I/O runs in a + thread so the event loop (streaming, adapters, in-flight turns) never stalls. + """ + from gateway.control_socket import resolve_client_socket_path + from hermes_constants import get_hermes_home + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if proc.poll() is not None: + logger.warning("Blue-green handover: successor (PID %d) exited during startup", proc.pid) + return False + path = resolve_client_socket_path(get_hermes_home()) + if path is not None: + answer = await asyncio.to_thread(_identify_pid_and_state, path) + if answer is not None and answer[0] == proc.pid and answer[1] == "ready": + return True + await asyncio.sleep(_POLL_INTERVAL_S) + return False + + +def terminate_successor(proc: subprocess.Popen) -> None: + """Blue side: a successor that was never promoted must not outlive the fallback.""" + if proc.poll() is not None: + return + logger.warning("Blue-green handover: terminating unpromoted successor (PID %d)", proc.pid) + proc.terminate() + try: + proc.wait(timeout=FALLBACK_TERMINATE_S) + except subprocess.TimeoutExpired: + logger.error("Successor (PID %d) ignored SIGTERM; SIGKILL", proc.pid) + proc.kill() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + + +async def arm_after_predecessor_exit( + ctx: HandoverContext, + on_exit: Callable[[], Any], + timeout_s: float = PREDECESSOR_EXIT_TIMEOUT_S, +) -> None: + """Green side: run ``on_exit`` once the predecessor is gone (its stop path releases the + single-owner records). Arms anyway at the timeout — blue's exit path also cleans up.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + while loop.time() < deadline: + if not predecessor_alive(ctx): + break + await asyncio.sleep(_POLL_INTERVAL_S) + else: + logger.warning("Predecessor (PID %d) still alive after %.0fs; claiming deferred records anyway", + ctx.parent_pid, timeout_s) + outcome = on_exit() + if inspect.isawaitable(outcome): + await outcome + + +def _self_stop() -> None: + """Stop this gateway through its own signal path (clean drain, supervisor revives).""" + os.kill(os.getpid(), signal.SIGTERM) + + +def _claim_pid_record(ctx: HandoverContext) -> None: + """Claim the PID record once the predecessor is gone (the body of the deferred claim). + + A lost claim leaves this process as an unrecorded gateway — fatal by self-stop so the + supervisor's next boot takes over cleanly instead of two gateways racing the record. + """ + import atexit + from gateway.status import get_process_start_time, get_running_pid, remove_pid_file, write_pid_file + recorded = get_running_pid() + if recorded is not None and recorded != os.getpid(): + try: + os.kill(recorded, 0) + alive = get_process_start_time(recorded) is not None + except OSError: + alive = False + if alive: + logger.error("Deferred PID claim skipped: a live gateway (PID %d) holds the record; " + "this successor stops itself", recorded) + _self_stop() + return + logger.warning("Replacing stale predecessor PID record (PID %d)", recorded) + remove_pid_file() + try: + write_pid_file() + except FileExistsError as exc: + logger.error("Deferred PID claim lost the PID-file race (%s); this successor stops itself", exc) + _self_stop() + return + atexit.register(remove_pid_file) + from gateway.run import _claim_host_gateway_role + try: + # The predecessor held the host role; with its exit the role is free, but claim it + # explicitly (force = the handover contract) so host-attach resolution points at us. + _claim_host_gateway_role(force=True) + except SystemExit as exc: # the role claim refuses via SystemExit(75) — degrade, not die + logger.error("Deferred host-role claim refused after handover: %s", exc) + except Exception as exc: # noqa: BLE001 -- role claim failure must not undo the PID claim + logger.error("Deferred host-role claim failed after handover: %s", exc) + from hermes_cli.gateway_multiplex_mode import publish_pending_multiplex_decision + publish_pending_multiplex_decision() + logger.info("Blue-green handover complete: PID record claimed (predecessor PID %d exited)", + ctx.parent_pid) + + +def deferred_pid_claim(ctx: HandoverContext) -> None: + """Green side: schedule ``_claim_pid_record`` to run once blue exits.""" + + async def _run() -> None: + await arm_after_predecessor_exit(ctx, lambda: _claim_pid_record(ctx)) + + asyncio.get_running_loop().create_task(_run()) diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py index b6f0ec9f62f43..dec71c0254d1f 100644 --- a/gateway/run_bootstrap.py +++ b/gateway/run_bootstrap.py @@ -371,6 +371,9 @@ def _rescan_profiles_handler() -> dict: # Hosted-room transports install onto this listener; bind it here so every launcher # that starts the control socket (not only start_gateway) exposes it. runner.session_control_server = _control_server + # NOTE: a handover successor does NOT notify MAINPID itself — under NotifyAccess=main + # systemd drops notify messages from any pid but the current main. Blue sends the + # switch after green reports ready (gateway.run_shutdown.request_restart). except Exception as _cs_exc: logger.error("Control socket startup failed; the gateway cannot start without it: %s", _cs_exc) _control_server = None @@ -795,7 +798,14 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = # Only --force skips the host-lock refusal. Every generated unit carries --replace, so reading it as # --force there disabled the one arbiter of the two-units-at-once race; a replace that took the # owner over already freed the lock with that process. - if not _start_gateway_claim_pid_file(force=force): + # A blue-green handover successor defers the claim instead: the predecessor owns the record + # until it exits, and the deferred task (armed after the control socket serves) takes it over. + from gateway import handover as _handover + _handover_ctx = _handover.handover_from_env() + if _handover_ctx is not None: + logger.info("Blue-green handover: this boot succeeds gateway PID %d", _handover_ctx.parent_pid) + _handover.deferred_pid_claim(_handover_ctx) + elif not _start_gateway_claim_pid_file(force=force): release_gateway_runtime_lock() return False @@ -817,6 +827,9 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool = _enable_multiplex_log_routing(runner.config) # ``--replace`` is explicit startup authority, not a durable reconnect policy: GatewayRunner scopes # it to cold adapter connects and clears it before the background reconnect watcher starts. + # Deliberately NOT given to a blue-green handover successor: takeover SIGTERMs the holder, + # which would kill blue mid-drain. Green's adapters fail the lock while blue holds it and + # reconnect once blue's stop releases it. runner._platform_lock_takeover_on_start = bool(replace) # Unexpected signals exit non-zero so service managers revive us; planned stops write a marker first. diff --git a/gateway/run_config_loaders.py b/gateway/run_config_loaders.py index 5230924b18411..87ae9cbf68281 100644 --- a/gateway/run_config_loaders.py +++ b/gateway/run_config_loaders.py @@ -380,6 +380,12 @@ def _warn_unparsable_timeout(cfg_key: str, raw: object, default: float) -> None: except (TypeError, ValueError): logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default) + @classmethod + def _load_blue_green_restart(cls) -> bool: + """Blue-green restart handover opt-in (successor-first restart; default off).""" + raw = cls._env_or_cfg_str("HERMES_BLUE_GREEN_RESTART", "agent", "blue_green_restart") + return raw.strip().lower() in {"1", "true", "yes", "on"} + @classmethod def _load_restart_drain_timeout(cls) -> float: """Graceful gateway restart/stop drain timeout in seconds.""" diff --git a/gateway/run_runtime_init.py b/gateway/run_runtime_init.py index 3e75e19c4b044..c800afe81cbff 100644 --- a/gateway/run_runtime_init.py +++ b/gateway/run_runtime_init.py @@ -23,6 +23,7 @@ def _init_runtime_settings(self) -> None: self._human_delay = self._human_delay_from_config(_load_gateway_config()) self._human_delay_by_profile: Dict[str, Optional[tuple[int, int]]] = {} self._restart_drain_timeout = self._load_restart_drain_timeout() + self._blue_green_restart = self._load_blue_green_restart() # Live launchd ``ExitTimeOut`` for this job (None when not launchd-owned). Read once at # boot — launchd fixes it at load — and applied only to signal-driven stops, which are the # only stops launchd times. See _load_launchd_exit_timeout(). diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index 1606a343e27d4..c29658e923d41 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1697,9 +1697,35 @@ def request_restart(self, *, detached: bool = False, via_service: bool = False) self._mark_api_runs_shutdown_requested() async def _run_restart() -> None: + # Blue-green handover: the successor boots FIRST, so new sessions land on it while + # blue keeps serving established connections through its drain. Blue sends the + # MAINPID switch only after green reports ready (NotifyAccess=main drops green's + # own notify), and falls back to the classic drain-exit restart otherwise. + handover_child: subprocess.Popen | None = None + from gateway import handover as _handover + if not self._stop_requested_by_signal and _handover.enabled(self._blue_green_restart): + handover_child = _handover.spawn_successor() + ready = handover_child is not None and await _handover.wait_successor_ready(handover_child) + if ready: + from gateway import systemd_notify + if not systemd_notify.notify_mainpid(handover_child.pid): + logger.warning("Blue-green handover: MAINPID notify failed (no systemd notify " + "socket); falling back to the drain-exit restart") + ready = False + if ready: + self._handover_promoted = True + else: + logger.warning("Blue-green handover did not become ready (pid=%s); falling back " + "to the drain-exit restart", getattr(handover_child, "pid", None)) + if handover_child is not None: + await asyncio.to_thread(_handover.terminate_successor, handover_child) + handover_child = None + self._handover_child = handover_child await self._await_active_work_before_restart() # Detached helper only AFTER the after-turn wait, or its drain_timeout+5 deadline fires mid-turn. - if detached: + # Never when the handover succeeded: systemd's MainPID is already the successor, and a + # post-exit `gateway restart` would bounce the new gateway for nothing. + if detached and handover_child is None: with _log_suppressed(logging.ERROR, "Failed to launch detached gateway restart helper: %s"): await self._launch_detached_restart_command() await asyncio.sleep(0.05) @@ -2244,6 +2270,13 @@ async def stop( ) -> None: """Stop the gateway and disconnect all adapters.""" from gateway.run import GatewayRunner + # A handover successor never promoted to MainPID (signal stop during the handover + # window, or a fallback race) must not outlive this stop as a duplicate gateway. + _handover_child = getattr(self, "_handover_child", None) + if _handover_child is not None and not getattr(self, "_handover_promoted", False): + from gateway import handover as _handover_mod + await asyncio.to_thread(_handover_mod.terminate_successor, _handover_child) + self._handover_child = None # getattr-guard: shutdown-path tests build bare runners via object.__new__ that lack the # liveness-guard machinery. _stop_guards = getattr(self, "_stop_loop_liveness_guards", None) diff --git a/gateway/systemd_notify.py b/gateway/systemd_notify.py index da66753533f02..1909c14f56a0f 100644 --- a/gateway/systemd_notify.py +++ b/gateway/systemd_notify.py @@ -28,6 +28,13 @@ def notify(message: str) -> bool: return False +def notify_mainpid(pid: int) -> bool: + """Point systemd's MainPID at ``pid`` (Type=notify units). The predecessor's exit then + looks like routine cgroup cleanup instead of a MainPID crash, so ``Restart=always`` + does not respawn the unit behind the successor's back.""" + return notify(f"MAINPID={pid}") + + def watchdog_interval_seconds() -> float | None: try: interval = float(os.environ.get("WATCHDOG_USEC", "") if _notify_socket() else "") / 1e6 diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 63ff490ee732e..a1716ed87bc83 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -109,6 +109,11 @@ def _aux(timeout, *, reasoning_effort=True, **extra): # TimeoutStopSec or risk SIGKILL mid-cleanup; for /restart prefer restart_after_turn_timeout # so turns finish BEFORE stop(). "restart_drain_timeout": 0, + # Blue-green restart: the successor gateway boots before the predecessor drains, new + # sessions land on it immediately, and sd_notify(MAINPID=...) keeps Restart=always + # quiet. Requires a Type=notify unit (NOTIFY_SOCKET); otherwise restarts keep the + # classic drain-exit shape. + "blue_green_restart": False, # Floor under the stop()/drain wait (seconds) for cron jobs and api_server runs. Interrupted # chat turns resume on the next message, but an interrupted cron run is recorded as a # permanent failure and an interrupted /v1 run fails its waiting caller, so neither may diff --git a/tests/gateway/test_handover.py b/tests/gateway/test_handover.py new file mode 100644 index 0000000000000..b5cd821069f97 --- /dev/null +++ b/tests/gateway/test_handover.py @@ -0,0 +1,220 @@ +"""Blue-green handover primitives: env contract, gates, readiness probe, deferred claim.""" + +from __future__ import annotations + +import asyncio +import json +import os +import socket +import subprocess +import sys +import threading + +import pytest + +from gateway import handover, systemd_notify + + +def _clear_env(monkeypatch): + for key in (handover.HANDOVER_ENV, handover.PARENT_PID_ENV, handover.PARENT_START_ENV, + "NOTIFY_SOCKET"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setattr(handover, "_parsed_context", None) + + +def test_env_roundtrip_and_scrub(monkeypatch): + _clear_env(monkeypatch) + monkeypatch.setenv(handover.HANDOVER_ENV, "1") + monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid() + 12345)) + monkeypatch.delenv(handover.PARENT_START_ENV, raising=False) + ctx = handover.handover_from_env() + assert ctx is not None and ctx.parent_pid == os.getpid() + 12345 + assert ctx.parent_start_ticks is None + # Parsed-and-scrubbed: the authority must not leak to this process's own children, and + # bootstrap's second parse returns the cached context. + assert os.environ.get(handover.HANDOVER_ENV) is None + assert handover.handover_from_env() is ctx + env = handover.successor_env() + assert env[handover.HANDOVER_ENV] == "1" + + +def test_env_rejects_self_and_missing(monkeypatch): + _clear_env(monkeypatch) + assert handover.handover_from_env() is None + monkeypatch.setattr(handover, "_parsed_context", None) + monkeypatch.setenv(handover.HANDOVER_ENV, "1") + monkeypatch.setenv(handover.PARENT_PID_ENV, str(os.getpid())) + assert handover.handover_from_env() is None # predecessor cannot be ourselves + monkeypatch.setattr(handover, "_parsed_context", None) + monkeypatch.setenv(handover.PARENT_PID_ENV, "not-a-pid") + assert handover.handover_from_env() is None + + +def test_enabled_gate(monkeypatch): + monkeypatch.delenv("NOTIFY_SOCKET", raising=False) + assert handover.enabled(False) is False # no NOTIFY_SOCKET -> cannot switch MainPID + monkeypatch.setenv("NOTIFY_SOCKET", "/run/user/1/notify") + assert handover.enabled(True) is True + assert handover.enabled(False) is False + + +def test_notify_mainpid_payload(tmp_path): + sock_path = tmp_path / "notify.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + server.bind(str(sock_path)) + server.setblocking(False) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(systemd_notify, "_notify_socket", lambda: str(sock_path)) + assert systemd_notify.notify_mainpid(4242) is True + assert server.recv(4096) == b"MAINPID=4242" + server.close() + with pytest.MonkeyPatch.context() as mp: + mp.setattr(systemd_notify, "_notify_socket", lambda: "") + assert systemd_notify.notify_mainpid(1) is False + + +def _serve_identify(server, pid, state, replies=64): + server.setblocking(True) + + def serve(): + for _ in range(replies): + try: + conn, _ = server.accept() + except OSError: + return + with conn: + conn.recv(4096) + conn.sendall(json.dumps({"ok": True, "result": {"pid": pid, "state": state}}).encode() + b"\n") + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + return thread + + +def test_wait_successor_ready_requires_ready_state(tmp_path, monkeypatch): + sock_path = tmp_path / "gateway.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(sock_path)) + server.listen(8) + successor_pid = os.getpid() + 777 + thread = _serve_identify(server, successor_pid, "ready") + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.05) + monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + try: + loop = asyncio.new_event_loop() + try: + ready = loop.run_until_complete( + asyncio.wait_for(handover.wait_successor_ready(proc, timeout=5.0), 8)) + assert ready is True + finally: + loop.close() + finally: + proc.kill() + proc.wait() + server.close() + thread.join(timeout=2) + + +def test_wait_successor_ready_rejects_draining_and_wrong_pid(tmp_path, monkeypatch): + sock_path = tmp_path / "gateway.sock" + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(str(sock_path)) + server.listen(8) + # A draining blue answering on its own socket must never satisfy the probe. + thread = _serve_identify(server, os.getpid(), "draining") + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) + monkeypatch.setattr("gateway.control_socket.resolve_client_socket_path", lambda home: sock_path) + proc = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"]) + try: + loop = asyncio.new_event_loop() + try: + ready = loop.run_until_complete(handover.wait_successor_ready(os.getpid() + 999, timeout=0.3)) + assert ready is False + finally: + loop.close() + finally: + proc.kill() + proc.wait() + server.close() + thread.join(timeout=2) + + +def test_terminate_successor_fallback(): + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + handover.terminate_successor(child) + assert child.poll() is not None + already = subprocess.Popen([sys.executable, "-c", "pass"]) + already.wait() + handover.terminate_successor(already) # no-op on an exited child + + +def test_arm_after_predecessor_exit(monkeypatch): + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.2)"]) + ctx = handover.HandoverContext(parent_pid=child.pid, parent_start_ticks=None) + fired = [] + monkeypatch.setattr(handover, "_POLL_INTERVAL_S", 0.02) + loop = asyncio.new_event_loop() + try: + loop.run_until_complete( + asyncio.wait_for(handover.arm_after_predecessor_exit(ctx, lambda: fired.append(1)), 10)) + finally: + loop.close() + child.wait() + assert fired == [1] + + +def test_predecessor_alive_pid_reuse_guard(monkeypatch): + # A recycled pid with mismatched start ticks is not our predecessor. + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(0.5)"]) + try: + ctx = handover.HandoverContext(parent_pid=child.pid, parent_start_ticks=1) + monkeypatch.setattr("gateway.status.get_process_start_time", lambda pid: 999) + assert handover.predecessor_alive(ctx) is False + finally: + child.kill() + child.wait() + + +def _patch_claim_status(monkeypatch, recorded_pid, recorded_alive, writes): + import gateway.status as status + monkeypatch.setattr(status, "get_running_pid", lambda *a, **k: recorded_pid) + if recorded_alive: + monkeypatch.setattr(status, "get_process_start_time", lambda pid: 12345) + + def _kill(pid, sig): + return None + else: + def _kill(pid, sig): + raise ProcessLookupError() + monkeypatch.setattr(os, "kill", _kill) + monkeypatch.setattr(status, "write_pid_file", lambda: writes.append(1)) + monkeypatch.setattr(status, "remove_pid_file", lambda: writes.append("removed")) + import hermes_cli.gateway_multiplex_mode as gmm + monkeypatch.setattr(gmm, "publish_pending_multiplex_decision", lambda: None) + import gateway.run as run_mod + monkeypatch.setattr(run_mod, "_claim_host_gateway_role", lambda force=False: None) + + +def test_claim_skips_live_owner(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=True, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == [] # a live recorded gateway blocks the claim + + +def test_claim_replaces_stale_owner(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=4321, recorded_alive=False, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == ["removed", 1] + + +def test_claim_when_record_free(monkeypatch): + writes: list = [] + _patch_claim_status(monkeypatch, recorded_pid=None, recorded_alive=False, writes=writes) + ctx = handover.HandoverContext(parent_pid=os.getpid() + 555, parent_start_ticks=None) + handover._claim_pid_record(ctx) + assert writes == [1]