diff --git a/Dockerfile b/Dockerfile index 6109679..082acd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN pip install --no-cache-dir \ "cryptography>=42.0" # Copy agent -COPY watchdog.py . +COPY watchdog.py lifecycle.py . # UID 1000 account for install.sh's non-root hardening path (docker-compose.yaml # defaults to root; install.sh instead sets WATCHDOG_UID/GID to run as this user). diff --git a/README.md b/README.md index c8c3ce3..48cc51e 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,11 @@ Alerts are deduplicated via a **cooldown** — once an alert fires for a contain Two additional deduplication rules suppress redundant alerts at the event level: - **OOM → die suppression**: when a container is OOM-killed, Docker emits both an `oom` event and an immediately following `die` event. The watchdog fires the `oom` CRITICAL alert and suppresses the subsequent `crashed` alert to avoid a duplicate for the same incident. -- **Restart-loop incident coalescing**: Docker `die`/OOM events provide an immediate first failure alert. Only periodic Docker `RestartCount` increments determine the number of automatic restarts and whether the `restart_threshold` has been reached within `restart_window_minutes`; event deliveries are **never** added to the restart total. One HIGH `restart-loop` alert then suppresses per-cycle crashes, OOMs and failed Docker, manual or auto-discovered probes until sustained recovery, regardless of cooldown. Docker IDs identify distinct container instances even when names are reused. +- **Restart-loop incident coalescing**: Docker `die`/OOM events initiate the first failure alert (nonzero `die` events may be held for the short stop-correlation window). Only periodic Docker `RestartCount` increments determine the number of automatic restarts and whether the `restart_threshold` has been reached within `restart_window_minutes`; event deliveries are **never** added to the restart total. One HIGH `restart-loop` alert then suppresses per-cycle crashes, OOMs and failed Docker, manual or auto-discovered probes until sustained recovery, regardless of cooldown. Docker IDs identify distinct container instances even when names are reused. + +**Docker-requested stops and exit 137:** An exit status of 137 means SIGKILL; it does **not** prove an OOM. Docker may deliver a stop event shortly after die, so nonzero exits are briefly held for lifecycle_detection.correlation_window_seconds (default 3 seconds). A matching stop from the same container ID suppresses that instance's misleading CRASH and is logged instead; a direct docker kill, an unconfirmed stop, or a genuine OOM still produces the normal CRITICAL incident. A start event prevents the following run's stop from masking an earlier crash. This correlation only provides evidence of a **Docker-requested stop**, not evidence that stopping the service was authorized. The existing restart-loop counting, recovery rules, and alert delivery remain authoritative. Set lifecycle_detection.enabled to false to disable the correlation. + +Planned-stop correlation does not implement an availability policy: long-running services stopped intentionally will not produce a downtime alert unless another monitoring rule detects their outage. Optional expected-running monitoring and HA-specific planned-shutdown handling are tracked separately. **Recovery alerts**: once a previously-alarmed container is healthy and running, the watchdog sends a one-time INFO `recovered` notification (if `alert_on_recovery: true`). For an auto-restarting container recovering from `crashed`, `oom`, or `restart-loop`, a fleeting `running` state is *not* recovery: Docker's `State.StartedAt` must prove at least `restart_recovery_stability_seconds` continuous uptime (default 120 seconds), with no intervening failure. On recovery, restart history is reset to avoid immediate re-alerting. Docker health events cannot bypass this check. Other container recovery checks preserve their existing behavior. @@ -72,6 +76,7 @@ Two additional deduplication rules suppress redundant alerts at the event level: ``` . ├── watchdog.py # Main monitor script +├── lifecycle.py # Bounded Docker stop/die evidence correlation ├── watchdog-config.yaml # All configuration (no secrets) ├── docker-compose.yaml # Production / offline runtime (image only) ├── docker-compose.build.yaml # Developer / online build diff --git a/lifecycle.py b/lifecycle.py new file mode 100644 index 0000000..c13b1f9 --- /dev/null +++ b/lifecycle.py @@ -0,0 +1,86 @@ +"""Short-lived Docker stop/die evidence, not another alert-state machine. + +Docker can deliver the final stop event just after die. Buffer only unexpected +nonzero exits for a bounded time; when no stop evidence arrives, report the +failure through Watchdog's existing incident engine. +""" +from dataclasses import dataclass +import threading +import time + + +@dataclass +class PendingDeath: + container_id: str + name: str + container: object + exit_code: int + event_time: float | None + happened_at: float + stopped: bool = False + oom: bool = False + new_run_started: bool = False + + +class DockerLifecycleCorrelator: + def __init__(self, window_seconds: float = 3.0): + self.window = float(window_seconds) + self._lock = threading.Lock() + self._pending: list[PendingDeath] = [] + self._recent_stops: dict[str, float] = {} + + def record(self, action: str, container_id: str, *, name: str = "", + container: object = None, exit_code: int = 1, + event_time: float | None = None, now: float | None = None) -> bool: + """Return whether an OOM event matched an already pending death.""" + if not container_id: + return False + now = time.monotonic() if now is None else now + with self._lock: + self._recent_stops = { + cid: when for cid, when in self._recent_stops.items() + if 0 <= now - when <= self.window + } + if action == "start": + self._recent_stops.pop(container_id, None) + # A later run's stop cannot excuse a prior run's failure. + for death in self._pending: + if death.container_id == container_id: + death.new_run_started = True + elif action == "stop": + self._recent_stops[container_id] = now + for death in reversed(self._pending): + if (death.container_id == container_id and not death.new_run_started + and 0 <= now - death.happened_at <= self.window): + death.stopped = True + break # One stop must not mask several restart-cycle deaths. + elif action == "oom": + matched = False + for death in reversed(self._pending): + if (death.container_id == container_id and not death.new_run_started + and 0 <= now - death.happened_at <= self.window): + death.oom = True + matched = True + break + return matched + elif action == "die": + stopped_at = self._recent_stops.get(container_id) + stopped = stopped_at is not None and 0 <= now - stopped_at <= self.window + self._pending.append(PendingDeath( + container_id, name, container, exit_code, event_time, now, + stopped=stopped, + )) + return False + + def drain_ready(self, *, now: float | None = None, force: bool = False) -> list[PendingDeath]: + now = time.monotonic() if now is None else now + with self._lock: + ready, waiting = [], [] + for death in self._pending: + if (force or death.stopped or death.oom or death.new_run_started + or now - death.happened_at >= self.window): + ready.append(death) + else: + waiting.append(death) + self._pending = waiting + return ready diff --git a/tests/test_event_missing.py b/tests/test_event_missing.py index 6687daf..e2d9e93 100644 --- a/tests/test_event_missing.py +++ b/tests/test_event_missing.py @@ -12,6 +12,7 @@ def make_subject(self): monitor = object.__new__(watchdog.Watchdog) monitor._excluded = set() monitor._missing_events = {} + monitor._lifecycle_enabled = False monitor.client = SimpleNamespace(containers=Mock()) monitor.client.containers.get.side_effect = docker.errors.NotFound("removed") return monitor diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py new file mode 100644 index 0000000..dd644bb --- /dev/null +++ b/tests/test_lifecycle.py @@ -0,0 +1,254 @@ +"""Integration regression: Docker stop evidence feeds ONE PR #5 incident engine.""" +import threading +import time +import unittest +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import Mock + +from lifecycle import DockerLifecycleCorrelator +import watchdog + +CID = "a" * 64 +OTHER = "b" * 64 + + +class CorrelationTests(unittest.TestCase): + def test_die_before_stop(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=137, now=10) + tracker.record("stop", CID, now=10.1) + [death] = tracker.drain_ready(now=10.2) + self.assertTrue(death.stopped) + + def test_stop_before_die(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("stop", CID, now=10) + tracker.record("die", CID, name="svc", exit_code=137, now=10.1) + [death] = tracker.drain_ready(now=10.2) + self.assertTrue(death.stopped) + + def test_late_stop_not_suppressed(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=137, now=10) + tracker.record("stop", CID, now=14) + [death] = tracker.drain_ready(now=14) + self.assertFalse(death.stopped) + + def test_wrong_id_not_suppressed(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=137, now=10) + tracker.record("stop", OTHER, now=10.1) + [death] = tracker.drain_ready(now=14) + self.assertFalse(death.stopped) + + def test_start_clears_old_stop(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("stop", CID, now=10) + tracker.record("start", CID, now=10.1) + tracker.record("die", CID, name="svc", exit_code=137, now=10.2) + [death] = tracker.drain_ready(now=14) + self.assertFalse(death.stopped) + + def test_next_run_stop_cannot_hide_previous_run_crash(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=1, now=10) + tracker.record("start", CID, now=10.1) + tracker.record("stop", CID, now=10.2) + [death] = tracker.drain_ready(now=10.3) + self.assertFalse(death.stopped) + + def test_single_stop_can_suppress_only_latest_die(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=2, now=10) + tracker.record("die", CID, name="svc", exit_code=137, now=10.2) + tracker.record("stop", CID, now=10.3) + deaths = tracker.drain_ready(force=True) + self.assertEqual([d.stopped for d in deaths], [False, True]) + + def test_oom_after_die_is_correlated(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=137, now=10) + self.assertTrue(tracker.record("oom", CID, now=10.1)) + [death] = tracker.drain_ready(now=10.2) + self.assertTrue(death.oom) + + def test_oom_without_pending_die_is_not_matched(self): + tracker = DockerLifecycleCorrelator(3) + self.assertFalse(tracker.record("oom", CID, now=10)) + + def test_no_premature_resolution(self): + tracker = DockerLifecycleCorrelator(3) + tracker.record("die", CID, name="svc", exit_code=1, now=10) + self.assertEqual(tracker.drain_ready(now=12.99), []) + self.assertEqual(len(tracker.drain_ready(now=13.01)), 1) + + +class FakeContainer: + def __init__(self, cid=CID, name="svc"): + self.id, self.name = cid, name + self.status = "running" + self.labels = {} + self.attrs = { + "Config": {"Image": "test/svc", "Labels": {}}, + "HostConfig": {"AutoRemove": False, "RestartPolicy": {"Name": "always"}}, + "State": {"OOMKilled": False, + "StartedAt": (datetime.now(timezone.utc)-timedelta(seconds=5)).isoformat()}, + "Mounts": [], "RestartCount": 0, + "NetworkSettings": {"Networks": {}}, + } + self.image = SimpleNamespace(tags=["test/svc:latest"]) + + def logs(self, **kwargs): + return b"failure in test container" + + def stats(self, **kwargs): + return {"memory_stats": {}} + + def reload(self): + pass + + +class WatchdogLifecycleTests(unittest.TestCase): + def setUp(self): + self.container = FakeContainer() + wd = object.__new__(watchdog.Watchdog) + wd.cfg = {"alert_on_recovery": True} + wd.state = watchdog.WatchdogState("") + wd.host = "lab" + wd.runbook_base = "" + wd._excluded = set() + wd._ignored_events = [] + wd._discovered_checks = {} + wd._oom_containers = set() + wd._missing_events = {} + wd._lifecycle_enabled = True + wd._lifecycle = DockerLifecycleCorrelator(3) + wd._lifecycle_wake = threading.Event() + wd._restart_threshold = 3 + wd._restart_window_secs = 600 + wd._restart_recovery_stability_secs = 120 + wd._unhealthy_threshold = 3 + wd._cooldown_minutes = 5 + wd._auto_cfg = {"enabled": False} + wd._manual_checks = {} + wd._boot_time = 0 + wd._grace_until = 0 + wd._folded = [] + wd._folded_lock = threading.Lock() + wd.client = SimpleNamespace( + containers=SimpleNamespace(get=lambda cid: self.container, + list=lambda all=True: [self.container])) + self.alerts = [] + wd._dispatch = self.alerts.append + self.wd = wd + + def event(self, action, code=None, cid=CID, timeNano=None): + attrs = {"name": self.container.name} + if code is not None: + attrs["exitCode"] = str(code) + event = {"Action": action, "Actor": {"ID": cid, "Attributes": attrs}} + if timeNano: + event["timeNano"] = timeNano + return event + + def resolve(self): + for death in self.wd._lifecycle.drain_ready(force=True): + self.wd._resolve_lifecycle_death(death) + + def reasons(self): + return [alert.failure_type for alert in self.alerts] + + def test_october_forced_stop_does_not_crash(self): + self.wd._handle_event(self.event("die", 137)) + self.wd._handle_event(self.event("stop")) + self.resolve() + self.assertEqual(self.reasons(), []) + self.assertEqual(self.wd.state.get("svc").alerted_for, "") + + def test_stop_first_suppresses_forced_exit(self): + self.wd._handle_event(self.event("stop")) + self.wd._handle_event(self.event("die", 137)) + self.resolve() + self.assertEqual(self.reasons(), []) + + def test_direct_kill_without_stop_is_a_crash(self): + self.wd._handle_event(self.event("die", 137)) + self.resolve() + self.assertEqual(self.reasons(), ["crashed"]) + self.assertNotIn("likely OOM", self.alerts[0].recommended_action) + + def test_nonzero_exit_without_stop_is_a_crash(self): + self.wd._handle_event(self.event("die", 2)) + self.resolve() + self.assertEqual(self.reasons(), ["crashed"]) + + def test_clean_exit_is_not_a_crash(self): + self.wd._handle_event(self.event("die", 0)) + self.resolve() + self.assertEqual(self.reasons(), []) + + def test_oom_before_die_no_duplicate(self): + self.wd._handle_event(self.event("oom")) + self.wd._handle_event(self.event("die", 137)) + self.resolve() + self.assertEqual(self.reasons(), ["oom"]) + + def test_oom_after_die_no_duplicate_or_stale_marker(self): + self.wd._handle_event(self.event("die", 137)) + self.wd._handle_event(self.event("oom")) + self.resolve() + self.assertEqual(self.reasons(), ["oom"]) + self.assertEqual(self.wd._oom_containers, set()) + + def test_pending_die_does_not_erase_active_restart_loop(self): + state = self.wd.state.get("svc") + state.alerted_for = "restart-loop" + self.wd._handle_event(self.event("die", 2)) + self.resolve() + self.assertEqual(self.reasons(), []) + self.assertEqual(state.alerted_for, "restart-loop") + + def test_multiple_pending_dies_still_one_crash(self): + for _ in range(4): + self.wd._handle_event(self.event("die", 2)) + self.resolve() + self.assertEqual(self.reasons(), ["crashed"]) + + def test_pending_death_from_replaced_container_is_ignored(self): + self.wd._handle_event(self.event("die", 137)) + new = FakeContainer(cid=OTHER) + self.wd._sync_container_identity(self.wd.state.get("svc"), new) + self.resolve() + self.assertEqual(self.reasons(), []) + + def test_delayed_old_event_after_recovery_is_ignored(self): + state = self.wd.state.get("svc") + state.last_recovery_time = time.time() + self.wd._handle_event(self.event("die", 2, timeNano=int((time.time()-5)*1e9))) + self.resolve() + self.assertEqual(self.reasons(), []) + + def test_stop_evidence_skips_container_lookup(self): + self.wd.client.containers.get = Mock(side_effect=AssertionError("unexpected inspect")) + self.wd._handle_event(self.event("stop")) + self.wd.client.containers.get.assert_not_called() + + def test_lifecycle_can_be_disabled(self): + self.wd._lifecycle_enabled = False + self.wd._handle_event(self.event("die", 2)) + self.assertEqual(self.reasons(), ["crashed"]) + + def test_event_subscription_does_not_need_kill(self): + events = [] + self.wd._stop_event = threading.Event() + self.wd.client.events = lambda **kw: events.append(kw) or iter([]) + self.wd._event_listener() + filters = events[0]["filters"]["event"] + self.assertIn("start", filters) + self.assertIn("stop", filters) + self.assertNotIn("kill", filters) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_restart_loop.py b/tests/test_restart_loop.py index 9c533c0..f39d8e6 100644 --- a/tests/test_restart_loop.py +++ b/tests/test_restart_loop.py @@ -48,6 +48,7 @@ def make_watchdog(container, threshold=3, stable=120): wd._discovered_checks = {} wd._oom_containers = set() wd._missing_events = {} # PR #6 event cache (normal Watchdog init owns it) + wd._lifecycle_enabled = False wd._auto_cfg = {"enabled": False} wd._manual_checks = {} wd._boot_time = 1 diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index cdf9429..05dbba7 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -24,6 +24,13 @@ restart_recovery_stability_seconds: 120 # require 2 min continuous uptime befor unhealthy_cycles_threshold: 3 # consecutive unhealthy cycles before alerting alert_on_recovery: true # send an INFO "recovered" alert when a container returns to normal +# A Docker-requested stop can end in exit 137 when the process ignores SIGTERM. +# Wait briefly for stop evidence (Docker may emit stop after die) before a CRASH. +# Without that evidence, unexpected nonzero exits still generate CRITICAL alerts. +lifecycle_detection: + enabled: true + correlation_window_seconds: 3 # valid range 0.5-30 seconds + # Containers to never alert on (debug, test, or one-shot containers) excluded_containers: [] # - my-temp-container diff --git a/watchdog.py b/watchdog.py index 83738d1..c4be80e 100644 --- a/watchdog.py +++ b/watchdog.py @@ -26,6 +26,8 @@ import socket import yaml +from lifecycle import DockerLifecycleCorrelator + # ── Logging setup ───────────────────────────────────────────────────────────── LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" SYSLOG_FORMAT = "watchdog[%(process)d]: %(levelname)s %(name)s: %(message)s" @@ -87,6 +89,7 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = No "restart_threshold": 5, "restart_window_minutes": 10, "restart_recovery_stability_seconds": 120, # prevent false recovery between restarts + "lifecycle_detection": {"enabled": True, "correlation_window_seconds": 3}, "unhealthy_cycles_threshold": 3, "alert_on_recovery": True, "excluded_containers": [], @@ -973,7 +976,7 @@ def _suppress_invalid_state(loop, context): "crashed": ( "CRITICAL", "Check exit code and logs. Run: docker logs --tail 50. " - "If exit code 137, likely OOM — increase memory limit.", + "Exit 137 means SIGKILL, not necessarily OOM. Check Docker stop/oom events and OOMKilled.", ), "oom": ( "CRITICAL", @@ -1278,6 +1281,17 @@ def __init__(self, cfg: dict): self._discovered_checks: dict[str, dict | None] = {} # auto-discovery cache self._oom_containers: set[str] = set() # suppress die alert when oom already fired self._missing_events: dict[str, tuple[float, set[str]]] = {} + lc = cfg.get("lifecycle_detection") or {} + self._lifecycle_enabled = lc.get("enabled", True) is True + try: + window = float(lc.get("correlation_window_seconds", 3)) + if not 0.5 <= window <= 30: + raise ValueError("out of range") + except (TypeError, ValueError): + log.warning("Invalid lifecycle correlation window; using 3 seconds") + window = 3.0 + self._lifecycle = DockerLifecycleCorrelator(window) + self._lifecycle_wake = threading.Event() self._auto_cfg: dict = cfg.get("auto_health_check", {}) _manual = cfg.get("container_health_checks") or {} self._manual_checks: dict = _manual if isinstance(_manual, dict) else {} @@ -1482,10 +1496,97 @@ def _maybe_alert_recovery(self, container, state: ContainerState) -> None: probe_detail=f"Recovered from '{previous}'") self._dispatch(payload) + + def _process_die(self, container, state: ContainerState, name: str, + exit_code: int | None, event_time: float | None) -> None: + """Existing PR #5 incident rules: the ONLY place that emits a die/crash.""" + # Consume paired OOM/die events, but never count event deliveries + # as restarts: Docker RestartCount is authoritative. + oom_die = name in self._oom_containers + if oom_die: + self._oom_containers.discard(name) + automatic = has_automatic_restart_policy(container) + if exit_code != 0: + with state.lock: + failure_at = event_time if event_time is not None else time.time() + state.last_failure_time = max(state.last_failure_time, failure_at) + previous = state.alerted_for + else: + with state.lock: + previous = state.alerted_for + + if previous == "restart-loop": + log.info("%s: die event during active restart-loop — suppressing crashed alert", name) + return + + # OOM kills also emit a die; count the restart above but do not + # downgrade the OOM incident into a redundant crashed alert. + if oom_die: + log.info("%s: die event follows OOM — suppressing duplicate crashed alert", name) + return + + if exit_code == 0: + log.info("%s exited cleanly (code 0) — no crashed alert", name) + return + + if exit_code == 137: + log.warning("%s: exit 137 (SIGKILL) without a confirmed Docker stop; " + "check OOMKilled and Docker event history", name) + # One CRASH for the initial incident; subsequent crashes coalesce + # until restart-loop escalation or sustained recovery. + if previous == "crashed" or (automatic and previous == "oom"): + log.debug("%s: additional die within open %s incident — suppressing", + name, previous) + return + if self._claim_alert(state, "crashed"): + payload = build_payload( + container, "crashed", self.host, self.runbook_base, exit_code, + probe_detail=f"Crash probe failed - container exited with exit code {exit_code}") + self._dispatch(payload) + + def _resolve_lifecycle_death(self, death) -> None: + state = self.state.get(death.name) + with state.lock: + # A replacement using the same name must never inherit the old death. + replaced = bool(state.last_container_id and + state.last_container_id != death.container_id) + stale = bool(death.event_time is not None and state.last_recovery_time + and death.event_time <= state.last_recovery_time) + if replaced or stale: + log.debug("%s: discard outdated pending death (replaced=%s stale=%s)", + death.name, replaced, stale) + return + if death.oom: + log.debug("%s: die matched OOM; no duplicate CRASH", death.name) + return + if death.stopped: + if death.exit_code == 137: + log.warning("%s: Docker-requested stop ended with SIGKILL (137), " + "not proof of OOM", death.name) + else: + log.info("%s: Docker-requested stop (exit %s)", death.name, death.exit_code) + return + self._process_die(death.container, state, death.name, death.exit_code, + death.event_time) + + def _lifecycle_worker(self) -> None: + """Resolve bounded stop/die evidence outside the Docker event reader.""" + while not self._stop_event.is_set(): + self._lifecycle_wake.wait(0.2) + self._lifecycle_wake.clear() + for death in self._lifecycle.drain_ready(): + try: + self._resolve_lifecycle_death(death) + except Exception: + log.exception("Failed to handle pending death for %s", death.name) + # ── Event stream listener ───────────────────────────────────────────────── def _event_listener(self) -> None: log.info("Docker event listener started") - filters = {"type": "container", "event": ["die", "oom", "health_status"]} + events = ["die", "oom", "health_status"] + if self._lifecycle_enabled: + events += ["stop", "start"] + filters = {"type": "container", "event": events} try: for event in self.client.events(decode=True, filters=filters): if self._stop_event.is_set(): @@ -1507,6 +1608,14 @@ def _handle_event(self, event: dict) -> None: log.debug("Docker event: %s on %s", action, name) + # Start and stop are evidence, not incidents; even removed containers + # can produce these events, so do not inspect them through Docker. + if action in ("start", "stop") and self._lifecycle_enabled: + cid = event.get("Actor", {}).get("ID") or event.get("id") + self._lifecycle.record(action, cid) + self._lifecycle_wake.set() + return + # Keep recent missing IDs so late events do not repeatedly hit Docker. # An ID-less legacy event must still be looked up: its name can be reused. container_id = event.get("Actor", {}).get("ID") or event.get("id") @@ -1583,49 +1692,18 @@ def _handle_event(self, event: dict) -> None: ) return - # Consume paired OOM/die events, but never count event deliveries - # as restarts: Docker RestartCount is authoritative. - oom_die = name in self._oom_containers - if oom_die: - self._oom_containers.discard(name) - automatic = has_automatic_restart_policy(container) - if exit_code != 0: - with state.lock: - failure_at = event_time if event_time is not None else time.time() - state.last_failure_time = max(state.last_failure_time, failure_at) - previous = state.alerted_for - else: + if self._lifecycle_enabled and container_id and exit_code not in (0, None): + # Record failure time immediately so a concurrent poll cannot + # declare stable recovery while this die is awaiting stop evidence. with state.lock: - previous = state.alerted_for - - if previous == "restart-loop": - log.info("%s: die event during active restart-loop — suppressing crashed alert", name) - return - - # OOM kills also emit a die; count the restart above but do not - # downgrade the OOM incident into a redundant crashed alert. - if oom_die: - log.info("%s: die event follows OOM — suppressing duplicate crashed alert", name) - return - - if exit_code == 0: - log.info("%s exited cleanly (code 0) — no crashed alert", name) + failed_at = event_time if event_time is not None else time.time() + state.last_failure_time = max(state.last_failure_time, failed_at) + self._lifecycle.record("die", container_id, name=name, + container=container, exit_code=exit_code, + event_time=event_time) + self._lifecycle_wake.set() return - - if exit_code == 137: - log.warning("%s: exit 137 not suppressed — signals: %s", - name, _mariadb_helper_signals(container, attributes)) - # One CRASH for the initial incident; subsequent crashes coalesce - # until restart-loop escalation or sustained recovery. - if previous == "crashed" or (automatic and previous == "oom"): - log.debug("%s: additional die within open %s incident — suppressing", - name, previous) - return - if self._claim_alert(state, "crashed"): - payload = build_payload( - container, "crashed", self.host, self.runbook_base, exit_code, - probe_detail=f"Crash probe failed - container exited with exit code {exit_code}") - self._dispatch(payload) + self._process_die(container, state, name, exit_code, event_time) elif action == "oom": failure_type = "oom" @@ -1634,7 +1712,14 @@ def _handle_event(self, event: dict) -> None: failure_at = event_time if event_time is not None else time.time() state.last_failure_time = max(state.last_failure_time, failure_at) active_loop = state.alerted_for == "restart-loop" - self._oom_containers.add(name) # mark so the imminent die event is suppressed + # A late OOM paired to pending die must not leave a marker that + # accidentally suppresses the next, unrelated die event. + matched_pending = (self._lifecycle.record("oom", container_id) + if self._lifecycle_enabled and container_id else False) + if matched_pending: + self._lifecycle_wake.set() + else: + self._oom_containers.add(name) if active_loop: log.info("%s: OOM during active restart-loop — suppressing per-cycle OOM alert", name) return @@ -2143,6 +2228,9 @@ def start(self) -> None: target=self._event_listener, name="event-listener", daemon=True ) event_thread.start() + if self._lifecycle_enabled: + threading.Thread(target=self._lifecycle_worker, name="lifecycle-worker", + daemon=True).start() # Poll runs in the main thread try: self._poll_loop() @@ -2154,6 +2242,7 @@ def start(self) -> None: def stop(self) -> None: log.info("Watchdog shutting down") self._stop_event.set() + self._lifecycle_wake.set() self.state.save(self._boot_time)