fix: Rate-limit removed container event warnings and resolve by ID

tests/test_event_missing.py

@@ -0,0 +1,108 @@
+import logging
+import unittest
+from types import SimpleNamespace
+from unittest.mock import Mock, patch
+
+import docker
+import watchdog
+
+
+class MissingContainerEventTests(unittest.TestCase):
+    def make_subject(self):
+        monitor = object.__new__(watchdog.Watchdog)
+        monitor._excluded = set()
+        monitor._missing_events = {}
+        monitor.client = SimpleNamespace(containers=Mock())
+        monitor.client.containers.get.side_effect = docker.errors.NotFound("removed")
+        return monitor
+
+    def test_flood_logs_once_per_event_type_and_uses_id(self):
+        monitor = self.make_subject()
+        events = []
+        for _ in range(55):
+            for action in ("health_status: unhealthy", "die"):
+                events.append({"Action": action, "Actor": {"ID": "old-instance-123", "Attributes": {"name": "health-demo", "exitCode": "2"}}})
+        with self.assertLogs("watchdog", level="WARNING") as logs:
+            for event in events:
+                monitor._handle_event(event)
+        self.assertEqual(len(logs.output), 2)
+        self.assertEqual(monitor.client.containers.get.call_count, 1)
+        self.assertTrue(all(call.args[0] == "old-instance-123" for call in monitor.client.containers.get.call_args_list))
+
+    def test_different_instance_not_suppressed(self):
+        monitor = self.make_subject()
+        with self.assertLogs("watchdog", level="WARNING") as logs:
+            for cid in ("old-id", "new-id"):
+                monitor._handle_event({"Action": "die", "Actor": {"ID": cid, "Attributes": {"name": "same-name"}}})
+        self.assertEqual(len(logs.output), 2)
+
+    def test_legacy_event_falls_back_to_name(self):
+        monitor = self.make_subject()
+        with self.assertLogs("watchdog", level="WARNING"):
+            monitor._handle_event({"Action": "die", "Actor": {"Attributes": {"name": "old-service"}}})
+        monitor.client.containers.get.assert_called_once_with("old-service")
+
+
+    def test_expiry_retries_missing_container(self):
+        monitor = self.make_subject()
+        event = {"Action": "die", "Actor": {"ID": "expired-id", "Attributes": {"name": "expired"}}}
+        with patch("watchdog.time.monotonic", side_effect=[100, 101, 161]):
+            with self.assertLogs("watchdog", level="WARNING") as logs:
+                for _ in range(3):
+                    monitor._handle_event(event)
+        self.assertEqual(monitor.client.containers.get.call_count, 2)
+        self.assertEqual(len(logs.output), 2)
+
+    def test_cache_bounded_during_high_churn(self):
+        monitor = self.make_subject()
+        with patch("watchdog.time.monotonic", return_value=100):
+            with patch("watchdog.log.warning"):
+                for n in range(1030):
+                    monitor._handle_event({"Action": "die", "Actor": {
+                        "ID": f"old-{n}", "Attributes": {"name": f"old-{n}"}
+                    }})
+                self.assertEqual(len(monitor._missing_events), 1024)
+                self.assertNotIn("old-0", monitor._missing_events)
+                monitor._handle_event({"Action": "die", "Actor": {
+                    "ID": "old-0", "Attributes": {"name": "old-0"}
+                }})
+        self.assertEqual(monitor.client.containers.get.call_count, 1031)
+        self.assertEqual(len(monitor._missing_events), 1024)
+
+    def test_idless_legacy_name_is_reinspected(self):
+        monitor = self.make_subject()
+        event = {"Action": "die", "Actor": {"Attributes": {"name": "reusable-name"}}}
+        with self.assertLogs("watchdog", level="WARNING") as logs:
+            monitor._handle_event(event)
+            monitor._handle_event(event)
+        self.assertEqual(monitor.client.containers.get.call_count, 2)
+        self.assertEqual(len(logs.output), 1)
+
+    def test_rename_and_name_reuse_records_old_container_not_new(self):
+        monitor = self.make_subject()
+        old = SimpleNamespace(name="service-old", attrs={"Config": {"Labels": {}}})
+        monitor.client.containers.get.side_effect = None
+        monitor.client.containers.get.return_value = old
+        monitor._ignored_events = []
+        monitor._discovered_checks = {}
+        monitor._oom_containers = set()
+        old_state = watchdog.ContainerState("service-old")
+        monitor.state = SimpleNamespace(get=Mock(return_value=old_state))
+        monitor.host = "test"
+        monitor.runbook_base = "test"
+        monitor._claim_alert = Mock(return_value=True)
+        monitor._dispatch = Mock()
+        event = {"Action": "die", "Actor": {"ID": "old-id", "Attributes": {
+            "name": "service", "exitCode": "3"
+        }}}
+        with patch("watchdog.build_payload", return_value=object()):
+            monitor._handle_event(event)
+        monitor.client.containers.get.assert_called_once_with("old-id")
+        monitor.state.get.assert_called_once_with("service-old")
+        monitor._claim_alert.assert_called_once_with(old_state, "crashed")
+        monitor._dispatch.assert_called_once()
+
+
+
+if __name__ == "__main__":
+    unittest.main()

watchdog.py

@@ -1232,6 +1232,7 @@ def __init__(self, cfg: dict):
         self._ignored_events: list[dict] = cfg.get("ignored_container_events") or []
         self._discovered_checks: dict[str, dict | None] = {}  # auto-discovery cache
         self._oom_containers: set[str] = set()  # suppress die alert when oom already fired
+        self._missing_events: dict[str, tuple[float, set[str]]] = {}
         self._auto_cfg: dict = cfg.get("auto_health_check", {})
         _manual = cfg.get("container_health_checks") or {}
         self._manual_checks: dict = _manual if isinstance(_manual, dict) else {}
@@ -1406,10 +1407,43 @@ def _handle_event(self, event: dict) -> None:
 
         log.debug("Docker event: %s on %s", action, name)
 
+        # Keep recent missing IDs so late events do not repeatedly hit Docker.
+        # An ID-less legacy event must still be looked up: its name can be reused.
+        container_id = event.get("Actor", {}).get("ID") or event.get("id")
+        cache_key = container_id or f"name:{name}"
+        now = time.monotonic()
+        missing = self._missing_events.get(cache_key)
+        if missing is not None and now - missing[0] >= 60:
+            self._missing_events.pop(cache_key)
+            missing = None
+
+        if container_id and missing is not None:
+            if action not in missing[1]:
+                missing[1].add(action)
+                log.warning("Container %s (%s) already removed before %s event could be processed",
+                            name, container_id, action)
+            return
+
         try:
-            container = self.client.containers.get(name)
+            container = self.client.containers.get(container_id or name)
         except docker.errors.NotFound:
-            log.warning("Container %s not found for event %s", name, action)
+            if missing is None:
+                missing = (now, set())
+                self._missing_events[cache_key] = missing
+                # Insertion-ordered dict: evict the oldest entry even during a burst.
+                if len(self._missing_events) > 1024:
+                    self._missing_events.pop(next(iter(self._missing_events)))
+            if action not in missing[1]:
+                missing[1].add(action)
+                log.warning("Container %s (%s) already removed before %s event could be processed",
+                            name, container_id or "unknown ID", action)
+            return
+
+        # Events may carry the previous name of a renamed container. Incident
+        # tracking, recovery, OOM suppression, and probe caches use the name
+        # reported by the resolved container, consistent with the poll loop.
+        name = container.name
+        if name in self._excluded:
             return
 
         if is_ignored_container_event(container, attributes, action, self._ignored_events):