From 94eb57516f28f29fce82344fbecbe7cff43aedaa Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 15:35:28 +0100 Subject: [PATCH 01/12] fix(reolink): default timeout on every camera request --- .../camera/adapters/reolink.py | 15 ++++++++++++++- pyro_camera_api/tests/test_reolink_tls.py | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py b/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py index 2a35fe4d..9b35bc9c 100644 --- a/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py +++ b/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py @@ -30,7 +30,19 @@ logger = logging.getLogger(__name__) -class _LegacyTLSAdapter(HTTPAdapter): +# A PTZ call without timeout once hung a patrol thread forever (pyro-engine#422): +# every request gets this unless the call site passes its own. +_DEFAULT_TIMEOUT = 10 + + +class _TimeoutAdapter(HTTPAdapter): + def send(self, request: Any, *args: Any, **kwargs: Any) -> Any: + if kwargs.get("timeout") is None: + kwargs["timeout"] = _DEFAULT_TIMEOUT + return super().send(request, *args, **kwargs) + + +class _LegacyTLSAdapter(_TimeoutAdapter): """Old Reolink firmwares only offer RSA-key-exchange TLS ciphers, which Python 3.10+ dropped from its default cipher list: restore OpenSSL's own.""" @@ -51,6 +63,7 @@ def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: # hands back a stale socket (matches the per-call behaviour of requests.get). _session.headers["Connection"] = "close" _session.mount("https://", _LegacyTLSAdapter()) +_session.mount("http://", _TimeoutAdapter()) # Zoom/focus reads and the zoom restore run under the stream lock in # routes_stream, so a camera that stops answering must not hang them forever. diff --git a/pyro_camera_api/tests/test_reolink_tls.py b/pyro_camera_api/tests/test_reolink_tls.py index 14faf654..b7c359ce 100644 --- a/pyro_camera_api/tests/test_reolink_tls.py +++ b/pyro_camera_api/tests/test_reolink_tls.py @@ -3,6 +3,9 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. +import pytest +import requests + from pyro_camera_api.camera.adapters.reolink import _session @@ -15,3 +18,19 @@ def test_https_session_offers_rsa_key_exchange_ciphers(): assert "AES128-GCM-SHA256" in names # Proxy env vars must not reroute camera calls around the TLS adapter assert not _session.trust_env + + +def test_session_applies_default_timeout(monkeypatch): + # A call without timeout must not wait forever on a silent camera (pyro-engine#422) + seen = [] + + def fake_send(self, request, **kwargs): + seen.append(kwargs["timeout"]) + raise requests.ConnectionError + + monkeypatch.setattr("requests.adapters.HTTPAdapter.send", fake_send) + for url in ("https://192.168.1.10", "http://192.168.1.10"): + for kwargs in ({}, {"timeout": 2}): + with pytest.raises(requests.ConnectionError): + _session.post(url, **kwargs) + assert seen == [10, 2, 10, 2] From f4c18d1df80dd991ab3e9a2c92d3b202c84e4b40 Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 15:36:03 +0100 Subject: [PATCH 02/12] fix(camera-api): return 204 from latest_image when the frame is stale --- .../pyro_camera_api/api/routes_cameras.py | 11 ++++++-- .../pyro_camera_api/camera/base.py | 7 +++++ .../pyro_camera_api/camera/patrol.py | 4 +-- pyro_camera_api/tests/test_stale_patrol.py | 27 +++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 pyro_camera_api/tests/test_stale_patrol.py diff --git a/pyro_camera_api/pyro_camera_api/api/routes_cameras.py b/pyro_camera_api/pyro_camera_api/api/routes_cameras.py index 744d4102..abe250a9 100644 --- a/pyro_camera_api/pyro_camera_api/api/routes_cameras.py +++ b/pyro_camera_api/pyro_camera_api/api/routes_cameras.py @@ -238,13 +238,19 @@ def get_latest_image( le=100, description="JPEG quality that controls compression level. Higher means better quality and larger file size.", ), + max_age_s: int = Query( + default=300, + ge=1, + description="Images stored longer ago than this are treated as missing.", + ), ): """ Return the last stored image for a given camera and pose. The camera adapter may cache captured frames per preset pose in its last_images mapping. This endpoint exposes that cache. If there is - no image for the requested pose, the endpoint returns HTTP 204 with + no image for the requested pose, or it is older than max_age_s (for + example a hung patrol thread), the endpoint returns HTTP 204 with an empty body. """ cam = CAMERA_REGISTRY.get(camera_ip) @@ -252,7 +258,8 @@ def get_latest_image( raise HTTPException(status_code=404, detail="Unknown camera") image = cam.last_images.get(pose) - if image is None: + stored_at = cam.last_image_at.get(pose) + if image is None or stored_at is None or time.monotonic() - stored_at > max_age_s: return Response(status_code=status.HTTP_204_NO_CONTENT) buffer = BytesIO() diff --git a/pyro_camera_api/pyro_camera_api/camera/base.py b/pyro_camera_api/pyro_camera_api/camera/base.py index a12bd162..63ba1200 100644 --- a/pyro_camera_api/pyro_camera_api/camera/base.py +++ b/pyro_camera_api/pyro_camera_api/camera/base.py @@ -6,6 +6,7 @@ from __future__ import annotations +import time from abc import ABC, abstractmethod from typing import Dict, List, Optional @@ -36,6 +37,12 @@ def __init__(self, camera_id: str, cam_type: str = "static") -> None: # PTZ cameras can use pose -> image # Static cameras can use -1 -> image self.last_images: Dict[int, Optional[Image.Image]] = {} + # time.monotonic() of each stored image, so stale frames can be refused + self.last_image_at: Dict[int, float] = {} + + def store_image(self, pose: int, image: Image.Image) -> None: + self.last_images[pose] = image + self.last_image_at[pose] = time.monotonic() @abstractmethod def capture(self, **kwargs) -> Optional[Image.Image]: diff --git a/pyro_camera_api/pyro_camera_api/camera/patrol.py b/pyro_camera_api/pyro_camera_api/camera/patrol.py index 90f0f8db..4222ec92 100644 --- a/pyro_camera_api/pyro_camera_api/camera/patrol.py +++ b/pyro_camera_api/pyro_camera_api/camera/patrol.py @@ -99,7 +99,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: image = cam.capture() if image: - cam.last_images[pose] = image + cam.store_image(pose, image) logger.info("[%s] Stored image for pose %s", camera_ip, pose) except Exception as exc: @@ -161,7 +161,7 @@ def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: settle_until = opened_at + 1.0 if image and now >= settle_until: - cam.last_images[-1] = image + cam.store_image(-1, image) logger.info("[%s] Updated static image (pose -1)", camera_ip) # success reset failure counter and clear skip FAILURE_COUNT[camera_ip] = 0 diff --git a/pyro_camera_api/tests/test_stale_patrol.py b/pyro_camera_api/tests/test_stale_patrol.py new file mode 100644 index 00000000..fda3a703 --- /dev/null +++ b/pyro_camera_api/tests/test_stale_patrol.py @@ -0,0 +1,27 @@ +# Copyright (C) 2022-2026, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from PIL import Image + +from pyro_camera_api.api.routes_cameras import get_latest_image +from pyro_camera_api.camera.adapters.mock import MockCamera +from pyro_camera_api.camera.registry import CAMERA_REGISTRY + + +def test_latest_image_refuses_stale_frame(monkeypatch): + # A hung patrol thread leaves an old frame in the cache: it must not be served + cam_id = "stale-cam" + cam = MockCamera(camera_id=cam_id, cam_type="ptz", cam_poses=[0]) + CAMERA_REGISTRY[cam_id] = cam + try: + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).status_code == 204 + + cam.store_image(0, Image.new("RGB", (8, 8))) + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).media_type == "image/jpeg" + + monkeypatch.setattr("time.monotonic", lambda: cam.last_image_at[0] + 301) + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).status_code == 204 + finally: + CAMERA_REGISTRY.pop(cam_id, None) From f1c8ef787cedc95f111296e050ffd5dc5e48212e Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 15:37:51 +0100 Subject: [PATCH 03/12] fix(camera-api): restart the container when a patrol thread hangs --- .../pyro_camera_api/camera/patrol.py | 37 ++++++++++++++++++- .../pyro_camera_api/camera/registry.py | 2 + pyro_camera_api/pyro_camera_api/main.py | 3 +- pyro_camera_api/tests/test_reolink_tls.py | 2 +- pyro_camera_api/tests/test_stale_patrol.py | 26 +++++++++++++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/pyro_camera_api/pyro_camera_api/camera/patrol.py b/pyro_camera_api/pyro_camera_api/camera/patrol.py index 4222ec92..15df35bd 100644 --- a/pyro_camera_api/pyro_camera_api/camera/patrol.py +++ b/pyro_camera_api/pyro_camera_api/camera/patrol.py @@ -7,11 +7,12 @@ from __future__ import annotations import logging +import os import threading import time -from typing import Any, Dict +from typing import Any, Dict, List -from pyro_camera_api.camera.registry import CAMERA_REGISTRY +from pyro_camera_api.camera.registry import CAMERA_REGISTRY, PATROL_PROGRESS, PATROL_THREADS from pyro_camera_api.services.stream import is_camera_streaming logger = logging.getLogger(__name__) @@ -23,6 +24,11 @@ # seconds between stream re-checks while the patrol is paused by a live stream STREAM_CHECK_INTERVAL = 5.0 +# A loop iteration takes well under a minute even when every camera call times +# out, so this much silence from a live patrol thread means it is hung. +STALL_LIMIT = 10 * 60.0 +WATCHDOG_INTERVAL = 60.0 + # per camera state for the static loop FAILURE_COUNT: Dict[str, int] = {} SKIP_UNTIL: Dict[str, float] = {} @@ -68,7 +74,31 @@ def is_stream_running_for(app: Any, camera_ip: str) -> bool: return False +def stalled_patrols() -> List[str]: + now = time.monotonic() + return [ + ip + for ip, thread in PATROL_THREADS.items() + if thread.is_alive() and now - PATROL_PROGRESS.get(ip, now) > STALL_LIMIT + ] + + +def patrol_watchdog() -> None: + """Exit the process when a patrol thread hangs, so Docker restarts the container. + + A thread blocked in a call cannot be killed from Python, and a Docker + healthcheck cannot restart the container either (uvicorn is PID 1). + """ + while True: + time.sleep(WATCHDOG_INTERVAL) + stalled = stalled_patrols() + if stalled: + logger.critical("No patrol progress for %ds on %s, exiting to restart", int(STALL_LIMIT), stalled) + os._exit(1) + + def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: + PATROL_PROGRESS[camera_ip] = time.monotonic() cam = CAMERA_REGISTRY[camera_ip] poses = getattr(cam, "cam_poses", []) or [] @@ -79,6 +109,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: logger.info("[%s] Starting patrol cycle with %d poses", camera_ip, len(poses)) while not stop_flag.is_set(): + PATROL_PROGRESS[camera_ip] = time.monotonic() # Live stream has priority: hold the patrol while a stream is active # so the camera never moves under the viewer, and resume automatically # once the stream stops (the thread keeps running, patrol stays "on"). @@ -133,6 +164,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: + PATROL_PROGRESS[camera_ip] = time.monotonic() cam = CAMERA_REGISTRY[camera_ip] logger.info("[%s] Starting static camera loop", camera_ip) @@ -143,6 +175,7 @@ def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: settle_until = 0.0 while not stop_flag.is_set(): + PATROL_PROGRESS[camera_ip] = time.monotonic() now = time.time() # skip window diff --git a/pyro_camera_api/pyro_camera_api/camera/registry.py b/pyro_camera_api/pyro_camera_api/camera/registry.py index 23e8f124..75a78c8a 100644 --- a/pyro_camera_api/pyro_camera_api/camera/registry.py +++ b/pyro_camera_api/pyro_camera_api/camera/registry.py @@ -55,6 +55,8 @@ def repl(match: re.Match[str]) -> str: # Patrol threading state, later managed in camera.patrol PATROL_THREADS: Dict[str, threading.Thread] = {} PATROL_FLAGS: Dict[str, threading.Event] = {} +# time.monotonic() of the last loop iteration of each patrol thread +PATROL_PROGRESS: Dict[str, float] = {} # Stuck-detector threading state, later managed in camera.stuck_detector STUCK_CHECK_THREADS: Dict[str, threading.Thread] = {} diff --git a/pyro_camera_api/pyro_camera_api/main.py b/pyro_camera_api/pyro_camera_api/main.py index e5081d98..26cb188c 100644 --- a/pyro_camera_api/pyro_camera_api/main.py +++ b/pyro_camera_api/pyro_camera_api/main.py @@ -20,7 +20,7 @@ from pyro_camera_api.api.routes_health import router as health_router from pyro_camera_api.api.routes_patrol import router as patrol_router from pyro_camera_api.api.routes_stream import router as stream_router -from pyro_camera_api.camera.patrol import patrol_loop, static_loop +from pyro_camera_api.camera.patrol import patrol_loop, patrol_watchdog, static_loop from pyro_camera_api.camera.pose_azimuths import azimuth_sync_loop from pyro_camera_api.camera.registry import ( CAMERA_REGISTRY, @@ -92,6 +92,7 @@ async def lifespan(app: FastAPI): logger.info("Stuck detector disabled by ENABLE_STUCK_DETECTOR for %s", cam_id) threading.Thread(target=stop_stream_if_idle, daemon=True).start() + threading.Thread(target=patrol_watchdog, daemon=True).start() # Resolve pose azimuths from the platform API in the background (retries # until the API is reachable; azimuth stays unknown in the meantime). diff --git a/pyro_camera_api/tests/test_reolink_tls.py b/pyro_camera_api/tests/test_reolink_tls.py index b7c359ce..4f4b7fa1 100644 --- a/pyro_camera_api/tests/test_reolink_tls.py +++ b/pyro_camera_api/tests/test_reolink_tls.py @@ -24,7 +24,7 @@ def test_session_applies_default_timeout(monkeypatch): # A call without timeout must not wait forever on a silent camera (pyro-engine#422) seen = [] - def fake_send(self, request, **kwargs): + def fake_send(_self, _request, **kwargs): seen.append(kwargs["timeout"]) raise requests.ConnectionError diff --git a/pyro_camera_api/tests/test_stale_patrol.py b/pyro_camera_api/tests/test_stale_patrol.py index fda3a703..c12f8e53 100644 --- a/pyro_camera_api/tests/test_stale_patrol.py +++ b/pyro_camera_api/tests/test_stale_patrol.py @@ -3,9 +3,13 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. +import threading +import time + from PIL import Image from pyro_camera_api.api.routes_cameras import get_latest_image +from pyro_camera_api.camera import patrol from pyro_camera_api.camera.adapters.mock import MockCamera from pyro_camera_api.camera.registry import CAMERA_REGISTRY @@ -25,3 +29,25 @@ def test_latest_image_refuses_stale_frame(monkeypatch): assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).status_code == 204 finally: CAMERA_REGISTRY.pop(cam_id, None) + + +def test_watchdog_flags_only_hung_patrol_threads(monkeypatch): + # A live thread silent for too long is hung, a finished one is not + hung, done = threading.Event(), threading.Event() + threads = { + "hung": threading.Thread(target=hung.wait, daemon=True), + "busy": threading.Thread(target=hung.wait, daemon=True), + "done": threading.Thread(target=done.set), + } + for thread in threads.values(): + thread.start() + threads["done"].join() + monkeypatch.setattr(patrol, "PATROL_THREADS", threads) + now = time.monotonic() + monkeypatch.setattr( + patrol, "PATROL_PROGRESS", {"hung": now - patrol.STALL_LIMIT - 1, "busy": now, "done": now - 3600} + ) + try: + assert patrol.stalled_patrols() == ["hung"] + finally: + hung.set() From 158b7546a3bcf1e113ca3639005acbda017f7361 Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 17:01:56 +0100 Subject: [PATCH 04/12] refactor(camera-api): restart on a hung patrol from the compose healthcheck --- docker-compose.yml | 10 ++++++++-- .../pyro_camera_api/api/routes_health.py | 12 +++++++++--- pyro_camera_api/pyro_camera_api/camera/patrol.py | 16 ---------------- pyro_camera_api/pyro_camera_api/main.py | 3 +-- pyro_camera_api/tests/test_stale_patrol.py | 8 ++++++++ 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 28bd1265..0fa57f20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,12 +17,18 @@ services: - ./data:/usr/src/app/data restart: always network_mode: host + # tini as PID 1, so the healthcheck's kill -9 -1 also reaches uvicorn + init: true healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8081/health || exit 1"] + # Docker never restarts an unhealthy container by itself: on a hung patrol (503) or an API + # that stopped answering, kill everything but PID 1 so `restart: always` brings it back. + # curl retries twice so a single slow answer does not restart it. Connection refused (rc 7) + # only means uvicorn is still starting, as it binds after camera setup. + test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] interval: 30s retries: 3 start_period: 20s - timeout: 5s + timeout: 30s logging: driver: json-file options: diff --git a/pyro_camera_api/pyro_camera_api/api/routes_health.py b/pyro_camera_api/pyro_camera_api/api/routes_health.py index d4b5c125..d32f07eb 100644 --- a/pyro_camera_api/pyro_camera_api/api/routes_health.py +++ b/pyro_camera_api/pyro_camera_api/api/routes_health.py @@ -7,6 +7,9 @@ from __future__ import annotations from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from pyro_camera_api.camera.patrol import stalled_patrols router = APIRouter() @@ -14,9 +17,12 @@ @router.get("/health") def health(): """ - Return a basic heartbeat response to confirm that the API is running. + Return 200 while the API answers and every running patrol makes progress. - This endpoint does not perform any camera or service check. - It is intended for uptime monitoring and infrastructure-level health probes. + Returns 503 with the hung cameras when a patrol thread is stuck, which the + compose healthcheck turns into a container restart. """ + stalled = stalled_patrols() + if stalled: + return JSONResponse(status_code=503, content={"status": "patrol_stalled", "cameras": stalled}) return {"status": "ok"} diff --git a/pyro_camera_api/pyro_camera_api/camera/patrol.py b/pyro_camera_api/pyro_camera_api/camera/patrol.py index 15df35bd..ad946c37 100644 --- a/pyro_camera_api/pyro_camera_api/camera/patrol.py +++ b/pyro_camera_api/pyro_camera_api/camera/patrol.py @@ -7,7 +7,6 @@ from __future__ import annotations import logging -import os import threading import time from typing import Any, Dict, List @@ -27,7 +26,6 @@ # A loop iteration takes well under a minute even when every camera call times # out, so this much silence from a live patrol thread means it is hung. STALL_LIMIT = 10 * 60.0 -WATCHDOG_INTERVAL = 60.0 # per camera state for the static loop FAILURE_COUNT: Dict[str, int] = {} @@ -83,20 +81,6 @@ def stalled_patrols() -> List[str]: ] -def patrol_watchdog() -> None: - """Exit the process when a patrol thread hangs, so Docker restarts the container. - - A thread blocked in a call cannot be killed from Python, and a Docker - healthcheck cannot restart the container either (uvicorn is PID 1). - """ - while True: - time.sleep(WATCHDOG_INTERVAL) - stalled = stalled_patrols() - if stalled: - logger.critical("No patrol progress for %ds on %s, exiting to restart", int(STALL_LIMIT), stalled) - os._exit(1) - - def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: PATROL_PROGRESS[camera_ip] = time.monotonic() cam = CAMERA_REGISTRY[camera_ip] diff --git a/pyro_camera_api/pyro_camera_api/main.py b/pyro_camera_api/pyro_camera_api/main.py index 26cb188c..e5081d98 100644 --- a/pyro_camera_api/pyro_camera_api/main.py +++ b/pyro_camera_api/pyro_camera_api/main.py @@ -20,7 +20,7 @@ from pyro_camera_api.api.routes_health import router as health_router from pyro_camera_api.api.routes_patrol import router as patrol_router from pyro_camera_api.api.routes_stream import router as stream_router -from pyro_camera_api.camera.patrol import patrol_loop, patrol_watchdog, static_loop +from pyro_camera_api.camera.patrol import patrol_loop, static_loop from pyro_camera_api.camera.pose_azimuths import azimuth_sync_loop from pyro_camera_api.camera.registry import ( CAMERA_REGISTRY, @@ -92,7 +92,6 @@ async def lifespan(app: FastAPI): logger.info("Stuck detector disabled by ENABLE_STUCK_DETECTOR for %s", cam_id) threading.Thread(target=stop_stream_if_idle, daemon=True).start() - threading.Thread(target=patrol_watchdog, daemon=True).start() # Resolve pose azimuths from the platform API in the background (retries # until the API is reachable; azimuth stays unknown in the meantime). diff --git a/pyro_camera_api/tests/test_stale_patrol.py b/pyro_camera_api/tests/test_stale_patrol.py index c12f8e53..e0a81ebd 100644 --- a/pyro_camera_api/tests/test_stale_patrol.py +++ b/pyro_camera_api/tests/test_stale_patrol.py @@ -8,7 +8,9 @@ from PIL import Image +from pyro_camera_api.api import routes_health from pyro_camera_api.api.routes_cameras import get_latest_image +from pyro_camera_api.api.routes_health import health from pyro_camera_api.camera import patrol from pyro_camera_api.camera.adapters.mock import MockCamera from pyro_camera_api.camera.registry import CAMERA_REGISTRY @@ -51,3 +53,9 @@ def test_watchdog_flags_only_hung_patrol_threads(monkeypatch): assert patrol.stalled_patrols() == ["hung"] finally: hung.set() + + +def test_health_reports_stalled_patrols(monkeypatch): + assert health() == {"status": "ok"} + monkeypatch.setattr(routes_health, "stalled_patrols", lambda: ["192.168.1.12"]) + assert health().status_code == 503 From 3a258566835e719350f4bd493386aca83c102c80 Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 19:43:22 +0100 Subject: [PATCH 05/12] fix(camera-api): keep /health as liveness, check patrols on /health/patrols The Shelly watchdog reads /health to decide whether to power-cycle the Pi. --- docker-compose.yml | 2 +- .../pyro_camera_api/api/routes_health.py | 16 +++++++++++++--- pyro_camera_api/tests/test_stale_patrol.py | 10 ++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0fa57f20..951773ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: # that stopped answering, kill everything but PID 1 so `restart: always` brings it back. # curl retries twice so a single slow answer does not restart it. Connection refused (rc 7) # only means uvicorn is still starting, as it binds after camera setup. - test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] + test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health/patrols; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] interval: 30s retries: 3 start_period: 20s diff --git a/pyro_camera_api/pyro_camera_api/api/routes_health.py b/pyro_camera_api/pyro_camera_api/api/routes_health.py index d32f07eb..b7150324 100644 --- a/pyro_camera_api/pyro_camera_api/api/routes_health.py +++ b/pyro_camera_api/pyro_camera_api/api/routes_health.py @@ -17,10 +17,20 @@ @router.get("/health") def health(): """ - Return 200 while the API answers and every running patrol makes progress. + Return a basic heartbeat response to confirm that the API is running. - Returns 503 with the hung cameras when a patrol thread is stuck, which the - compose healthcheck turns into a container restart. + Kept free of camera checks: the Shelly watchdog reads it as Pi liveness and + power-cycles the Pi and the router when it fails. + """ + return {"status": "ok"} + + +@router.get("/health/patrols") +def health_patrols(): + """ + Return 200 while every running patrol makes progress, 503 with the hung cameras otherwise. + + Used by the compose healthcheck, which restarts the container on a 503. """ stalled = stalled_patrols() if stalled: diff --git a/pyro_camera_api/tests/test_stale_patrol.py b/pyro_camera_api/tests/test_stale_patrol.py index e0a81ebd..330d7133 100644 --- a/pyro_camera_api/tests/test_stale_patrol.py +++ b/pyro_camera_api/tests/test_stale_patrol.py @@ -10,7 +10,7 @@ from pyro_camera_api.api import routes_health from pyro_camera_api.api.routes_cameras import get_latest_image -from pyro_camera_api.api.routes_health import health +from pyro_camera_api.api.routes_health import health, health_patrols from pyro_camera_api.camera import patrol from pyro_camera_api.camera.adapters.mock import MockCamera from pyro_camera_api.camera.registry import CAMERA_REGISTRY @@ -55,7 +55,9 @@ def test_watchdog_flags_only_hung_patrol_threads(monkeypatch): hung.set() -def test_health_reports_stalled_patrols(monkeypatch): - assert health() == {"status": "ok"} +def test_health_patrols_reports_stalled_patrols(monkeypatch): + assert health_patrols() == {"status": "ok"} monkeypatch.setattr(routes_health, "stalled_patrols", lambda: ["192.168.1.12"]) - assert health().status_code == 503 + assert health_patrols().status_code == 503 + # The Shelly reads /health as Pi liveness: a hung patrol must not power-cycle the Pi + assert health() == {"status": "ok"} From 3bc0f60023f4776df6368b729e13e393db13dd8a Mon Sep 17 00:00:00 2001 From: Mateo Lostanlen Date: Wed, 28 Oct 2026 12:26:09 +0100 Subject: [PATCH 06/12] docs(compose): say why connection refused does not kill the camera API The healthcheck kills on every curl failure but rc 7, which looked like a hole: a uvicorn that never binds would stay unhealthy forever. Killing on rc 7 would loop at startup instead, as the kill runs during start_period too, and a hang before the bind is bounded by the 10 s timeout on every camera call. --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 951773ec..9089d05c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,9 @@ services: # Docker never restarts an unhealthy container by itself: on a hung patrol (503) or an API # that stopped answering, kill everything but PID 1 so `restart: always` brings it back. # curl retries twice so a single slow answer does not restart it. Connection refused (rc 7) - # only means uvicorn is still starting, as it binds after camera setup. + # only means uvicorn is still starting, as it binds after camera setup: killing on it would + # loop at startup (the kill runs during start_period too). A hang before the bind is bounded + # by the 10 s timeout on every camera call, so rc 7 cannot last. test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health/patrols; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] interval: 30s retries: 3 From c61e73240d636b683b91a7b93b52980817ad39cd Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 19:02:42 +0100 Subject: [PATCH 07/12] fix(engine): stop running patrols at night --- pyroengine/core.py | 2 +- tests/test_core.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pyroengine/core.py b/pyroengine/core.py index 92744a63..e72331f4 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -270,7 +270,7 @@ def main_loop(self, period: int, send_alerts: bool = True) -> None: for ip in self.camera_data: try: patrol_status = self.camera_api_client.get_patrol_status(ip) - if not patrol_status.get("patrol_running", True): + if patrol_status.get("patrol_running", False): self.camera_api_client.stop_patrol(ip) logger.info(f"Stopped patrol for camera {ip} due to night") except Exception as e: diff --git a/tests/test_core.py b/tests/test_core.py index fc322dab..b259e8fa 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -121,3 +121,23 @@ def test_inference_loop_skips_when_stream_active(mock_client_class, mock_engine, assert not mock_client.get_latest_image.called assert not mock_engine.predict.called + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_stops_running_patrols(mock_client_class, mock_engine, mock_camera_data, monkeypatch): + mock_client = mock_client_class.return_value + mock_client.get_patrol_status.return_value = {"patrol_running": True} + controller = SystemController(mock_engine, mock_camera_data, "http://fake.url") + controller.is_day = False + + class NightSleep(Exception): + pass + + def sleep(_s): + raise NightSleep + + monkeypatch.setattr("pyroengine.core.time.sleep", sleep) + with pytest.raises(NightSleep): + controller.main_loop(period=30) + + mock_client.stop_patrol.assert_called_once_with("192.168.1.1") From 297e136710b8e68a668778ba15c61a17d966b795 Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 19:03:18 +0100 Subject: [PATCH 08/12] fix(engine): decide day or night per frame, sleep only when every camera is at night --- CLAUDE.md | 2 +- pyroengine/core.py | 23 ++++++++++++++++++----- tests/test_core.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ba9f8a5d..47fa5852 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,7 @@ For each `cam_id`, the engine periodically fetches a JSON file at `{bbox_mask_ur ### Night / day handling -Day is determined by IR-channel analysis (`is_day_time(strategy="ir")`): if `max(R - G) == 0` the image is considered infrared/night. At night the main loop sleeps for 1 hour, stops all patrols, then re-checks before resuming inference. +Day is determined by IR-channel analysis (`is_day_time(strategy="ir")`): if `max(R - G) == 0` the image is considered infrared/night. Night frames are not analyzed. Once every frame of an inference pass is a night frame, the main loop stops all patrols, sleeps for 1 hour, then re-checks before resuming inference. ### Environment Variables (`.env`) diff --git a/pyroengine/core.py b/pyroengine/core.py index e72331f4..07f217b8 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -188,6 +188,7 @@ def inference_loop(self) -> None: logger.info("Stream detected, skipping inference on all cameras") return + day_frames = [] for ip, cam in self.camera_data.items(): camera_name = cam["name"] @@ -201,8 +202,7 @@ def inference_loop(self) -> None: frame = self._safe_get_latest_image(ip, pose) if frame is not None: logger.info(f"Captured image for {ip}, pose {pose}") - self.is_day = is_day_time(None, frame, "ir") - self.engine.predict(frame, cam_id) + day_frames.append(self._analyze(frame, cam_id)) except requests.HTTPError as e: logger.error( f"HTTP error for {camera_name}, pose {pose}: " @@ -220,13 +220,26 @@ def inference_loop(self) -> None: frame = self._safe_get_latest_image(ip, -1) if frame is not None: logger.info(f"Captured image for {ip}") - self.is_day = is_day_time(None, frame, "ir") - self.engine.predict(frame, cam_id) + day_frames.append(self._analyze(frame, cam_id)) except requests.HTTPError as e: logger.error(f"HTTP error for {camera_name}: {e.response.text if e.response is not None else e}") except Exception as e: logger.error(f"Error for {camera_name}: {e}") + # The site sleeps only once every camera reports night, so one camera in infrared + # (fog, a dark area) does not blind the others. + if day_frames: + self.is_day = any(day_frames) + + def _analyze(self, frame: Image.Image, cam_id: str) -> bool: + """Run inference on a day frame and return whether the frame is a day frame.""" + if not is_day_time(None, frame, "ir"): + # The model is trained on day images, infrared frames only give false positives + logger.info(f"Night frame for {cam_id}, skipping inference") + return False + self.engine.predict(frame, cam_id) + return True + def check_and_restart_patrol(self) -> None: """ Check stream activity and ensure patrol is running when no stream is active. @@ -276,7 +289,7 @@ def main_loop(self, period: int, send_alerts: bool = True) -> None: except Exception as e: logger.error(f"Failed to stop patrol on camera {ip}: {e}") - logger.info("Nighttime detected by at least one camera, sleeping for 1 hour") + logger.info("Nighttime detected by every camera, sleeping for 1 hour") time.sleep(3600) try: diff --git a/tests/test_core.py b/tests/test_core.py index b259e8fa..bb8e34a7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -141,3 +141,32 @@ def sleep(_s): controller.main_loop(period=30) mock_client.stop_patrol.assert_called_once_with("192.168.1.1") + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_frames_are_skipped_and_site_sleeps_only_when_all_are_night(mock_client_class, mock_engine): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "ptz", "poses": [1]}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + # One camera in infrared: it is not analyzed, the other one keeps the site awake + mock_client.get_latest_image.side_effect = [night_img, day_img] + controller.inference_loop() + mock_engine.predict.assert_called_once_with(day_img, "192.168.1.2") + assert controller.is_day + + mock_client.get_latest_image.side_effect = [night_img, night_img] + controller.inference_loop() + assert mock_engine.predict.call_count == 1 + assert not controller.is_day + + # No frame at all (stale cache): keep the previous state + mock_client.get_latest_image.side_effect = [None, None] + controller.inference_loop() + assert not controller.is_day From f0b04d59428a5f416b52ae20c300f40ad939729f Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 19:04:02 +0100 Subject: [PATCH 09/12] fix(engine): skip a frame identical to the previous one for the same pose --- pyroengine/core.py | 9 +++++++++ tests/test_core.py | 25 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pyroengine/core.py b/pyroengine/core.py index 07f217b8..6b6235a6 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -4,6 +4,7 @@ # See LICENSE or go to for full license details. +import hashlib import logging import sys import time @@ -81,6 +82,7 @@ def __init__( self.camera_data = camera_data self.is_day = True self.last_autofocus: Optional[datetime] = None + self._last_frame_digest: Dict[str, bytes] = {} # Wait for the camera API to be available time.sleep(self.API_INITIAL_WAIT) @@ -237,6 +239,13 @@ def _analyze(self, frame: Image.Image, cam_id: str) -> bool: # The model is trained on day images, infrared frames only give false positives logger.info(f"Night frame for {cam_id}, skipping inference") return False + # A live camera never returns the exact same pixels twice: an identical frame is frozen, + # and analyzing it again would re-upload the same detection every pass. + digest = hashlib.md5(frame.tobytes(), usedforsecurity=False).digest() + if self._last_frame_digest.get(cam_id) == digest: + logger.warning(f"Frame identical to the previous one for {cam_id}, skipping inference") + return True + self._last_frame_digest[cam_id] = digest self.engine.predict(frame, cam_id) return True diff --git a/tests/test_core.py b/tests/test_core.py index bb8e34a7..6a7ac4de 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -130,14 +130,14 @@ def test_night_stops_running_patrols(mock_client_class, mock_engine, mock_camera controller = SystemController(mock_engine, mock_camera_data, "http://fake.url") controller.is_day = False - class NightSleep(Exception): + class NightSleepError(Exception): pass def sleep(_s): - raise NightSleep + raise NightSleepError monkeypatch.setattr("pyroengine.core.time.sleep", sleep) - with pytest.raises(NightSleep): + with pytest.raises(NightSleepError): controller.main_loop(period=30) mock_client.stop_patrol.assert_called_once_with("192.168.1.1") @@ -170,3 +170,22 @@ def test_night_frames_are_skipped_and_site_sleeps_only_when_all_are_night(mock_c mock_client.get_latest_image.side_effect = [None, None] controller.inference_loop() assert not controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_identical_frame_is_not_analyzed_twice(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.side_effect = [ + Image.new("RGB", (100, 100), (255, 200, 200)), + Image.new("RGB", (100, 100), (255, 200, 200)), + Image.new("RGB", (100, 100), (255, 201, 200)), + ] + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + for _ in range(3): + controller.inference_loop() + + assert mock_engine.predict.call_count == 2 + assert controller.is_day From 78af4324949fe80b5f6fecca3a0246b0f44c1a4b Mon Sep 17 00:00:00 2001 From: Mateo Date: Mon, 26 Oct 2026 19:09:39 +0100 Subject: [PATCH 10/12] fix(engine): keep day evidence on prediction errors and drop night history --- pyroengine/core.py | 26 ++++++++++++++++---------- tests/test_core.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/pyroengine/core.py b/pyroengine/core.py index 6b6235a6..f8ca7656 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -204,7 +204,9 @@ def inference_loop(self) -> None: frame = self._safe_get_latest_image(ip, pose) if frame is not None: logger.info(f"Captured image for {ip}, pose {pose}") - day_frames.append(self._analyze(frame, cam_id)) + frame_is_day = is_day_time(None, frame, "ir") + day_frames.append(frame_is_day) + self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error( f"HTTP error for {camera_name}, pose {pose}: " @@ -222,7 +224,9 @@ def inference_loop(self) -> None: frame = self._safe_get_latest_image(ip, -1) if frame is not None: logger.info(f"Captured image for {ip}") - day_frames.append(self._analyze(frame, cam_id)) + frame_is_day = is_day_time(None, frame, "ir") + day_frames.append(frame_is_day) + self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error(f"HTTP error for {camera_name}: {e.response.text if e.response is not None else e}") except Exception as e: @@ -233,21 +237,23 @@ def inference_loop(self) -> None: if day_frames: self.is_day = any(day_frames) - def _analyze(self, frame: Image.Image, cam_id: str) -> bool: - """Run inference on a day frame and return whether the frame is a day frame.""" - if not is_day_time(None, frame, "ir"): - # The model is trained on day images, infrared frames only give false positives + def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: + """Run inference on a day frame that differs from the previous one of this camera.""" + if not frame_is_day: + # The model is trained on day images, infrared frames only give false positives. + # Drop the camera's detection history so evening frames are not combined with the + # next morning's ones. logger.info(f"Night frame for {cam_id}, skipping inference") - return False + self.engine._states.pop(cam_id, None) + return # A live camera never returns the exact same pixels twice: an identical frame is frozen, # and analyzing it again would re-upload the same detection every pass. digest = hashlib.md5(frame.tobytes(), usedforsecurity=False).digest() if self._last_frame_digest.get(cam_id) == digest: logger.warning(f"Frame identical to the previous one for {cam_id}, skipping inference") - return True - self._last_frame_digest[cam_id] = digest + return self.engine.predict(frame, cam_id) - return True + self._last_frame_digest[cam_id] = digest def check_and_restart_patrol(self) -> None: """ diff --git a/tests/test_core.py b/tests/test_core.py index 6a7ac4de..312bd22a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -189,3 +189,35 @@ def test_identical_frame_is_not_analyzed_twice(mock_client_class, mock_engine): assert mock_engine.predict.call_count == 2 assert controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_failed_prediction_keeps_day_and_is_retried(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.return_value = day_img + mock_engine.predict.side_effect = [RuntimeError("upload failed"), None] + controller = SystemController(mock_engine, camera_data, "http://fake.url") + controller.is_day = True + + controller.inference_loop() + assert controller.is_day + + controller.inference_loop() + assert mock_engine.predict.call_count == 2 + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_frame_drops_detection_history(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.return_value = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_engine._states = {"192.168.1.2": {"ongoing": True}} + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + controller.inference_loop() + + assert "192.168.1.2" not in mock_engine._states From c2bc588fe1e5d03a7715c60e9774bb324ef7c492 Mon Sep 17 00:00:00 2001 From: Mateo Lostanlen Date: Wed, 28 Oct 2026 12:26:12 +0100 Subject: [PATCH 11/12] fix(engine): a missing frame never proves night, wake-up probes every camera - The pass only collected the frames it received, so one night frame next to a camera answering 204 (no fresh image, PR #458) read as "every camera reports night" and put the site to sleep for an hour. The site now sleeps only when every expected frame came back and all of them are night frames; one day frame still wakes it. - After the night sleep, only the first camera was probed: in infrared or down, it kept the whole site asleep in daylight. Every camera is probed until one shows daylight, and the site stays asleep only when all of them answer night. --- pyroengine/core.py | 41 +++++++++++++++++++++++++++++++++-------- tests/test_core.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/pyroengine/core.py b/pyroengine/core.py index f8ca7656..4064e2af 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -191,6 +191,7 @@ def inference_loop(self) -> None: return day_frames = [] + expected = 0 for ip, cam in self.camera_data.items(): camera_name = cam["name"] @@ -201,6 +202,7 @@ def inference_loop(self) -> None: return try: cam_id = f"{ip}_{pose}" + expected += 1 frame = self._safe_get_latest_image(ip, pose) if frame is not None: logger.info(f"Captured image for {ip}, pose {pose}") @@ -221,6 +223,7 @@ def inference_loop(self) -> None: return try: cam_id = ip + expected += 1 frame = self._safe_get_latest_image(ip, -1) if frame is not None: logger.info(f"Captured image for {ip}") @@ -232,10 +235,13 @@ def inference_loop(self) -> None: except Exception as e: logger.error(f"Error for {camera_name}: {e}") - # The site sleeps only once every camera reports night, so one camera in infrared - # (fog, a dark area) does not blind the others. - if day_frames: - self.is_day = any(day_frames) + # One day frame keeps the site awake, so one camera in infrared (fog, a dark area) does + # not blind the others. It sleeps only when every expected frame came back and all are + # night: a camera without a fresh image (204, error) says nothing about the light. + if any(day_frames): + self.is_day = True + elif day_frames and len(day_frames) == expected: + self.is_day = False def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: """Run inference on a day frame that differs from the previous one of this camera.""" @@ -255,6 +261,27 @@ def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: self.engine.predict(frame, cam_id) self._last_frame_digest[cam_id] = digest + def _probe_day(self) -> bool: + """Capture one frame per camera until one shows daylight. + + Stays night only when every camera answered with a night frame: a camera that + fails to answer says nothing, so the site wakes up and lets the inference loop + decide from the next fresh frames. + """ + night = 0 + for ip in self.camera_data: + try: + frame = self.camera_api_client.capture_image(ip) + except Exception as e: + logger.error(f"Failed to capture {ip} after sleep: {e}") + continue + if frame is None: + continue + if is_day_time(None, frame, "ir"): + return True + night += 1 + return night < len(self.camera_data) + def check_and_restart_patrol(self) -> None: """ Check stream activity and ensure patrol is running when no stream is active. @@ -308,10 +335,8 @@ def main_loop(self, period: int, send_alerts: bool = True) -> None: time.sleep(3600) try: - ip = next(iter(self.camera_data.keys())) - frame = self.camera_api_client.capture_image(ip) - self.is_day = is_day_time(None, frame, "ir") - logger.info(f"Re checked day and night using camera {ip}, result is_day={self.is_day}") + self.is_day = self._probe_day() + logger.info(f"Re checked day and night after sleep, result is_day={self.is_day}") if self.is_day: logger.info("Day detected, restarting patrols") diff --git a/tests/test_core.py b/tests/test_core.py index 312bd22a..f40a862b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -171,6 +171,39 @@ def test_night_frames_are_skipped_and_site_sleeps_only_when_all_are_night(mock_c controller.inference_loop() assert not controller.is_day + # One night frame and one camera without a fresh image (204): the missing camera + # says nothing about the light, the site must not go to sleep on one night frame + controller.is_day = True + mock_client.get_latest_image.side_effect = [night_img, None] + controller.inference_loop() + assert controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_wake_up_probes_every_camera(mock_client_class, mock_engine): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "static"}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + # The first camera stays in infrared (or is down), the second one sees daylight + mock_client.capture_image.side_effect = [night_img, day_img] + assert controller._probe_day() + mock_client.capture_image.side_effect = [RuntimeError("down"), day_img] + assert controller._probe_day() + + # Every camera reports night: stay asleep + mock_client.capture_image.side_effect = [night_img, night_img] + assert not controller._probe_day() + + # One night frame and one camera down: not proven night, wake up + mock_client.capture_image.side_effect = [night_img, RuntimeError("down")] + assert controller._probe_day() + @patch("pyroengine.core.PyroCameraAPIClient") def test_identical_frame_is_not_analyzed_twice(mock_client_class, mock_engine): From 10cc832969f4537dd339a69dade311a15beb17d6 Mon Sep 17 00:00:00 2001 From: Mateo Date: Tue, 3 Nov 2026 17:49:54 +0100 Subject: [PATCH 12/12] fix(engine): confirm night over several frames, count only cameras that answered A single infrared frame dropped the camera's detection history, and with it an ongoing event: cameras flicker between colour and infrared at dusk. The history is now dropped after 3 night frames in a row, and a camera in infrared while the others see daylight is warned once, as it sends no heartbeat. A camera without a fresh image never counted as night, so one dead camera kept the site awake all night: PTZ patrols ran, autofocus ran on infrared images, and the stuck detector spent its reboot budget overnight. Only cameras with a fresh frame count now, and the site sleeps after 3 all-night passes. The frozen frame case is already handled by the API, which drops it after max_age_s. Same rule for the wake-up probe. The identical frame log goes to DEBUG: the engine reads the API cache, which the patrol refreshes slower than the loop runs, so the same frame twice is the normal case. The API's stuck detector diagnoses frozen cameras. The healthcheck killed the API after 25 s of failed checks, as the kill runs inside the check and `retries` gives no margin. curl now retries for about 2 min, still well under the 10 min stall limit. --- docker-compose.yml | 14 ++++--- pyroengine/core.py | 69 ++++++++++++++++++++++++---------- tests/test_core.py | 92 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 142 insertions(+), 33 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9089d05c..b71895f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,15 +22,17 @@ services: healthcheck: # Docker never restarts an unhealthy container by itself: on a hung patrol (503) or an API # that stopped answering, kill everything but PID 1 so `restart: always` brings it back. - # curl retries twice so a single slow answer does not restart it. Connection refused (rc 7) - # only means uvicorn is still starting, as it binds after camera setup: killing on it would - # loop at startup (the kill runs during start_period too). A hang before the bind is bounded - # by the 10 s timeout on every camera call, so rc 7 cannot last. - test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health/patrols; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] + # The kill runs on the first failed check, so `retries` gives no margin: curl retries for + # 80 s (fast 503) to 125 s (9 tries of 5 s, 10 s apart) so a saturated Pi does not restart + # the API, with every camera and live stream. `timeout` stays above that, or Docker kills + # the check before the kill runs. Connection refused (rc 7) only means uvicorn is still starting, as it binds + # after camera setup: killing on it would loop at startup (the kill runs during start_period + # too). A hang before the bind is bounded by the 10 s timeout on every camera call. + test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 8 --retry-delay 10 http://localhost:8081/health/patrols; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] interval: 30s retries: 3 start_period: 20s - timeout: 30s + timeout: 140s logging: driver: json-file options: diff --git a/pyroengine/core.py b/pyroengine/core.py index 4064e2af..26c661be 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -31,6 +31,11 @@ logging.basicConfig(format="%(asctime)s | %(levelname)s: %(message)s", level=logging.INFO, force=True) logger = logging.getLogger(__name__) +# Cameras flicker between colour and infrared at dusk, and one frame must not restart an ongoing +# event or put a site to sleep in daylight: both need this many night frames or passes in a row. +NIGHT_FRAMES_BEFORE_RESET = 3 +NIGHT_PASSES_BEFORE_SLEEP = 3 + def is_day_time(cache: Optional[Path], frame: Image.Image, strategy: str, delta: int = 0) -> bool: """ @@ -83,6 +88,8 @@ def __init__( self.is_day = True self.last_autofocus: Optional[datetime] = None self._last_frame_digest: Dict[str, bytes] = {} + self._night_frames: Dict[str, int] = {} # consecutive night frames per cam_id + self._night_passes = 0 # consecutive passes where every fresh frame was a night one # Wait for the camera API to be available time.sleep(self.API_INITIAL_WAIT) @@ -191,7 +198,7 @@ def inference_loop(self) -> None: return day_frames = [] - expected = 0 + night_cams = [] for ip, cam in self.camera_data.items(): camera_name = cam["name"] @@ -202,12 +209,13 @@ def inference_loop(self) -> None: return try: cam_id = f"{ip}_{pose}" - expected += 1 frame = self._safe_get_latest_image(ip, pose) if frame is not None: logger.info(f"Captured image for {ip}, pose {pose}") frame_is_day = is_day_time(None, frame, "ir") day_frames.append(frame_is_day) + if not frame_is_day: + night_cams.append(cam_id) self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error( @@ -223,12 +231,13 @@ def inference_loop(self) -> None: return try: cam_id = ip - expected += 1 frame = self._safe_get_latest_image(ip, -1) if frame is not None: logger.info(f"Captured image for {ip}") frame_is_day = is_day_time(None, frame, "ir") day_frames.append(frame_is_day) + if not frame_is_day: + night_cams.append(cam_id) self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error(f"HTTP error for {camera_name}: {e.response.text if e.response is not None else e}") @@ -236,27 +245,41 @@ def inference_loop(self) -> None: logger.error(f"Error for {camera_name}: {e}") # One day frame keeps the site awake, so one camera in infrared (fog, a dark area) does - # not blind the others. It sleeps only when every expected frame came back and all are - # night: a camera without a fresh image (204, error) says nothing about the light. + # not blind the others. Only cameras with a fresh frame count: a camera down says nothing + # about the light, and a frozen frame is already dropped by the API after max_age_s. The + # site sleeps once every fresh frame has been a night one for several passes in a row. if any(day_frames): self.is_day = True - elif day_frames and len(day_frames) == expected: - self.is_day = False + self._night_passes = 0 + # A camera in infrared while the site sees daylight is blind (stuck IR cut, dark + # area) and sends no heartbeat, as predict is not called. Warned once per episode. + stuck = [c for c in night_cams if self._night_frames[c] == NIGHT_FRAMES_BEFORE_RESET] + if stuck: + logger.warning(f"Cameras in infrared while others see daylight: {stuck}") + elif day_frames: + self._night_passes += 1 + if self._night_passes >= NIGHT_PASSES_BEFORE_SLEEP: + self.is_day = False def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: """Run inference on a day frame that differs from the previous one of this camera.""" if not frame_is_day: # The model is trained on day images, infrared frames only give false positives. - # Drop the camera's detection history so evening frames are not combined with the - # next morning's ones. logger.info(f"Night frame for {cam_id}, skipping inference") - self.engine._states.pop(cam_id, None) + self._night_frames[cam_id] = self._night_frames.get(cam_id, 0) + 1 + if self._night_frames[cam_id] == NIGHT_FRAMES_BEFORE_RESET: + # Drop the camera's detection history so evening frames are not combined with + # the next morning's ones. Not on the first frame: it would also drop an ongoing + # event and its crop boxes on one isolated infrared frame. + self.engine._states.pop(cam_id, None) return - # A live camera never returns the exact same pixels twice: an identical frame is frozen, - # and analyzing it again would re-upload the same detection every pass. + self._night_frames[cam_id] = 0 + # The engine reads the API cache, which the patrol refreshes slower than the loop runs: + # the same frame read twice is normal, not a frozen camera (the API detects those). It + # is skipped so a detection is not scored and uploaded again on every pass. digest = hashlib.md5(frame.tobytes(), usedforsecurity=False).digest() if self._last_frame_digest.get(cam_id) == digest: - logger.warning(f"Frame identical to the previous one for {cam_id}, skipping inference") + logger.debug(f"Frame identical to the previous one for {cam_id}, skipping inference") return self.engine.predict(frame, cam_id) self._last_frame_digest[cam_id] = digest @@ -264,10 +287,11 @@ def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: def _probe_day(self) -> bool: """Capture one frame per camera until one shows daylight. - Stays night only when every camera answered with a night frame: a camera that - fails to answer says nothing, so the site wakes up and lets the inference loop - decide from the next fresh frames. + Only cameras that answer count: one down says nothing about the light and must not + keep the site awake all night. When none answers, the site wakes up and lets the + inference loop decide from the next fresh frames. """ + day = False night = 0 for ip in self.camera_data: try: @@ -278,9 +302,16 @@ def _probe_day(self) -> bool: if frame is None: continue if is_day_time(None, frame, "ir"): - return True + day = True + break night += 1 - return night < len(self.camera_data) + if day or night == 0: + # Wake up with a clean count: the night streak must not put the site back to sleep + # on the first pass, and a camera still in infrared at dawn must be warned again. + self._night_frames.clear() + self._night_passes = 0 + return True + return False def check_and_restart_patrol(self) -> None: """ @@ -331,7 +362,7 @@ def main_loop(self, period: int, send_alerts: bool = True) -> None: except Exception as e: logger.error(f"Failed to stop patrol on camera {ip}: {e}") - logger.info("Nighttime detected by every camera, sleeping for 1 hour") + logger.info("Nighttime detected by every camera with a fresh image, sleeping for 1 hour") time.sleep(3600) try: diff --git a/tests/test_core.py b/tests/test_core.py index f40a862b..271eab30 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,3 +1,4 @@ +import logging import pathlib from datetime import datetime from unittest.mock import MagicMock, patch @@ -6,7 +7,7 @@ import requests from PIL import Image -from pyroengine.core import SystemController, is_day_time +from pyroengine.core import NIGHT_FRAMES_BEFORE_RESET, NIGHT_PASSES_BEFORE_SLEEP, SystemController, is_day_time # Disable sleeps in SystemController during tests to make them fast @@ -161,22 +162,61 @@ def test_night_frames_are_skipped_and_site_sleeps_only_when_all_are_night(mock_c mock_engine.predict.assert_called_once_with(day_img, "192.168.1.2") assert controller.is_day - mock_client.get_latest_image.side_effect = [night_img, night_img] + # Every camera at night: the site sleeps after NIGHT_PASSES_BEFORE_SLEEP passes, not one + mock_client.get_latest_image.side_effect = lambda *_: night_img + for _ in range(NIGHT_PASSES_BEFORE_SLEEP - 1): + controller.inference_loop() + assert controller.is_day controller.inference_loop() assert mock_engine.predict.call_count == 1 assert not controller.is_day # No frame at all (stale cache): keep the previous state - mock_client.get_latest_image.side_effect = [None, None] + mock_client.get_latest_image.side_effect = lambda *_: None controller.inference_loop() assert not controller.is_day - # One night frame and one camera without a fresh image (204): the missing camera - # says nothing about the light, the site must not go to sleep on one night frame + # One day frame wakes the site and restarts the count controller.is_day = True - mock_client.get_latest_image.side_effect = [night_img, None] + mock_client.get_latest_image.side_effect = lambda *_: night_img + controller.inference_loop() + controller.inference_loop() + mock_client.get_latest_image.side_effect = [night_img, day_img] controller.inference_loop() assert controller.is_day + mock_client.get_latest_image.side_effect = lambda *_: night_img + controller.inference_loop() + assert controller.is_day + + # One camera down (204) says nothing about the light: the other one puts the site to sleep + controller.is_day = True + mock_client.get_latest_image.side_effect = lambda _ip, pose: night_img if pose == 1 else None + for _ in range(NIGHT_PASSES_BEFORE_SLEEP): + controller.inference_loop() + assert not controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_camera_in_infrared_during_day_is_warned_once(mock_client_class, mock_engine, caplog): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "static"}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.side_effect = lambda ip, _pose: night_img if ip == "192.168.1.1" else day_img + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + with caplog.at_level(logging.WARNING): + for _ in range(NIGHT_FRAMES_BEFORE_RESET + 2): + controller.inference_loop() + + warnings = [r for r in caplog.records if "infrared while others" in r.message] + assert len(warnings) == 1 + assert "192.168.1.1" in warnings[0].message + assert controller.is_day @patch("pyroengine.core.PyroCameraAPIClient") @@ -200,9 +240,40 @@ def test_wake_up_probes_every_camera(mock_client_class, mock_engine): mock_client.capture_image.side_effect = [night_img, night_img] assert not controller._probe_day() - # One night frame and one camera down: not proven night, wake up + # One night frame and one camera down: the camera down says nothing, stay asleep mock_client.capture_image.side_effect = [night_img, RuntimeError("down")] + assert not controller._probe_day() + + # Nobody answers: wake up and let the inference loop decide + mock_client.capture_image.side_effect = [RuntimeError("down"), None] + assert controller._probe_day() + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_wake_up_restarts_the_night_counts(mock_client_class, mock_engine): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "static"}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + # Every camera at night for the whole evening, then the site slept + mock_client.get_latest_image.side_effect = lambda *_: night_img + for _ in range(NIGHT_PASSES_BEFORE_SLEEP + 2): + controller.inference_loop() + assert not controller.is_day + + # Dawn: nobody answers the probe, the site wakes up. One all-night pass must not put it + # back to sleep, and the camera still in infrared is warned once others see daylight. + mock_client.capture_image.side_effect = [RuntimeError("down"), None] assert controller._probe_day() + controller.is_day = True + controller.inference_loop() + assert controller.is_day + assert controller._night_frames["192.168.1.1"] == 1 @patch("pyroengine.core.PyroCameraAPIClient") @@ -250,7 +321,12 @@ def test_night_frame_drops_detection_history(mock_client_class, mock_engine): mock_client.get_latest_image.return_value = Image.new("RGB", (100, 100), (255, 255, 255)) mock_engine._states = {"192.168.1.2": {"ongoing": True}} controller = SystemController(mock_engine, camera_data, "http://fake.url") + controller._night_frames.clear() # __init__ already ran one pass - controller.inference_loop() + # Not on one frame: cameras flicker at dusk and an ongoing event must survive it + for _ in range(NIGHT_FRAMES_BEFORE_RESET - 1): + controller.inference_loop() + assert "192.168.1.2" in mock_engine._states + controller.inference_loop() assert "192.168.1.2" not in mock_engine._states