diff --git a/CLAUDE.md b/CLAUDE.md index ba9f8a5d..47fa5852 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,7 +87,7 @@ For each `cam_id`, the engine periodically fetches a JSON file at `{bbox_mask_ur ### Night / day handling -Day is determined by IR-channel analysis (`is_day_time(strategy="ir")`): if `max(R - G) == 0` the image is considered infrared/night. At night the main loop sleeps for 1 hour, stops all patrols, then re-checks before resuming inference. +Day is determined by IR-channel analysis (`is_day_time(strategy="ir")`): if `max(R - G) == 0` the image is considered infrared/night. Night frames are not analyzed. Once every frame of an inference pass is a night frame, the main loop stops all patrols, sleeps for 1 hour, then re-checks before resuming inference. ### Environment Variables (`.env`) diff --git a/docker-compose.yml b/docker-compose.yml index 28bd1265..9089d05c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,12 +17,20 @@ services: - ./data:/usr/src/app/data restart: always network_mode: host + # tini as PID 1, so the healthcheck's kill -9 -1 also reaches uvicorn + init: true healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8081/health || exit 1"] + # Docker never restarts an unhealthy container by itself: on a hung patrol (503) or an API + # that stopped answering, kill everything but PID 1 so `restart: always` brings it back. + # curl retries twice so a single slow answer does not restart it. Connection refused (rc 7) + # only means uvicorn is still starting, as it binds after camera setup: killing on it would + # loop at startup (the kill runs during start_period too). A hang before the bind is bounded + # by the 10 s timeout on every camera call, so rc 7 cannot last. + test: ["CMD-SHELL", "curl -sf --max-time 5 --retry 2 --retry-delay 5 http://localhost:8081/health/patrols; rc=$$?; [ $$rc -eq 0 ] && exit 0; [ $$rc -eq 7 ] && exit 1; kill -9 -1"] interval: 30s retries: 3 start_period: 20s - timeout: 5s + timeout: 30s logging: driver: json-file options: diff --git a/pyro_camera_api/pyro_camera_api/api/routes_cameras.py b/pyro_camera_api/pyro_camera_api/api/routes_cameras.py index 744d4102..abe250a9 100644 --- a/pyro_camera_api/pyro_camera_api/api/routes_cameras.py +++ b/pyro_camera_api/pyro_camera_api/api/routes_cameras.py @@ -238,13 +238,19 @@ def get_latest_image( le=100, description="JPEG quality that controls compression level. Higher means better quality and larger file size.", ), + max_age_s: int = Query( + default=300, + ge=1, + description="Images stored longer ago than this are treated as missing.", + ), ): """ Return the last stored image for a given camera and pose. The camera adapter may cache captured frames per preset pose in its last_images mapping. This endpoint exposes that cache. If there is - no image for the requested pose, the endpoint returns HTTP 204 with + no image for the requested pose, or it is older than max_age_s (for + example a hung patrol thread), the endpoint returns HTTP 204 with an empty body. """ cam = CAMERA_REGISTRY.get(camera_ip) @@ -252,7 +258,8 @@ def get_latest_image( raise HTTPException(status_code=404, detail="Unknown camera") image = cam.last_images.get(pose) - if image is None: + stored_at = cam.last_image_at.get(pose) + if image is None or stored_at is None or time.monotonic() - stored_at > max_age_s: return Response(status_code=status.HTTP_204_NO_CONTENT) buffer = BytesIO() diff --git a/pyro_camera_api/pyro_camera_api/api/routes_health.py b/pyro_camera_api/pyro_camera_api/api/routes_health.py index d4b5c125..b7150324 100644 --- a/pyro_camera_api/pyro_camera_api/api/routes_health.py +++ b/pyro_camera_api/pyro_camera_api/api/routes_health.py @@ -7,6 +7,9 @@ from __future__ import annotations from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from pyro_camera_api.camera.patrol import stalled_patrols router = APIRouter() @@ -16,7 +19,20 @@ def health(): """ Return a basic heartbeat response to confirm that the API is running. - This endpoint does not perform any camera or service check. - It is intended for uptime monitoring and infrastructure-level health probes. + Kept free of camera checks: the Shelly watchdog reads it as Pi liveness and + power-cycles the Pi and the router when it fails. + """ + return {"status": "ok"} + + +@router.get("/health/patrols") +def health_patrols(): + """ + Return 200 while every running patrol makes progress, 503 with the hung cameras otherwise. + + Used by the compose healthcheck, which restarts the container on a 503. """ + stalled = stalled_patrols() + if stalled: + return JSONResponse(status_code=503, content={"status": "patrol_stalled", "cameras": stalled}) return {"status": "ok"} diff --git a/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py b/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py index 2a35fe4d..9b35bc9c 100644 --- a/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py +++ b/pyro_camera_api/pyro_camera_api/camera/adapters/reolink.py @@ -30,7 +30,19 @@ logger = logging.getLogger(__name__) -class _LegacyTLSAdapter(HTTPAdapter): +# A PTZ call without timeout once hung a patrol thread forever (pyro-engine#422): +# every request gets this unless the call site passes its own. +_DEFAULT_TIMEOUT = 10 + + +class _TimeoutAdapter(HTTPAdapter): + def send(self, request: Any, *args: Any, **kwargs: Any) -> Any: + if kwargs.get("timeout") is None: + kwargs["timeout"] = _DEFAULT_TIMEOUT + return super().send(request, *args, **kwargs) + + +class _LegacyTLSAdapter(_TimeoutAdapter): """Old Reolink firmwares only offer RSA-key-exchange TLS ciphers, which Python 3.10+ dropped from its default cipher list: restore OpenSSL's own.""" @@ -51,6 +63,7 @@ def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: # hands back a stale socket (matches the per-call behaviour of requests.get). _session.headers["Connection"] = "close" _session.mount("https://", _LegacyTLSAdapter()) +_session.mount("http://", _TimeoutAdapter()) # Zoom/focus reads and the zoom restore run under the stream lock in # routes_stream, so a camera that stops answering must not hang them forever. diff --git a/pyro_camera_api/pyro_camera_api/camera/base.py b/pyro_camera_api/pyro_camera_api/camera/base.py index a12bd162..63ba1200 100644 --- a/pyro_camera_api/pyro_camera_api/camera/base.py +++ b/pyro_camera_api/pyro_camera_api/camera/base.py @@ -6,6 +6,7 @@ from __future__ import annotations +import time from abc import ABC, abstractmethod from typing import Dict, List, Optional @@ -36,6 +37,12 @@ def __init__(self, camera_id: str, cam_type: str = "static") -> None: # PTZ cameras can use pose -> image # Static cameras can use -1 -> image self.last_images: Dict[int, Optional[Image.Image]] = {} + # time.monotonic() of each stored image, so stale frames can be refused + self.last_image_at: Dict[int, float] = {} + + def store_image(self, pose: int, image: Image.Image) -> None: + self.last_images[pose] = image + self.last_image_at[pose] = time.monotonic() @abstractmethod def capture(self, **kwargs) -> Optional[Image.Image]: diff --git a/pyro_camera_api/pyro_camera_api/camera/patrol.py b/pyro_camera_api/pyro_camera_api/camera/patrol.py index 90f0f8db..ad946c37 100644 --- a/pyro_camera_api/pyro_camera_api/camera/patrol.py +++ b/pyro_camera_api/pyro_camera_api/camera/patrol.py @@ -9,9 +9,9 @@ import logging import threading import time -from typing import Any, Dict +from typing import Any, Dict, List -from pyro_camera_api.camera.registry import CAMERA_REGISTRY +from pyro_camera_api.camera.registry import CAMERA_REGISTRY, PATROL_PROGRESS, PATROL_THREADS from pyro_camera_api.services.stream import is_camera_streaming logger = logging.getLogger(__name__) @@ -23,6 +23,10 @@ # seconds between stream re-checks while the patrol is paused by a live stream STREAM_CHECK_INTERVAL = 5.0 +# A loop iteration takes well under a minute even when every camera call times +# out, so this much silence from a live patrol thread means it is hung. +STALL_LIMIT = 10 * 60.0 + # per camera state for the static loop FAILURE_COUNT: Dict[str, int] = {} SKIP_UNTIL: Dict[str, float] = {} @@ -68,7 +72,17 @@ def is_stream_running_for(app: Any, camera_ip: str) -> bool: return False +def stalled_patrols() -> List[str]: + now = time.monotonic() + return [ + ip + for ip, thread in PATROL_THREADS.items() + if thread.is_alive() and now - PATROL_PROGRESS.get(ip, now) > STALL_LIMIT + ] + + def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: + PATROL_PROGRESS[camera_ip] = time.monotonic() cam = CAMERA_REGISTRY[camera_ip] poses = getattr(cam, "cam_poses", []) or [] @@ -79,6 +93,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: logger.info("[%s] Starting patrol cycle with %d poses", camera_ip, len(poses)) while not stop_flag.is_set(): + PATROL_PROGRESS[camera_ip] = time.monotonic() # Live stream has priority: hold the patrol while a stream is active # so the camera never moves under the viewer, and resume automatically # once the stream stops (the thread keeps running, patrol stays "on"). @@ -99,7 +114,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: image = cam.capture() if image: - cam.last_images[pose] = image + cam.store_image(pose, image) logger.info("[%s] Stored image for pose %s", camera_ip, pose) except Exception as exc: @@ -133,6 +148,7 @@ def patrol_loop(camera_ip: str, stop_flag: threading.Event) -> None: def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: + PATROL_PROGRESS[camera_ip] = time.monotonic() cam = CAMERA_REGISTRY[camera_ip] logger.info("[%s] Starting static camera loop", camera_ip) @@ -143,6 +159,7 @@ def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: settle_until = 0.0 while not stop_flag.is_set(): + PATROL_PROGRESS[camera_ip] = time.monotonic() now = time.time() # skip window @@ -161,7 +178,7 @@ def static_loop(camera_ip: str, stop_flag: threading.Event) -> None: settle_until = opened_at + 1.0 if image and now >= settle_until: - cam.last_images[-1] = image + cam.store_image(-1, image) logger.info("[%s] Updated static image (pose -1)", camera_ip) # success reset failure counter and clear skip FAILURE_COUNT[camera_ip] = 0 diff --git a/pyro_camera_api/pyro_camera_api/camera/registry.py b/pyro_camera_api/pyro_camera_api/camera/registry.py index 23e8f124..75a78c8a 100644 --- a/pyro_camera_api/pyro_camera_api/camera/registry.py +++ b/pyro_camera_api/pyro_camera_api/camera/registry.py @@ -55,6 +55,8 @@ def repl(match: re.Match[str]) -> str: # Patrol threading state, later managed in camera.patrol PATROL_THREADS: Dict[str, threading.Thread] = {} PATROL_FLAGS: Dict[str, threading.Event] = {} +# time.monotonic() of the last loop iteration of each patrol thread +PATROL_PROGRESS: Dict[str, float] = {} # Stuck-detector threading state, later managed in camera.stuck_detector STUCK_CHECK_THREADS: Dict[str, threading.Thread] = {} diff --git a/pyro_camera_api/tests/test_reolink_tls.py b/pyro_camera_api/tests/test_reolink_tls.py index 14faf654..4f4b7fa1 100644 --- a/pyro_camera_api/tests/test_reolink_tls.py +++ b/pyro_camera_api/tests/test_reolink_tls.py @@ -3,6 +3,9 @@ # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. +import pytest +import requests + from pyro_camera_api.camera.adapters.reolink import _session @@ -15,3 +18,19 @@ def test_https_session_offers_rsa_key_exchange_ciphers(): assert "AES128-GCM-SHA256" in names # Proxy env vars must not reroute camera calls around the TLS adapter assert not _session.trust_env + + +def test_session_applies_default_timeout(monkeypatch): + # A call without timeout must not wait forever on a silent camera (pyro-engine#422) + seen = [] + + def fake_send(_self, _request, **kwargs): + seen.append(kwargs["timeout"]) + raise requests.ConnectionError + + monkeypatch.setattr("requests.adapters.HTTPAdapter.send", fake_send) + for url in ("https://192.168.1.10", "http://192.168.1.10"): + for kwargs in ({}, {"timeout": 2}): + with pytest.raises(requests.ConnectionError): + _session.post(url, **kwargs) + assert seen == [10, 2, 10, 2] diff --git a/pyro_camera_api/tests/test_stale_patrol.py b/pyro_camera_api/tests/test_stale_patrol.py new file mode 100644 index 00000000..330d7133 --- /dev/null +++ b/pyro_camera_api/tests/test_stale_patrol.py @@ -0,0 +1,63 @@ +# Copyright (C) 2022-2026, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +import threading +import time + +from PIL import Image + +from pyro_camera_api.api import routes_health +from pyro_camera_api.api.routes_cameras import get_latest_image +from pyro_camera_api.api.routes_health import health, health_patrols +from pyro_camera_api.camera import patrol +from pyro_camera_api.camera.adapters.mock import MockCamera +from pyro_camera_api.camera.registry import CAMERA_REGISTRY + + +def test_latest_image_refuses_stale_frame(monkeypatch): + # A hung patrol thread leaves an old frame in the cache: it must not be served + cam_id = "stale-cam" + cam = MockCamera(camera_id=cam_id, cam_type="ptz", cam_poses=[0]) + CAMERA_REGISTRY[cam_id] = cam + try: + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).status_code == 204 + + cam.store_image(0, Image.new("RGB", (8, 8))) + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).media_type == "image/jpeg" + + monkeypatch.setattr("time.monotonic", lambda: cam.last_image_at[0] + 301) + assert get_latest_image(cam_id, 0, quality=95, max_age_s=300).status_code == 204 + finally: + CAMERA_REGISTRY.pop(cam_id, None) + + +def test_watchdog_flags_only_hung_patrol_threads(monkeypatch): + # A live thread silent for too long is hung, a finished one is not + hung, done = threading.Event(), threading.Event() + threads = { + "hung": threading.Thread(target=hung.wait, daemon=True), + "busy": threading.Thread(target=hung.wait, daemon=True), + "done": threading.Thread(target=done.set), + } + for thread in threads.values(): + thread.start() + threads["done"].join() + monkeypatch.setattr(patrol, "PATROL_THREADS", threads) + now = time.monotonic() + monkeypatch.setattr( + patrol, "PATROL_PROGRESS", {"hung": now - patrol.STALL_LIMIT - 1, "busy": now, "done": now - 3600} + ) + try: + assert patrol.stalled_patrols() == ["hung"] + finally: + hung.set() + + +def test_health_patrols_reports_stalled_patrols(monkeypatch): + assert health_patrols() == {"status": "ok"} + monkeypatch.setattr(routes_health, "stalled_patrols", lambda: ["192.168.1.12"]) + assert health_patrols().status_code == 503 + # The Shelly reads /health as Pi liveness: a hung patrol must not power-cycle the Pi + assert health() == {"status": "ok"} diff --git a/pyroengine/core.py b/pyroengine/core.py index 92744a63..4064e2af 100644 --- a/pyroengine/core.py +++ b/pyroengine/core.py @@ -4,6 +4,7 @@ # See LICENSE or go to for full license details. +import hashlib import logging import sys import time @@ -81,6 +82,7 @@ def __init__( self.camera_data = camera_data self.is_day = True self.last_autofocus: Optional[datetime] = None + self._last_frame_digest: Dict[str, bytes] = {} # Wait for the camera API to be available time.sleep(self.API_INITIAL_WAIT) @@ -188,6 +190,8 @@ def inference_loop(self) -> None: logger.info("Stream detected, skipping inference on all cameras") return + day_frames = [] + expected = 0 for ip, cam in self.camera_data.items(): camera_name = cam["name"] @@ -198,11 +202,13 @@ def inference_loop(self) -> None: return try: cam_id = f"{ip}_{pose}" + expected += 1 frame = self._safe_get_latest_image(ip, pose) if frame is not None: logger.info(f"Captured image for {ip}, pose {pose}") - self.is_day = is_day_time(None, frame, "ir") - self.engine.predict(frame, cam_id) + frame_is_day = is_day_time(None, frame, "ir") + day_frames.append(frame_is_day) + self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error( f"HTTP error for {camera_name}, pose {pose}: " @@ -217,16 +223,65 @@ def inference_loop(self) -> None: return try: cam_id = ip + expected += 1 frame = self._safe_get_latest_image(ip, -1) if frame is not None: logger.info(f"Captured image for {ip}") - self.is_day = is_day_time(None, frame, "ir") - self.engine.predict(frame, cam_id) + frame_is_day = is_day_time(None, frame, "ir") + day_frames.append(frame_is_day) + self._analyze(frame, cam_id, frame_is_day) except requests.HTTPError as e: logger.error(f"HTTP error for {camera_name}: {e.response.text if e.response is not None else e}") except Exception as e: logger.error(f"Error for {camera_name}: {e}") + # One day frame keeps the site awake, so one camera in infrared (fog, a dark area) does + # not blind the others. It sleeps only when every expected frame came back and all are + # night: a camera without a fresh image (204, error) says nothing about the light. + if any(day_frames): + self.is_day = True + elif day_frames and len(day_frames) == expected: + self.is_day = False + + def _analyze(self, frame: Image.Image, cam_id: str, frame_is_day: bool) -> None: + """Run inference on a day frame that differs from the previous one of this camera.""" + if not frame_is_day: + # The model is trained on day images, infrared frames only give false positives. + # Drop the camera's detection history so evening frames are not combined with the + # next morning's ones. + logger.info(f"Night frame for {cam_id}, skipping inference") + self.engine._states.pop(cam_id, None) + return + # A live camera never returns the exact same pixels twice: an identical frame is frozen, + # and analyzing it again would re-upload the same detection every pass. + digest = hashlib.md5(frame.tobytes(), usedforsecurity=False).digest() + if self._last_frame_digest.get(cam_id) == digest: + logger.warning(f"Frame identical to the previous one for {cam_id}, skipping inference") + return + self.engine.predict(frame, cam_id) + self._last_frame_digest[cam_id] = digest + + def _probe_day(self) -> bool: + """Capture one frame per camera until one shows daylight. + + Stays night only when every camera answered with a night frame: a camera that + fails to answer says nothing, so the site wakes up and lets the inference loop + decide from the next fresh frames. + """ + night = 0 + for ip in self.camera_data: + try: + frame = self.camera_api_client.capture_image(ip) + except Exception as e: + logger.error(f"Failed to capture {ip} after sleep: {e}") + continue + if frame is None: + continue + if is_day_time(None, frame, "ir"): + return True + night += 1 + return night < len(self.camera_data) + def check_and_restart_patrol(self) -> None: """ Check stream activity and ensure patrol is running when no stream is active. @@ -270,20 +325,18 @@ def main_loop(self, period: int, send_alerts: bool = True) -> None: for ip in self.camera_data: try: patrol_status = self.camera_api_client.get_patrol_status(ip) - if not patrol_status.get("patrol_running", True): + if patrol_status.get("patrol_running", False): self.camera_api_client.stop_patrol(ip) logger.info(f"Stopped patrol for camera {ip} due to night") except Exception as e: logger.error(f"Failed to stop patrol on camera {ip}: {e}") - logger.info("Nighttime detected by at least one camera, sleeping for 1 hour") + logger.info("Nighttime detected by every camera, sleeping for 1 hour") time.sleep(3600) try: - ip = next(iter(self.camera_data.keys())) - frame = self.camera_api_client.capture_image(ip) - self.is_day = is_day_time(None, frame, "ir") - logger.info(f"Re checked day and night using camera {ip}, result is_day={self.is_day}") + self.is_day = self._probe_day() + logger.info(f"Re checked day and night after sleep, result is_day={self.is_day}") if self.is_day: logger.info("Day detected, restarting patrols") diff --git a/tests/test_core.py b/tests/test_core.py index fc322dab..f40a862b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -121,3 +121,136 @@ def test_inference_loop_skips_when_stream_active(mock_client_class, mock_engine, assert not mock_client.get_latest_image.called assert not mock_engine.predict.called + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_stops_running_patrols(mock_client_class, mock_engine, mock_camera_data, monkeypatch): + mock_client = mock_client_class.return_value + mock_client.get_patrol_status.return_value = {"patrol_running": True} + controller = SystemController(mock_engine, mock_camera_data, "http://fake.url") + controller.is_day = False + + class NightSleepError(Exception): + pass + + def sleep(_s): + raise NightSleepError + + monkeypatch.setattr("pyroengine.core.time.sleep", sleep) + with pytest.raises(NightSleepError): + controller.main_loop(period=30) + + mock_client.stop_patrol.assert_called_once_with("192.168.1.1") + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_frames_are_skipped_and_site_sleeps_only_when_all_are_night(mock_client_class, mock_engine): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "ptz", "poses": [1]}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + # One camera in infrared: it is not analyzed, the other one keeps the site awake + mock_client.get_latest_image.side_effect = [night_img, day_img] + controller.inference_loop() + mock_engine.predict.assert_called_once_with(day_img, "192.168.1.2") + assert controller.is_day + + mock_client.get_latest_image.side_effect = [night_img, night_img] + controller.inference_loop() + assert mock_engine.predict.call_count == 1 + assert not controller.is_day + + # No frame at all (stale cache): keep the previous state + mock_client.get_latest_image.side_effect = [None, None] + controller.inference_loop() + assert not controller.is_day + + # One night frame and one camera without a fresh image (204): the missing camera + # says nothing about the light, the site must not go to sleep on one night frame + controller.is_day = True + mock_client.get_latest_image.side_effect = [night_img, None] + controller.inference_loop() + assert controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_wake_up_probes_every_camera(mock_client_class, mock_engine): + camera_data = { + "192.168.1.1": {"name": "cam1", "type": "static"}, + "192.168.1.2": {"name": "cam2", "type": "static"}, + } + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + night_img = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_client = mock_client_class.return_value + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + # The first camera stays in infrared (or is down), the second one sees daylight + mock_client.capture_image.side_effect = [night_img, day_img] + assert controller._probe_day() + mock_client.capture_image.side_effect = [RuntimeError("down"), day_img] + assert controller._probe_day() + + # Every camera reports night: stay asleep + mock_client.capture_image.side_effect = [night_img, night_img] + assert not controller._probe_day() + + # One night frame and one camera down: not proven night, wake up + mock_client.capture_image.side_effect = [night_img, RuntimeError("down")] + assert controller._probe_day() + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_identical_frame_is_not_analyzed_twice(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.side_effect = [ + Image.new("RGB", (100, 100), (255, 200, 200)), + Image.new("RGB", (100, 100), (255, 200, 200)), + Image.new("RGB", (100, 100), (255, 201, 200)), + ] + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + for _ in range(3): + controller.inference_loop() + + assert mock_engine.predict.call_count == 2 + assert controller.is_day + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_failed_prediction_keeps_day_and_is_retried(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + day_img = Image.new("RGB", (100, 100), (255, 200, 200)) + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.return_value = day_img + mock_engine.predict.side_effect = [RuntimeError("upload failed"), None] + controller = SystemController(mock_engine, camera_data, "http://fake.url") + controller.is_day = True + + controller.inference_loop() + assert controller.is_day + + controller.inference_loop() + assert mock_engine.predict.call_count == 2 + + +@patch("pyroengine.core.PyroCameraAPIClient") +def test_night_frame_drops_detection_history(mock_client_class, mock_engine): + camera_data = {"192.168.1.2": {"name": "cam2", "type": "static"}} + mock_client = mock_client_class.return_value + mock_client.get_stream_status.return_value = {"active_streams": 0} + mock_client.get_latest_image.return_value = Image.new("RGB", (100, 100), (255, 255, 255)) + mock_engine._states = {"192.168.1.2": {"ongoing": True}} + controller = SystemController(mock_engine, camera_data, "http://fake.url") + + controller.inference_loop() + + assert "192.168.1.2" not in mock_engine._states