diff --git a/README.md b/README.md index 4a6a8e1..01524bd 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,36 @@ pyro-engine checkout, point `credentials.json` at them with the host and port as and use the same `CAM_USER` / `CAM_PWD` as in this repo's `.env`. +### Robustness tests on a Pi + +`scripts/replay-robustness-tests.sh` replays the failures of +[roadmap#16](https://github.com/pyronear/roadmap/issues/16) on a test Pi running the real +engine stack, with the fake cameras standing in for broken ones. It runs on the Pi itself: + +```bash +scp scripts/replay-robustness-tests.sh pi@:~ && ssh pi@ +./replay-robustness-tests.sh setup # pyro-engine branch, images, fake cameras, scenarios +./replay-robustness-tests.sh all # every scenario below, about 90 min +./replay-robustness-tests.sh real # back to the real cameras +``` + +The Pi needs Docker, passwordless sudo and a `~/pyro-engine` checkout with its `.env` and a +two-camera `data/credentials.json`. Point `API_URL` at preprod: the engine sends heartbeats +and images with the real camera tokens. Do not deploy with pi-manager on the Pi during a run, +it recreates the containers. + +| Scenario | Fault injected | Expected | Time | +|---|---|---|---| +| `camera-timeout` | camera HTTPS redirected to a listener that never answers | `Read timed out (read timeout=10)` | 2 min | +| `api-frozen` | camera API process frozen | healthcheck restarts the API | 3 min | +| `night-per-frame` | one camera in grayscale (infrared), then one without a fresh image (204), then both grayscale | night frames skipped, no sleep on 204, sleep when every camera is at night | 15 min | +| `camera-silent` | both cameras redirected to the silent listener | API reboot attempt, no password in logs, budget kept after a container restart | 18 min | +| `camera-frozen` | one fake camera holding one frame, one serving noisy variants of one view | `frozen` and `not_moving` reboots, one "budget spent" line, second reboot after the 30 min cooldown | 50 min | + +Each scenario prints `OK:` or `FAIL:` and exits non-zero on failure. The redirect rule and +its listener are removed on exit. Timings are the detector's own (5 min checks, 30 min +cooldown), so there is no fast mode. + --- ## 📂 Data Usage diff --git a/scripts/replay-robustness-tests.sh b/scripts/replay-robustness-tests.sh new file mode 100755 index 0000000..f6d12b2 --- /dev/null +++ b/scripts/replay-robustness-tests.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Replay the pyronear/roadmap#16 robustness tests on a test Pi. +# +# Runs ON the Pi (user with sudo, Docker installed), from any directory: +# scp replay-robustness-tests.sh pi@:~ && ssh pi@ +# ./replay-robustness-tests.sh setup # branch, images, fake cameras, scenarios (20 to 30 min on a Pi 5) +# ./replay-robustness-tests.sh scenarios # only regenerate the fake credentials and scenarios +# ./replay-robustness-tests.sh camera-timeout # Reolink call that hangs -> 10 s timeout (~2 min) +# ./replay-robustness-tests.sh api-frozen # camera API frozen -> healthcheck restarts it (~3 min) +# ./replay-robustness-tests.sh night-per-frame # night per frame, 204 never proves night (~15 min) +# ./replay-robustness-tests.sh camera-silent # no fresh image -> API reboot, budget on disk (~18 min) +# ./replay-robustness-tests.sh camera-frozen # frozen + not_moving -> reboot, 30 min cooldown (~50 min) +# ./replay-robustness-tests.sh all # everything above, in that order (~90 min) +# ./replay-robustness-tests.sh real # back to the real cameras +# +# Needs: passwordless sudo, ~/pyro-engine checkout with a .env (CAM_USER, CAM_PWD, API_URL...) and +# data/credentials.json holding two cameras. Everything else is built by `setup`. +# The engine talks to API_URL with the real camera tokens: point it at preprod. +# Scenario timings follow the detector's own constants (5 min checks, 30 min cooldown). +set -euo pipefail + +ENGINE_DIR=${ENGINE_DIR:-$HOME/pyro-engine} +ENVDEV_DIR=${ENVDEV_DIR:-$HOME/pyro-envdev} +TAG=${TAG:-robustness} +# pyro-engine PR branches merged on top of develop; BRANCHES= once they are merged +BRANCHES=${BRANCHES-"feat/camera-recovery"} +FAKE_NET=fakecams +FAKE1=172.30.0.11 +FAKE2=172.30.0.12 +SINK_PORT=4443 +SCN=$ENVDEV_DIR/data/scenarios + +D() { sudo -n docker "$@"; } +say() { printf '\n[%s] %s\n' "$(date +%H:%M:%S)" "$*"; } +die() { say "FAIL: $*"; exit 1; } +ok() { say "OK: $*"; } + +# wait_log [since]: succeed when the pattern shows up +wait_log() { + local c=$1 pat=$3 since=${4:-$STARTED} end=$(( $(date +%s) + $2 )) + while [ "$(date +%s)" -lt "$end" ]; do + D logs --since "$since" "$c" 2>&1 | grep -qE "$pat" && return 0 + sleep 10 + done + return 1 +} +count_log() { D logs --since "${3:-$STARTED}" "$1" 2>&1 | grep -cE "$2" || true; } +restarts() { D inspect "$1" --format '{{.RestartCount}}'; } +status() { curl -s "localhost:8081/patrol/patrol_status?camera_ip=$1"; } +mark() { STARTED=$(date +%Y-%m-%dT%H:%M:%S); } # docker --since reads local time + +cams() { python3 -c "import json;print(' '.join(json.load(open('$ENGINE_DIR/data/credentials.real.json'))))"; } +set_env() { grep -q "^$1=" "$ENGINE_DIR/.env" && sed -i -E "s/^$1=.*/$1=$2/" "$ENGINE_DIR/.env" || echo "$1=$2" >> "$ENGINE_DIR/.env"; } +stack_up() { (cd "$ENGINE_DIR" && D compose up -d --force-recreate >/dev/null 2>&1); mark; sleep 20; } + +use_real() { cp "$ENGINE_DIR/data/credentials.real.json" "$ENGINE_DIR/data/credentials.json"; D stop fakecam-1 fakecam-2 >/dev/null 2>&1 || true; } +use_fake() { cp "$ENGINE_DIR/data/credentials.fake.json" "$ENGINE_DIR/data/credentials.json"; } + +# fakecam <1|2> : (re)start one fake Reolink camera serving that scenario +fakecam() { + local n=$1 ip name + [ "$n" = 1 ] && ip=$FAKE1 || ip=$FAKE2 + name=$(python3 -c "import json;c=json.load(open('$ENGINE_DIR/data/credentials.fake.json'));print(c['$ip']['name'])") + D rm -f "fakecam-$n" >/dev/null 2>&1 || true + set -a; . "$ENGINE_DIR/.env"; set +a + D run -d --name "fakecam-$n" --network $FAKE_NET --ip "$ip" -e CAMERA="$name" -e SCENARIO="$2" \ + -e CAM_USER="$CAM_USER" -e CAM_PWD="$CAM_PWD" -v "$SCN:/scenarios:ro" pyronear/fake-camera:local >/dev/null +} + +# sink : redirect each camera's HTTPS to a local listener that accepts and never answers +sink_on() { + python3 -c " +import socket; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) +s.bind(('0.0.0.0',$SINK_PORT)); s.listen(100); held=[] +while True: held.append(s.accept()[0])" >/dev/null 2>&1 & + echo $! > /tmp/sink.pid + for _ in 1 2 3 4 5 6 7 8 9 10; do ss -ltn | grep -q ":$SINK_PORT " && break; sleep 0.5; done + ss -ltn | grep -q ":$SINK_PORT " || die "sink listener did not start" + local me; me=$(ip -4 route get 1.1.1.1 | awk '{print $7; exit}') + for ip in "$@"; do sudo -n iptables -t nat -I OUTPUT -d "$ip" -p tcp --dport 443 -m comment --comment pyro-robustness-sink -j DNAT --to-destination "$me:$SINK_PORT"; done + echo "$*" > /tmp/sink.ips + # Probe: a connection to the camera must now hang (curl rc 28), not fail or answer + for ip in "$@"; do + local rc=0; curl -sk -m 3 -o /dev/null "https://$ip/" || rc=$? + [ "$rc" = 28 ] || die "sink not effective for $ip (curl rc=$rc)" + done +} +sink_off() { + [ -f /tmp/sink.ips ] || return 0 + local me; me=$(ip -4 route get 1.1.1.1 | awk '{print $7; exit}') + for ip in $(cat /tmp/sink.ips); do sudo -n iptables -t nat -D OUTPUT -d "$ip" -p tcp --dport 443 -m comment --comment pyro-robustness-sink -j DNAT --to-destination "$me:$SINK_PORT" 2>/dev/null || true; done + kill "$(cat /tmp/sink.pid)" 2>/dev/null || true + rm -f /tmp/sink.ips /tmp/sink.pid +} +trap sink_off EXIT INT TERM + +# ---------------------------------------------------------------- setup +setup() { + say "branch test/$TAG = develop + $BRANCHES" + cd "$ENGINE_DIR" + git config user.email >/dev/null || git config user.email "test@pyronear.org" + git config user.name >/dev/null || git config user.name "pi-test" + git stash -q 2>/dev/null || true + git fetch -q origin + git checkout -q -B "test/$TAG" origin/develop + for b in $BRANCHES; do git merge -q --no-edit "origin/$b" || die "conflict merging $b"; done + say "pyro-engine at $(git rev-parse --short HEAD)" + set_env PYRO_ENGINE_VERSION "$TAG" + set_env ENABLE_STUCK_DETECTOR true + set_env ALLOW_STILL_CAMERA false + [ -f data/credentials.real.json ] || cp data/credentials.json data/credentials.real.json + + say "building images (develop's compose build context is broken, build like the Makefile)" + D build -q pyro_camera_api -t "pyronear/pyro-camera-api:$TAG" + D build -q . -t "pyronear/pyro-engine:$TAG" + + say "fake Reolink cameras (pyro-envdev#65)" + [ -d "$ENVDEV_DIR" ] || git clone -q --depth 1 -b feat/fake-reolink-camera https://github.com/pyronear/pyro-envdev.git "$ENVDEV_DIR" + cd "$ENVDEV_DIR" + [ -d data/scenarios/never-smoke/nemours-02 ] || { + curl -fsSL https://github.com/pyronear/pyro-envdev/releases/download/scenarios/never-smoke.zip -o /tmp/ns.zip + unzip -oq /tmp/ns.zip -d data/scenarios; rm /tmp/ns.zip; } + D build -q --build-context scenarios=data/scenarios -f containers/fake_camera/Dockerfile -t pyronear/fake-camera:local containers/fake_camera/ + D network inspect $FAKE_NET >/dev/null 2>&1 || D network create --subnet 172.30.0.0/24 $FAKE_NET >/dev/null + + make_scenarios + ok "setup done, images tagged $TAG" +} + +# fake credentials pointing at the fake cameras, and the day / night / noisy scenarios +make_scenarios() { + say "scenarios: day (one colour frame), night (same frame in grayscale = IR), noisy (40 variants of one view)" + read -r REAL1 REAL2 <<<"$(cams)" + python3 - "$ENGINE_DIR/data/credentials.real.json" "$ENGINE_DIR/data/credentials.fake.json" "$REAL1" "$REAL2" "$FAKE1" "$FAKE2" <<'EOF' +import json, sys +src, dst, r1, r2, f1, f2 = sys.argv[1:] +c = json.load(open(src)); out = {} +for old, new in ((r1, f1), (r2, f2)): + d = dict(c[old]); d["adapter"] = "reolink"; out[new] = d +json.dump(out, open(dst, "w"), indent=2) +EOF + D run --rm -i -v "$SCN:/s" -v "$ENGINE_DIR/data/credentials.fake.json:/creds.json:ro" "pyronear/pyro-camera-api:$TAG" python - <<'EOF' +import json, pathlib +import numpy as np +from PIL import Image +creds = json.load(open("/creds.json")) +src = sorted(pathlib.Path("/s/never-smoke").glob("*/000.jpg")) +rng = np.random.default_rng(0) +for scn in ("day", "night", "noisy"): + d = pathlib.Path(f"/s/pi-{scn}"); d.mkdir(exist_ok=True) + cameras = {c["name"]: {"poses": {str(p): i * 90.0 for i, p in enumerate(c["poses"])}} for c in creds.values()} + d.joinpath("scenario.json").write_text(json.dumps({"description": scn, "cameras": cameras})) + for cam, frame in zip(creds.values(), src): + out = d / cam["name"]; out.mkdir(exist_ok=True) + img = Image.open(frame).convert("RGB") + if scn == "day": + img.save(out / "000.jpg", quality=90) + elif scn == "night": + img.convert("L").save(out / "000.jpg", quality=90) + else: + base = np.asarray(img, dtype=np.int16) + for i in range(40): + Image.fromarray(np.clip(base + rng.integers(-6, 7, base.shape, dtype=np.int16), 0, 255).astype(np.uint8)).save(out / f"{i:03d}.jpg", quality=90) +print("scenarios written") +EOF + sudo -n chown -R "$USER" "$SCN" + for scn in day night noisy; do [ -d "$SCN/pi-$scn" ] || die "scenario pi-$scn not generated"; done + ok "scenarios written under $SCN" +} + +# ---------------------------------------------------------------- tests +test_camera_timeout() { + say "black-hole the first camera's HTTPS, expect 'Read timed out (read timeout=10)'" + use_real; stack_up + read -r REAL1 _ <<<"$(cams)" + sink_on "$REAL1"; mark + wait_log pyro-camera-api 180 "\[$REAL1\].*Read timed out.*read timeout=10" || die "no 10 s timeout seen" + sink_off + ok "Reolink call timed out after 10 s instead of hanging" +} + +test_api_frozen() { + say "freeze the camera API, expect the healthcheck to restart it" + use_real; stack_up + local before; before=$(restarts pyro-camera-api) + D top pyro-camera-api -o pid,comm | awk '/uvicorn|python/ {print $1}' | xargs sudo -n kill -STOP + local end=$(( $(date +%s) + 180 )) + while [ "$(date +%s)" -lt "$end" ]; do [ "$(restarts pyro-camera-api)" -gt "$before" ] && break; sleep 5; done + [ "$(restarts pyro-camera-api)" -gt "$before" ] || die "camera API not restarted within 3 min" + ok "camera API killed by the healthcheck and restarted" +} + +test_night_per_frame() { + say "camera 1 grayscale (IR), camera 2 colour: night frames skipped, site awake" + fakecam 1 pi-night; fakecam 2 pi-day; use_fake; stack_up + wait_log engine 240 "Night frame for ${FAKE1}_" || die "no night frame seen on camera 1" + wait_log engine 120 "Frame identical to the previous one for ${FAKE2}_" || die "identical frame not skipped on camera 2" + [ "$(count_log engine 'Nighttime detected')" = 0 ] || die "site went to sleep with a day camera" + ok "phase 1: night per frame, identical frames skipped, no sleep" + + say "stop camera 2 so it answers 204 after 5 min: one night frame + one missing must not sleep" + D stop fakecam-2 >/dev/null; mark + sleep 480 + [ "$(count_log engine 'Nighttime detected')" = 0 ] || die "site went to sleep on night + 204" + curl -s -o /dev/null -w '%{http_code}' "localhost:8081/cameras/latest_image?camera_ip=$FAKE2&pose=$(python3 -c "import json;print(json.load(open('$ENGINE_DIR/data/credentials.json'))['$FAKE2']['poses'][0])")" | grep -q 204 || die "camera 2 not in 204" + ok "phase 2: 204 never proves night" + + say "both cameras grayscale: patrols stopped, site sleeps" + fakecam 2 pi-night; mark + wait_log engine 240 "Nighttime detected by every camera" || die "site did not sleep with every camera at night" + [ "$(count_log engine 'Stopped patrol for camera')" -ge 1 ] || die "patrols not stopped" + ok "phase 3: every camera at night, sleep" +} + +test_camera_silent() { + say "black-hole both cameras: no_fresh_image -> API reboot attempt, budget kept across a restart" + use_real; sudo -n rm -f "$ENGINE_DIR/data/camera_recovery.json"; stack_up + read -r REAL1 REAL2 <<<"$(cams)" + sink_on "$REAL1" "$REAL2"; mark + wait_log pyro-camera-api 1000 "\[$REAL1\] Rebooting camera, no_fresh_image" || die "no reboot attempt within 16 min" + sleep 15 + [ "$(count_log pyro-camera-api 'Reboot raised.*password=')" = 0 ] || die "camera password leaked in the log" + status "$REAL1" | grep -q '"reboot_attempts_24h":1' || die "status does not show 1 attempt" + grep -q "$REAL1" "$ENGINE_DIR/data/camera_recovery.json" || die "attempt not saved on disk" + D restart pyro-camera-api >/dev/null; sleep 25 + status "$REAL1" | grep -q '"reboot_attempts_24h":1' || die "attempt lost after a container restart" + sink_off + ok "silent camera: reboot attempted, no password in logs, budget survives a restart" +} + +test_camera_frozen() { + say "camera 1 frozen (same pixels), camera 2 not moving (fresh frames, same view), ALLOW_STILL_CAMERA=false" + fakecam 1 pi-day; fakecam 2 pi-noisy; use_fake; sudo -n rm -f "$ENGINE_DIR/data/camera_recovery.json"; stack_up + wait_log pyro-camera-api 1000 "\[$FAKE1\] Rebooting camera, frozen" || die "camera 1 not rebooted for frozen within 16 min" + wait_log pyro-camera-api 60 "\[$FAKE2\] Rebooting camera, not_moving" || die "camera 2 not rebooted for not_moving" + [ "$(count_log pyro-camera-api 'Reboot command rejected')" = 0 ] || die "fake camera rejected the reboot" + ok "frozen and not_moving detected, reboot sent" + + say "cooldown: 'budget spent' once, then a second reboot 30 min after the first" + wait_log pyro-camera-api 1000 "\[$FAKE1\].*reboot budget spent" || die "no budget message at the next hit 2/2" + sleep 1000 + [ "$(count_log pyro-camera-api "\[$FAKE1\].*reboot budget spent")" = 1 ] || die "budget message logged more than once" + status "$FAKE1" | grep -q '"reboot_attempts_24h":2' || die "second attempt not made after the cooldown" + ok "cooldown: one budget line, second reboot after 30 min, 2 attempts on disk" +} + +# ---------------------------------------------------------------- main +STARTED=$(date +%Y-%m-%dT%H:%M:%S) +case "${1:-}" in + setup) setup ;; + scenarios) make_scenarios ;; + camera-timeout) test_camera_timeout ;; + api-frozen) test_api_frozen ;; + night-per-frame) test_night_per_frame ;; + camera-silent) test_camera_silent ;; + camera-frozen) test_camera_frozen ;; + all) test_camera_timeout; test_api_frozen; test_night_per_frame; test_camera_silent; test_camera_frozen; use_real; stack_up; ok "all scenarios passed" ;; + real) use_real; stack_up; ok "back on the real cameras" ;; + *) sed -n '2,19p' "$0"; exit 1 ;; +esac