diff --git a/.env.example b/.env.example index c826a1a..e209345 100644 --- a/.env.example +++ b/.env.example @@ -3,8 +3,8 @@ # cp .env.example .env # chmod 600 .env # secrets — restrict to owner only # -# Only credentials/secrets belong here. -# All other settings (hosts, ports, recipients, thresholds) go in watchdog-config.yaml. +# Credentials/secrets and installer-managed deployment values live here. +# Monitoring thresholds, channel hosts, ports, and recipients belong in watchdog-config.yaml. # ── Slack ───────────────────────────────────────────────────────────────────── # Required if alert_channels includes "slack" @@ -48,8 +48,5 @@ SMTP_PASSWORD=your-smtp-password/api key value # ── Image version ───────────────────────────────────────────────────────────── # Release tag the container runs (see the repository VERSION file). install.sh -# sets this to the version it loaded from watchdog.tar; unset means watchdog:latest. +# sets this to the verified release image version loaded from packaged watchdog.tar. # WATCHDOG_VERSION= - -# ── Tuning ──────────────────────────────────────────────────────────────────── -LOG_LEVEL=INFO diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 950e08f..3da3888 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - "release/**" jobs: validate: @@ -17,10 +18,18 @@ jobs: run: | bash -n install.sh bash -n uninstall.sh + bash -n tests/release-smoke.sh + bash -n scripts/package_release.sh - name: Validate Python syntax run: | python3 -m py_compile watchdog.py + python3 - <<'PYCODE' + from pathlib import Path + script = Path("tests/release-smoke.sh").read_text() + code = script.split("<<'PY'", 1)[1].rsplit("\nPY", 1)[0] + compile(code, "release-smoke", "exec") + PYCODE - name: Install test dependencies run: | diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml new file mode 100644 index 0000000..ba06a62 --- /dev/null +++ b/.github/workflows/release-package.yml @@ -0,0 +1,86 @@ +name: Publish Container Alert release + +# Publish only a reviewed, tagged commit already merged into main. +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write + +jobs: + package: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify tag, version, and main ancestry + shell: bash + run: | + set -euo pipefail + version="$(tr -d '[:space:]' < VERSION)" + test "$GITHUB_REF_NAME" = "v$version" + git fetch origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + echo "WATCHDOG_VERSION=$version" >> "$GITHUB_ENV" + + - name: Validate release scripts and Python tests + shell: bash + run: | + set -euo pipefail + bash -n install.sh uninstall.sh + python3 -m pip install docker==7.1.0 requests==2.32.3 PyYAML==6.0.2 + python3 -m unittest discover -s tests -q + + - name: Build version-labelled release image + shell: bash + run: | + set -euo pipefail + docker build --build-arg "WATCHDOG_VERSION=$WATCHDOG_VERSION" \ + -t "watchdog:$WATCHDOG_VERSION" . + image_version="$(docker image inspect "watchdog:$WATCHDOG_VERSION" \ + --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" + test "$image_version" = "$WATCHDOG_VERSION" + docker run --rm --network none --entrypoint python3 \ + "watchdog:$WATCHDOG_VERSION" -c \ + "import docker, requests, yaml, pysnmp, cryptography, watchdog, lifecycle, ha_monitor" + + - name: Assemble and validate single customer package + shell: bash + run: | + set -euo pipefail + docker save "watchdog:$WATCHDOG_VERSION" -o watchdog.tar + bash scripts/package_release.sh watchdog.tar dist + (cd dist && sha256sum --check SHA256SUMS) + package="dist/container-alert-v$WATCHDOG_VERSION.tar.gz" + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/watchdog.tar' + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/SHA256SUMS' + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/install.sh' + + - name: Publish GitHub Release + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + notes=(--generate-notes) + if [ -f "docs/operations/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then + notes=(--notes-file "docs/operations/RELEASE_NOTES_V$WATCHDOG_VERSION.md") + fi + + # Check both assets before making this public release visible. + gh release create "$GITHUB_REF_NAME" \ + dist/SHA256SUMS "dist/container-alert-v$WATCHDOG_VERSION.tar.gz" \ + --verify-tag --draft --title "Container Alert $GITHUB_REF_NAME" \ + "${notes[@]}" + + expected=$(printf '%s\n' SHA256SUMS \ + "container-alert-v$WATCHDOG_VERSION.tar.gz" | sort) + actual=$(gh release view "$GITHUB_REF_NAME" --json assets --jq '.assets[].name' | sort) + test "$actual" = "$expected" || { echo "Release assets incomplete" >&2; exit 1; } + gh release edit "$GITHUB_REF_NAME" --draft=false diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml new file mode 100644 index 0000000..c3a4bb9 --- /dev/null +++ b/.github/workflows/release-smoke.yml @@ -0,0 +1,105 @@ +name: Release smoke (isolated) + +# Runs only on disposable GitHub-hosted workers; never on operational engines. +# PR trigger permits validation before release; manual reruns after first merge. +on: + pull_request: + branches: + - main + paths: + - "watchdog.py" + - "lifecycle.py" + - "ha_monitor.py" + - "install.sh" + - "VERSION" + - "docker-compose.yaml" + - "docker-compose.build.yaml" + - "tests/release-smoke.sh" + - ".github/workflows/release-smoke.yml" + - ".github/workflows/release-package.yml" + - "scripts/package_release.sh" + - "Dockerfile" + workflow_dispatch: + +permissions: + contents: read + +jobs: + real-docker: + name: Real Docker (disposable GitHub runner) + runs-on: ubuntu-24.04 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Build release candidate on disposable runner + run: docker build --pull=false --build-arg "WATCHDOG_VERSION=$(cat VERSION)" --tag "watchdog:$(cat VERSION)" . + + - name: Create empty, labelled isolated Docker Engine + shell: bash + run: | + set -euo pipefail + marker="ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "SMOKE_ENGINE_MARKER=${marker}" >> "${GITHUB_ENV}" + docker run -d --name ca-smoke-engine --restart no \ + --privileged --cpus 2 --memory 2g --pids-limit 768 \ + --network bridge -p 127.0.0.1::2375 \ + -e DOCKER_TLS_CERTDIR="" \ + docker:27.5-dind \ + --storage-driver=vfs --iptables=false --ip6tables=false \ + --bridge=none \ + --label "com.radware.container-alert.smoke-engine=${marker}" + port="$(docker port ca-smoke-engine 2375/tcp | tail -1 | awk -F: '{print $NF}')" + test -n "${port}" + echo "DOCKER_HOST=tcp://127.0.0.1:${port}" >> "${GITHUB_ENV}" + for i in $(seq 1 40); do + if DOCKER_HOST="tcp://127.0.0.1:${port}" docker info >/dev/null 2>&1; then + echo "SMOKE_DOCKER_SOCKET_GID=$(docker exec ca-smoke-engine stat -c '%g' /var/run/docker.sock)" >> "${GITHUB_ENV}" + echo "Disposable Engine ready." + exit 0 + fi + sleep 2 + done + echo "Disposable Engine failed to start" >&2 + exit 1 + + - name: Load exact built image into isolated Engine + shell: bash + run: | + set -euo pipefail + DOCKER_HOST=unix:///var/run/docker.sock docker save "watchdog:$(cat VERSION)" | DOCKER_HOST="${DOCKER_HOST}" docker load + DOCKER_HOST="${DOCKER_HOST}" docker image inspect "watchdog:$(cat VERSION)" >/dev/null + + - name: Real Docker acceptance (15 scenarios, no operational access) + shell: bash + env: + SMOKE_RUNNER_KIND: ${{ runner.environment }} + run: | + set -euo pipefail + SMOKE_IMAGE="watchdog:$(cat VERSION)" bash tests/release-smoke.sh + + - name: Real OOM then external restart (full 120-second recovery) + shell: bash + env: + SMOKE_RUNNER_KIND: ${{ runner.environment }} + SMOKE_ONLY: T15_EXTERNAL_OOM + run: | + set -euo pipefail + SMOKE_IMAGE="watchdog:$(cat VERSION)" bash tests/release-smoke.sh + + - name: Upload test reports (test-only synthetic data) + if: always() + uses: actions/upload-artifact@v4 + with: + name: container-alert-release-smoke-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/container-alert-smoke-*/ + if-no-files-found: warn + retention-days: 30 + + - name: Remove disposable test Engine + if: always() + shell: bash + run: | + DOCKER_HOST=unix:///var/run/docker.sock docker rm -f ca-smoke-engine 2>/dev/null || true diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md deleted file mode 100644 index f6588fc..0000000 --- a/DEPLOYMENT.md +++ /dev/null @@ -1,842 +0,0 @@ -# CyberController Container Watchdog — Deployment Guide - -The **CyberController Container Watchdog** is an autonomous monitoring service that provides continuous health visibility across all Docker containers running on the host. It detects failures — including crashes, out-of-memory kills, prolonged unhealthy states, and restart loops — and dispatches real-time alerts through one or more configurable channels (**Slack**, **SMTP**, **SNMP Traps**, or **Syslog**). It also sends a one-time INFO **recovered** alert once a previously-alarmed container returns to normal operation. - -This guide covers initial deployment, alert channel configuration, and ongoing operational management of the watchdog service. - ---- - -## Table of Contents - -1. [Prerequisites](#1-prerequisites) - - [1.1 Pre-flight Checks](#11-pre-flight-checks) -2. [Environment Setup](#2-environment-setup) - - [2.1 Clone the Repository](#21-clone-the-repository) - - [2.2 Configure Environment Variables (.env)](#22-configure-environment-variables-env) - - [2.3 Configure Watchdog Global Settings](#23-configure-watchdog-global-settings) - - [2.4 Configure Alert Channels](#24-configure-alert-channels) -3. [Deployment Steps](#3-deployment-steps) - - [3.1 Option A: Automated Installation (install.sh) (Recommended)](#31-option-a-automated-installation-installsh-recommended) - - [3.2 Option B: Manual Installation](#32-option-b-manual-installation) -4. [Verification](#4-verification) - - [4.1 Confirm Container Status and Health](#41-confirm-container-status-and-health) - - [4.2 Reviewing Logs](#42-reviewing-logs) -5. [Rollback](#5-rollback) - - [5.1 Roll Back a Failed Deployment](#51-roll-back-a-failed-deployment) - - [5.2 Roll Back an Image Upgrade](#52-roll-back-an-image-upgrade) - - [5.3 Roll Back an Application Upgrade (Git)](#53-roll-back-an-application-upgrade-git) -6. [Operations and Maintenance](#6-operations-and-maintenance) - - [6.1 Lifecycle Management](#61-lifecycle-management) - - [6.2 Applying Configuration Changes](#62-applying-configuration-changes) - - [6.3 Upgrade Guide](#63-upgrade-guide) - - [6.4 Uninstalling](#64-uninstalling) -7. [Troubleshooting](#7-troubleshooting) -8. [Appendix and Reference](#8-appendix-and-reference) - - [8.1 Repository Structure](#81-repository-structure) - - [8.2 Configuration Reference](#82-configuration-reference) - - [8.3 SNMP Trap Var-Binds](#83-snmp-trap-var-binds) - - [8.4 Support Contacts](#84-support-contacts) - ---- - -## 1. Prerequisites - -| Requirement | Minimum Version / Notes | -|---|---| -| Docker Engine | 20.10+ | -| Docker Compose | v2 plugin (`docker compose`) or standalone `docker-compose` | -| Linux host | Ubuntu 20.04 / RHEL 8 or later (required for Docker socket access) | -| Git | Required to clone the repository | -| Host permissions | Root, or membership in the `docker` group (to read `/var/run/docker.sock`) | - -> By default the container runs as root so a manual install needs no host-specific group setup. The automated installer ([3.1 Option A](#31-option-a-automated-installation-installsh-recommended)) instead hardens it to run non-root (UID/GID 1000, joined to the docker.sock group) automatically, with no input required. Either way it's still contained by a read-only root filesystem, dropped Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID limits (see [Container Hardening in the README](README.md#container-hardening)). - -```bash -docker compose version -``` - -### 1.1 Pre-flight Checks - -Run these checks before starting the deployment. Do not proceed if any of them fail. - -```bash -# Docker daemon is running and reachable -docker info - -# Current user can talk to the Docker socket without sudo -docker ps - -# Compose plugin is installed -docker compose version - -# Sufficient free disk space for the image and log files -df -h . -``` - -Image and container size is documented in [README.md § Image Size](README.md#image-size). - -If you plan to use an alert channel that requires outbound network access (Slack webhook, SMTP relay, or SNMP trap receiver), confirm the host can reach that endpoint before deployment. - -> Repository layout is documented in [8.1 Repository Structure](#81-repository-structure). - ---- - -## 2. Environment Setup - -> If you plan to use the automated installer ([3.1 Option A: Automated Installation (install.sh)](#31-option-a-automated-installation-installsh-recommended), recommended), the configuration wizard creates `.env` and `watchdog-config.yaml` for you interactively — you may skip 2.2–2.4 and go directly to [3. Deployment Steps](#3-deployment-steps). Complete 2.2–2.4 only if you are installing manually ([3.2 Option B: Manual Installation](#32-option-b-manual-installation)) or want to pre-stage configuration before running the installer. - -### 2.1 Clone the Repository - -# Navigate to your target installation directory -cd /path/to/Container_Alert - -```bash -git clone https://github.com/Radware/CyberController-Container-Alert Container_Alert -cd Container_Alert -``` - -All remaining commands in this guide are run from inside the `Container_Alert/` directory. - ---- -*Note*: If you use the automated installation method in Section 2.1, skip Sections 2.2, 2.3 and 2.4 and proceed directly to Section 3. - -### 2.2 Configure Environment Variables (.env) - -```bash -cp .env.example .env -chmod 600 .env # restrict to owner only — contains secrets -``` - -Open `.env` and fill in every value: - -```dotenv -# Slack -SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T.../B.../... - -# SMTP (only if SMTP channel is enabled) -SMTP_USERNAME=your-smtp-username/login email -SMTP_PASSWORD=your-smtp-password/api key value - -# Tuning (optional — defaults shown) -LOG_LEVEL=INFO -``` - -Only credentials and secrets belong in `.env`. Hosts, ports, recipients, and thresholds are configured in `watchdog-config.yaml`, never hardcoded in this guide or in scripts. - -The container runs as root by default for a zero-configuration manual install. If you want the non-root hardening that [Option A](#31-option-a-automated-installation-installsh-recommended) applies automatically, uncomment the `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` lines in `.env.example` and fill in `DOCKER_GID` (`stat -c '%g' /var/run/docker.sock`). - ---- - -### 2.3 Configure Watchdog Global Settings - -```yaml -# ── Watchdog global settings ──────────────────────────────────────────────────────── -check_interval_seconds: 60 # poll interval -cooldown_minutes: 5 # no duplicate alerts within this window -restart_threshold: 5 # restarts before "restart-loop" alert -restart_window_minutes: 10 # rolling window for restart counting -unhealthy_cycles_threshold: 3 # consecutive unhealthy polls before alert -alert_on_recovery: true # send an INFO "recovered" alert when a container returns to normal - -excluded_containers: - - debug-shell # add any containers you don't want monitored - -ignored_container_events: - - label: - com.radware.cybercontroller.role: mariadb-ha-syntax-check - events: - - die # ignore only R&D-designated temporary syntax-check dies -``` - -Radware R&D to add `--label com.radware.cybercontroller.role=mariadb-ha-syntax-check` when creating the temporary MariaDB HA syntax-check container. That explicit label is the preferred contract: normal service container dies alert; R&D-designated temporary helper dies are ignored for the configured event. - -Until that label exists, the watchdog also has a built-in fallback for dynamic MariaDB dump helper crashes. It suppresses the alert only when image `kvision_infra_mariadb`, container name is not `config_kvision-infra-mariadb_1`, Docker `AutoRemove` is `true`, a mount contains `/mnt/cli/tmp/mariadb_dump.sql.gz`, and the exit code is `137` all match together. - ---- - -### 2.4 Configure Alert Channels - -| Channel | `alert_channels` identifier | What you need | -|---|---|---| -| Slack Webhook | `slack` | Slack incoming webhook URL | -| Syslog | `syslog` | Syslog server reachable from the Docker host | -| Email (SMTP) | `smtp` | SMTP relay and credentials | -| SNMP Traps | `snmp_trap` | SNMP trap receiver (NMS / SIEM) | - -At least one communication channel must be configured before the watchdog can send alerts. Choose from Slack, SMTP, SNMP Trap, or Syslog, then obtain the required credentials or endpoint details for your chosen channel(s) and configure them in `watchdog-config.yaml` and `.env`. - -For each channel you want to use, follow the relevant option below. Ensure its identifier is listed under `alert_channels` in `watchdog-config.yaml`. - -#### 2.4.1 Slack Webhook - -1. Go to [https://api.slack.com/apps](https://api.slack.com/apps) → **Create New App** → **From scratch** -2. Name it `Container Watchdog`, select your workspace -3. In the left menu: **Incoming Webhooks** → toggle **On** -4. Click **Add New Webhook to Workspace** → choose an alert channel (e.g. `#container-alerts`) -5. Copy the webhook URL (starts with `https://hooks.slack.com/services/...`) -6. Paste it into `.env` as `SLACK_WEBHOOK_URL` - -Edit `watchdog-config.yaml` to match your environment: -```yaml -# ── Slack ───────────────────────────────────────────────────────────────────── -slack: - enabled: true # set to true to enable - webhook_url_env: SLACK_WEBHOOK_URL # env var: https://hooks.slack.com/services/... - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - slack -``` - - -#### 2.4.2 Syslog Forwarding - -Enable the syslog block in `watchdog-config.yaml`: - -```yaml -# ── Syslog (remote / central log server) ───────────────────────────────────── -syslog: - enabled: true # set to true to enable - host: 155.1.1.4 # IP or hostname of your syslog server - port: 514 # 514 = standard syslog (UDP); 601 = reliable syslog (TCP) - protocol: udp # udp (fire-and-forget) or tcp (reliable) - facility: local0 # syslog facility: local0–local7, daemon, user, etc. - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - syslog -``` - ---- - -#### 2.4.3 SMTP (Email) - -Add `smtp` to `alert_channels` in `watchdog-config.yaml` and configure the block: - -```yaml -# ── Email (SMTP) ───────────────────────────────────────────────────────────── -# Add "smtp" to alert_channels above to enable. -smtp: - enabled: true # set to true to enable - host: smtp.radware.com - port: 587 - sender: noc-alerts@radware.com - username_env: SMTP_USERNAME # env var: SMTP username or app token (Mailtrap, Gmail app password, etc.) - recipients: - - ops-team@radware.com - - oncall@radware.com - tls: true - password_env: SMTP_PASSWORD # env var: SMTP password or app token - -#### SMTP Relay (No Authentication) -# ── Email (SMTP Relay - No Authentication) ─────────────────────────────────── -# Use this configuration only when your SMTP relay accepts unauthenticated -# connections from trusted hosts. - -smtp: - enabled: true - - # SMTP server settings - host: your-relay.internal - port: 25 - tls: false # Enable only if your relay requires STARTTLS - - # Authentication - auth: false - - # Email settings - sender: watchdog@yourdomain.com - recipients: - - ops@yourdomain.com - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - smtp -``` - -Set SMTP credentials in `.env`: - -```dotenv -SMTP_USERNAME=your-smtp-username/ login email -SMTP_PASSWORD=your-smtp-password/ api key value -``` - ---- - -#### 2.4.4 SNMP Traps - -Add `snmp_trap` to `alert_channels` in `watchdog-config.yaml` and configure the block: - -```yaml -# ── SNMP Traps ──────────────────────────────────────────────────────────────── -# Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. -# Add "snmp_trap" to alert_channels above to enable. -# Requires: pip install pysnmp (already in requirements-watchdog.txt) -snmp_trap: - enabled: false # set to true to enable - host: 155.1.1.1 # IP or hostname of your SNMP trap receiver (NMS) - port: 162 # UDP port — 162 is the standard SNMP trap port - version: v2c # SNMP version: v1, v2c, or v3 - community: public # SNMPv1/v2c community string (ignored for v3) - # trap_oid is not set here — code defaults to Radware enterprise OID 1.3.6.1.4.1.89.110.0.1 - # SNMPv3 settings (used only when version: v3 — add secrets to .env) - # v3_username: watchdog-user - # v3_auth_protocol: SHA # MD5 | SHA | SHA256 | SHA384 | SHA512 - # v3_auth_key_env: SNMP_V3_AUTH_KEY # env var in .env holding the auth passphrase - # v3_priv_protocol: AES # AES (AES-128, recommended) | AES192 | AES256 | DES (weak — logs warning) - # v3_priv_key_env: SNMP_V3_PRIV_KEY # env var in .env holding the priv passphrase - # v3_local_engine_id: "" # Pin the sender engine ID (hex, with or without 0x). - # The watchdog logs the auto-derived engine ID at startup (INFO level) — copy it here. - # Then set in snmptrapd.conf: createUser -e 0x watchdog-user SHA "..." AES "..." - # This prevents snmptrapd from needing reconfiguration after container restarts. - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - snmp_trap -``` - -> **Security note:** `public` is the well-known default SNMPv1/v2c community string and provides no real access control — anyone who can reach the trap receiver on UDP/162 with that string can be trusted implicitly. Change it to a non-default value, restrict network access to the trap receiver, or use SNMPv3 with authentication and encryption for production deployments. - -Trap var-binds and SNMPv3 setup notes are documented in [8.3 SNMP Trap Var-Binds](#83-snmp-trap-var-binds). - ---- - -## 3. Deployment Steps - -### 3.1 Option A: Automated Installation (install.sh) (Recommended) - -Run the installer from the `Container_Alert/` directory created in [2.1 Clone the Repository](#21-clone-the-repository). It guides you through every step interactively — including environment and alert-channel configuration — and requires no manual file editing. - -> **Before running:** If internet access is available, no additional steps are required as the installer will automatically build the Docker image from source; otherwise, download the pre-built Docker image archive [watchdog.tar](https://radwareil.sharepoint.com/:f:/s/NAResidentEngineers/IgC4d3ALtPwvSrz4is6QWmnPATRCQ7g6BXbnbWO3Q0FFMSk?e=oj4WGS) and place it in the same directory as install.sh before starting the installation. -If you have trouble accessing the download link, see [8.4 Support Contacts](#84-support-contacts). - -```bash -bash install.sh -``` - -The script walks through the following stages in order: - -| Stage | What happens | -|---|---| -| **Prerequisites** | Verifies Docker and Docker Compose are installed and running | -| **Log directory** | Creates `./watchdog/` for persistent log storage | -| **Load image** | Loads `watchdog.tar` and requires it to contain `watchdog:` (from the root `VERSION` file); aborts on a version mismatch, and builds from source only when the archive is absent | -| **Host identification** | `WATCHDOG_HOST` is taken from `.env` (the installer prompts for it), defaulting to `CyberController-Server` | -| **Configuration wizard** | Selects channels and prompts for Slack/SMTP/SNMP/Syslog values interactively | -| **Credentials + config write** | Generates/updates `.env` (including `WATCHDOG_VERSION`) and `watchdog-config.yaml` from wizard answers | -| **Start** | Runs `docker compose up -d` in the background, aborting if Compose reports a failure | -| **Verify** | Fails the installation unless the container is running **and** uses `watchdog:` | - -> **Re-running `install.sh` on an existing installation is safe.** If both configuration files exist, the installer asks whether to reconfigure from scratch. Choose `N` to keep them unchanged — the packaged image is still loaded and the deployment is still moved to the new version. If only one of the two files is present, the wizard regenerates both, since a half-present configuration cannot start. - -Continue to [4. Verification](#4-verification). - ---- - -### 3.2 Option B: Manual Installation - -Use this path when you need full control over configuration files before starting. This procedure is optional — skip it if you completed Option A. It assumes you already completed [2.1 Clone the Repository](#21-clone-the-repository) and [2.2–2.4 Environment Setup](#2-environment-setup). - -#### Online Installation - -*Note* - If internet access is available, build the image and skip to [Start the container](#start-the-container) below. - -```bash -WATCHDOG_VERSION="$(cat VERSION)" docker compose -f docker-compose.build.yaml build -``` - -#### Offline Installation - -*Note* - If internet access is unavailable, load the pre-built image instead of building it. - -Download [`watchdog.tar` — pre-built Docker image for offline installation](https://radwareil.sharepoint.com/:u:/s/NAResidentEngineers/IQASPcHrYh6ATqPzBOvz_TgnAYi1knv7GbmwctN7m1JWtOM?e=YKrQP0). If you have trouble accessing the download link, see [8.4 Support Contacts](#84-support-contacts). -```bash -docker load -i watchdog.tar -``` - -#### Start the container - -```bash -docker compose up -d -``` - -Continue to [4. Verification](#4-verification). - ---- - -## 4. Verification - -### 4.1 Confirm Container Status and Health - -```bash -docker compose ps -docker compose logs -f docker-container-watchdog -``` - -Expected output from `docker compose ps`: - -``` -NAME STATUS -docker-container-watchdog Up (healthy) -``` - -If the status does not reach `Up (healthy)` within a couple of minutes, see [7. Troubleshooting](#7-troubleshooting), or follow [5. Rollback](#5-rollback) to back out the deployment. - ---- - -### 4.2 Reviewing Logs - -| Location | Description | -|---|---| -| `docker compose logs docker-container-watchdog` | Live stdout logs | -| `./watchdog/watchdog.log` | Bind-mounted log file on the host (inside the `Container_Alert/` directory) | -| `/var/log/watchdog/watchdog.log` | Inside the container (rotated, max 10 MB × 5 files) | - -To follow logs with timestamps: - -```bash -docker compose logs -f --timestamps docker-container-watchdog -``` - -To read the log file directly from the volume: - -```bash -docker exec docker-container-watchdog tail -f /var/log/watchdog/watchdog.log -``` - ---- - -### Optional HA role-change verification - -This check is only for **two-node CyberController HA** installations and is -disabled by default. On **each HA host**: - -1. Verify the local exporter is active: curl -fsS http://127.0.0.1:9664/metrics. -2. Confirm metrics for Pacemaker resources, quorum, collector scrape success, - and local node identity are present. Port 9664 is the verified lab setting; - read /etc/ha_cluster_exporter.yaml if different. -3. Enable ha_monitor.enabled: true in watchdog-config.yaml on **both** - nodes; restart only the watchdog service using the normal documented - configuration-change procedure. -4. Verify the baseline is stored under ha_monitor.active_node in the persistent - watchdog state.json; startup should send **no** HA event. -5. During an approved HA maintenance window, perform one controlled switchover - and failback using the supported CyberController operations procedure. - Confirm exactly one notification from the newly active node for each - transition; check docker logs docker-container-watchdog on both hosts. -6. Disable by setting ha_monitor.enabled: false and restarting the watchdog. - This clears its saved HA baseline; re-enabling starts with a fresh, quiet - baseline (no notification for transitions that occurred while disabled). - An ordinary restart while HA remains enabled keeps the confirmed baseline. - No HA service or Pacemaker configuration changes are necessary. - -Delivery is best-effort. Test the actual alert receiver on **both** nodes: -an HA alert in watchdog logs does not prove a syslog or SMTP destination -received it. This release has no persisted notification retry queue. - -Do not attempt a network-partition simulation on a cluster with STONITH -disabled: split-brain and database inconsistency are possible. The native -Prometheus port mismatch (9002 vs exporter 9664) is a separate appliance -configuration issue; it does not block the watchdog's direct localhost check. - ---- - -## 5. Rollback - -### 5.1 Roll Back a Failed Deployment - -If verification fails immediately after `docker compose up -d` — the container will not start, restarts in a loop, or the health check never turns healthy: - -```bash -docker compose down -``` - -Diagnose the cause using [4.2 Reviewing Logs](#42-reviewing-logs) and [7. Troubleshooting](#7-troubleshooting), correct the configuration, then redeploy: - -```bash -docker compose up -d -``` - -Re-run [4.1 Confirm Container Status and Health](#41-confirm-container-status-and-health) before considering the deployment complete. - -### 5.2 Roll Back an Image Upgrade - -Each release is tagged `watchdog:` and the deployed tag is pinned by `WATCHDOG_VERSION` in `.env`. As long as the previous image is still on the host, rolling back is a version change: - -```bash -docker images watchdog # list the versions available locally -sed -i 's/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=1.5.3/' .env -docker compose up -d -``` - -If the previous image was only ever tagged `watchdog:latest`, preserve it under a rollback tag before loading a new archive: - -```bash -docker tag watchdog:latest watchdog:rollback -``` - -### 5.3 Roll Back an Application Upgrade (Git) - -Before running `git pull` (see [6.3 Upgrade Guide](#63-upgrade-guide)), record the current commit so you can return to it: - -```bash -git rev-parse HEAD > backup/previous-commit.txt -``` - -To restore the prior source checkout: - -```bash -git checkout "$(cat backup/previous-commit.txt)" -``` - -This **does not roll back the running container**. To revert deployed behavior, restore the previous `WATCHDOG_VERSION` and redeploy the corresponding image as described in [5.2 Image Rollback](#52-roll-back-an-image-upgrade). - -Also restore the `.env` and `watchdog-config.yaml` files backed up before the upgrade — see [6.3 Upgrade Guide](#63-upgrade-guide). - ---- - -## 6. Operations and Maintenance - -### 6.1 Lifecycle Management - -#### Stopping the Service - -```bash -# Pause the container — restartable with `docker compose start` -docker compose stop - -# Stop and remove the container, keep the image and log files -docker compose down -``` - -#### Restarting After Configuration Changes - -```bash -# Pick up watchdog-config.yaml changes (no rebuild needed) -docker compose restart docker-container-watchdog - -# Re-read .env credential changes -docker compose up -d -``` - -#### Updating the Watchdog Image - -```bash -# Rebuild the image (requires internet) -WATCHDOG_VERSION="$(cat VERSION)" docker compose -f docker-compose.build.yaml build - -# Redeploy using the production runtime file -docker compose up -d -``` - ---- - -### 6.2 Applying Configuration Changes - -No image rebuild is required for any configuration change. The type of change determines the required command: - -| Changed file | What to run | -|---|---| -| `watchdog-config.yaml` (thresholds, channels, exclusions) | `docker compose restart docker-container-watchdog` | -| `.env` (secrets, credentials) | `docker compose up -d` | -| `docker-compose.yaml` (host name, resource limits) | `docker compose up -d` | - -#### Apply watchdog-config.yaml changes - -```bash -docker compose restart docker-container-watchdog -``` - -#### Apply .env changes (credentials / secrets) - -```bash -docker compose up -d -``` - -#### Apply docker-compose.yaml changes (container settings) - -For example, change the hostname shown in alerts by editing `WATCHDOG_HOST` in `.env`: - -```bash -WATCHDOG_HOST=my-new-server-name -``` - -Then redeploy to take effect: - -```bash -docker compose up -d -``` - ---- - -### 6.3 Upgrade Guide - -Updating the source checkout and upgrading the running Docker image are different operations. Only deploying a new image changes the running watchdog code. - -#### Before You Begin (Recommended) - -Back up your deployment-specific configuration files before upgrading. These backups are also used for [5. Rollback](#5-rollback) if the upgrade fails. - -```bash -mkdir -p backup - -cp .env backup/.env.bak -cp watchdog-config.yaml backup/watchdog-config.yaml.bak -``` - -To restore the configuration later (if required): - -```bash -cp backup/.env.bak .env -cp backup/watchdog-config.yaml.bak watchdog-config.yaml -``` - ---- - -#### Update Source Code (Not a Running Image) - -The application (`watchdog.py`, `lifecycle.py`, `ha_monitor.py`) is **built into the Docker image**, not bind-mounted. Updating the Git checkout alone **does not deploy code**, and `docker compose up -d` with the same pinned image will not apply the latest changes. Do not stop the running watchdog just to update source files. - -To update the repository without changing the running container: - -```bash -mkdir -p backup -git rev-parse HEAD > backup/previous-commit.txt -git fetch origin -git pull --ff-only origin main -``` - -To deploy new code, first prepare a **new numbered release**: update `VERSION` and the README version history, build and test the matching `watchdog:` image using `docker-compose.build.yaml`, and verify it contains all required Python modules. Keep the previous versioned image for rollback; update `.env`'s `WATCHDOG_VERSION` only when deploying the verified new image. For offline deployments, distribute the new `VERSION` and matching `watchdog.tar`, then use `install.sh`. See [Deployment Steps](#3-deployment-steps), [Image Rollback](#52-roll-back-an-image-upgrade), and [README Version History](README.md#10-version-history). - ---- - -#### Upgrade Docker Image Only - -Automatic upgrades are **not enabled**. Preserve the existing versioned `watchdog:` image for [rollback](#52-roll-back-an-image-upgrade). Only if the old image exists solely as `watchdog:latest`, save that image as `watchdog:rollback` before loading the replacement: - -```bash -docker tag watchdog:latest watchdog:rollback -``` - -There are two ways to upgrade the image — online or offline. Follow only one of the options. - -##### Option A: Online Image Upgrade - -If a new versioned image has been published to the image registry configured for this deployment, pull that tag. A local-only `watchdog:` image is not available via `docker compose pull`; build it or load the release archive instead: - -```bash -docker compose pull -docker compose up -d -``` - -##### Option B: Offline Image Upgrade - -1. Request the Radware RE team to create an updated pre-built Docker image for offline installation. See [8.4 Support Contacts](#84-support-contacts). -2. Load the image. The archive carries the release tag `watchdog:` matching the package's `VERSION` file. - - ```bash - docker load -i watchdog.tar - sed -i "s/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=$(cat VERSION)/" .env - docker compose up -d - ``` - - Running `bash install.sh` performs all three steps (and verifies the result) automatically. - ---- - -#### Checking the Installed Version - -View the image used by the running container: - -```bash -docker inspect docker-container-watchdog --format='{{.Config.Image}}' -``` - -View locally available Watchdog images: - -```bash -docker images watchdog -``` - ---- - -### 6.4 Uninstalling - -Run `uninstall.sh` from the `Container_Alert/` directory. Choose the option that matches how much you want removed: - -| Scenario | Container removed | Image removed | Logs removed | Command | -|---|:---:|:---:|:---:|---| -| Standard (container removed; image + logs prompted) | ✓ | Prompted | Prompted | `bash uninstall.sh` | -| Remove everything without prompts | ✓ | ✓ | ✓ | `bash uninstall.sh --remove-all --force` | -| Remove container, keep image + logs | ✓ | ✗ | ✗ | `bash uninstall.sh --keep-image --keep-logs` | -| Remove container + image, keep logs | ✓ | ✓ | ✗ | `bash uninstall.sh --keep-logs --force` | - -#### Standard interactive uninstall - -```bash -bash uninstall.sh -``` - -Shows a removal plan (container name, image size, log directory size), asks for confirmation, then prompts whether to delete log files. Safe default for manual runs. - -#### Remove container and image, preserve logs - -```bash -bash uninstall.sh --keep-logs -``` - -Stops and removes the `watchdog` container and the `watchdog:` and `watchdog:latest` Docker image tags. The `./watchdog/` log directory is left intact so you can review historical logs later. - -#### Remove everything including logs - -```bash -bash uninstall.sh --remove-all -``` - -Removes the container, image, and the entire `./watchdog/` log directory. Prompts for confirmation before proceeding. - -#### Fully non-interactive - -```bash -bash uninstall.sh --remove-all --force -``` - -Skips all confirmation prompts. Use in automated pipelines or when scripting repeated clean installs. - -> `--keep-logs` and `--remove-all` are mutually exclusive and cannot be combined. - ---- - -## 7. Troubleshooting - -### Service Fails to Start - -```bash -docker compose logs docker-container-watchdog -``` - -Common causes: -- `/var/run/docker.sock` is not accessible — ensure the host socket exists and Docker is running. Manual installs run the container as root, so no host-side group configuration is required; if you opted into non-root hardening (`WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env`), verify `DOCKER_GID` matches `stat -c '%g' /var/run/docker.sock`. -- Missing `.env` file — run `cp .env.example .env` and fill in values - -### Alert Notifications Not Received - -1. Verify `SLACK_WEBHOOK_URL` is set correctly in `.env` -2. Test the webhook manually: - ```bash - curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"Watchdog test"}' \ - "$SLACK_WEBHOOK_URL" - ``` - Expected response: `ok` - -### Duplicate Alert Notifications - -Increase `cooldown_minutes` in `watchdog-config.yaml`. The default is 5 minutes per container per failure type. - -### Excessive Alert Volume - -Add noisy containers to `excluded_containers` in `watchdog-config.yaml`: - -```yaml -excluded_containers: - - debug-shell - - load-test - - my-noisy-container -``` - -If none of the above resolves the issue, see [5. Rollback](#5-rollback) to back out a recent change and redeploy from a known-good state. - ---- - -## 8. Appendix and Reference - -### 8.1 Repository Structure - -``` -Container_Alert/ -├── docker-compose.yaml # Production stack — uses pre-built image -├── docker-compose.build.yaml # Developer build stack — requires internet -├── Dockerfile # Watchdog image definition -├── requirements-watchdog.txt # Python deps: docker, requests, PyYAML -├── watchdog.py # Watchdog agent -├── watchdog-config.yaml # Watchdog behaviour & channel settings -├── install.sh # Install helper script -├── uninstall.sh # Uninstall helper script -├── README.md # Quick-start reference -└── .env.example # Template — copy to .env and fill in -``` - ---- - -### 8.2 Configuration Reference - -All settings are in `watchdog-config.yaml`. No image rebuild is required for configuration changes — restart the watchdog container to apply updates: - -```bash -docker compose restart docker-container-watchdog -``` - -| Setting | Default | Description | -|---|---|---| -| `check_interval_seconds` | `60` | How often to poll all containers | -| `cooldown_minutes` | `5` | Suppress duplicate alerts per container | -| `restart_threshold` | `5` | Restarts within the window to trigger restart-loop alert | -| `restart_window_minutes` | `10` | Rolling window for restart counting | -| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy polls before alerting | -| `alert_on_recovery` | `true` | Send an INFO "recovered" alert once a previously-alarmed container returns to normal | -| `excluded_containers` | `[]` | Containers that are never alerted on | -| `log_level` | `INFO` | Logging verbosity: `INFO` (recommended for production) or `DEBUG` | - -#### Host Identification in Alerts - -Set `WATCHDOG_HOST` in `.env` (written by `install.sh` from the hostname prompt), then run `docker compose up -d`: - -```bash -WATCHDOG_HOST=my-server-name -``` - -When it is unset, `docker-compose.yaml` falls back to `CyberController-Server`. - ---- - -### 8.3 SNMP Trap Var-Binds - -Configuration for the SNMP Traps channel is in [2.4.4 SNMP Traps](#244-snmp-traps). - -**Trap var-binds (enterprise OID arc `1.3.6.1.4.1.89.110`):** - -| OID | Object | Value | -|-----|--------|-------| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Hostname of the alerting node | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Human-readable alert summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Probe failure detail | - -**HA trap mapping:** this release uses the existing Container Watchdog MIB. -For HA, cwContainerName is (HA cluster), cwFailureType is ha-failover, and the -summary/detail fields contain previous and new active nodes. NMS rules should -match cwFailureType instead of treating all traps as container failures. No -new trap OID is introduced. - -**SNMPv3 notes:** - -- **Engine ID:** On the first run the watchdog logs an auto-stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...` in the logs). Copy it into `v3_local_engine_id` and register it in `snmptrapd.conf` with `createUser -e 0x watchdog-user SHA "..." AES "..."`. This keeps the ID stable across container restarts. -- **Key length:** Both auth and priv passphrases must be at least 8 characters (RFC 3414 §11.2). The watchdog rejects shorter keys with a clear error. -- **No privacy-only mode:** Setting a priv key without an auth key is rejected — SNMPv3 has no `privNoAuth` security level. Both keys must be set together for encrypted traps. -- **Protocol names:** Auth: `MD5`, `SHA`, `SHA256`, `SHA384`, `SHA512`. Priv: `AES` (AES-128, recommended), `AES192`, `AES256`, `DES` (weak — logs a warning). An unrecognised name is rejected with an error listing the supported values. - ---- - -### 8.4 Support Contacts - -If you have trouble accessing the `watchdog.tar` download link, or need an updated offline image from the Radware RE team, contact: - -- rahulku@radware.com -- Egore@radware.com -- northamericare@radware.com diff --git a/Dockerfile b/Dockerfile index d906b30..2d6ec32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,10 @@ FROM python:3.11-slim WORKDIR /app +# Version is set by the release build and checked by install.sh. +ARG WATCHDOG_VERSION=unknown +LABEL org.opencontainers.image.version="$WATCHDOG_VERSION" + # Install dependencies RUN pip install --no-cache-dir \ docker==7.1.0 \ diff --git a/README.md b/README.md index 67593ee..0228ed3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitoring service that continuously tracks the health of all Docker containers on the host. It detects failures — including crashes, out-of-memory kills, prolonged unhealthy states, and restart loops — and dispatches real-time alerts through one or more configurable channels (**Slack**, **SMTP**, **SNMP Traps**, or **Syslog**). It also sends a one-time INFO **recovered** alert through the same channels once a previously-alarmed container returns to normal operation. +**Documentation:** [Customer installation and operations](docs/operations/INSTALLATION_AND_OPERATIONS.md) · [v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). + +**For maintainers (source repository):** [Development and release workflow](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md). Historical design, test reports, and CI procedures live under `docs/dev/` and are **not included** in customer installation packages. + --- ## Table of Contents @@ -12,27 +16,12 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitori - [Image Size](#image-size) - [Python Dependencies](#python-dependencies) 3. [Configuration](#3-configuration) - - [Configuration Keys](#configuration-keys) - - [Environment Variables](#environment-variables) 4. [Supported Alert Channels](#4-supported-alert-channels) - - [Slack](#slack) - - [Syslog](#syslog) - - [SMTP (Email)](#smtp-email) - - [SNMP Traps](#snmp-traps) 5. [Deployment Instructions](#5-deployment-instructions) 6. [Failure Types & Severities](#6-failure-types--severities) -7. [Testing](#7-testing) - - [OOM Simulation](#oom-simulation) - - [Probe Testing](#probe-testing) - - [Recovery Alert Test](#recovery-alert-test) +7. [Verify Watchdog Operation](#7-verify-watchdog-operation) 8. [Troubleshooting](#8-troubleshooting) - - [Service Fails to Start](#service-fails-to-start) - - [Alert Notifications Not Received](#alert-notifications-not-received) - - [Duplicate and Excessive Alert Notifications](#duplicate-and-excessive-alert-notifications) - - [Probe Type Not as Expected](#probe-type-not-as-expected) - - [Viewing Logs](#viewing-logs) -9. [Container Probe Map (This Deployment)](#9-container-probe-map-this-deployment) -10. [Version History](#10-version-history) +9. [Version History](#9-version-history) --- @@ -79,11 +68,11 @@ existing Docker container health checks. Enable the following block on **both** HA nodes, after verifying that each node has a local Pacemaker HA cluster exporter: -~~~yaml +```yaml ha_monitor: enabled: true exporter_url: http://127.0.0.1:9664/metrics -~~~ +``` The watchdog reads HA cluster resource metrics and validates local-node identity, cluster quorum, and Pacemaker scrape success. The first valid reading @@ -124,7 +113,7 @@ CyberController services have recovered. The exporter may use port **9664** even when the built-in CyberController Prometheus configuration references **9002**. This optional check reads the exporter directly and makes **no changes** to native Prometheus configuration. -See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). +See [customer HA configuration and expected behavior](docs/operations/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). --- @@ -132,19 +121,24 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ### Project Structure +The following is the **source repository** layout, not the customer archive. The published package includes the installer, runtime Compose file, version, templates, customer documentation, and prebuilt `watchdog.tar` — **not** Python source, Dockerfile, or engineering reports. + ``` . ├── watchdog.py # Main monitor script ├── lifecycle.py # Bounded Docker stop/die evidence correlation -├── watchdog-config.yaml # All configuration (no secrets) +├── watchdog-config.yaml.example # Configuration template (active file is generated) ├── docker-compose.yaml # Production / offline runtime (image only) -├── docker-compose.build.yaml # Developer / online build +├── docker-compose.build.yaml # Developer-only source build ├── Dockerfile # Image build definition (python:3.11-slim) ├── requirements-watchdog.txt # Python dependencies (reference; Dockerfile pip-installs inline) -├── install.sh # Offline install script +├── install.sh # Guided installation from complete release package ├── uninstall.sh # Stop + remove script +├── scripts/package_release.sh # CI release-package assembly (maintainers only) +├── docs/operations/ # Installation/operations and release notes +├── docs/dev/ # Development, release, HA and test evidence ├── VERSION # Release version — drives the image tag and installer -├── watchdog.tar # Pre-built Docker image (provided, no internet needed) +├── watchdog.tar # Prebuilt image only in published package └── .env # Secrets — NOT committed to git ``` @@ -156,13 +150,13 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). |------|------| | Docker image (`watchdog:`) | ~180 MB (python:3.11-slim base + dependencies) | | Running container (memory) | ~50–80 MB baseline; capped at `mem_limit: 256m` in `docker-compose.yaml` | -| `watchdog.tar` export | ~170 MB | +| `watchdog.tar` export | Approximate; inspect the actual published release | -> **Note:** The values above are approximate and may vary depending on the host operating system, Docker version, and installed dependencies. If usage approaches the 256 MB limit (check with `docker stats docker-container-watchdog`), raise `mem_limit`/`mem_reservation` in `docker-compose.yaml` rather than removing the limit — and raise `memswap_limit` to at least the new `mem_limit` in **both** `docker-compose.yaml` and `docker-compose.build.yaml`, since Docker rejects a config where `memswap_limit` is lower than `mem_limit`. +> **Note:** Size and memory figures above are prior estimates, not guaranteed release specifications. Actual image size varies by build. If resource usage is abnormal, collect Docker health, logs and resource metrics. Do not change resource limits based only on a monitoring alert. ### Container Hardening -By default the container runs as **root** so `docker compose up` works with zero configuration — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [DEPLOYMENT.md](DEPLOYMENT.md#22-configure-environment-variables-env)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. +When running Compose manually without installer-generated `.env`, the default container user is **root** — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/operations/INSTALLATION_AND_OPERATIONS.md#41-configuration-files-and-editing)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. ### Python Dependencies @@ -172,6 +166,7 @@ docker==7.1.0 requests==2.32.3 PyYAML==6.0.2 pysnmp>=6.2 +cryptography>=42.0 ``` @@ -183,134 +178,44 @@ pysnmp>=6.2 ## 3. Configuration -All non-secret settings live in `watchdog-config.yaml`. Secrets (webhook URLs, passwords) are stored in `.env` and never committed to version control. Restart the container after editing either file — no image rebuild is needed. - -### Configuration Keys - -| Key | Default | Description | -|-----|---------|-------------| -| `alert_channels` | `["slack"]` | Active destinations: `slack`, `smtp`, `snmp_trap` | -| `check_interval_seconds` | `60` | How often the poll loop runs | -| `cooldown_minutes` | `5` | Suppress duplicate alerts per container per failure type | -| `restart_threshold` | `5` | Restart count within window that triggers a restart-loop alert | -| `restart_window_minutes` | `10` | Rolling window for restart counting | -| `restart_recovery_stability_seconds` | `120` | Minimum uninterrupted running time before a recovered alert for automatically restarted crash/OOM/restart-loop incidents | -| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy cycles before alerting | -| `alert_on_recovery` | `true` | Send an INFO "recovered" alert once a previously-alarmed container returns to normal | -| `excluded_containers` | `[]` | Container names to never alert on | -| `ignored_container_events` | MariaDB syntax-check label rule | Label-based event suppressions for intentional temporary containers | -| `auto_health_check` | enabled | Probe containers reporting `health=none` by discovering an HTTP/TCP endpoint | -| `container_health_checks` | `{}` | Per-container probe overrides (`type: http` or `type: exec`) for `health=none` containers; takes precedence over `auto_health_check` | -| `log_level` | `INFO` | `DEBUG` / `INFO` / `WARNING` / `ERROR` | -| `log_file` | `/var/log/watchdog/watchdog.log` | Bind-mounted to `./watchdog/watchdog.log` on host. Rotates at 10 MB, 5 backups. Set to `null` to disable | -| `state_file` | `/var/log/watchdog/state.json` | Persists open alerts and the host boot time so recovery and reboot reporting survive a restart. Set to `""` to disable | -| `boot_grace_seconds` | `180` | After a detected host reboot, hold alerts this long while services start, then send one `reboot-summary`. `0` alerts immediately and sends the summary on the first poll | -| `runbook_base_url` | — | URL included in every alert | - -Preferred suppression: Temporary MariaDB HA syntax-check containers should be created with Docker label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The watchdog ignores only configured events for containers carrying that explicit label; normal service containers still alert. - -Temporary fallback: until that label exists, dynamic MariaDB dump helper containers are not alerted as crashes only when all identifying signals match together: image `kvision_infra_mariadb`, container name is not `config_kvision-infra-mariadb_1`, Docker `AutoRemove` is `true`, a mount contains `/mnt/cli/tmp/mariadb_dump.sql.gz`, and the exit code is `137`. The primary MariaDB container remains monitored. - -### Environment Variables - -| Variable | Required | Description | -|----------|----------|-------------| -| `SLACK_WEBHOOK_URL` | Yes (if Slack enabled) | Slack incoming webhook URL | -| `SMTP_USERNAME` | Yes (if SMTP enabled) | SMTP username or login email used for login | -| `SMTP_PASSWORD` | Yes (if SMTP enabled) | SMTP account password or api key value | -| `SNMP_V3_AUTH_KEY` | Yes (if SNMPv3 enabled) | SNMPv3 authentication passphrase — minimum 8 characters (RFC 3414) | -| `SNMP_V3_PRIV_KEY` | Yes (if SNMPv3 authPriv) | SNMPv3 privacy passphrase — minimum 8 characters; requires auth key also set | -| `WATCHDOG_CONFIG` | No | Path to config file (default: `/etc/watchdog/watchdog-config.yaml`) | -| `WATCHDOG_HOST` | No | Hostname shown in alerts (default: system hostname) | - ---- - -## 4. Supported Alert Channels - -Configure at least one channel and add its identifier to `alert_channels` in `watchdog-config.yaml`. Multiple channels can be active simultaneously. - -### Slack - -Add `slack` to `alert_channels` and set `SLACK_WEBHOOK_URL` in `.env`: - -```yaml -slack: - enabled: true - webhook_url_env: SLACK_WEBHOOK_URL -``` - -### Syslog - -```yaml -syslog: - enabled: true - host: - port: 514 - protocol: udp # udp or tcp - facility: local0 -``` +The guided installer creates `watchdog-config.yaml` (monitoring policy, probes, logging and notification endpoints) and `.env` (credentials and runtime identity). The configuration is preserved on an upgrade unless the operator explicitly chooses to reconfigure it. -### SMTP (Email) +**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](docs/operations/INSTALLATION_AND_OPERATIONS.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. -Add `smtp` to `alert_channels` to enable. +A YAML-only change takes effect after restarting the watchdog. A changed `.env` value requires recreating **only** the watchdog container. Neither action rebuilds Python code into the image. Thresholds, exclusions, custom exec probes, and monitoring suppressions require authorized review; do not use them merely to silence alerts. -```yaml -smtp: - enabled: true - host: smtp.radware.com - port: 587 - sender: noc-alerts@radware.com - username_env: SMTP_USERNAME # env var: SMTP username or app token - recipients: - - ops-team@radware.com - - oncall@radware.com - tls: true - password_env: SMTP_PASSWORD # env var: SMTP password or app token -``` - -### SNMP Traps - -Add `snmp_trap` to `alert_channels` to enable. Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. +--- -```yaml -snmp_trap: - enabled: true - host: 155.1.1.4 # IP or hostname of your SNMP trap receiver - port: 162 # Standard SNMP trap port - version: v2c # SNMP version: v1, v2c, or v3 - community: public # SNMPv1/v2c community string (ignored for v3) - trap_oid: "1.3.6.1.4.1.89.110.0.1" # Radware enterprise container-alert notification OID - # SNMPv3 only — add SNMP_V3_AUTH_KEY / SNMP_V3_PRIV_KEY to .env - # v3_username: watchdog-user - # v3_auth_protocol: SHA # MD5 | SHA | SHA256 | SHA384 | SHA512 - # v3_auth_key_env: SNMP_V3_AUTH_KEY # min 8 chars - # v3_priv_protocol: AES # AES (recommended) | AES192 | AES256 | DES (weak) - # v3_priv_key_env: SNMP_V3_PRIV_KEY # min 8 chars; requires auth key also set - # v3_local_engine_id: "" # hex engine ID pinned from first-run log; see DEPLOYMENT.md -``` +## 4. Supported Alert Channels -Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3.6.1.4.1.89.110`): +| Channel | YAML identifier | Credentials or other setup | +|---|---|---| +| Syslog | `syslog` | Syslog destination, port, protocol and facility | +| SMTP (email) | `smtp` | Mail server, sender, recipients, TLS/auth; credentials in `.env` if applicable | +| SNMP Trap | `snmp_trap` | NMS destination, SNMP version; SNMPv3 keys in `.env` | +| Slack | `slack` | Webhook URL stored in `.env` | -| OID | Object | Value | -|-----|--------|-------| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Hostname of the alerting node | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Human-readable alert summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Probe failure detail | +At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/operations/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [SNMP trap receiver details](docs/operations/INSTALLATION_AND_OPERATIONS.md#snmp-traps) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. --- ## 5. Deployment Instructions -For full deployment instructions — including alert channel setup, offline installation, developer builds, and troubleshooting — see [DEPLOYMENT.md](DEPLOYMENT.md). +Install **one complete versioned package** from a published [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), or transfer the same package from another machine. No Git clone, separate Docker image download, or source build is required on the customer host. -**Quick start (automated):** +Download/transfer both `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (substitute the version you selected). From the directory containing those two files: ```bash +sha256sum --check SHA256SUMS +tar -xzf container-alert-v1.6.0.tar.gz +cd CyberController-Container-Alert bash install.sh ``` +The wizard verifies and loads the included Docker image, asks for the selected notification channels, and deploys **only** the watchdog. **The v1.6.0 release archive becomes downloadable after the release is published.** + +Read the full [Deployment Guide](docs/operations/INSTALLATION_AND_OPERATIONS.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md) and the [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). + --- ## 6. Failure Types & Severities @@ -344,298 +249,39 @@ Every alert includes two probe fields that identify exactly how and why the fail --- -## 7. Testing - -The sections below describe optional manual tests that validate the watchdog's detection and alerting paths. Each creates a temporary Docker container and cleans up after itself. - -### OOM Simulation - -Two options — choose based on whether the host has internet access. - -#### Option A — `polinux/stress` *(requires internet access)* - -> Pulls `polinux/stress` from Docker Hub. - -```bash -# --memory-swap=32m disables swap so the kernel is forced to OOM-kill rather than swap out -# Note: the polinux/stress entrypoint IS the stress binary — do not repeat "stress" in the args -docker run -d --name oom-test --memory=32m --memory-swap=32m polinux/stress \ - --vm 1 --vm-bytes 64M --vm-keep -``` - -```bash -# Clean up -docker rm -f oom-test -docker rmi polinux/stress -``` - -#### Option B — `watchdog:latest` *(no internet required)* - -> Uses the Watchdog image that is already present on the host, avoiding the need to pull the image during deployment. - -```bash -docker run -d --name oom-test --memory=32m --memory-swap=32m watchdog:latest \ - python3 -c "x = b'\xff' * (64 * 1024 * 1024)" -``` - -```bash -# Clean up — container only (keep the watchdog image) -docker rm -f oom-test -``` - -Docker kills the container with exit code `137` and emits an `oom` event. The watchdog captures memory stats and the last 20 log lines, then dispatches a CRITICAL alert — all within seconds of the kill. - -> **If the container exits cleanly (exit code 0) instead of being OOM-killed**, cgroup memory accounting may be disabled on this host. Verify with: -> ```bash -> docker info 2>&1 | grep -i "memory limit" -> ``` -> `WARNING: No memory limit support` means the `--memory` flag is silently ignored. Enable cgroup memory accounting by adding `cgroup_enable=memory swapaccount=1` to `GRUB_CMDLINE_LINUX` in `/etc/default/grub`, then run `sudo update-grub` and reboot. - ---- - -### Probe Testing - -> **Optional — requires internet access.** These tests pull images from Docker Hub and create temporary containers on the host. -> Each section includes full cleanup instructions (container + image). - -#### HTTP Probe - -```bash -# Start a container where every HTTP path returns 503 -# Note: the full server { } block is required — a bare location { } directive is not valid nginx config -docker run -d --name http-probe-test \ - nginx:alpine \ - sh -c "printf 'server { listen 80; location / { return 503; } }' > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" - -# Verify the container is running: -docker ps | grep http-probe-test - -# Wait one poll cycle (60s), then check watchdog detected it: -grep "http-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f http-probe-test -docker rmi nginx:alpine -``` - -During auto-discovery the watchdog sends an HTTP GET to `/-/healthy` on port 80. Nginx returns 503, so the watchdog immediately caches that endpoint as failing and reports unhealthy. After `unhealthy_cycles_threshold` consecutive unhealthy cycles an alert fires with Detection value `HTTP probe failed — error response 503 Service Unavailable, URI http://:80/-/healthy`. - ---- - -#### TCP Connect Probe - -```bash -# Expose port 9999 but run nothing on it — TCP connect will be refused -docker run -d --name tcp-probe-test \ - --expose 9999 \ - alpine \ - sh -c "sleep infinity" - -# Watch the watchdog fall through HTTP → TCP and alert: -grep "tcp-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f tcp-probe-test -docker rmi alpine -``` - -After `unhealthy_cycles_threshold` consecutive failed cycles an alert fires with Detection value `TCP probe failed — connection refused or timed out on :9999`. - ---- - -#### `/proc` Alive Check - -```bash -# No EXPOSE — watchdog falls straight to /proc alive check -docker run -d --name proc-probe-test \ - alpine \ - sleep infinity - -# Verify the container is running: -docker ps | grep proc-probe-test - -# After one poll cycle, confirm the watchdog selected /proc as the probe type: -grep "proc-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f proc-probe-test -docker rmi alpine -``` +## 7. Verify Watchdog Operation -With no exposed ports, auto-discovery skips HTTP and TCP and falls back to reading `/proc/1/status` inside the container via `exec_run`. Since `sleep` is PID 1 in a live `S` (sleeping) state, no alert fires. The log will show `falling back to /proc alive-check` confirming the probe type was selected. In production, this probe fires when PID 1 enters zombie (`Z`) or stopped (`T`) state. - ---- - -### Recovery Alert Test - -Validates the INFO `recovered` alert fires once a previously-alarmed container is healthy again. Use any real, already-monitored container — no throwaway container needed. +On an installed system, use these read-only checks: ```bash -# 1. Force a failure (SIGKILL → non-zero exit code → CRITICAL "crashed" alert) -docker kill - -# 2. Confirm the crashed alert was logged/dispatched -grep "" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log | grep CRITICAL - -# 3. Bring it back -docker start - -# 4. Wait for it to report healthy again, then confirm the recovery alert: -grep "" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log | grep -i recovered +docker ps --filter name=docker-container-watchdog +docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' +docker logs --tail 50 docker-container-watchdog ``` -Expected log line: `ALERT [INFO] — recovered (channels: [...])`. +Expected: the watchdog is running/healthy and monitoring logs are current. Check the configured notification receiver for any existing alerts. A healthy system does not need to generate a new alert just for verification. -- Containers with a Docker `HEALTHCHECK` recover via the `health_status: healthy` **event** — near-instant once Docker reports healthy (subject to the image's own `start_period`/`interval`). -- Containers on HTTP/TCP/`/proc` active probes recover on the **next poll cycle** (`check_interval_seconds`). -- No alert fires if `alert_on_recovery: false` is set, or if the container was never in an alarmed state (`state.alerted_for` empty). - ---- +Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_SMOKE_TESTING.md). ## 8. Troubleshooting -### Service Fails to Start - -```bash -docker compose logs docker-container-watchdog -``` - -Common causes: -- **Missing `.env` file** — run `cp .env.example .env` and fill in credentials -- **Docker socket not accessible** — ensure `/var/run/docker.sock` exists and the container has read access -- **Image not loaded** — run `docker images watchdog`; if empty, set `WATCHDOG_VERSION=$(cat VERSION)` and build with `WATCHDOG_VERSION="$WATCHDOG_VERSION" docker compose -f docker-compose.build.yaml build` (see [DEPLOYMENT.md](DEPLOYMENT.md) for details) - -### Alert Notifications Not Received - -1. Check `alert_channels` in `watchdog-config.yaml` — ensure at least one channel is listed and configured -2. Check watchdog logs for errors: - ```bash - grep -E "ERROR|CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - ``` -3. For Slack: verify `SLACK_WEBHOOK_URL` is set in `.env` and test manually: - ```bash - curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"Watchdog test"}' "$SLACK_WEBHOOK_URL" - ``` - Expected response: `ok` - -**Recovery alert (`recovered`) specifically not received:** -- Confirm `alert_on_recovery` is not set to `false` in the live `watchdog-config.yaml`. -- Confirm the container was actually alarmed first — `state.alerted_for` must be non-empty; a container that was never flagged won't generate a recovery alert. -- For containers with a Docker `HEALTHCHECK`, confirm Docker actually reports `healthy` — `docker inspect --format '{{json .State.Health}}'`. If it's stuck on `starting`, the watchdog is waiting on Docker, not the other way around. -- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; add a manual entry under `container_health_checks` instead. - -### Duplicate and Excessive Alert Notifications - -Increase `cooldown_minutes` in `watchdog-config.yaml` (default: `5` minutes per container per failure type). To silence a noisy container entirely, add it to `excluded_containers`: - -```yaml -excluded_containers: - - debug-shell - - load-test - - my-noisy-container -``` - -### Probe Type Not as Expected +For installation failures, alert delivery, missing recovery, probe selection, +persistent logs, maintenance, upgrades or rollback, use the +[Customer Installation and Operations Guide](docs/operations/INSTALLATION_AND_OPERATIONS.md#6-troubleshooting). +Use read-only status/log checks on operational hosts, and do not simulate +failures on application containers. -Check which probe was cached at startup: - -```bash -grep -E "auto-discover|falling back" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log -``` - -### Viewing Logs - -Log destinations (simultaneous): -- **stdout** — always on; captured by `docker compose logs` -- **file** — `./watchdog/watchdog.log` on the host (bind-mounted), rotates at 10 MB, keeps 5 backups -- **syslog** — remote server if `syslog.enabled: true` in config - -```bash -# Live tail (preferred — host path, no need to exec into the container) -tail -f /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Last 100 lines -tail -100 /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Alerts and errors only -grep -E "ALERT|ERROR|CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Filter by container name -grep "my-container" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Search across rotated logs -grep "CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log \ - /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log.1 +--- -# Via Docker (stdout only) -docker compose logs -f docker-container-watchdog -``` +## 9. Version History ---- +**Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -## 9. Container Probe Map (This Deployment) - -Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitored passively via events; all others get an active probe auto-discovered on the first poll and cached for the container's lifetime. - -| Container | Probe Type | Endpoint | Why | -|-----------|-----------|----------|-----| -| `config_kvision-infra-fluentd_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-dp-inline-config_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-configuration-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-anomaly-detection-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-persist-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-rt-alert_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_policy-editor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-alerts_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-reporter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-vrm_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-redis_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-health_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-webui_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-tor-feed_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-efk_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-polling-mgr_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-formatter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-polling-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-cadvisor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-vdirect_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-help_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-ted_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_postgres_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-snmp-trap-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-auto-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-lls_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-assist-service_1` | HTTP | `:3005` (path auto-discovered) | No Docker HEALTHCHECK; Node.js service; path cached on first successful response | -| `config_kvision-infra-rabbitmq_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-ha-operator_1` | HTTP | `:8080` (path auto-discovered) | No Docker HEALTHCHECK; Java HTTP service; path cached on first successful response | -| `config_kvision-dc-nginx_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `process-exporter` | HTTP | `:9256/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `prometheus` | HTTP | `:9090/-/ready` | No Docker HEALTHCHECK; standard Prometheus readiness endpoint | -| `alertmanager` | HTTP | `:9093/-/healthy` | No Docker HEALTHCHECK; standard Alertmanager health endpoint | -| `postgres-exporter` | HTTP | `:9187/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `mysql-exporter` | HTTP | `:9104/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `es-exporter` | HTTP | `:9114/metrics` | No Docker HEALTHCHECK; port 7979 HTTP fails; port 9114 serves `/metrics` | -| `node-exporter` | HTTP | `:9100/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `grafana` | HTTP | `:3000/api/health` | No Docker HEALTHCHECK; Grafana health endpoint returns 200 | -| `config_kvision-infra-mariadb_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | - -> **Verify what was actually cached after startup:** -> ```bash -> grep -E "auto-discover|falling back" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log -> ``` - -## 10. Version History - -**Release status (2026-10-14):** `VERSION` remains `1.5.4`. The changes below are merged into `main` but **unreleased**: the running `watchdog:1.5.4` image does not acquire them from `git pull`. - -**Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. +**Release procedure:** Merge reviewed release changes into `main`, then push the matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md); v1.6.0 publication checks are in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). | Version | Date | Author | Changes | |---------|------------|--------|---------| -| Unreleased (post-1.5.4) | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts. | +| 1.6.0 | 2026-10-15 | — | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, revised informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | @@ -644,7 +290,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor | 1.4.0 | 2026-08-17 | Rahul Kumar | Added INFO "recovered" alert | | 1.3.3 | 2026-08-05 | Rahul Kumar | Added Auth True/false | | 1.3.2 | 2026-07-28 | Rahul Kumar | Fixed SMTP Auth issue | -| 1.3.1 | 2026-07-21 | Egor Egorov | Updated Readme and Deployment guides | +| 1.3.1 | 2026-07-21 | — | Updated Readme and Deployment guides | | 1.3.0 | 2026-07-16 | Rahul Kumar | Added Upgrade Guide | | 1.2.0 | 2026-07-16 | Rahul Kumar | Removed sudo and renamed the container | | 1.1.0 | 2026-07-09 | Rahul Kumar | Added SNMPv3 support | diff --git a/VERSION b/VERSION index 94fe62c..dc1e644 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.4 +1.6.0 diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index c2fe83e..a181e24 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -1,22 +1,15 @@ -# ── DEVELOPER / ONLINE BUILD ────────────────────────────────────────────────── -# Use this file when you have internet access and want to build a fresh image. -# -# Usage: -# export WATCHDOG_VERSION=$(cat VERSION) -# docker compose -f docker-compose.build.yaml build -# docker compose -f docker-compose.build.yaml up -d -# -# After a successful build, export the image for offline customers — always -# ship the versioned tag so install.sh can pin the deployment to it: -# docker tag watchdog:"$WATCHDOG_VERSION" watchdog:latest -# docker save watchdog:"$WATCHDOG_VERSION" watchdog:latest -o watchdog.tar -# ───────────────────────────────────────────────────────────────────────────── +# DEVELOPER-ONLY SOURCE BUILD (not used during customer installation). +# export WATCHDOG_VERSION="$(cat VERSION)" +# docker compose -f docker-compose.build.yaml build +# Official offline bundles and images are built by the tagged release workflow. services: docker-container-watchdog: build: context: . + args: + WATCHDOG_VERSION: "$WATCHDOG_VERSION" image: watchdog:${WATCHDOG_VERSION:?Set WATCHDOG_VERSION from the VERSION file before building} container_name: docker-container-watchdog restart: always diff --git a/docker-compose.yaml b/docker-compose.yaml index 9ac162d..cc33cd5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,15 +1,7 @@ -# ── PRODUCTION / OFFLINE RUNTIME ───────────────────────────────────────────── -# Uses a pre-built image. No internet access required. -# The image must be loaded first: -# docker load -i watchdog.tar -# -# Usage: -# cp .env.example .env # fill in SLACK_WEBHOOK_URL and any other channel secrets -# docker compose up -d -# -# To build a new image (requires internet): -# docker compose -f docker-compose.build.yaml build -# ───────────────────────────────────────────────────────────────────────────── +# PRODUCTION RUNTIME — use one verified, prebuilt versioned Docker image. +# The complete customer release archive includes watchdog.tar and its checksum. +# install.sh verifies/loads that image; customers do not build source or manually load images. +# For authorized manual operations: docker compose up -d --no-deps docker-container-watchdog services: diff --git a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md b/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md deleted file mode 100644 index 8d3b261..0000000 --- a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md +++ /dev/null @@ -1,173 +0,0 @@ -# Optional CyberController HA Active-Node Alert — Design Proposal - -**Status:** Feature under review; manual switchover verified, failback and alert delivery pending -**Date:** 2026-10-09 -**Target:** Radware/CyberController-Container-Alert, baseline v1.5.4 (main: a4ddfed) -**Delivery:** One small implementation PR, with release blocked on two-node QA - -## 1. Goal and decision - -Customer request: alert when CyberController HA changes its active node, including a manual switchover, automatic failover, or failback. The alert should identify the old and new active nodes, not speculate about the cause. - -**Decision:** Add one opt-in HA role-change check to the existing Container Watchdog. Read the existing HA exporter directly over host-local HTTP. Reuse the watchdog's polling interval, persistent state, and existing syslog/SMTP/SNMP/Slack delivery. Do not introduce another daemon, Docker container, metric exporter, firewall port, or Pacemaker control capability. - -This is an **observed active-node change** alert, not a claim to identify why a failover happened or that all CyberController applications are healthy. - -## 2. Verified appliance evidence (read-only inspection) - -On the two-node HA environment ha-test-admin-ea-iad1, Pacemaker reports nodes 10.199.0.241 and 10.199.0.242 online. The HA Operator, OpenSearch, PostgreSQL, MariaDB, and Failback promotable resources are currently **Promoted on .241** and **Unpromoted on .242**. The Pacemaker DC is .242; **DC is not the CyberController active node**. - -- HA exporter: **ha_cluster_exporter.service**, active on .241, serving port **9664** and reachable at **http://127.0.0.1:9664/metrics** (HTTP 200). -- Promoted role metric: **ha_cluster_pacemaker_resources** includes resource=HAOperatorRep-Res, role=promoted, status=active, node=10.199.0.241, value 1. -- Host identity metric: **ha_cluster_corosync_member_votes** includes local=true, node=10.199.0.241. -- Data-quality signals: **ha_cluster_scrape_success{collector="pacemaker"} = 1** and **ha_cluster_corosync_quorate = 1**. -- Existing Watchdog v1.5.4 runs on .241 in **host network** mode, polls every 60 seconds, has persistent state, and currently delivers syslog. -- Prometheus's native HA target is misconfigured: it scrapes **localhost:9002**, while the exporter serves **9664**. The existing rule **ha_failover_occured** compares per-node location-constraint sum vs average and is not reliable for an actual role change. Correcting the native configuration is a **separate Radware/R&D action**, not part of this feature. -- Standby .242 did **not** accept cross-node connections to 9664/9099/9093. It has not been inspected locally, so its exporter/watchdog installation is **unverified**. Release testing must cover both nodes. - -## 3. Small architecture - -~~~text -Per CyberController node: - Existing ha_cluster_exporter :9664/metrics (localhost only) - | - | bounded HTTP GET from independent HA polling thread (60s) - v - Optional HA role check: - identify local node + unique promoted HA Operator - validate quorum / scrape health - compare with persisted last confirmed active node - confirm role change on 2 consecutive successful polls - | - | only the newly promoted node sends - v - Existing watchdog alert dispatcher - syslog / SMTP / SNMP / Slack - -Both HA nodes must run the watchdog and have their own local exporter. -No inter-node polling or new firewall opening is required. -~~~ - -The HA check runs in its own lightweight thread with a dedicated HTTP session. Slow/blocking Docker probes cannot delay HA polling, and failed HA reads cannot break Docker monitoring. This is a single additional thread within the existing process, not a new service, container, queue, or worker framework. - -### Signal selection - -**Active owner:** accept exactly one node for which the HA exporter reports: - -~~~text -ha_cluster_pacemaker_resources{ - resource="HAOperatorRep-Res", - role="promoted", - status="active", - node="" -} 1 -~~~ - -The check must not use the Pacemaker DC, host virtual IP alone, or location-constraint scores as evidence of an actual promotion. This resource was verified in the current CC version; release testing must confirm the resource name is consistent on the targeted CC versions. - -**Sender identity:** derive the local node from the exporter's **ha_cluster_corosync_member_votes{local="true",node=...}** series. Do not guess local identity from WATCHDOG_HOST, container hostname, or the Prometheus instance label. - -**Validity:** require a successful HTTP response, valid/parseable metrics, successful Pacemaker collection, quorum present, exactly one promoted HA Operator resource, and a unique local-node identity. Otherwise retain previous state, log a bounded diagnostic warning, and do not announce a completed failover. - -### Proposed configuration (disabled by default) - -~~~yaml -ha_monitor: - enabled: false - exporter_url: http://127.0.0.1:9664/metrics -~~~ - -Inherit **check_interval_seconds** (60 seconds in current deployments). Keep the implementation constants small: HTTP timeout 3 seconds, a bounded response size, and two consecutive good polls to confirm a new owner. Avoid extra user-configurable knobs unless validation demonstrates a need. - -Use only the local exporter in v1. No firewall rule, container exec, host bind mount, database query, REST credential, or Pacemaker write command is required. - -## 4. State and alert behavior - -Retain a **small optional ha_monitor section in the existing state.json** (backward-compatible with current container alert state). Track the last confirmed active node and the last notified transition. This state must survive watchdog restart, use existing atomic/serialized persistence, and must not be treated as a Docker container entry. - -| Observation | Required behavior | -|---|---| -| Disabled, or no HA configured | Existing watchdog behavior unchanged; no HA requests | -| First valid poll without prior HA state | Record baseline silently; no historical or startup alert | -| Same active node | Refresh/retain baseline; no notification | -| Candidate active node changes A -> B | Confirm on next good poll (two consecutive observations) | -| Confirmed A -> B | Update persistent baseline; **only watchdog on B** sends one WARNING | -| Confirmed B -> A (failback) | Same behavior; notify once from A | -| Watchdog restarted while HA stays enabled | Restore confirmed owner; do not repeat previous transition | -| HA intentionally disabled and watchdog restarted | Clear saved HA owner; re-enable establishes a silent new baseline | -| Exporter timeout, 404/500, invalid/incomplete series, lost quorum, zero/multiple promoted owners | Do not change baseline or claim a completed failover; log a rate-limited local warning | -| Watchdog newly installed after the transition | Baseline silently; historical detection not guaranteed | -| Host reboot/grace window | Respect existing boot-grace behavior; do not mix HA alerts into the container reboot-summary. Reconfirm role after grace | -| Both nodes running watchdog | Both can observe role state, but only the newly promoted node emits the role-change notification | - -**Operator notification** (example, not an observed event): - -~~~text -[WARNING] CyberController HA active node changed -Previous active: 10.199.0.241 -New active: 10.199.0.242 -Observed at: -Evidence: HAOperatorRep-Res promoted on 10.199.0.242 (Pacemaker exporter) -Next step: Confirm expected switchover and application health. -~~~ - -Treat this as an event, **not a persistent container failure**: do not send a spurious "container recovered" alert. Keep the normal container alert subject/fields unchanged. Make only the minimum HA-specific display adjustment needed so email/Slack/syslog/SNMP do not misleadingly label it as a crashed container. Preserve existing notification formats for container alerts. - -**Delivery semantics (accepted v1 limitation):** one best-effort notification per confirmed observed transition, with state-based deduplication across restarts. The confirmed owner is persisted before dispatch; a delivery failure is logged but is not automatically retried. External syslog/SNMP delivery acknowledgement and exactly-once semantics are not available; adding a persisted retry queue would require changing the existing dispatcher contract and is deferred. A failed state-file save can lead to a duplicate after restart. Verify alert routing on both nodes. Detection time is an observation timestamp, not necessarily the exact Pacemaker event time. - -**SNMP compatibility (accepted v1 mapping):** the existing Container Watchdog MIB is reused. For HA notifications, cwContainerName is `(HA cluster)`, cwFailureType is `ha-failover`, and cwSummary/cwProbeDetail contain the old/new active nodes. An NMS should distinguish the HA event by cwFailureType; no new trap identity or OIDs are introduced in this release. - -## 5. Implementation scope — one development PR, after approval - -| Existing component | Small change | -|---|---| -| **watchdog.py / DEFAULT_CONFIG** | Optional ha_monitor block, disabled by default | -| **watchdog.py / Watchdog.start, _ha_poll_loop** | Start independent HA polling thread with its own HTTP session and stop event; Docker polling remains unchanged | -| **HA reader/transition helper** | Bounded local HTTP read; parse only required metrics; validate; confirm active change; deduplicate | -| **WatchdogState persistence** | Optional HA baseline and notified transition, without breaking existing v1 state files or container recovery | -| **AlertPayload/dispatch_alert** | Reuse channels; ensure HA event is clearly labelled and not folded into container reboot/recovery logic | -| **watchdog-config.yaml.example, README.md** | Add brief opt-in example, behavior, limitations, and both-node setup requirements | -| **tests/test_ha_monitor.py + existing regression suite** | Unit tests, mocks, notification checks | - -No changes to **install.sh**, Compose files, Python dependencies, port mappings, Docker privileges, Pacemaker services, or Prometheus are expected. If validation reveals a real need for one, bring it back for design review rather than expanding scope silently. - -## 6. Tests and Definition of Done (single implementation increment) - -### Automated tests (CI, no live HA action) - -- [ ] Feature disabled by default: zero exporter requests and no changes to existing container alerts. -- [ ] Parse metrics with different label orders; select only HAOperatorRep-Res that is active and promoted (value 1), and identify local=true node. -- [ ] Initial valid sample establishes baseline without alert. -- [ ] Stable A -> B across two polls sends **one** HA alert from B; the old node A sends none. -- [ ] B -> A produces one new notification; stable subsequent polls do not repeat. -- [ ] Duplicate/wrong-role resource series, missing local identity, missing Pacemaker-success metric, lost quorum, and zero/multiple promoted owners never create false failover alerts. -- [ ] HTTP timeout, connection refused, malformed metrics and empty response are bounded, leave baseline intact, and do not prevent Docker polling. -- [ ] Persist and restore role and notification deduplication; existing state v1 loads unchanged, and concurrent state writes remain valid. -- [ ] Restart/reboot/grace-window behavior does not generate stale duplicate alerts or contaminate the container reboot-summary. -- [ ] Notification shows previous/new node and UTC observation, is formatted correctly for the configured channels, and does not report fake container failure/recovery. -- [ ] Existing regression tests, Python syntax checks, and Compose/CI checks pass with no new dependencies. - -### Controlled two-node validation (manual QA, authorized window) - -- [ ] Watchdog and local HA exporter are **running and reachable locally on both nodes**; active role is visible from both local metric endpoints. -- [ ] Confirm normal steady state emits no failover alerts; initial installation/restart remains quiet. -- [ ] With an **explicitly authorized controlled switchover**, verify one notification from newly active node with correct A/B details and no duplicate from old active. -- [ ] Controlled failback produces one opposite-direction notification. -- [ ] Verify a loss-of-active-node scenario, or an approved equivalent fault simulation, demonstrates the surviving node can alert without the original node being reachable. -- [ ] Temporarily unavailable local exporter (test fixture or approved fault scenario) does not produce false failover alerts and Docker monitoring continues. -- [ ] Verify syslog end-to-end; test other configured alert channels as applicable. -- [ ] Observe bounded CPU/memory/HTTP impact and document expected limitations, rollback, and compatibility. - -### Definition of Done - -The implementation is ready for review/release when all automated tests and CI pass, controlled two-node role-change and failback tests produce the expected one-per-transition notifications, an active-node loss is covered by the surviving watchdog, resource use remains within current watchdog limits, documentation shows how to enable/disable the option on both nodes, and existing non-HA functionality is unchanged. No HA commands that modify cluster state or customer production services are introduced. - -## 7. Release gates, risks, and intentional exclusions - -**Before release:** confirm both HA nodes can send notifications end-to-end. Both watchdogs and local exporters were confirmed running during the manual .242 -> .241 test. The standby's earlier *remote* exporter connection refusal reflected an inaccessible cross-node port, not a missing local service. No inter-node exporter access is required. - -**Known limitations:** If the surviving node has no running watchdog/exporter or has never recorded the original active node, this check cannot reconstruct the failover after the fact. Polling can miss a transition that changes twice between samples. Metrics cannot distinguish planned switchover vs unexpected failover, and promotion does not guarantee the entire application is healthy. - -**Out of scope:** HA orchestration/control, restarting or promoting resources, automatic remediation, logging every Pacemaker transition, live log parsing, new monitoring dashboards/exporters, historical HA event reconstruction, generalized multi-service health monitoring, and remediation of the native Prometheus 9002 vs 9664 mismatch. The latter should be handled as a separate CyberController native monitoring correction. - -**Implementation status:** Approved by the user on 2026-10-09 and implemented on the feature branch. Do not merge or release until the two-node acceptance gates in [HA_VALIDATION.md](HA_VALIDATION.md) are satisfied. diff --git a/docs/HA_VALIDATION.md b/docs/HA_VALIDATION.md deleted file mode 100644 index 1682f8f..0000000 --- a/docs/HA_VALIDATION.md +++ /dev/null @@ -1,39 +0,0 @@ -# Optional HA active-node alert — validation status - -**Branch:** feature/optional-ha-failover-alert -**Status:** Draft PR #4; tested in the two-node lab, not approved for merge -**Updated:** 2026-10-13 - -## Automated tests and build - -- **26 tests pass** after the PR review fixes. Added regression coverage for an HA thread running while the Docker poll is blocked and raises, and disable -> role change -> re-enable with a quiet baseline. -- The new HA thread owns its HTTP session, uses the existing stop event and global polling interval, and does not change Docker health checks or require new dependencies. -- Earlier PR #4 CI passed, including the clean Docker build. **CI must pass again on the updated commit** before merging. -- The existing state.json v1 format and Docker container alert state are retained. - -## Real two-node environment — user-initiated test (2026-10-10 UTC) - -- **Both nodes have HA exporters and watchdog builds running** during the user test, not just the original active node. These installations were done by the user, not by this code review. -- At about 02:53:46 UTC the user initiated a manual switchover on .242, which moved the promoted HA Operator resource to .241. -- .242 logged at **02:55:17**: "HA role changed 10.199.0.242 -> 10.199.0.241; new owner will notify." -- .241 logged at **02:55:37**: one WARNING, "CyberController HA active node changed: 10.199.0.242 -> 10.199.0.241". -- Both nodes stayed online and .241's promoted HA resources became active. The actual switch was detected correctly, without duplicate HA events observed. -- **Actual notification delivery still needs verification.** .241 had syslog configured to 127.0.0.1:514; receiving NMS confirmation was not established. .242 used SMTP, but smtp.radware.com did not resolve, so its container failure messages were not emailed. -- During the planned demotion, the HA Operator explicitly removed vDirect, TED and Assist Service on .242. The watchdog generated separate CRITICAL crash alerts for vDirect and Assist Service. Narrowly scoped suppression is **out of scope** for PR #4 and should have a separate code review. -- A **reverse failback (.241 -> .242)**, end-to-end receiver confirmation, and a safe surviving-node test after failure of the original active node have not been confirmed. - -## Release gates still open - -1. CI passes for the current head, with new concurrency/re-enable tests. -2. Test a controlled **failback** from .241 to .242, verifying a single reverse alert. -3. Verify the configured receiving systems, not just local watchdog logs; resolve .242 SMTP DNS or configure a working syslog route. -4. Validate a supported active-node-loss scenario without deliberately partitioning a cluster where STONITH is disabled. -5. Confirm targeted CC-version compatibility and ordinary watchdog restart/disabled option behavior. - -## Accepted v1 limitations - -- One-shot best-effort dispatch: transition state is persisted **before** notification. Failed SMTP/Slack/SNMP/syslog delivery does not automatically retry. A failed state write may cause a duplicate after restart. Do not add a durable delivery queue to this PR without changing the dispatch API. -- HA SNMP events use the existing Container Watchdog MIB: cwContainerName=(HA cluster), cwFailureType=ha-failover. NMS integrations should classify by failure type. A separate MIB identity is not included in v1. -- Planned HA service removals currently produce container failure alerts. Suppression is a separate, narrowly scoped enhancement. - -**No HA host, Docker service, Pacemaker, firewall, or monitoring configuration changes were made during this code-review update.** diff --git a/docs/dev/DEVELOPMENT_AND_RELEASE.md b/docs/dev/DEVELOPMENT_AND_RELEASE.md new file mode 100644 index 0000000..7556941 --- /dev/null +++ b/docs/dev/DEVELOPMENT_AND_RELEASE.md @@ -0,0 +1,221 @@ +# Container Alert — Development and Release Guide + +**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. To deploy, follow [Installation and Operations Guide](../operations/INSTALLATION_AND_OPERATIONS.md). + +## Table of contents + +1. [What we publish](#1-what-we-publish) +2. [Make and test a change](#2-make-and-test-a-change) +3. [Prepare the next version](#3-prepare-the-next-version) +4. [Review, merge and tag](#4-review-merge-and-tag) +5. [Verify the published package](#5-verify-the-published-package) +6. [Customer delivery and rollback](#6-customer-delivery-and-rollback) +7. [Safety and best practices](#7-safety-and-best-practices) +8. [Developer-only builds and monitoring notes](#8-developer-only-builds-and-monitoring-notes) + +## 1. What we publish + +The source repository contains the Python application, tests, installer and release automation. **Customer hosts get none of the application Python source.** A tagged release builds one versioned image and creates two public release assets: + +| Asset | Contents | +|---|---| +| `container-alert-v.tar.gz` | Installer, VERSION, Compose, templates, operator docs, prebuilt `watchdog.tar` and the image's internal `SHA256SUMS` | +| `SHA256SUMS` | SHA-256 checksum of the complete release archive | + +All versions use the **same workflow, package format and installer**. A release such as `1.6.1` uses `VERSION=1.6.1`, Git tag `v1.6.1`, image `watchdog:1.6.1` and package `container-alert-v1.6.1.tar.gz`. These are examples, not claims that a version has been published. + +Changes in Git do **not** upgrade running customer containers. Publication and installation are separate actions. + +**End-to-end sequence:** change code → open a development PR → review/test → merge the development PR → prepare and review a release PR → **merge the release PR into `main`** → tag the resulting `main` commit → GitHub publishes the package → verify the release. + +If a development PR targets an existing `release/vX.Y.Z` branch, it merges into that branch first. The release PR later brings those changes into `main`. Do not merge the same changes twice. + +## 2. Make and test a change + +**Step 1 — Create a development branch in a clean checkout.** + +```bash +git switch main +git pull --ff-only origin main +git switch -c fix/describe-change +``` + +Use `feature/...`, `fix/...` or `docs/...`. If there is an active stabilization branch for an upcoming release, check whether your PR should target that release branch instead of main. Multiple worktrees are supported. + +**Step 2 — Change code, tests and documentation.** Typical files include `watchdog.py`, `lifecycle.py`, `ha_monitor.py`, `install.sh`, `scripts/package_release.sh`, and `tests/`. Don't reuse a previously published Docker image version for changed Python code. + +**Step 3 — Run non-destructive tests on a development workstation.** + +```bash +bash -n install.sh uninstall.sh scripts/package_release.sh +python3 -m unittest discover -s tests -q +git diff --check +``` + +The Python test environment needs the repo's test dependencies (including Docker SDK, Requests, PyYAML). Installer/package tests use fake Docker. **Never** run real failure-injection tests or `tests/release-smoke.sh` on operational/shared Docker hosts. + +**Step 4 — Commit, push, request review.** + +```bash +git add +git commit -m "fix: describe the change" +git push -u origin fix/describe-change +gh pr create --base main --head fix/describe-change --fill +``` + +Substitute your branch/files. PRs should cover behavior, tests, customer impact and safety. Review results: + +```bash +gh pr view -R Radware/CyberController-Container-Alert +gh pr checks -R Radware/CyberController-Container-Alert +``` + +Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. + +**Step 5 — Merge the development PR.** Once the latest changes have been reviewed and checks pass, open the PR on GitHub, confirm its **base branch** (`main` or the active release branch), and choose **Merge pull request**. GitHub applies any branch protection rules. + +A PR targeting `main` puts its changes directly into `main`; a PR targeting `release/vX.Y.Z` updates that release branch. **Do not merge the release PR at this step**—its final merge happens in [Section 4](#4-review-merge-and-tag), after release preparation and testing. + +## 3. Prepare the next version + +Several changes may ship together. When ready to publish, choose a **new** version (e.g. `1.6.1` for a compatible fix; `1.7.0` for a compatible feature). + +```bash +NEXT_VER=1.6.1 # set the desired NEW version here each time +git switch main +git pull --ff-only origin main +git switch -c "release/v$NEXT_VER" +printf '%s\n' "$NEXT_VER" > VERSION +``` + +Update the following to match the chosen version: + +1. `VERSION` — set the new number (the command above does this). +2. `README.md` — add the matching version-history row; CI checks this. +3. `docs/operations/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. +4. Deployment and examples if operator procedures/settings changed; keep private lab/customer identifiers and secrets out. +5. Tests and hosted smoke results for the release PR. + +```bash +git add VERSION README.md "docs/operations/RELEASE_NOTES_V$NEXT_VER.md" +git commit -m "release: prepare v$NEXT_VER" +git push -u origin "release/v$NEXT_VER" +gh pr create --base main --head "release/v$NEXT_VER" --title "Release v$NEXT_VER" --fill +``` + +Change only `NEXT_VER` at the start of the release commands and use the same shell session, or set it again later. Write actual release notes before staging; the quoted filename in `git add` must exist. Stage all other modified files explicitly. If the release branch already exists, update it and its PR instead of creating a second one. + +**Leave this release PR open** while you review and run CI and isolated smoke. The next step is the final merge into `main`. + +## 4. Review, merge and tag + +**This is where you merge the release PR into `main`—after final validation, before tagging.** + +**Step 1 — Review the release PR.** Confirm its base branch is `main`, the release notes and rollback instructions are ready, and regular CI plus isolated smoke passed on the **latest PR head commit**. GitHub release assets are public in this repository, so check the contents before publishing. + +**Step 2 — Merge the release PR.** On GitHub, open the release PR created in Section 3, review **Files changed** and **Checks**, then choose **Merge pull request** and confirm. You do not need to merge earlier feature PRs again. + +This merges `release/vX.Y.Z` into `main`. **It does not publish the release yet.** + +**Step 3 — Sync your local `main`.** Use a clean checkout; commit, set aside, or finish any uncommitted work before switching branches: + +```bash +git switch main +git pull --ff-only origin main +git status --short # must be empty +cat VERSION # check intended release version +git log -1 --oneline # record merged main commit +``` + +**Step 4 — Check for an existing tag.** The following tag query should print **nothing**: + +```bash +VER="$(cat VERSION)" +git ls-remote --tags origin "refs/tags/v$VER" +``` + +**Step 5 — Create and push the release tag.** Tag the **merged `main` commit**, not the release branch: + +```bash +git tag -a "v$VER" -m "Container Alert v$VER" +git push origin "v$VER" +``` + +The tag triggers [Publish Container Alert release](../../.github/workflows/release-package.yml). Its GitHub-hosted runner checks version/ancestry, runs tests, builds the prebuilt image, creates the package and checksum, uploads both assets as a **draft release**, checks the assets, then publishes. PR merges do not automatically create customer packages. + +## 5. Verify the published package + +Confirm the tagged workflow passed and exactly two assets exist: + +```bash +gh run list -R Radware/CyberController-Container-Alert --workflow release-package.yml --limit 5 +gh release view "v$VER" -R Radware/CyberController-Container-Alert +``` + +Download and inspect **without touching Docker**: + +```bash +mkdir -p "/tmp/container-alert-check-$VER" +cd "/tmp/container-alert-check-$VER" +gh release download "v$VER" -R Radware/CyberController-Container-Alert +sha256sum --check SHA256SUMS +tar -tzf "container-alert-v$VER.tar.gz" | grep -E 'VERSION|install.sh|watchdog.tar|SHA256SUMS' +mkdir -p extracted +tar -xzf "container-alert-v$VER.tar.gz" -C extracted +(cd extracted/CyberController-Container-Alert && sha256sum --check SHA256SUMS) +cat extracted/CyberController-Container-Alert/VERSION +``` + +Record the Git tag/commit, CI/smoke URLs, release link, package digest and final image digest. The tagged build may not be byte-identical to the PR build while image bases/dependencies can float; pinning/signing is a separate supply-chain improvement. + +If the workflow fails or the release is incomplete, do not distribute it. Inspect the failed workflow and draft release before retrying; **never rewrite an existing published tag**. + +## 6. Customer delivery and rollback + +Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or through file transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. + +YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the previous release image and a compatible configuration; follow [Deployment — Rollback](../operations/INSTALLATION_AND_OPERATIONS.md#54-rollback). + +## 7. Safety and best practices + +- Never run destructive Docker tests on customer, production or shared operational hosts. Use mock Docker unit tests locally and isolated GitHub-hosted Docker acceptance. +- Do not stop or alter CyberController application containers during installation or tests. +- The SHA-256 manifest verifies contents, not the publisher's identity. Use the expected release source and verify the package checksum. +- Never publish personal lab hostnames, IP addresses, passwords or customer-specific data. +- Review the code, test results and public visibility of release assets before tagging. Image builds use prebuilt, versioned artifacts rather than compiling on customer systems. + +References: [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), [Managing Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [About releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). + + +## 8. Developer-only builds and monitoring notes + +### Local image build (never a customer installation) + +For development on a disposable build environment: + +```bash +export WATCHDOG_VERSION="$(cat VERSION)" +docker compose -f docker-compose.build.yaml build +``` + +The Dockerfile packages `watchdog.py`, `lifecycle.py`, and `ha_monitor.py` +inside the image. A restart alone does **not** apply Python source edits. +Production deployment requires a published, prebuilt release package. + +### Narrow CyberController event exceptions + + +The intended MariaDB HA syntax-check helper carries the explicit label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The configured `ignored_container_events` policy ignores only the designated temporary helper `die` event; normal service container failures remain monitored. + +A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Do not alter these narrow exceptions simply to suppress alerts; validate changes with regression tests. + + +These exceptions are not blanket MariaDB or OOM alert suppression. Changes to monitoring exceptions require regression validation. + +### Historical design and validation + +The [HA monitoring design and verification notes](HA_MONITORING.md) retain the reasons behind +the HA decisions and the validation that supports them. The +[release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md) and +[earlier test findings](RELEASE_VALIDATION_V1.6.0.md) are release-specific test +records, not alternative installation instructions. Use the diff --git a/docs/dev/HA_MONITORING.md b/docs/dev/HA_MONITORING.md new file mode 100644 index 0000000..1b5ce4e --- /dev/null +++ b/docs/dev/HA_MONITORING.md @@ -0,0 +1,57 @@ +# HA Monitoring — Design Decisions and Verification + +**Purpose:** Explain why the optional active-node change alert works as it does, what it detects, and what its limits are. This is an engineering reference, not an installation procedure or a release checklist. Operator settings are in [Installation and Operations](../operations/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). + +## Decision summary + +| Decision | Why | +|---|---| +| Observe the **promoted/active HA Operator** (`HAOperatorRep-Res`), not the Pacemaker Designated Controller (DC) | The DC is the Pacemaker coordinator; it can be on the standby node and is **not** evidence of the active CyberController role | +| Use the existing **local Pacemaker HA exporter** at `http://127.0.0.1:9664/metrics` | No additional daemon, privileged host command, container, firewall opening, cross-node API call, or exporter is needed | +| Run HA polling in a **separate thread and HTTP session** from Docker polling | Slow or blocked Docker API calls must not delay HA observations; HA exporter failures must not interrupt container monitoring | +| Require **quorum, a successful Pacemaker scrape, one local node and exactly one promoted owner** | Missing or conflicting telemetry is not proof of a role change; fail closed instead of inventing a failover | +| Require **two consecutive valid observations** of a new active node | Avoid reporting transient/promoted-resource ambiguity while the cluster is transitioning | +| **Only the newly active node** emits the alert | Watchdogs run on both nodes; duplicate notifications from old and new active nodes would be misleading | +| Persist the confirmed active node using the existing state file | Survive watchdog restarts without replaying an old role change or creating a second state store | +| Reuse the existing notification channels and SNMP MIB | Treat HA as another monitored event without adding a new alert transport or breaking container notifications | + +## How the monitor decides + +1. Disabled by default. When enabled, each node reads its **own** exporter using the normal `check_interval_seconds` (default 60 seconds). Requests have a 3-second timeout, a 256 KiB response cap, and permit only loopback HTTP `/metrics` URLs with no redirects. +2. It reads `ha_cluster_pacemaker_resources{resource="HAOperatorRep-Res",role="promoted",status="active"} == 1` to identify the active owner and `ha_cluster_corosync_member_votes{local="true"}` to identify the local node. The Pacemaker scrape-success and quorum metrics must both be 1. +3. The **first valid reading** establishes a silent baseline. An active-node change must appear in **two valid consecutive checks** before it is confirmed. +4. On a confirmed transition A → B, both nodes can update their saved baseline, but **only the watchdog local to B** issues one `WARNING` `ha-failover` event with previous/new node details. +5. Invalid/unavailable metrics leave the last confirmed baseline unchanged and reset a pending candidate. Exporter errors are logged at most once every five minutes; they never trigger a speculative failover alert. +6. HA state is kept separately from container incidents in `watchdog/state.json`. Restart with monitoring enabled restores that state. Running the watchdog with HA monitoring disabled clears its baseline, so a later re-enable starts silently. +7. During the host reboot grace interval, HA role checks are deferred. HA events are not included in container recovery or reboot-summary logic. + +The alert means **a promoted active role was observed to change**. It does **not** identify the cause (planned switchover versus unexpected failover) or guarantee all applications are healthy. The detection timestamp is the observation time, not necessarily the Pacemaker event time. + +## Notification and state trade-offs + +- Existing Syslog, SMTP, Slack and SNMP delivery paths are reused. The alert is `ha-failover`, not a container crash, and must not produce a fake container recovery. +- SNMP retains the existing notification OID and five var-binds: `cwContainerName=(HA cluster)` and `cwFailureType=ha-failover` distinguish HA events; previous/new owners appear in the summary and details. +- Delivery is **best effort**: the confirmed baseline is persisted before dispatch, and failed delivery is logged but not automatically retried. This avoids duplicates from repeated polls; it does **not** guarantee exactly-once receiver delivery. Failed persistence can allow a duplicate after restart. Adding a durable delivery queue would change the notification architecture and is intentionally out of scope. + +## What verification established + +| Check | Result / implication | +|---|---| +| Parser, state and failure-mode unit tests | Covered correct promotion/local identity, bad metrics, no quorum, startup silence, two-poll confirmation, failback, disabled/re-enabled baseline, persistence, and notification formatting | +| Docker polling deliberately blocked in a mocked test | The independent HA thread continued; Docker and HA checks do not depend on each other's progress | +| Real two-node **switchover and failback** | Observed one new-owner WARNING per direction and no old-owner duplicate | +| Actual Syslog delivery on both nodes | HA failover and failback records reached the configured Syslog receivers | +| Transient ambiguous exporter reading during a transition | An invalid reading did not cause a false HA event; later valid readings produced the correct notification | + +The [release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md#ha-acceptance-carried-forward-actual-pacemaker-operations) contains the supporting lab results. Future regressions are in [`tests/test_ha_monitor.py`](../../tests/test_ha_monitor.py). These are prior validation results, not evidence that every delivery channel or later code/image version has been validated. + +## Limits and lessons retained + +- Both HA nodes need a running watchdog, accessible **local** exporter, and configured notification path. No access from one node to the other's exporter is necessary. +- If a surviving node has never learned the earlier active owner, it starts with a silent baseline. The monitor cannot reconstruct older transitions. Fast changes that happen entirely between polls can also be missed. +- Unexpected active-node loss, network partitions, fencing/STONITH and SMTP/Slack/physical-NMS SNMP delivery were **not** demonstrated end to end by the recorded two-node acceptance. Do not infer coverage from successful switchover/failback tests. +- During a planned HA demotion, removal of dependent application containers previously produced separate CRITICAL container alerts. HA role-change detection is intentionally separate from Docker failure detection; suppressing those application events requires a distinct, narrowly scoped rule and its own tests. +- The older native Prometheus HA rule used location-score information that did not reliably prove promotion; its exporter scrape also targeted a different port. Rather than modifying that native monitoring, this feature reads the validated local HA Operator metrics directly. +- No HA control, resource promotion, remediation, Pacemaker writes, cluster configuration changes, or new ports are part of this feature. + +**When revisiting the design:** first confirm exporter metric names and HA Operator resource identity on the target CyberController version. Preserve fail-closed checks, independent polling and quiet baseline. Run mocked regression tests and use only a controlled, separate HA test environment for real switchovers; never induce failover in an operational cluster to test documentation. diff --git a/docs/dev/PROBE_MAP_SNAPSHOT.md b/docs/dev/PROBE_MAP_SNAPSHOT.md new file mode 100644 index 0000000..a6bafdf --- /dev/null +++ b/docs/dev/PROBE_MAP_SNAPSHOT.md @@ -0,0 +1,59 @@ +# Historical Container Probe Map — example only + +**Engineering reference, not an installation configuration or a current product guarantee.** +This is a snapshot from one earlier CyberController environment. Image HEALTHCHECK +definitions, service ports and probe discovery can differ by product version and +deployment. Never copy these entries into monitoring overrides without confirming +the current Docker state and obtaining the normal authorization. + + +Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitored passively via events; all others get an active probe auto-discovered on the first poll and cached for the container's lifetime. + +| Container | Probe Type | Endpoint | Why | +|-----------|-----------|----------|-----| +| `config_kvision-infra-fluentd_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-dp-inline-config_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-configuration-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-anomaly-detection-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-persist-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-rt-alert_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_policy-editor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-alerts_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-reporter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-vrm_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-redis_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-health_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-webui_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-tor-feed_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-efk_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-polling-mgr_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-formatter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-polling-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-cadvisor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-vdirect_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-help_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-ted_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_postgres_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-snmp-trap-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-auto-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-lls_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-assist-service_1` | HTTP | `:3005` (path auto-discovered) | No Docker HEALTHCHECK; Node.js service; path cached on first successful response | +| `config_kvision-infra-rabbitmq_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-ha-operator_1` | HTTP | `:8080` (path auto-discovered) | No Docker HEALTHCHECK; Java HTTP service; path cached on first successful response | +| `config_kvision-dc-nginx_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `process-exporter` | HTTP | `:9256/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `prometheus` | HTTP | `:9090/-/ready` | No Docker HEALTHCHECK; standard Prometheus readiness endpoint | +| `alertmanager` | HTTP | `:9093/-/healthy` | No Docker HEALTHCHECK; standard Alertmanager health endpoint | +| `postgres-exporter` | HTTP | `:9187/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `mysql-exporter` | HTTP | `:9104/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `es-exporter` | HTTP | `:9114/metrics` | No Docker HEALTHCHECK; port 7979 HTTP fails; port 9114 serves `/metrics` | +| `node-exporter` | HTTP | `:9100/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `grafana` | HTTP | `:3000/api/health` | No Docker HEALTHCHECK; Grafana health endpoint returns 200 | +| `config_kvision-infra-mariadb_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | + +> **Verify what was actually cached after startup:** +> ```bash +> grep -E "auto-discover|falling back" watchdog/watchdog.log +> ``` diff --git a/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md b/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md new file mode 100644 index 0000000..73e39e7 --- /dev/null +++ b/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md @@ -0,0 +1,108 @@ +# Container Alert v1.6.0 - Final release-candidate acceptance + + +> **Historical acceptance evidence (October 14, 2026), not current test instructions.** +> These records predate the v1.6.0 hosted-only smoke-test safety guard and final +> customer-facing message updates. Do **not** rerun their Docker commands on +> any shared or operational Docker Engine. Earlier nested testing triggered +> real notifications on its parent host. +> See [current release notes](../operations/RELEASE_NOTES_V1.6.0.md) and +> [safe smoke workflow](RELEASE_SMOKE_TESTING.md). + +**Validation date:** October 14, 2026 (UTC) +**Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) +**Source basis:** release branch including merged [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10), plus the targeted recovery fix in this PR +**Verdict: PASS for the tested Docker, HA, Syslog, regression, and image-packaging acceptance criteria.** Live SMTP, SNMP, Slack, destructive HA injection, and host reboot are not included. + +## Executive results + +| Area | Result | Evidence | +|---|---|---| +| Automated Python regression tests | **PASS 109/109** | `python3 -m unittest discover -s tests -q` inside actual rebuilt `watchdog:1.6.0-rc-fix` image | +| Full real Docker integration suite | **PASS 15/15** | Isolated Docker Engine on a dedicated development host, run `261014224557488`; 22:45:57-22:52:41 UTC; 404.4 seconds | +| Real external restart after OOM | **PASS 1/1** | Separate targeted real OOM run `261014224323244`; RECOVERED after 120.6s | +| Regression reproduction before fix | **CONFIRMED FAIL** | Real crash followed by manual Docker start produced premature RECOVERED within about three seconds; run `261014223252035` | +| Raw UDP Syslog integration | **PASS 15/15** | 15 locally generated `watchdog_alert` records delivered as 15 actual UDP datagrams to isolated collector | +| Two-node HA failover and failback | **PASS** | Previous *live* lab testing: promoted HAOperator `.41 -> .42 -> .41`; one new-owner WARNING per transition, no duplicate, both DFC Syslog receivers confirmed | +| Installer source-build / offline image bundle | **PASS** | Source-build includes `watchdog.py`, `lifecycle.py`, `ha_monitor.py`; image save/load preserved ID | +| Disposable test cleanup | **PASS** | No labelled Docker test containers or volumes remained after each completed suite | +| Parent-host watchdog impact | **NONE** | Existing `watchdog:1.5.4` remained healthy, zero restarts, and saw no isolated Engine fixtures | + +## Reproducible environment + +- On a **dedicated development host**, ran a short-lived Docker-in-Docker engine `docker:27.5-dind`, exposed only at `127.0.0.1` using an ephemeral TCP port; it had a distinct Engine ID from the parent host's Docker Engine, memory limit 2 GiB, CPU limit 2 cores, no bridge/network provisioning inside the test engine, and its state was disposable. +- Real test observer ran the revised application code from `watchdog:1.6.0-rc-fix` with its own state volume and a 2s polling interval. Original 5-restart threshold and **120s recovery stability window** were preserved. The observer monitored **real Docker Engine events and cgroup OOMs**, not synthetic test events. +- Disposable failure containers were scoped with unique `ca-smoke-...` names/labels, limited CPU/memory, and no network access. Syslog receiver was a temporary UDP/514 collector on the isolated Engine. No production SMTP or external Syslog destinations were used. +- The built test image contained verified source hashes: + - `watchdog.py`: `4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa` + - `lifecycle.py`: `55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9` + - `ha_monitor.py`: `0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd` +- Test image ID within the isolated Engine: `sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9`. This was a locally built test candidate, **not** a published release artifact or Git tag. +- Real-event run captured **92 Docker events**, including 18 `die`, 22 `start`, 1 `oom`, 1 `stop`, native unhealthy/healthy transitions, and **zero unexpected `restart` actions**. The regular loop's expected internal restart behavior is tracked via Docker's actual `RestartCount`. + +## Full real Docker acceptance - final clean run + +| Test | Expected | Observed | Status | +|---|---|---|---| +| T01 CLEAN_EXIT | Exit 0, no incident | No alert | **PASS** | +| T02 UNEXPECTED_CRASH | Exit 2, one CRITICAL crash | Exactly one CRITICAL CRASH | **PASS** | +| T03 PLANNED_STOP_137 | Forced Docker stop/137 not mistaken for crash or OOM | No false alert | **PASS** | +| T04 DIRECT_SIGKILL | Direct signal 9/137 is crash, not OOM | `OOMKilled=false`, one CRITICAL CRASH | **PASS** | +| T05 REAL_OOM | Real kernel OOM, 137, single OOM alert, no redundant CRASH | `OOMKilled=true`, one CRITICAL OOM | **PASS** | +| T06 RESTART_LOOP | Repeated failures escalate after at least five real restarts | `RestartCount=5`, one CRASH, one HIGH RESTART-LOOP | **PASS** | +| T07 BRIEF_RECOVERY | 11 seconds of RUNNING is insufficient | Zero RECOVERED after 11 seconds | **PASS** | +| T08 INTERRUPTED_RECOVERY | A new actual failure restarts the stability clock | `RestartCount` 6 to 7; no early RECOVERED | **PASS** | +| T09 STABLE_RECOVERY | One INFO RECOVERED after 120s sustained uptime | One CRASH, one LOOP, one RECOVERED | **PASS** | +| T06-09 LIFECYCLE | One complete incident, no duplicate notifications | Correct alert order and totals | **PASS** | +| T14 EXTERNAL_RESTART | Manual start of policy-`no` crashed container must wait 120s | One RECOVERED near 120s (119.9s by second-resolution alert timestamp) | **PASS** | +| T10 NAME_REUSE | New Docker ID with reused name must start a new incident | Two distinct IDs, one CRASH per ID | **PASS** | +| T11 FAST_REMOVAL | Rapid `--rm` events do not flood missing-ID warnings or kill listener | Three ephemeral containers, zero observed warnings, observer alive | **PASS** | +| T12 NATIVE_HEALTHCHECK | Native Docker unhealthy then healthy | One HIGH UNHEALTHY, one INFO RECOVERED | **PASS** | +| T13 SYSLOG | Every generated alert reaches receiver with correct typed severity, no planned-stop false alarm | **15/15** structured UDP datagrams; no planned-stop false alarm | **PASS** | + +The isolated collector captured actual raw Syslog priorities: **CRITICAL `<130>` (10), HIGH `<131>` (2), INFO `<134>` (3)** with local0 facility. This verifies PR #10's intended priority mapping for Docker event classes. Previous real HA testing captured and stored HA WARNING through DFC (local0.WARNING `<132>`). + +### Additional OOM-then-manual-start case + +The test `T15_EXTERNAL_OOM` deliberately exceeded a 64 MiB cgroup memory limit, confirmed exit 137 with Docker `OOMKilled=true`, received one CRITICAL OOM alert, then manually started the same policy-`no` container with a healthy process. No early recovery was emitted; precisely one INFO RECOVERED occurred **120.6 seconds** after actual Docker uptime began. **PASS.** + +## Defect, fix, and historical HA-lab failures + +The earlier HA-lab smoke run (`261014214436754`, originally **11 PASS / 3 FAIL**) observed containers with Docker restart policy `no` starting again after a failure despite not being controlled by the test suite. The captured events prove this external restart behavior; the initiator was **not positively identified**. Do not attribute it to a particular CyberController host service without direct evidence. + +We reproduced the **application bug independently** on the isolated test Docker Engine: the old code emitted RECOVERED just a few seconds after a manual `docker start` of a crashed policy-`no` container. Original `_maybe_alert_recovery` applied the 120-second stability gate to CRASH/OOM only when the Docker restart policy was automatic. That assumption is invalid when an external manager manually restarts failed containers. + +**Minimal code correction:** for any open `crashed`, `oom`, or `restart-loop` incident, require sustained Docker uptime and no recent failure for the existing recovery-stability interval. The existing state machine, Docker event correlation, HA monitoring, and notification APIs were not redesigned. Two policy-`no` crash/OOM regressions now protect this path. All **109** automated tests pass. + +The original lab's external Docker restarter remains **unattributed and unchanged**. Its behavior is an integration-environment consideration, not proof the new application fix failed. The isolated Engine recorded zero unrelated restart actions and its full suite passed. + +## HA acceptance carried forward (actual Pacemaker operations) + +The user manually performed real switchover and failback of the existing two-node CyberController HA cluster while both real watchdogs ran the PR #10 1.6.0 candidate. + +| HA case | Evidence and outcome | +|---|---| +| `192.0.2.41 -> 192.0.2.42` | **PASS**: promoted operator moved to .42; one WARNING alert on newly active .42 at 20:56:16 UTC, no duplicate on .41 | +| .42 DFC delivery | **PASS**: structured `ha-failover` record received/stored in `config_dfc_1:/var/log/syslog` | +| `192.0.2.42 -> 192.0.2.41` | **PASS**: one WARNING alert on .41 at 21:09:47 UTC; .42 deferred, no duplicate | +| .41 DFC delivery | **PASS**: structured reverse alert received/stored in its DFC Syslog receiver | +| Final HA health | **PASS**: quorum healthy, .41 promoted, both watchdogs healthy and baselines .41 | + +The new Docker recovery fix does not change the HA monitoring code. Nevertheless, **the currently running HA-node images still contain the previous pre-fix candidate**, so they are not proof that the final recovery-fixed image has been installed on those nodes. + +## Packaging, exclusions, and release decision + +Installer source-build and offline archive `docker save`/`docker load` procedures were validated earlier in this release cycle; CI covered shell syntax, Compose, code compilation, unit tests, and image builds. The test suite itself runs without installing packages on operational HA nodes. + +**Not validated end to end:** SNMP traps to a physical NMS, Slack delivery, SMTP delivery (explicitly excluded by user), host reboot, network partition, STONITH, destructive exporter injection. These are not represented as passed. Failover/failback, DFC Syslog, ordinary Docker incidents, and the fixed recovery path have direct live evidence. + +**Release recommendation:** The previously blocking Docker recovery bug is fixed and independently validated. The release branch is ready for final PR review and fresh CI validation. Do **not** claim that the Git tag, GitHub Release, or final v1.6.0 package was published; do not deploy or merge without the user's chosen release step. + +## Evidence and replay + +- [Final full suite JSON](evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json) - 15/15 and cleanup evidence +- [Before-fix real failure reproduction](evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json) +- [Real OOM/manual-restart 120.6s proof](evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json) +- [Earlier HA-lab conditional report](RELEASE_VALIDATION_V1.6.0.md) +- [Reusable real Docker smoke runner](../../tests/release-smoke.sh) +- Historical runtime evidence: temporary smoke-run directories containing test events, Syslog records and observer state were retained during validation. diff --git a/docs/dev/RELEASE_SMOKE_TESTING.md b/docs/dev/RELEASE_SMOKE_TESTING.md new file mode 100644 index 0000000..de5e01c --- /dev/null +++ b/docs/dev/RELEASE_SMOKE_TESTING.md @@ -0,0 +1,72 @@ +# Release Smoke Tests — Mandatory Isolation + +**Scope:** Container Alert v1.6.0 release validation. The 15 core cases and a separate true kernel-OOM/manual-restart test use real Docker events but must never access an operational or shared Docker Engine. + +## Safety rule + +**NEVER run `tests/release-smoke.sh` on production CyberController, customer production systems, or any shared Docker host.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. + +Instead run the workflow [Release smoke (isolated)](../../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to any operational host. All test-created resources are disposable and contained within that hosted worker. + +The workflow has no corporate customer credentials, no HA exporter dependency, no CyberController Docker socket, no external notification destinations, and no network access from the disposable failure fixtures. + +## Launch + +The hosted smoke workflow triggers on in-repository release PR changes targeting +`main`. The first run can occur before the workflow is merged into the +default branch. A manual `workflow_dispatch` rerun is available once the +workflow file exists on the default branch. + +1. Review [release PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) and select its current `release/v1.6.0` branch. +2. For the current PR, verify the automatically triggered **Release smoke (isolated)** + check. After the workflow is on `main`, an authorized maintainer can + also manually rerun it from Actions or use: + + ```bash + gh workflow run release-smoke.yml --ref release/v1.6.0 + ``` + +3. Confirm the workflow's commit SHA matches the release PR head. It builds the watchdog image on the **hosted runner only**, loads it into a freshly created nested Docker Engine, and checks that the image files match the checked-out source. +4. Download the attached structured evidence artifact (Docker events, alerts, raw UDP receiver packets, state snapshot, run reports). The workflow runs both the full acceptance suite and the additional kernel OOM/manual restart scenario. +5. All cases must PASS; the final cleanup must report zero test-labelled containers and volumes. Keep the actual workflow URL/commit in the release verification record. + +## Automatic fail-closed checks + +Before even inspecting an image or creating a test resource, the smoke runner requires all of the following: + +- GitHub Actions execution on a GitHub-hosted runner (`GITHUB_ACTIONS=true` and `SMOKE_RUNNER_KIND=github-hosted`). +- An explicit localhost-only TCP Docker Engine endpoint `DOCKER_HOST=tcp://127.0.0.1:`; no `DOCKER_CONTEXT` override. +- A unique run marker matching the Docker **daemon-side** label `com.radware.container-alert.smoke-engine`. +- **Zero existing containers** on the target Docker Engine. + +The runner has **no live-host override**. If preflight fails it must exit FAIL **without calling Docker to create/inspect any application containers, and without performing cleanup commands against the unverified Engine**. + +Within the verified Engine, each fixture uses unique names and a per-run ownership label. Cleanup re-verifies ownership before removal and fails if a command fails or a container/volume is left over. The fast-removal test verifies real exit codes and actual Docker `die` events, not only observer liveness. + +## Acceptance criteria + +| Scenario | Expected | +|---|---| +| Clean exit | Exit 0 → no alert | +| Nonzero crash | One CRITICAL CRASH | +| Planned forced stop | Docker-requested stop/137 → no false CRASH or OOM | +| Direct SIGKILL | CRITICAL CRASH, not kernel OOM | +| Genuine cgroup OOM | `OOMKilled=true`, exactly one CRITICAL OOM | +| Restart loop | One CRASH → one HIGH RESTART-LOOP; 5 actual restarts | +| Brief or interrupted recovery | No early RECOVERED; actual new failure resets stability | +| Stable recovery | Exactly one INFO RECOVERED after configured uptime | +| Manual restart after crash | Policy `no`, no premature RECOVERED | +| Manual restart after real OOM | Policy `no`, actual kernel OOM and 120-second stability gate | +| Container identity reuse | New Docker ID with old name does not inherit old state | +| Rapid auto-removal | All short-lived test fixtures actually exited, no listener crash or warning flood | +| Native Docker HEALTHCHECK | HIGH UNHEALTHY → INFO RECOVERED | +| Syslog | Locally generated structured alerts received exactly once as real UDP Syslog | +| Cleanup | No leftover test-labelled containers/volumes, no unauthorized modifications | + +## Isolation lesson + +Earlier development tests showed that a nested test Docker Engine remains visible to its parent host. Destructive acceptance tests must run on a disposable hosted runner, not a shared or operational host. + +## Operator verification (read-only only) + +On an installed system, it is acceptable for authorized personnel to review the watchdog's own health, log records and configured alert destinations. It is **not** acceptable to intentionally kill/restart a service to "test notifications." Real failure injection occurs exclusively through the disposable hosted workflow. diff --git a/docs/dev/RELEASE_V1.6.0_CHECKLIST.md b/docs/dev/RELEASE_V1.6.0_CHECKLIST.md new file mode 100644 index 0000000..a3de0c7 --- /dev/null +++ b/docs/dev/RELEASE_V1.6.0_CHECKLIST.md @@ -0,0 +1,64 @@ +# Container Alert v1.6.0 — Final Release Checklist + +**This checklist applies only to v1.6.0.** For every subsequent release, use the permanent [Development and Release Guide](DEVELOPMENT_AND_RELEASE.md) and create a corresponding version-specific approval checklist only if needed. + +**Status on October 16, 2026:** `main` still contains v1.5.4; `release/v1.6.0` contains `VERSION=1.6.0` and is reviewed through [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9), which is open/unmerged. The v1.6.0 GitHub Release has **not** been published by this work. + +## Approval and verification — before merge + +- [ ] Final engineering review of PR #9: alert/incident lifecycle, notifications, HA behavior, security and image packaging. +- [ ] Normal GitHub CI **and** [Release smoke (isolated)](RELEASE_SMOKE_TESTING.md) both pass against the **latest PR head SHA**. The smoke is a disposable GitHub-hosted test; never run it against an operational Docker host. +- [ ] Document all changes since v1.5.4 in [release notes](../operations/RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. +- [ ] Confirm exact supported customer message text, severity mappings, and downstream Syslog/SNMP compatibility. +- [ ] Confirm there are no private lab hostnames, IP addresses, personal information or credentials in the **published installer archive**, documentation and release notes. +- [ ] Approve distributing the Docker image and complete release package through this **public** GitHub repository. +- [ ] Confirm upgrade/rollback procedure and identify any additional authorized HA/customer-delivery verification needed. CI success alone does not establish real customer SMTP/Slack/SNMP delivery. + +## Finalize and publish + +After **all** approvals above: + +1. Merge PR #9 to `main` through the reviewed GitHub process. +2. In a clean checkout, run: + + ```bash + git switch main + git pull --ff-only origin main + cat VERSION # expect 1.6.0 + git status --short # expect empty + git ls-remote --tags origin refs/tags/v1.6.0 + ``` + + The final command must show **no existing tag**. If it exists, investigate; don't overwrite tags. + +3. With approval to publish, tag the approved `main` commit: + + ```bash + test "$(cat VERSION)" = "1.6.0" || exit 1 + git tag -a v1.6.0 -m "Container Alert v1.6.0" + git push origin v1.6.0 + ``` + +4. Confirm the tagged [release workflow](../../.github/workflows/release-package.yml) passed: + + ```bash + gh run list -R Radware/CyberController-Container-Alert --workflow release-package.yml --limit 5 + gh release view v1.6.0 -R Radware/CyberController-Container-Alert + ``` + +5. Verify the **two published assets**, `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`, without installing or starting containers: + + ```bash + mkdir -p /tmp/container-alert-v1.6.0-verify + cd /tmp/container-alert-v1.6.0-verify + gh release download v1.6.0 -R Radware/CyberController-Container-Alert + sha256sum --check SHA256SUMS + mkdir -p unpacked + tar -xzf container-alert-v1.6.0.tar.gz -C unpacked + (cd unpacked/CyberController-Container-Alert && sha256sum --check SHA256SUMS) + cat unpacked/CyberController-Container-Alert/VERSION + ``` + +6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../operations/INSTALLATION_AND_OPERATIONS.md). + +**Stop if a release gate fails.** Don't distribute a draft, incomplete or unapproved release. Do not run installation, Docker failure injection or HA failover tests on production systems as part of this checklist. diff --git a/docs/dev/RELEASE_VALIDATION_V1.6.0.md b/docs/dev/RELEASE_VALIDATION_V1.6.0.md new file mode 100644 index 0000000..65bddd9 --- /dev/null +++ b/docs/dev/RELEASE_VALIDATION_V1.6.0.md @@ -0,0 +1,117 @@ +# Container Alert v1.6.0 — Release Acceptance Validation + + +> **Historical acceptance evidence (October 14, 2026), not current test instructions.** +> These records predate the v1.6.0 hosted-only smoke-test safety guard and final +> customer-facing message updates. Do **not** rerun their Docker commands on +> any shared or operational Docker Engine. Earlier nested testing triggered +> real notifications on its parent host. +> See [current release notes](../operations/RELEASE_NOTES_V1.6.0.md) and +> [safe smoke workflow](RELEASE_SMOKE_TESTING.md). + +> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on a dedicated development host, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). Retain this document as evidence of the original findings, not the current release verdict. + +**Date:** October 14, 2026 (UTC) +**Repository:** `Radware/CyberController-Container-Alert` +**Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) — open +**Notification fix:** [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10) — merged into release branch +**Source revision:** `3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05` +**Lab image:** `watchdog:1.6.0`, image ID `sha256:e3d02f6ef56cde0bc7021919def9e240505434a15769aac3884cf15ac8f0ccff` +**Verdict: CONDITIONAL / NOT FULLY ACCEPTED.** HA passes. Most real Docker cases pass; some checks are confounded by out-of-band Docker restarts and expose premature recovery / extra alert behavior. Do not represent the entire release as green. + +## Test methods and safeguards + +- Tested against a real Docker Engine in the HA test environment (`192.0.2.41`). No simulated Docker events. +- A separate resource-limited v1.6.0 observer used its own Docker volume/state, 2-second Docker polling, real 5-restart threshold, and **real 120-second** recovery requirement. SMTP disabled only on the observer; Syslog sent to the **existing** DFC UDP/514 receiver. Production-style main watchdog and cluster were not reconfigured. +- Used uniquely prefixed, labeled disposable containers, restricted to 0.2 CPU and 64–128 MiB each, with `--network none`. Verified actual Docker state, events, watchdog structured alert records, and DFC `/var/log/syslog` deliveries. +- No host packages, new Docker networks, Pacemaker operations, or host service modifications. Cleanup removed all smoke-owned containers and volumes. Original watchdog stayed healthy. +- Runner: [`tests/release-smoke.sh`](../../tests/release-smoke.sh). It saves `report.json`, `report.md`, `watchdog.log`, `docker-events-live.jsonl`, and `dfc-syslog.txt` to `/tmp/container-alert-smoke-/` inside the authorized RDC client. + +## 1. Automated, packaging, and release checks + +| Check | Acceptance | Result | +|---|---|---| +| Automated regression suite | All tests pass against final built source | **PASS — 108/108** inside a test Docker image | +| GitHub CI for PR #9 head | Green | **PASS** on `3b1de153` (before adding this documentation) | +| Release code/image integrity | Embedded `watchdog.py`, `lifecycle.py`, `ha_monitor.py` hashes match Git | **PASS** | +| `install.sh` / smoke script syntax | `bash -n`, smoke embedded Python compiles | **PASS** | +| Docker Compose build configuration | Valid config with `WATCHDOG_VERSION=1.6.0` | **PASS** | +| Installer source-build fallback | Dockerfile and three Python modules build/import | **PASS** with normal cached/online Docker build | +| Offline archive | `docker save` / SHA-256 / `docker load`; same release image ID | **PASS** | +| Source-build without network | All `pip` dependencies available offline | **NOT SUPPORTED**. No-network source build failed because pip must download dependencies; use the tested `watchdog.tar` for offline installations | + +## 2. Real two-node HA acceptance (completed before Docker smoke) + +| Scenario | Acceptance | Actual result | +|---|---|---| +| Baseline | Both nodes agree `.41` active, no startup failover | **PASS** | +| Failover `.41 → .42` | One WARNING on new owner, zero on previous | **PASS** — alert on `.42` at **20:56:16 UTC**; `.41` logged "new owner will notify" | +| HA Syslog on `.42` | Structured failover record delivered to DFC | **PASS** — stored in DFC `/var/log/syslog` | +| Failback `.42 → .41` | One WARNING on new owner, zero on previous | **PASS** — alert on `.41` at **21:09:47 UTC**; `.42` logged owner defer | +| HA Syslog on `.41` | Structured failback record delivered to DFC | **PASS** | +| Final cluster status | One promoted HA Operator, quorum, both watchdogs healthy | **PASS** — `.41` active; both watchdogs image `1.6.0`, healthy, zero restarts | +| Transient exporter ambiguity during transition | No fabricated failover | **PASS** — `.42` logged one invalid-promoted-owner observation at 20:54:16 UTC, then recovered and sent correct alert | +| SMTP delivery | Excluded by test owner | **EXCLUDED** — SMTP network timeouts are not part of this verdict | + +## 3. Real Docker acceptance — final run + +**Final run ID:** `261014214436754` +**Time:** 21:44:36–21:48:30 UTC (233.9 seconds) +**Final summary: 11 PASS / 3 FAIL** across 14 runner assertions (includes a grouped restart lifecycle check). + +| ID | Scenario / acceptance criteria | Result and evidence | +|---|---|---| +| T01 | Exit 0 yields no crash alert | **PASS** | +| T02 | Nonzero exit 2 yields exactly one CRITICAL crash | **PASS** | +| T03 | Planned `docker stop`, SIGKILL/137, yields no downstream false crash | **FAIL (interference)**. Initial Docker stop had matching `stop/die` evidence; the lab later restarted the stopped fixture, and the subsequent forced cleanup generated an additional CRASH | +| T04 | Direct SIGKILL/137 has `OOMKilled=false`, one CRITICAL crash, no false OOM | **PASS** in final run | +| T05 | Genuine cgroup OOM; Docker confirms `OOMKilled=true`; one CRITICAL OOM, no duplicate | **FAIL (unstable state)** in final run. Earlier run confirmed true `OOMKilled`; final live event capture proved **12 real OOM events** after unexpected restarts, so the subsequent Docker state was no longer the first killed instance | +| T06 | Restart loop escalates after 5 actual restarts, with one CRASH and one HIGH RESTART-LOOP | **PASS** — Docker `RestartCount=6` | +| T07 | Short period running does not prematurely recover restart loop | **PASS** — 11 seconds healthy, zero RECOVERED | +| T08 | New actual failure during recovery interrupts stability clock | **PASS** — Docker restart count incremented 6→7; no early RECOVERED | +| T09 | After at least 120 seconds sustained recovery, exactly one INFO RECOVERED | **PASS** — one CRASH, one RESTART-LOOP, one RECOVERED | +| T10 | Reuse exact container name with a different ID gets a fresh incident | **PASS** — 2 unique IDs, one crash per instance | +| T11 | Three `--rm` containers do not flood missing-container warnings or kill event listener | **PASS for no-flood** — 0 warnings observed; missing-ID race branch not independently forced | +| T12 | Native Docker health check transitions unhealthy→healthy | **PASS** — one HIGH UNHEALTHY, one INFO RECOVERED | +| T13 | All correct structured alert records arrive at DFC; no false planned-stop alert | **FAIL strict semantic criterion** — **26/26** actual watchdog records reached DFC, but one unwanted planned-fixture CRASH also reached it | +| Grouped T06–T09 | Correct restart-loop lifecycle as a whole | **PASS** | + +### Three attempts and reproducibility + +- **First run** (`261014212952944`): core real-event cases, name reuse, health, and DFC transport passed. A **test-runner formatting error** (missing grouped-test return value) prevented a clean report; fixed afterward. Unexpected out-of-band restarts began causing extra OOM/recovery alerts. +- **Second run** (`261014213819904`): **12 PASS / 2 FAIL**. Unexpected SIGKILL fixture restart and false recovery of an OOM fixture were observed. The runner was corrected to record these explicitly. +- **Final run** (above): **11 PASS / 3 FAIL**; includes live capture of **206 Docker events**. The failure conditions vary with the lab's externally initiated restarts, not a deterministic source-code change. + +### Investigation: out-of-band restarts + +The live event capture on `.41` included **37 `restart` actions, 57 starts, 54 dies, and 12 `oom` actions** for test-tagged containers. Importantly, some affected fixtures had `HostConfig.RestartPolicy.Name=no`, yet their Docker IDs were started/restarted repeatedly. The initiator of these restarts was **not established**; do not attribute them to Container Alert without additional evidence. + +The planned-stop fixture initially emitted real `kill(SIGTERM)`, `kill(SIGKILL)`, `stop`, and `die(137)` events. The unwanted CRASH appeared **later**, when that fixture had already been restarted and the test suite used force-remove to clean it up; that final kill had no `stop` evidence. This is not evidence that the original `docker stop` was misclassified. + +The OOM fixture produced genuine kernel `oom` and `die(137)` events. When another actor immediately restarts a container whose Docker policy is `no`, the current recovery rule can declare an OOM/CRASH incident recovered after only a brief running interval. Repeated OOM/recovery alert pairs were observed in the earlier run. **This is a real edge case worth addressing or accepting explicitly**, separate from the one-shot OOM detector. + +The DFC Syslog pipeline itself had **100% message-count correspondence in the final run (26/26)** and preserved structured severity/type fields. However, transport delivery is not the same as correct incident classification. + +## 4. Additional coverage and explicit exclusions + +- PR #10 automated tests check alert severity mapping, SMTP/Slack formatting, missing container fields on HA alerts, and mocked SNMP alert dispatch. The existing five-var-bind SNMP MIB mapping was kept unchanged, but it was **not verified by a live SNMP trap receiver**. Lab smoke verified real Syslog emission/receipt of CRITICAL, HIGH, and INFO severity fields; it did **not** independently capture raw UDP PRI for every severity. +- Live SNMP trap delivery, Slack delivery, and SMTP delivery were **not** exercised end-to-end; these require configured destinations (SMTP explicitly excluded by the owner). +- Real network partition, forced host reboot, Pacemaker failure injection, and blocked Docker daemon were intentionally **not** performed. HA independence and exporter failure handling are covered by automated tests; the transient exporter error was also observed during actual switchover. + +## 5. Cleanup, impact, and release decision + +- Both HA nodes remain on `watchdog:1.6.0` with their existing configs/state and healthy watchdogs; cluster role and quorum remain intact. +- All test-labelled temporary containers and named volumes were removed. No host packages, new networks, firewall rules, or HA resources were installed/changed. Report/evidence files reside in the RDC admin client's `/tmp` (ephemeral, not a host install). +- **Do not mark complete release acceptance yet.** HA, regression suite, packaging, and Syslog transport are verified. Real Docker repeated OOM/recovery and forced-cleanup false-alert behavior need a controlled-engine rerun or an explicit scoped acceptance. +- Minimal next investigation: isolate the lab's external Docker restarter, reproduce crash/OOM with exactly one failure, and decide whether all crash/OOM recoveries should require the already-existing 120-second continuous-uptime check (not just containers with Docker auto-restart). Avoid broad lifecycle redesign before validating that decision. +- PR #9 remains **unmerged**; do not tag or publish `v1.6.0` solely on these conditional results. + +## Test evidence pointers + +- Lab RDC container: `/tmp/container-alert-smoke-261014214436754/` +- Raw files: `report.json`, `report.md`, `docker-events-live.jsonl`, `watchdog.log`, `dfc-syslog.txt`, and `observer-state.json`. +- Source-controlled runner: [`tests/release-smoke.sh`](../../tests/release-smoke.sh). The historical lab smoke command must **not be repeated**; current destructive +acceptance is permitted only through the +[disposable GitHub-hosted workflow](RELEASE_SMOKE_TESTING.md). + +Machine-readable final run: [`RELEASE_SMOKE_V1.6.0_20261014.json`](evidence/RELEASE_SMOKE_V1.6.0_20261014.json). diff --git a/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_20261014.json new file mode 100644 index 0000000..c465df7 --- /dev/null +++ b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_20261014.json @@ -0,0 +1,276 @@ +{ + "run_id": "261014214436754", + "started_utc": "2026-10-14T21:44:36.266768Z", + "finished_utc": "2026-10-14T21:48:30.206757Z", + "status": "FAIL", + "image": "watchdog:1.6.0", + "results": [ + { + "id": "T01_CLEAN_EXIT", + "status": "PASS", + "expected": "exit=0 means no alert", + "observed": "real exit=0, no alerts", + "elapsed_seconds": 12.7 + }, + { + "id": "T02_UNEXPECTED_CRASH", + "status": "PASS", + "expected": "exit=2 sends one CRITICAL crash", + "observed": "Docker exit=2, exactly one CRITICAL crashed alert", + "elapsed_seconds": 15.7 + }, + { + "id": "T03_PLANNED_STOP_137", + "status": "FAIL", + "expected": "Docker stop exit 137 is suppressed", + "observed": "unexpected planned fixture restart", + "elapsed_seconds": 26.6 + }, + { + "id": "T04_DIRECT_SIGKILL", + "status": "PASS", + "expected": "SIGKILL 137 is crash, not OOM", + "observed": "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash", + "elapsed_seconds": 43.6 + }, + { + "id": "T05_REAL_OOM", + "status": "FAIL", + "expected": "OOMKilled=true sends only CRITICAL OOM", + "observed": "Docker cgroup did not confirm OOMKilled", + "elapsed_seconds": 45.1 + }, + { + "id": "T06_RESTART_LOOP", + "status": "PASS", + "expected": "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "observed": "real Docker RestartCount=6; crash=1; restart-loop=1", + "elapsed_seconds": 57.8 + }, + { + "id": "T07_BRIEF_RECOVERY", + "status": "PASS", + "expected": "no recovery before 120s stability", + "observed": "running after 11s, RECOVERED=0", + "elapsed_seconds": 69.5 + }, + { + "id": "T08_INTERRUPTED_RECOVERY", + "status": "PASS", + "expected": "new actual die/reset prevents premature recovery", + "observed": "Docker RestartCount 6 -> 7; RECOVERED=0 after 14s", + "elapsed_seconds": 86.2 + }, + { + "id": "T09_STABLE_RECOVERY", + "status": "PASS", + "expected": "one INFO RECOVERED after 120s sustained uptime", + "observed": "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery", + "elapsed_seconds": 198 + }, + { + "id": "T06_TO_T09_RESTART_LIFECYCLE", + "status": "PASS", + "expected": "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", + "observed": "one CRASH + one LOOP + one RECOVERED after real interrupted recovery", + "elapsed_seconds": 198 + }, + { + "id": "T10_NAME_REUSE", + "status": "PASS", + "expected": "replacement with same name gets new incident", + "observed": "two unique container IDs, same name, each gets one CRITICAL crash", + "elapsed_seconds": 207.9 + }, + { + "id": "T11_FAST_REMOVAL", + "status": "PASS", + "expected": "no event flood, event listener survives", + "observed": "3 real --rm containers, 0 bounded missing-ID warnings, observer alive", + "elapsed_seconds": 218.2 + }, + { + "id": "T12_NATIVE_HEALTHCHECK", + "status": "PASS", + "expected": "unhealthy -> recovered", + "observed": "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery", + "elapsed_seconds": 231.5 + }, + { + "id": "T13_DFC_SYSLOG", + "status": "FAIL", + "expected": "all expected structured alerts reach actual DFC", + "observed": "planned stop sent to DFC", + "elapsed_seconds": 231.7 + } + ], + "facts": { + "host": "ha-validation-host", + "revision": "3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05", + "image_id": "sha256:e3d02f6ef56cde0bc7021919def9e240505434a15769aac3884cf15ac8f0ccff", + "observer": "ca-smoke-261014214436754-observer", + "syslog_source_host": "release-smoke-261014214436754", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 50, + "clean": { + "name": "ca-smoke-261014214436754-clean", + "container_id": "98f25a334562103075cfc01df542f3f55fd8356565bbc6533b1922f9f2f640a3" + }, + "crash": { + "name": "ca-smoke-261014214436754-crash", + "container_id": "748a19f96463f79fcba026b9fcbe7d11aec540b728f04fd9012175888b9057d1" + }, + "planned": { + "name": "ca-smoke-261014214436754-planned", + "container_id": "34ac9faaa74856caebfe6f3d1e85975e03eb38104837162da0a3fdbffe10936c" + }, + "sigkill": { + "name": "ca-smoke-261014214436754-sigkill", + "container_id": "19f291e6c4186fb0a3d0c24ebdb503fee846a30c325c810a69f827422956e4a6" + }, + "sigkill_state": { + "exit": 137, + "OOMKilled": false + }, + "oom": { + "name": "ca-smoke-261014214436754-oom", + "container_id": "ddb952d6842830181d32a458d4fcac3cf4509a1b6d10fa4739e2d53dfb334bd7" + }, + "loop": { + "name": "ca-smoke-261014214436754-loop", + "container_id": "40700dcc7322b59cf86ffb4894c8cb946e31d2fcfee1f0afa2456002d7da59ad" + }, + "loop_restart_count_after": 7, + "reuse": { + "name": "ca-smoke-261014214436754-reuse", + "container_id": "e346b9f8a2282179ed00b4d9cc3bd55300b3bf517f76f138c63a2d6d525fb3f7" + }, + "fast_remove_warning_count": 0, + "health": { + "name": "ca-smoke-261014214436754-health", + "container_id": "4713718efb4607ffca32afdbac8ec44e77ad91dda0dc69d55ba7a3890b3c4bb1" + }, + "watchdog_alert_count": 26, + "dfc_received_count": 26, + "live_docker_events": 206, + "live_docker_action_counts": { + "create": 13, + "start": 57, + "die": 54, + "oom": 12, + "stop": 1, + "kill": 5, + "restart": 37, + "health_status: unhealthy": 1, + "health_status: healthy": 1, + "destroy": 7 + }, + "docker_event_lines": 53, + "final_fixture_states": { + "ca-smoke-261014214436754-clean": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:44:41.368218838Z", + "FinishedAt": "2026-10-14T21:44:41.820033442Z" + }, + "ca-smoke-261014214436754-crash": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 2, + "Error": "", + "StartedAt": "2026-10-14T21:48:22.990852752Z", + "FinishedAt": "2026-10-14T21:48:23.424342384Z" + }, + "ca-smoke-261014214436754-loop": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 2784034, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:45:46.731517222Z", + "FinishedAt": "2026-10-14T21:45:46.633484834Z" + }, + "ca-smoke-261014214436754-reuse": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 42, + "Error": "", + "StartedAt": "2026-10-14T21:48:22.984997284Z", + "FinishedAt": "2026-10-14T21:48:23.418274429Z" + }, + "ca-smoke-261014214436754-health": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 2797095, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:48:15.175556816Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-14T21:48:19.632370905Z", + "End": "2026-10-14T21:48:19.915665384Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:21.916497231Z", + "End": "2026-10-14T21:48:22.134371817Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:24.135145923Z", + "End": "2026-10-14T21:48:24.330071676Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:26.331087306Z", + "End": "2026-10-14T21:48:26.61488817Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:28.615762457Z", + "End": "2026-10-14T21:48:28.827588657Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "total_seconds": 233.9 + } +} diff --git a/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json new file mode 100644 index 0000000..cc7a168 --- /dev/null +++ b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json @@ -0,0 +1,324 @@ +{ + "run_id": "261014224557488", + "started_utc": "2026-10-14T22:45:57.106131Z", + "finished_utc": "2026-10-14T22:52:41.488066Z", + "status": "PASS", + "image": "watchdog:1.6.0-rc-fix", + "results": [ + { + "id": "T01_CLEAN_EXIT", + "status": "PASS", + "expected": "exit=0 means no alert", + "observed": "real exit=0, no alerts", + "elapsed_seconds": 21.9 + }, + { + "id": "T02_UNEXPECTED_CRASH", + "status": "PASS", + "expected": "exit=2 sends one CRITICAL crash", + "observed": "Docker exit=2, exactly one CRITICAL crashed alert", + "elapsed_seconds": 26.8 + }, + { + "id": "T03_PLANNED_STOP_137", + "status": "PASS", + "expected": "Docker stop exit 137 is suppressed", + "observed": "real docker stop -> SIGKILL/137; no crash or OOM alert", + "elapsed_seconds": 39.0 + }, + { + "id": "T04_DIRECT_SIGKILL", + "status": "PASS", + "expected": "SIGKILL 137 is crash, not OOM", + "observed": "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash", + "elapsed_seconds": 57.5 + }, + { + "id": "T05_REAL_OOM", + "status": "PASS", + "expected": "OOMKilled=true sends only CRITICAL OOM", + "observed": "kernel cgroup OOMKilled=true, exit=137, one CRITICAL OOM, zero crash", + "elapsed_seconds": 71.9 + }, + { + "id": "T06_RESTART_LOOP", + "status": "PASS", + "expected": "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "observed": "real Docker RestartCount=5; crash=1; restart-loop=1", + "elapsed_seconds": 83.7 + }, + { + "id": "T07_BRIEF_RECOVERY", + "status": "PASS", + "expected": "no recovery before 120s stability", + "observed": "running after 11s, RECOVERED=0", + "elapsed_seconds": 96.7 + }, + { + "id": "T08_INTERRUPTED_RECOVERY", + "status": "PASS", + "expected": "new actual die/reset prevents premature recovery", + "observed": "Docker RestartCount 6 -> 7; RECOVERED=0 after 14s", + "elapsed_seconds": 112.6 + }, + { + "id": "T09_STABLE_RECOVERY", + "status": "PASS", + "expected": "one INFO RECOVERED after 120s sustained uptime", + "observed": "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery", + "elapsed_seconds": 225.2 + }, + { + "id": "T06_TO_T09_RESTART_LIFECYCLE", + "status": "PASS", + "expected": "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", + "observed": "one CRASH + one LOOP + one RECOVERED after real interrupted recovery", + "elapsed_seconds": 225.3 + }, + { + "id": "T14_EXTERNAL_RESTART", + "status": "PASS", + "expected": "manual start after CRASH waits 120s before RECOVERED", + "observed": "real manual Docker start after crash; RECOVERED after 119.9s uptime", + "elapsed_seconds": 355.0 + }, + { + "id": "T10_NAME_REUSE", + "status": "PASS", + "expected": "replacement with same name gets new incident", + "observed": "two unique container IDs, same name, each gets one CRITICAL crash", + "elapsed_seconds": 367.1 + }, + { + "id": "T11_FAST_REMOVAL", + "status": "PASS", + "expected": "no event flood, event listener survives", + "observed": "3 real --rm containers, 0 bounded missing-ID warnings, observer alive", + "elapsed_seconds": 381.3 + }, + { + "id": "T12_NATIVE_HEALTHCHECK", + "status": "PASS", + "expected": "unhealthy -> recovered", + "observed": "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery", + "elapsed_seconds": 398.2 + }, + { + "id": "T13_SYSLOG", + "status": "PASS", + "expected": "all expected structured alerts reach test UDP collector", + "observed": "15 watchdog alerts delivered as 15 isolated UDP syslog messages; CRITICAL/HIGH/INFO fields correct", + "elapsed_seconds": 398.3 + } + ], + "facts": { + "embedded_source_sha256": { + "/app/watchdog.py": "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa", + "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", + "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" + }, + "host": "isolated-test-engine", + "revision": "", + "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", + "observer": "ca-smoke-261014224557488-observer", + "syslog_source_host": "release-smoke-261014224557488", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "clean": { + "name": "ca-smoke-261014224557488-clean", + "container_id": "93dc6ec3664fa2b472f7031cde519ecbd49549a82cce2ef1e85c8581657ed781" + }, + "crash": { + "name": "ca-smoke-261014224557488-crash", + "container_id": "59e4a2af2ecf9c225a09df23c6b59fc504ae4f2f0b147b13b9ca2b45c60120de" + }, + "planned": { + "name": "ca-smoke-261014224557488-planned", + "container_id": "ee156fae6fcd5bef6c53165886e04bf0988fe0061378fb496e76d9f488e3d4d9" + }, + "sigkill": { + "name": "ca-smoke-261014224557488-sigkill", + "container_id": "80e2215066163d64b2bd6a62ce5c6e43a57a7638d472539c53e8db9c7a539920" + }, + "sigkill_state": { + "exit": 137, + "OOMKilled": false + }, + "oom": { + "name": "ca-smoke-261014224557488-oom", + "container_id": "19482ebcd087d5d302ea6e707075e1a874524ef2a33e2f2db1c2b572de59e8a1" + }, + "oom_state": { + "exit": 137, + "OOMKilled": true + }, + "loop": { + "name": "ca-smoke-261014224557488-loop", + "container_id": "1016f94b5e91523c0358698cbe9b48edad0ac087ae4c332a5f55cc9690f9b02e" + }, + "loop_restart_count_after": 7, + "external": { + "name": "ca-smoke-261014224557488-external", + "container_id": "d30bdc108c3a194a9fe4d75a09e8ce313eca86a8c0c9f6af3d2f26812624f969" + }, + "reuse": { + "name": "ca-smoke-261014224557488-reuse", + "container_id": "c09027edaf62f85bb5babaf6aa38aee13d719866ba108f83a9cdeb9e52544c51" + }, + "fast_remove_warning_count": 0, + "health": { + "name": "ca-smoke-261014224557488-health", + "container_id": "23b9356951d88eb98fbd92d8bdd08be6ed29ffb293125338db78a8d35560a300" + }, + "watchdog_alert_count": 15, + "dfc_received_count": 15, + "live_docker_events": 92, + "live_docker_action_counts": { + "create": 14, + "start": 22, + "die": 18, + "oom": 1, + "stop": 1, + "kill": 3, + "restart": 0, + "health_status: unhealthy": 1, + "health_status: healthy": 1, + "destroy": 7 + }, + "docker_event_lines": 97, + "final_fixture_states": { + "ca-smoke-261014224557488-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 5484, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:46:02.126339022Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014224557488-clean": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:46:11.519112255Z", + "FinishedAt": "2026-10-14T22:46:11.888551005Z" + }, + "ca-smoke-261014224557488-crash": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 2, + "Error": "", + "StartedAt": "2026-10-14T22:46:20.387611646Z", + "FinishedAt": "2026-10-14T22:46:20.651109479Z" + }, + "ca-smoke-261014224557488-loop": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 6714, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:47:35.416361955Z", + "FinishedAt": "2026-10-14T22:47:35.299866533Z" + }, + "ca-smoke-261014224557488-external": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 6976, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:49:51.127967239Z", + "FinishedAt": "2026-10-14T22:49:45.881841779Z" + }, + "ca-smoke-261014224557488-reuse": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 42, + "Error": "", + "StartedAt": "2026-10-14T22:52:00.724185739Z", + "FinishedAt": "2026-10-14T22:52:00.970303463Z" + }, + "ca-smoke-261014224557488-health": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 7439, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:52:21.630416377Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-14T22:52:28.219276407Z", + "End": "2026-10-14T22:52:28.31610092Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:30.316786865Z", + "End": "2026-10-14T22:52:30.503706796Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:32.504440652Z", + "End": "2026-10-14T22:52:32.698663166Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:34.699078389Z", + "End": "2026-10-14T22:52:35.108639315Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:37.109706599Z", + "End": "2026-10-14T22:52:37.218394592Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "total_seconds": 404.4 + } +} diff --git a/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json new file mode 100644 index 0000000..620cf74 --- /dev/null +++ b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json @@ -0,0 +1,81 @@ +{ + "run_id": "261014224323244", + "started_utc": "2026-10-14T22:43:23.601963Z", + "finished_utc": "2026-10-14T22:45:47.277368Z", + "status": "PASS", + "image": "watchdog:1.6.0-rc-fix", + "results": [ + { + "id": "T15_EXTERNAL_OOM", + "status": "PASS", + "expected": "real OOM then manual start requires 120s stability", + "observed": "real kernel OOM -> external Docker start -> INFO RECOVERED after 120.6s", + "elapsed_seconds": 139.9 + } + ], + "facts": { + "embedded_source_sha256": { + "/app/watchdog.py": "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa", + "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", + "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" + }, + "host": "isolated-test-engine", + "revision": "", + "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", + "observer": "ca-smoke-261014224323244-observer", + "syslog_source_host": "release-smoke-261014224323244", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "externaloom": { + "name": "ca-smoke-261014224323244-externaloom", + "container_id": "8c4b398b1b98f0c13523ba16e67ddec1f44319f6cd5e10311c9a9215db8d5e03" + }, + "live_docker_events": 7, + "live_docker_action_counts": { + "create": 2, + "start": 3, + "die": 1, + "oom": 1, + "stop": 0, + "kill": 0, + "restart": 0, + "health_status: unhealthy": 0, + "health_status: healthy": 0, + "destroy": 0 + }, + "docker_event_lines": 9, + "final_fixture_states": { + "ca-smoke-261014224323244-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 4754, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:43:28.60988319Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014224323244-externaloom": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 5131, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:43:42.417354209Z", + "FinishedAt": "2026-10-14T22:43:40.28403892Z" + } + }, + "total_seconds": 143.7 + } +} diff --git a/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json new file mode 100644 index 0000000..032e142 --- /dev/null +++ b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json @@ -0,0 +1,76 @@ +{ + "run_id": "261014223252035", + "started_utc": "2026-10-14T22:32:52.613776Z", + "finished_utc": "2026-10-14T22:33:29.913178Z", + "status": "FAIL", + "image": "watchdog:1.6.0", + "results": [ + { + "id": "T14_EXTERNAL_RESTART", + "status": "FAIL", + "expected": "no recovery until 120 seconds after external start", + "observed": "ca-smoke-261014223252035-external recovered: expected 0 got 1", + "elapsed_seconds": 33.8 + } + ], + "facts": { + "host": "isolated-test-engine", + "revision": "", + "image_id": "sha256:3a9942e74f643b0c2b38c82bcd0ef2b805cd13dd0f7c100031080f2cf06e494a", + "observer": "ca-smoke-261014223252035-observer", + "syslog_source_host": "release-smoke-261014223252035", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "external": { + "name": "ca-smoke-261014223252035-external", + "container_id": "b78eb7be8c5464972808a9a3a85a2240a4b57aac347ca91b31a7c445015604af" + }, + "live_docker_events": 6, + "live_docker_action_counts": { + "create": 2, + "start": 3, + "die": 1, + "oom": 0, + "stop": 0, + "kill": 0, + "restart": 0, + "health_status: unhealthy": 0, + "health_status: healthy": 0, + "destroy": 0 + }, + "docker_event_lines": 8, + "final_fixture_states": { + "ca-smoke-261014223252035-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 790, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:32:57.598924455Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014223252035-external": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 1174, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:33:14.250958418Z", + "FinishedAt": "2026-10-14T22:33:09.081072575Z" + } + }, + "total_seconds": 37.3 + } +} diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md new file mode 100644 index 0000000..98983a6 --- /dev/null +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -0,0 +1,432 @@ +# Container Alert — Customer Installation and Operations + +This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. + +## Table of Contents + +1. [Prerequisites](#1-prerequisites) +2. [Installation](#2-installation) + - [Get and verify the release package](#21-get-and-verify-the-release-package) + - [Run the installer](#22-run-the-installer) +3. [Verify the installation](#3-verify-the-installation) +4. [Configuration reference](#4-configuration-reference) + - [Configuration files and editing](#41-configuration-files-and-editing) + - [Watchdog monitoring settings](#42-watchdog-monitoring-settings) + - [Notification channels](#43-notification-channels) + - [Syslog](#syslog) + - [SMTP (email)](#smtp-email) + - [Slack](#slack) + - [SNMP traps](#snmp-traps) + - [Health checks, exclusions, and HA](#44-health-checks-exclusions-and-ha) + - [Environment variables](#45-environment-variables) + - [Apply configuration changes](#46-apply-configuration-changes) +5. [Operations and maintenance](#5-operations-and-maintenance) + - [Common commands and logs](#51-common-commands-and-logs) + - [Backup and restore](#52-backup-and-restore) + - [Upgrade the watchdog](#53-upgrade-the-watchdog) + - [Rollback](#54-rollback) + - [Uninstall](#55-uninstall) +6. [Troubleshooting](#6-troubleshooting) +7. [Related documentation](#7-related-documentation) + +## 1. Prerequisites + +| Requirement | What is needed | +|---|---| +| Operating system | Supported Linux host with Docker (Ubuntu 20.04 / RHEL 8 or later are the documented baseline) | +| Docker and Compose | Docker Engine 20.10+ and Docker Compose (plugin or standalone) | +| Host permissions | Root or an operator with Docker access; `sudo` may be required to secure configuration files | +| Release package | Published `container-alert-v.tar.gz` and matching `SHA256SUMS` | +| Network access | Optional; download from [GitHub Releases](https://github.com/Radware/CyberController-Container-Alert/releases) or transfer both files from another machine | +| Alert delivery | Network connectivity to the chosen Syslog, SMTP, SNMP, or Slack destination | + + +## 2. Installation + +Download the installation package directly, or transfer the same package to a CyberController host `opt/radware/storage/scripts/` directory. The release package includes the installer, prebuilt Docker image and configuration templates. + +The examples use v1.6.0; download a version from [GitHub Releases](https://github.com/Radware/CyberController-Container-Alert/releases). + +### 2.1 Get and verify the release package + +**Step 1 — Choose a release version.** Open the [Container Alert Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) to see all published versions. The release marked **Latest** is usually the newest stable version. + +**Step 2 — Obtain the release package and checksum.** Use either method to get the **same two files** on the target host: + +**Host has Internet access:** From a root shell (or account with write access), download into the scripts directory: + +```bash +mkdir -p /opt/radware/storage/scripts +cd /opt/radware/storage/scripts +VER=1.6.0 # Example: GitHub tag v1.6.0 +URL="https://github.com/Radware/CyberController-Container-Alert/releases/download/v${VER}" +curl -fL -O "$URL/container-alert-v${VER}.tar.gz" -O "$URL/SHA256SUMS" +``` + +`curl` needs HTTPS access to GitHub Releases and its redirected download host. + +**Host without Internet access:** Download the chosen release package and `SHA256SUMS` on another computer. Transfer both files to `/opt/radware/storage/scripts/` on the CyberController host. + +**Step 3 — Verify and extract** the two files from the same directory: + +```bash +mkdir -p /opt/radware/storage/scripts +cd /opt/radware/storage/scripts +VER=1.6.0 # Use the version chosen in Step 1 +sha256sum --check SHA256SUMS +tar -xzf "container-alert-v${VER}.tar.gz" +cd /opt/radware/storage/scripts/CyberController-Container-Alert +``` + +The extracted directory contains `VERSION`, `install.sh`, `docker-compose.yaml`, `watchdog.tar` and its own image checksum. + +### 2.2 Run the installer + +**Step 4 — Start the configuration wizard:** + +```bash +bash install.sh +``` + +The installer automatically verifies the included Docker image checksum and version, loads the image, then asks you to select and configure the notification channels (Syslog, SMTP, SNMP and/or Slack) and alert hostname. On an upgrade, answer **No** to “Reconfigure from scratch?” to preserve existing channel settings. It recreates **only the watchdog**, not other application containers. + +## 3. Verify the installation + +Run these commands after installation: + +```bash +docker compose ps +docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' +docker inspect docker-container-watchdog --format '{{.Config.Image}}' +``` + +The watchdog should be `running` and become `healthy`; the image should be `watchdog:` matching the package. + +**Logs and alert delivery** + +| Location or command | Purpose | +|---|---| +| `docker logs -f docker-container-watchdog` | Live container output | +| `docker compose logs --tail 100 docker-container-watchdog` | Recent Compose logs | +| `./watchdog/watchdog.log` | Persistent host log (rotates at 10 MB, five backups) | +| `./watchdog/state.json` | Incident, HA and reboot state; do not edit manually | +| `grep 'watchdog_alert' watchdog/watchdog.log` | Find existing alert records | + +Check that the selected Syslog, SMTP, SNMP or Slack receiver can be reached and inspect any alerts already received. No artificial container failure is needed. + +**Optional HA check:** If `ha_monitor.enabled: true`, check the local exporter on **each HA node**: + +```bash +curl -fsS http://127.0.0.1:9664/metrics +``` + +Use the configured exporter URL if different. The first successful reading establishes a baseline; it does not send a failover alert. Test a real switchover only during planned HA maintenance. + + +## 4. Configuration reference + +The installer creates the active configuration files. Edit them only when needed; most installations require no manual changes. + +### 4.1 Configuration files and editing + +| File | What it contains | Apply changes | +|---|---|---| +| `watchdog-config.yaml` | Monitoring, alerts, health checks and HA | Restart watchdog | +| `.env` | Credentials, hostname and Docker image version | Recreate watchdog | +| `docker-compose.yaml` | Container runtime settings; generally managed by Radware | Recreate watchdog | +| `*.example` | Reference templates, not active settings | No effect on the running service | + +The installer creates `watchdog-config.yaml` and `.env`. From the installation directory, you can edit `watchdog-config.yaml` or `.env`. Back up first ([Section 5.2](#52-backup-and-restore)) + +### 4.2 Watchdog monitoring settings + +These settings are in `watchdog-config.yaml`. Defaults shown are the application defaults; the installer may set channel-specific values. + +| Setting | Default | Purpose | +|---|---|---| +| `alert_channels` | Installer selection (code default: `[slack]`) | Lists Slack, SMTP and SNMP notification channels | +| `check_interval_seconds` | `60` | Poll interval in seconds | +| `cooldown_minutes` | `5` | Reduce repeat alerts for the same incident | +| `restart_threshold` | `5` | Restarts needed to report a restart loop | +| `restart_window_minutes` | `10` | Time window for counting restarts | +| `restart_recovery_stability_seconds` | `120` | Stable running time before crash/OOM/restart-loop recovery | +| `unhealthy_cycles_threshold` | `3` | Consecutive failing health checks before alert | +| `alert_on_recovery` | `true` | Notify when an alerted condition clears | +| `lifecycle_detection.enabled` | `true` | Recognize intentional Docker stops | +| `lifecycle_detection.correlation_window_seconds` | `3` | Match stop and exit events (0.5–30 seconds) | +| `boot_grace_seconds` | `180` | Delay normal alerts after reboot; `0` disables the grace period | +| `log_level` | `INFO` | Local verbosity: DEBUG, INFO, WARNING or ERROR; forwarded Syslog priorities are separate | +| `log_file` | `/var/log/watchdog/watchdog.log` | Persistent log, mounted under `./watchdog/`; `null` disables file logging | +| `state_file` | `/var/log/watchdog/state.json` | Preserves incidents across restarts; empty value disables persistence | +| `runbook_base_url` | Empty | Optional runbook link in alerts | + +Changing thresholds, exclusions or probe types can hide failures. Check their impact carefully; do not change them just to suppress alerts. + +### 4.3 Notification channels + +Choose channels in the installer. For later changes, edit the following YAML settings. To enable SMTP, Slack or SNMP manually, add `smtp`, `slack` or `snmp_trap` to the `alert_channels` list and enable the corresponding channel. Syslog is enabled separately with `syslog.enabled`. + +| Channel | Settings in `watchdog-config.yaml` | Credentials in `.env` | +|---|---|---| +| **Syslog** | `syslog.enabled`, `syslog.host`, `syslog.port`, `syslog.protocol` (UDP/TCP), `syslog.facility` | None | +| **SMTP** | `smtp.enabled`, `smtp.auth`, `smtp.host`, `smtp.port`, `smtp.sender`, `smtp.recipients`, `smtp.tls`, `smtp.username_env` / `smtp.password_env` (**variable names only**) | `SMTP_USERNAME`, `SMTP_PASSWORD` if authentication is enabled | +| **Slack** | `slack.enabled`, `slack.webhook_url_env` | `SLACK_WEBHOOK_URL` | +| **SNMP traps** | `snmp_trap.enabled`, `snmp_trap.host`, `snmp_trap.port`, `snmp_trap.version`, `snmp_trap.community` (v1/v2c), optional `snmp_trap.trap_oid` | `SNMP_V3_AUTH_KEY`, `SNMP_V3_PRIV_KEY` for SNMPv3 | + +#### Syslog + +Set `syslog.enabled: true` and specify the receiver. Syslog forwards structured alerts, including INFO recoveries, regardless of local `log_level`. + +#### SMTP (email) + +Enable SMTP in `alert_channels` and set `smtp.enabled: true`. Set host, port, sender and recipients in YAML. `smtp.tls: true` uses STARTTLS; `smtp.auth: false` is for mail relays that do not require login. + +**Important:** `username_env` and `password_env` contain **environment variable names**, not your username or password. Example: + +```yaml +# watchdog-config.yaml — references only +smtp: + username_env: SMTP_USERNAME + password_env: SMTP_PASSWORD +``` + +```dotenv +# .env — actual values, never committed to Git +SMTP_USERNAME=your-mail-login +SMTP_PASSWORD=your-mail-password +``` + +The installer normally writes these settings for you. + +#### Slack + +Include `slack` in `alert_channels` and set `slack.enabled: true`. In YAML, `slack.webhook_url_env: SLACK_WEBHOOK_URL` names the variable; the **actual** webhook URL belongs in `.env`. + +#### SNMP traps + +Include `snmp_trap` in `alert_channels` and set `snmp_trap.enabled: true`. Enter the NMS host and UDP trap port (usually 162); versions v1/v2c use `community`. + +For **SNMPv3**, configure `v3_username`, `v3_auth_protocol` (MD5, SHA, SHA256, SHA384 or SHA512) and, for encrypted traps, `v3_priv_protocol` (AES, AES192 or AES256). The `v3_auth_key_env` and `v3_priv_key_env` fields reference passphrases stored in `.env`, not plaintext in YAML. Passphrases must be at least eight characters; encryption requires authentication. DES is weak and not recommended. + +Register the sender engine ID from the watchdog startup logs (`SNMP engine ID: 0x...`) with your NMS. Use `v3_local_engine_id` when a fixed sender identity is needed. + +The default notification OID is `1.3.6.1.4.1.89.110.0.1`. NMS integrations receive five fields: + +| Var-bind | OID | Value | +|---|---|---| +| `cwHost` | `1.3.6.1.4.1.89.110.1.1.0` | Hostname | +| `cwSummary` | `1.3.6.1.4.1.89.110.1.2.0` | Alert summary | +| `cwContainerName` | `1.3.6.1.4.1.89.110.1.3.0` | Container or HA cluster | +| `cwFailureType` | `1.3.6.1.4.1.89.110.1.4.0` | Failure type, recovery or `ha-failover` | +| `cwProbeDetail` | `1.3.6.1.4.1.89.110.1.5.0` | Detection details | + +HA traps use the same OID and var-binds with `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`. Check receiver-side SNMP user/engine-ID configuration if traps are rejected. Do not trigger application failures just to test notifications. + +### 4.4 Health checks, exclusions, and HA + +| Setting | Default | Purpose | +|---|---|---| +| `auto_health_check.enabled` | `true` | Probe containers without a Docker health check | +| `auto_health_check.paths` | `/-/healthy`, `/health`, `/healthz`, `/api/health`, `/metrics`, `/` | Discovery paths | +| `auto_health_check.timeout_seconds` | `2` | Probe timeout | +| `container_health_checks` | `{}` | Per-container `http` / `exec` probe overrides | +| `excluded_containers` | `[]` | Containers excluded from alerts | +| `ignored_container_events` | Radware-labelled temporary check | Narrow exceptions for known transient operations | +| `ha_monitor.enabled` | `false` | Optional active-node change alerts | +| `ha_monitor.exporter_url` | `http://127.0.0.1:9664/metrics` | Local HA exporter address | + +A custom probe can override discovery for a specific container. Example: + +```yaml +container_health_checks: + example-api: + type: http + internal_port: 8080 + path: /health + timeout_seconds: 5 +``` + +Fields are `type` (`http` or `exec`), `internal_port` and `path` for HTTP, `command` for exec, and optional `timeout_seconds` (default 5). Exec probes run commands **inside monitored containers**. Check the impact of probe, exclusion and ignore-rule changes before applying them. + +For HA alerts, enable `ha_monitor` on both nodes with a working local exporter. Check the configured exporter URL read-only (normally `http://127.0.0.1:9664/metrics`). The first valid reading sets a baseline without sending an alert. + +### 4.5 Environment variables + +The installer manages most `.env` values. SMTP passwords, Slack webhooks and SNMPv3 passphrases belong here, not in YAML. The SNMPv1/v2c community is configured in YAML, so protect that file too. + +| Variable | Purpose | +|---|---| +| `WATCHDOG_HOST` | Hostname shown in alerts | +| `WATCHDOG_VERSION` | Image version number only (for example `1.6.0`, without `watchdog:`) | +| `WATCHDOG_UID`, `WATCHDOG_GID`, `DOCKER_GID` | Runtime user and Docker socket permissions; installer-managed | +| `SLACK_WEBHOOK_URL` | Slack webhook | +| `SMTP_USERNAME`, `SMTP_PASSWORD` | SMTP credentials, when required | +| `SNMP_V3_AUTH_KEY`, `SNMP_V3_PRIV_KEY` | SNMPv3 passphrases | + +`WATCHDOG_CONFIG` is set in Compose, not normally in `.env`. Set logging with YAML `log_level`, not the obsolete `LOG_LEVEL` environment variable. + +### 4.6 Apply configuration changes + +Back up the configuration before editing. Run these from `/opt/radware/storage/scripts/CyberController-Container-Alert`: + +**YAML-only change — restart the watchdog:** + +```bash +docker compose restart docker-container-watchdog +``` + +**`.env` or Compose change — recreate the watchdog:** + +```bash +docker compose up -d --no-deps --force-recreate docker-container-watchdog +``` + +A Python code change requires a new versioned Docker image; restarting will not update code. Neither command restarts other application containers. Check the watchdog afterward using [Section 3](#3-verify-the-installation). + +## 5. Operations and maintenance + +Run these commands from `/opt/radware/storage/scripts/CyberController-Container-Alert`. They manage **only the watchdog**. + +### 5.1 Common commands and logs + +| Task | Command | +|---|---| +| Check status | `docker compose ps` | +| View recent logs | `docker logs --tail 100 docker-container-watchdog` | +| Follow logs live | `docker logs -f docker-container-watchdog` | +| Show running image | `docker inspect docker-container-watchdog --format '{{.Config.Image}}'` | +| List installed image versions | `docker image ls watchdog` | +| Start / stop | `docker compose start docker-container-watchdog` / `docker compose stop docker-container-watchdog` | +| Restart | `docker compose restart docker-container-watchdog` | +| Recreate | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | + +Stopping the watchdog **pauses monitoring** but does not stop application containers. Avoid `docker compose down` during routine maintenance. + +### 5.2 Backup and restore + +Before an upgrade or configuration change, back up `.env` and `watchdog-config.yaml`: + +```bash +sudo install -d -m 700 backup +cp -p .env backup/.env.bak +cp -p watchdog-config.yaml backup/watchdog-config.yaml.bak + +chmod 600 backup/.env.bak +chmod 600 backup/watchdog-config.yaml.bak +``` + +Backups contain credentials; keep the backup directory restricted. Preserve the `watchdog/` directory as it stores persistent logs and incident/HA state. Omit `sudo` if already running as root. + +To restore both files, then recreate **only** the watchdog: + +```bash +# Run from the watchdog installation directory +cp -p backup/.env.bak .env +cp -p backup/watchdog-config.yaml.bak watchdog-config.yaml + +# Protect credentials +chmod 600 .env + +# Validate Compose configuration +docker compose config --quiet + +# Recreate only the watchdog +docker compose up -d --no-deps --force-recreate docker-container-watchdog + +# Verify +docker compose ps docker-container-watchdog +docker compose logs --tail 50 docker-container-watchdog +``` + +### 5.3 Upgrade the watchdog + +The upgrade uses the same installer as a new installation. + +1. [Back up your configuration](#52-backup-and-restore). +2. Download or transfer the **new published version** and its `SHA256SUMS` into `/opt/radware/storage/scripts/`. +3. From that directory, verify and extract the new release **over the existing installation directory**, then run its installer: + + ```bash + cd /opt/radware/storage/scripts + VER=1.7.0 # Example: use the version you downloaded + sha256sum --check SHA256SUMS + tar -xzf "container-alert-v${VER}.tar.gz" + cd /opt/radware/storage/scripts/CyberController-Container-Alert + bash install.sh + ``` + +4. Choose **No** when asked `Reconfigure from scratch?` to keep existing settings. Check the watchdog health and image tag using [Section 3](#3-verify-the-installation). + +The package does not replace active `.env`, `watchdog-config.yaml` or `watchdog/`. Only the watchdog container is recreated. There are no automatic upgrades. + +### 5.4 Rollback + +If you need to return to an earlier version: + +1. Check that its image is still installed: `docker image ls watchdog`. +2. If the image exists, update `WATCHDOG_VERSION` in `.env`. For example, to return to v1.5.4: + + ```bash + cd /opt/radware/storage/scripts/CyberController-Container-Alert + sudo sed -i 's/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=1.5.4/' .env + grep '^WATCHDOG_VERSION=' .env + docker compose up -d --no-deps --force-recreate docker-container-watchdog + docker inspect docker-container-watchdog --format '{{.Config.Image}}' + ``` + + The `.env` value is `1.5.4`, **not** `watchdog:1.5.4`. Restore the matching configuration backup if necessary. + +3. If the earlier image is **not** installed, obtain its release package and follow the [upgrade procedure](#53-upgrade-the-watchdog) using that older version; its installer loads the image. Choose **No** to keep existing configuration. + +Verify status and logs after rollback. Restarting without selecting the older image does **not** change the code. Running a newer installer again will select the version packaged with that installer. + +### 5.5 Uninstall + +| Task | Command | +|---|---| +| Interactive uninstall | `bash uninstall.sh` | +| Remove watchdog, keep image and logs | `bash uninstall.sh --keep-image --keep-logs` | +| Remove watchdog and image, keep logs | `bash uninstall.sh --keep-logs` | +| Remove watchdog, image and logs | `bash uninstall.sh --remove-all` | + +Logs are preserved by default. `--force` skips confirmation; use it only when unattended removal is intended. Uninstalling the watchdog does not remove CyberController application containers. + +## 6. Troubleshooting + +Start with the watchdog status and logs; you do not need to disrupt an application container to investigate. + +```bash +docker inspect docker-container-watchdog --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' +docker logs --tail 100 docker-container-watchdog +``` + +| Problem | What to check | +|---|---| +| Release download fails | The version exists on the [Releases page](https://github.com/Radware/CyberController-Container-Alert/releases); host HTTPS access, or use an offline transfer | +| Image or checksum rejected | The archive and `SHA256SUMS` come from the same release; check disk space and re-download if needed | +| Docker/Compose errors | `docker info`, `docker compose version` (or `docker-compose --version`), Docker socket permissions | +| Watchdog exits or remains unhealthy | `docker logs --tail 100 docker-container-watchdog` and `docker inspect` health status; `starting` is not `healthy` | +| No notifications | Verify receiver address, port and connectivity. For Syslog check `syslog.enabled`; for SMTP/Slack/SNMP check `alert_channels` and the channel's `enabled` field | +| SMTP/Slack authentication fails | Check the expected variable names in YAML and the actual secret values in `.env`; recreate the watchdog after `.env` changes | +| SNMPv3 traps rejected | Match NMS USM user, auth/privacy settings and sender engine ID (logged at watchdog startup) | +| INFO recovery missing | Check `alert_on_recovery`, whether an initial alert occurred, and the required stable-running interval | +| Repeated crash/restart alerts | Examine container status, Docker events and watchdog logs; do not suppress or exclude containers just to silence alerts | +| Unexpected unhealthy alerts | Compare Docker's native HEALTHCHECK with auto/manual probe details before changing any monitoring settings | +| Missing HA change alert | Check both watchdogs and the local HA exporter. The first reading creates a quiet baseline; a change should alert only after a confirmed active-node transition | + +For additional information, inspect the persistent log in the installation directory: + +```bash +grep 'watchdog_alert' watchdog/watchdog.log +grep -E 'ERROR|CRITICAL' watchdog/watchdog.log +``` + +A `WARNING` local log level can hide INFO recovery messages from the local file/console, while forwarded Syslog alerts keep their own severity. Check the receiver when investigating missing INFO events. + +Do not stop, stress or restart monitored application containers merely to test notifications or silence alerts. Preserve the evidence and investigate the underlying problem before changing application settings. + +## 7. Related documentation + +- [Project overview](../../README.md) — product features and alert types. +- [v1.6.0 release notes](RELEASE_NOTES_V1.6.0.md) — changes and known limitations. +- [Configuration template](../../watchdog-config.yaml.example) and [environment template](../../.env.example) — settings examples. diff --git a/docs/operations/RELEASE_NOTES_V1.6.0.md b/docs/operations/RELEASE_NOTES_V1.6.0.md new file mode 100644 index 0000000..c55603a --- /dev/null +++ b/docs/operations/RELEASE_NOTES_V1.6.0.md @@ -0,0 +1,61 @@ +# v1.6.0 Release — Changes Since 1.5.4 + +This document describes version **1.6.0** and its changes since v1.5.4. Building, publishing, and deploying its image are separate release steps. + +## Feature and fix inventory + +| Area | Change since 1.5.4 | Change record | +|---|---|---| +| Baseline reliability and installation hardening | Serialize state persistence under concurrent event/poll writers; make manual exec probes bounded; consolidate post-reboot alert summaries; secure runtime config file as root-owned and group-readable; verify post-install health; improve image-archive validation and uninstall so unrelated tags are preserved | [PR #3](https://github.com/Radware/CyberController-Container-Alert/pull/3) | +| Docker event consistency | Resolve event identity by Docker ID; prevent historical names from corrupting a new container with the same name; rate-limit removed-container warnings; protect event processing from removed IDs | [PR #6](https://github.com/Radware/CyberController-Container-Alert/pull/6) | +| Restart-loop lifecycle | Count restarts using Docker's authoritative `RestartCount`; deduplicate CRASH / RESTART-LOOP notifications, prevent health checks from replacing active loop incidents; require sustained uptime before RECOVERED | [PR #5](https://github.com/Radware/CyberController-Container-Alert/pull/5) | +| Planned container shutdown | Correlate Docker stop intent with late / reversed `die` events, suppress false CRASH for legitimate Docker stops; distinguish SIGKILL exit 137 from confirmed kernel OOM; protect stop evidence across container starts | [PR #7](https://github.com/Radware/CyberController-Container-Alert/pull/7) | +| Optional CyberController HA change alert | Detect promoted/active HA Operator from existing Pacemaker exporter, not the Pacemaker DC. Separate HA polling from Docker; quiet baseline/rebaseline; only new active owner sends one WARNING on real failover/failback. Reuses existing notifications, state persistence and SNMP MIB | [PR #4](https://github.com/Radware/CyberController-Container-Alert/pull/4) | +| Release documentation | Clarify merged vs unreleased features and explicit image version/tag process | [PR #8](https://github.com/Radware/CyberController-Container-Alert/pull/8) | +| Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | +| Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Guided release installation | One complete versioned package includes the installer and prebuilt image; operators verify/extract and run one wizard regardless of network availability. Existing settings are preserved and only the watchdog is recreated. No customer-side source build, Git checkout or image download. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Operator documentation and configuration clarity | Restore one self-contained installation/upgrade path, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP receiver configuration alongside other notification channels; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Customer-facing alert descriptions | Factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | + +## Customer-facing message catalogue (v1.6.0) + +These strings are informational, not instructions to change production configuration. The existing incident severity and detection evidence remain intact. + +| Event | Severity | Context text | +|---|---|---| +| `crashed` | CRITICAL | Docker reported an unexpected container termination. Run: `docker logs --tail 50`. The exit code and detection details provide additional context. | +| `oom` | CRITICAL | Docker reported that the container was terminated due to an out-of-memory condition. | +| `unhealthy` | HIGH | The container is running, but a monitoring health check is failing. | +| `restart-loop` | HIGH | The container has restarted repeatedly within the configured monitoring window. This pattern may be associated with application errors, resource pressure, or unavailable dependencies. The underlying cause has not been determined. | +| `recovered` | INFO | The previously reported alert condition has cleared according to the available monitoring checks. | +| `reboot-summary` | INFO or HIGH | Monitoring has resumed following a host reboot. The reported summary includes container status, recovered incidents, and any conditions that remain unresolved. | +| `ha-failover` | WARNING | Verify whether the HA switchover was expected and check service health. **Unchanged.** | +| Unknown type | WARNING | A container monitoring event was reported. The failure type and available detection details provide additional context. | + +The CRASH log command is read-only. An alert alone is not a reason to modify Docker Compose, memory limits, restart policies or application resources. Investigate the cause before considering changes. + +**Compatibility:** No change to machine event identifiers, severities, HA identities, container IDs, OIDs/var-binds, probe details, or Syslog key `action`. Only human-facing text and email/Slack label `Context` change. No new alert transport, background service or message queue. + +## Verification and safety boundaries + +- Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. +- Destructive real Docker failure testing is **only** through the [Release smoke (isolated)](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/.github/workflows/release-smoke.yml) workflow, automatically triggered for qualifying release PR changes and manually rerunnable when available on the default branch. It uses an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_SMOKE_TESTING.md). +- Historical test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/dev/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. +- Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. +- Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the release commit, identified by image digest, and deployed through the existing versioned release process. + +## Image distribution and deployment + +The v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is reviewed and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. + +The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/DEVELOPMENT_AND_RELEASE.md) and [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/operations/INSTALLATION_AND_OPERATIONS.md). + +## Exclusions, limitations and release gating + +**Not established by testing:** live customer SMTP, Slack and SNMP delivery end-to-end; arbitrary host reboot, network partition, STONITH, or production recovery policy. SMTP was explicitly excluded during lab acceptance. HA failover/failback was tested on real lab nodes with a prior source candidate, without any further failover required for text-only changes. + +The HA notification dispatch path has no persistent retry queue (intentional v1 limitation). Docker planned-stop correlation is not an availability policy; a deliberately stopped long-running service does not automatically produce a downtime alert. Unexpected external Docker restarts were seen on a lab; the initiator was not determined. These are documented operational limitations, not release regressions. + +**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review completed, merge to `main`, build/verify exact `watchdog:1.6.0`, verify release checksums and packaged artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. Merging and tagging are manual steps; the package build and publication run automatically after the release tag is pushed. diff --git a/install.sh b/install.sh index 3228326..2e1cac1 100755 --- a/install.sh +++ b/install.sh @@ -323,73 +323,63 @@ detect_docker_gid() { # Docker Image ################################################################################ -load_docker_image() { - print_section "Loading Docker Image" - - # Always load the packaged archive first. The previous behaviour — skipping - # the load whenever any watchdog image already existed — meant an upgrade - # kept running the old code, because watchdog.py ships inside the image. - if [ -f "$IMAGE_ARCHIVE" ]; then - print_info "Loading image from archive: $(basename "$IMAGE_ARCHIVE")" - local load_output - if ! load_output=$(docker load -i "$IMAGE_ARCHIVE" 2>&1); then - print_error "Failed to load ${IMAGE_ARCHIVE}" - echo "$load_output" - exit 1 - fi - echo "$load_output" - - # The archive itself must carry the release tag. Merely inspecting the - # local tag after docker load is insufficient because a stale tag may - # have existed before this installer ran. - if ! printf '%s\n' "$load_output" | grep -Fxq "Loaded image: ${IMAGE_NAME}"; then - print_error "${IMAGE_ARCHIVE} did not load the required tag ${IMAGE_NAME}" - echo "" - echo " docker load reported:" - printf ' %s\n' "$load_output" - echo "" - echo " This package is version ${VERSION} (from ${VERSION_FILE})." - echo " Rebuild the archive with the versioned tag:" - echo " docker build -t ${IMAGE_NAME} ." - echo " docker tag ${IMAGE_NAME} ${IMAGE_LATEST}" - echo " docker save ${IMAGE_NAME} ${IMAGE_LATEST} -o watchdog.tar" - exit 1 - fi - if ! docker image inspect "$IMAGE_NAME" &>/dev/null; then - print_error "Required image ${IMAGE_NAME} is not available after docker load" - exit 1 - fi - else - print_warning "Image archive not found: ${IMAGE_ARCHIVE}" - - # No archive — build from source if the Dockerfile shipped. - if [ -f "${INSTALL_DIR}/Dockerfile" ]; then - print_info "Building ${IMAGE_NAME} from source — this may take a few minutes..." - # Use a clean temp dir so any .dockerignore in INSTALL_DIR is bypassed - local tmpdir - tmpdir=$(mktemp -d) - trap "rm -rf '${tmpdir}'" EXIT - cp "${INSTALL_DIR}/Dockerfile" "${tmpdir}/" - cp "${INSTALL_DIR}/watchdog.py" "${tmpdir}/" - if ! docker build -t "${IMAGE_NAME}" "${tmpdir}"; then - print_error "Build failed — cannot produce ${IMAGE_NAME}" - exit 1 - fi - print_success "Image built: ${IMAGE_NAME}" - else - print_error "Cannot proceed: ${IMAGE_NAME} is not available" - echo "" - echo " watchdog.tar not found and Dockerfile not found." - echo " Copy watchdog.tar into: $(dirname "$IMAGE_ARCHIVE") and re-run." - exit 1 - fi +# Confirm that an image carries the release version, not just an arbitrary tag. +verify_release_image() { + local reported_version + docker image inspect "$IMAGE_NAME" >/dev/null 2>&1 || return 1 + reported_version=$(docker image inspect "$IMAGE_NAME" \ + --format '{{index .Config.Labels "org.opencontainers.image.version"}}' 2>/dev/null) || return 1 + if [ "$reported_version" != "$VERSION" ]; then + print_warning "$IMAGE_NAME has version label '$reported_version', expected '$VERSION'" + return 1 fi +} - # Convenience alias only — docker-compose.yaml deploys ${IMAGE_NAME}. - docker tag "$IMAGE_NAME" "$IMAGE_LATEST" &>/dev/null || \ - print_warning "Could not update the ${IMAGE_LATEST} convenience tag" +# Verify the image archive checksum shipped in every release package. +load_release_archive() { + local archive="$1" sums="$2" expected actual loaded + if [ ! -r "$archive" ] || [ ! -r "$sums" ] || ! command -v sha256sum >/dev/null 2>&1; then + print_error "Release image or checksum is missing; image not loaded" + return 1 + fi + expected=$(awk '$2 == "watchdog.tar" || $2 == "*watchdog.tar" {print $1}' "$sums" | tail -n1) + if ! [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]]; then + print_error "SHA256SUMS has no valid digest for watchdog.tar" + return 1 + fi + actual=$(sha256sum "$archive" | awk '{print $1}') || return 1 + if [ "$actual" != "$expected" ]; then + print_error "Archive checksum mismatch; image not loaded" + return 1 + fi + print_success "Archive checksum verified" + if ! loaded=$(docker load -i "$archive" 2>&1); then + print_error "Docker could not load the image" + printf '%s\n' "$loaded" + return 1 + fi + if ! printf '%s\n' "$loaded" | grep -Fxq "Loaded image: $IMAGE_NAME"; then + print_error "Archive does not contain the expected tag $IMAGE_NAME" + return 1 + fi + if ! verify_release_image; then + print_error "Loaded image does not match release version $VERSION" + return 1 + fi + print_success "Loaded and verified $IMAGE_NAME" +} - print_success "Image ready: ${IMAGE_NAME}" +# One release bundle contains its version-labelled Docker image. +# A missing or invalid archive aborts before changing application configuration. +load_docker_image() { + print_section "Load Release Image" + if [ ! -f "$IMAGE_ARCHIVE" ] || [ ! -f "$INSTALL_DIR/SHA256SUMS" ]; then + print_error "Incomplete release package: watchdog.tar and SHA256SUMS are required" + echo " Download and extract the complete container-alert-v$VERSION.tar.gz package." + echo " No running containers have been changed." + return 1 + fi + load_release_archive "$IMAGE_ARCHIVE" "$INSTALL_DIR/SHA256SUMS" } ################################################################################ @@ -450,8 +440,9 @@ configure_all() { [ -f "$ENV_FILE" ] || missing+=" .env" [ -f "$CONFIG_FILE" ] || missing+=" watchdog-config.yaml" print_warning "Incomplete configuration — missing:${missing}" - print_warning "The wizard regenerates both files — re-enter your settings when prompted" - echo "" + print_error "Restore the missing configuration file from backup, then rerun." + print_info "Existing configuration was not changed." + return 1 # Offer to reconfigure if files already exist elif [ -f "$ENV_FILE" ] && [ -f "$CONFIG_FILE" ]; then print_info "Existing configuration detected: .env watchdog-config.yaml" @@ -488,6 +479,16 @@ configure_all() { harden_config_permissions "$CONFIG_FILE" || exit 1 return 0 fi + # Preserve both existing files before explicitly requested reconfiguration. + local backup_dir + backup_dir="$INSTALL_DIR/backup/$(date -u +%Y%m%dT%H%M%SZ)" + if ! _run_priv mkdir -p "$backup_dir" || ! _run_priv chmod 700 "$backup_dir" || \ + ! _run_priv cp -p "$ENV_FILE" "$backup_dir/.env" || \ + ! _run_priv cp -p "$CONFIG_FILE" "$backup_dir/watchdog-config.yaml"; then + print_error "Could not back up existing configuration; aborting reconfiguration" + return 1 + fi + print_success "Existing configuration backed up in: $backup_dir" echo "" fi @@ -536,16 +537,16 @@ configure_all() { fi # ── SMTP ────────────────────────────────────────────────────────────────── - local SMTP_HOST="smtp.radware.com" SMTP_PORT="587" - local SMTP_SENDER="noc-alerts@radware.com" SMTP_RECIPIENTS="ops-team@radware.com" + local SMTP_HOST="smtp.example.com" SMTP_PORT="587" + local SMTP_SENDER="sender@example.com" SMTP_RECIPIENTS="alerts@example.com" local SMTP_TLS="true" SMTP_AUTH="true" SMTP_USERNAME="" SMTP_PASSWORD="" SMTP_ENABLED="false" if _in_array "smtp" "${SELECTED_CHANNELS[@]}"; then SMTP_ENABLED="true" print_info "SMTP (email)" - SMTP_HOST=$(_prompt "Server host" "smtp.radware.com") + SMTP_HOST=$(_prompt "Server host" "smtp.example.com") SMTP_PORT=$(_prompt "Port" "587") - SMTP_SENDER=$(_prompt "Sender address" "noc-alerts@radware.com") - SMTP_RECIPIENTS=$(_prompt "Recipients (comma-separated)" "ops-team@radware.com") + SMTP_SENDER=$(_prompt "Sender address" "sender@example.com") + SMTP_RECIPIENTS=$(_prompt "Recipients (comma-separated)" "alerts@example.com") SMTP_TLS=$(_prompt_yn "Use TLS?" "yes") SMTP_AUTH=$(_prompt_yn "Server requires authentication (login)?" "yes") if [ "$SMTP_AUTH" = "true" ]; then @@ -613,7 +614,7 @@ configure_all() { chmod 600 "$ENV_TMP" { printf "# .env — generated by install.sh on %s\n" "$(date '+%Y-%m-%d %H:%M:%S')" - printf "# Secrets only. All other settings are in watchdog-config.yaml.\n\n" + printf "# Secrets and deployment settings; monitoring policy is in watchdog-config.yaml.\n\n" if [ "$SLACK_ENABLED" = "true" ]; then printf "# Slack\nSLACK_WEBHOOK_URL=%s\n\n" "$SLACK_WEBHOOK" fi @@ -629,7 +630,7 @@ configure_all() { "$WATCHDOG_UID" "$WATCHDOG_GID" "$DOCKER_GID" printf "# Deployed release — docker-compose.yaml pins watchdog:\${WATCHDOG_VERSION}\n" printf "WATCHDOG_VERSION=%s\n\n" "$VERSION" - printf "# Tuning\nLOG_LEVEL=INFO\n" + } > "$ENV_TMP" if ! _publish_file "$ENV_TMP" "$ENV_FILE" 600 "$(id -u):$(id -g)"; then rm -f "$ENV_TMP" @@ -651,7 +652,7 @@ configure_all() { r=$(echo "$r" | tr -d '[:space:]') [ -n "$r" ] && RECIPIENTS_YAML+=" - ${r}"$'\n' done - [ -z "$RECIPIENTS_YAML" ] && RECIPIENTS_YAML=" - ops-team@radware.com"$'\n' + [ -z "$RECIPIENTS_YAML" ] && RECIPIENTS_YAML=" - alerts@example.com"$'\n' # ── Write watchdog-config.yaml ──────────────────────────────────────────── local CONFIG_TMP @@ -672,7 +673,7 @@ configure_all() { printf " com.radware.cybercontroller.role: mariadb-ha-syntax-check\n" printf " events:\n" printf " - die\n" - printf "# Preferred: R&D labels temporary MariaDB HA syntax-check containers as above.\n" + printf "# Temporary MariaDB HA syntax-check containers should carry the label shown above.\n" printf "# Fallback: suppress kvision_infra_mariadb AutoRemove dump helpers exiting 137\n" printf "# only when /mnt/cli/tmp/mariadb_dump.sql.gz is mounted; primary MariaDB stays monitored.\n" printf "\nrunbook_base_url: \"\"\n" @@ -747,21 +748,14 @@ start_watchdog() { exit 1 fi - # Stop and remove existing container if present - if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - print_info "Stopping existing container..." - if ! $DC down; then - print_error "Failed to stop the existing container — aborting before redeploy" - exit 1 - fi + # Validate Compose first and recreate only our service, never the project. + if ! $DC config -q; then + print_error "Compose configuration invalid; running containers unchanged" + exit 1 fi - - print_info "Starting container with docker compose..." - if ! $DC up -d; then - print_error "docker compose up failed — the watchdog was not deployed" - echo "" - echo " Inspect the failure with:" - echo " $DC -f ${INSTALL_DIR}/docker-compose.yaml logs" + print_info "Deploying watchdog container only..." + if ! $DC up -d --no-deps --force-recreate docker-container-watchdog; then + print_error "Watchdog deployment failed; check its Compose logs" exit 1 fi print_success "Watchdog container started" @@ -804,8 +798,11 @@ verify_deployment() { # image left over from a previous version would otherwise look healthy. local RUNNING_IMAGE RUNNING_IMAGE=$(docker inspect --format '{{.Config.Image}}' "$CONTAINER_NAME" 2>/dev/null || echo "") - if [ "$RUNNING_IMAGE" = "$IMAGE_NAME" ]; then - print_success "Container is running the expected image: ${IMAGE_NAME}" + local RUNNING_IMAGE_ID EXPECTED_IMAGE_ID + RUNNING_IMAGE_ID=$(docker inspect --format '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true) + EXPECTED_IMAGE_ID=$(docker image inspect --format '{{.Id}}' "$IMAGE_NAME" 2>/dev/null || true) + if [ "$RUNNING_IMAGE" = "$IMAGE_NAME" ] && [ -n "$EXPECTED_IMAGE_ID" ] && [ "$RUNNING_IMAGE_ID" = "$EXPECTED_IMAGE_ID" ]; then + print_success "Container runs the verified image: $IMAGE_NAME" else print_error "Container is running '${RUNNING_IMAGE:-unknown}', expected '${IMAGE_NAME}'" echo "" @@ -866,13 +863,15 @@ display_usage_instructions() { main() { print_header check_prerequisites + load_docker_image || exit 1 setup_log_directory - load_docker_image - configure_all + configure_all || exit 1 start_watchdog verify_deployment display_usage_instructions } -main "$@" - +# Allow fake-Docker unit tests to source functions without invoking main. +if [[ "$BASH_SOURCE" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/package_release.sh b/scripts/package_release.sh new file mode 100644 index 0000000..58e2ef0 --- /dev/null +++ b/scripts/package_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build one customer package from an already-built release image. +# No Docker commands; safe to test with a disposable fake archive. +# Usage: bash scripts/package_release.sh +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$(dirname "$0")/.." && pwd)" +version="$(tr -d '[:space:]' < "$repo/VERSION")" +if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid VERSION: $version" >&2 + exit 1 +fi + +image="$(realpath "$1")" +if [ ! -s "$image" ]; then + echo "Prebuilt image archive is missing or empty" >&2 + exit 1 +fi + +mkdir -p "$2" +out="$(cd "$2" && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +dir="$work/CyberController-Container-Alert" +mkdir -p "$dir/docs/operations" +for f in install.sh uninstall.sh docker-compose.yaml VERSION \ + .env.example watchdog-config.yaml.example README.md; do + cp "$repo/$f" "$dir/$f" +done +cp "$repo"/docs/operations/*.md "$dir/docs/operations/" # only customer guides ship + +cp "$image" "$dir/watchdog.tar" +(cd "$dir" && sha256sum watchdog.tar > SHA256SUMS) + +package="container-alert-v$version.tar.gz" +tar -C "$work" -czf "$out/$package" CyberController-Container-Alert +(cd "$out" && sha256sum "$package" > SHA256SUMS) + +for asset in "$package" SHA256SUMS; do + if [ "$(stat -c '%s' "$out/$asset")" -ge 2147483648 ]; then + echo "Release asset exceeds the 2 GiB GitHub limit: $asset" >&2 + exit 1 + fi +done +echo "Packaged Container Alert v$version: $out/$package" diff --git a/tests/release-smoke.sh b/tests/release-smoke.sh new file mode 100755 index 0000000..7b4171a --- /dev/null +++ b/tests/release-smoke.sh @@ -0,0 +1,789 @@ +#!/usr/bin/env bash +# Container Alert v1.6.0: real Docker engine smoke tests, without host installations. +# Destructive tests are allowed ONLY on an empty, labelled Docker Engine launched +# on an ephemeral GitHub-hosted runner. Never use CyberController or shared hosts. +# Preflight fails closed before creating any containers/volumes. +set -euo pipefail +SMOKE_SOURCE_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" +export SMOKE_SOURCE_DIR +exec python3 - "$@" <<'PY' +import datetime as dt +import hashlib +import json +import os +import pathlib +from pathlib import Path +import re +import subprocess +import sys +import time +import traceback + +IMAGE = os.environ.get("SMOKE_IMAGE", "watchdog:1.6.0") +EXPECTED_REV = os.environ.get("SMOKE_EXPECTED_REV", "") +TAG = dt.datetime.now(dt.timezone.utc).strftime("%y%m%d%H%M%S") + str(os.getpid())[-3:] +PREFIX = "ca-smoke-" + TAG +LABEL = "container-alert.smoke=" + TAG +ISOLATED = True # no live-engine execution path exists +RECEIVER = PREFIX + "-syslog-receiver" +HOST = "release-smoke-" + TAG +OBS = PREFIX + "-observer" +STATE_VOL = PREFIX + "-state" +BASE = pathlib.Path(os.environ.get("SMOKE_OUTPUT_DIR", "/tmp")) / ("container-alert-smoke-" + TAG) +START = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") +containers, volumes, results, facts = [], [], [], {} +EVENT_PROC, EVENT_FILE = None, None +ENGINE_VERIFIED = False +BASE.mkdir(parents=True, exist_ok=False) +start_mono = time.monotonic() + +def arm_events(): + # Capture live to avoid Docker's short retrospective event-history buffer. + global EVENT_PROC, EVENT_FILE + EVENT_FILE = (BASE/"docker-events-live.jsonl").open("w") + EVENT_PROC = subprocess.Popen( + ["docker", "events", "--format", "{{json .}}", "--filter", "type=container", + "--filter", "label=" + LABEL], + stdout=EVENT_FILE, stderr=subprocess.DEVNULL) + time.sleep(0.5) + ensure(EVENT_PROC.poll() is None, "real-time docker events capture failed") + +def stop_events(): + global EVENT_PROC, EVENT_FILE + if EVENT_PROC is not None: + EVENT_PROC.terminate() + try: + EVENT_PROC.wait(timeout=4) + except subprocess.TimeoutExpired: + EVENT_PROC.kill() + EVENT_PROC.wait(timeout=4) + EVENT_PROC = None + if EVENT_FILE is not None: + EVENT_FILE.close() + EVENT_FILE = None + p = BASE/"docker-events-live.jsonl" + if p.exists(): + events = [json.loads(x) for x in p.read_text().splitlines() if x.strip()] + facts["live_docker_events"] = len(events) + facts["live_docker_action_counts"] = { + a: sum(e.get("Action") == a for e in events) + for a in ("create","start","die","oom","stop","kill","restart","health_status: unhealthy", + "health_status: healthy","destroy") + } + +def run(argv, timeout=30, check=True): + args = ["docker"] + [str(v) for v in argv] + p = subprocess.run(args, text=True, capture_output=True, timeout=timeout) + if check and p.returncode: + raise RuntimeError("docker " + " ".join(args[1:5]) + + " returned " + str(p.returncode) + ": " + (p.stderr or p.stdout)[-1500:]) + return p.stdout.strip() + +def log(s): + print(s, flush=True) + +def verify_engine_is_test_only(): + """Never issue any mutating Docker command without independent safeguards. + + A non-default, localhost-only endpoint is necessary but NOT sufficient: + require an ephemeral GitHub-hosted worker, a unique daemon-side test label, + and a completely empty target Engine. Do not support live-host opt-in flags. + """ + global ENGINE_VERIFIED + ensure(os.environ.get("GITHUB_ACTIONS") == "true" and + os.environ.get("SMOKE_RUNNER_KIND") == "github-hosted", + "Refusing destructive tests outside an ephemeral GitHub-hosted runner") + ensure(not os.environ.get("DOCKER_CONTEXT"), "DOCKER_CONTEXT could override DOCKER_HOST") + endpoint = os.environ.get("DOCKER_HOST", "") + ensure(re.fullmatch(r"tcp://127\.0\.0\.1:[0-9]{2,5}", endpoint) is not None, + "DOCKER_HOST must be the isolated localhost Docker TCP endpoint") + marker = os.environ.get("SMOKE_ENGINE_MARKER", "") + ensure(re.fullmatch(r"ci-[0-9]+-[0-9]+", marker) is not None, + "Missing unique CI test-engine marker") + info = json.loads(run(["info", "--format", "{{json .}}"], timeout=12)) + expected_label = "com.radware.container-alert.smoke-engine=" + marker + ensure(expected_label in (info.get("Labels") or []), + "Target Docker Engine is not the labelled disposable test Engine") + ensure(info.get("Containers") == 0, + "Refusing Engine with pre-existing containers: " + str(info.get("Containers"))) + ENGINE_VERIFIED = True + facts["isolated_engine_id"] = info.get("ID") + facts["isolated_engine_label"] = expected_label + facts["runner_kind"] = "github-hosted" + log("ISOLATION_VERIFIED " + str(info.get("ID"))) + + + +def mark(name, expected, evidence): + evidence = str(evidence) + elapsed = round(time.monotonic() - start_mono, 1) + results.append({"id": name, "status": "PASS", "expected": expected, + "observed": evidence, "elapsed_seconds": elapsed}) + log("PASS " + name + " - " + evidence[:190]) + +def issue(name, expected, err): + elapsed = round(time.monotonic() - start_mono, 1) + results.append({"id": name, "status": "FAIL", "expected": expected, + "observed": str(err), "elapsed_seconds": elapsed}) + log("FAIL " + name + " - " + str(err)[:350]) + +def checked(name, expected, fn): + try: + mark(name, expected, fn()) + return True + except Exception as exc: + issue(name, expected, exc) + return False + +def ensure(ok, message): + if not ok: + raise AssertionError(message) + +def fixture(suffix, program, options=(), memory="128m", control=None): + name = PREFIX + "-" + suffix + args = ["create", "--name", name, "--network", "none", "--label", LABEL, + # The image was built by Compose and carries service labels; do not + # let the lab reconciliation loop mistake test fixtures for watchdogs. + "--label", "com.docker.compose.project=release-smoke", + "--label", "com.docker.compose.service=fixture", + "--restart", "no", + "--cpus", "0.20", "--memory", memory, "--memory-swap", memory, + "--pids-limit", "64", "--security-opt", "no-new-privileges", + "--cap-drop", "ALL"] + if control: + args += ["--mount", "type=volume,source=" + control + ",target=/control"] + args += list(options) + ["--entrypoint", "python3", IMAGE, "-u", "-c", program] + cid = run(args) + containers.append(name) + facts[suffix] = {"name": name, "container_id": cid} + return name + +def start(name): + run(["start", name]) + +def inspect(name, value): + return run(["inspect", "--format", value, name]) + +def state(name): + return json.loads(inspect(name, "{{json .State}}")) + +def wait_exit(name, limit=30): + return int(run(["wait", name], timeout=limit).splitlines()[-1]) + +def count(name, kind): + a = alerts(name, kind) + return len(a) + +def alerts(name, kind=None): + raw = run(["logs", "--since", START, OBS], timeout=20, check=False) + lines = [x for x in raw.splitlines() if 'event="watchdog_alert"' in x + and ('container="' + name + '"') in x] + if kind is not None: + lines = [x for x in lines if ('failure_type="' + kind + '"') in x] + return lines + +def wait_alert(name, kind, n=1, limit=45): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + if count(name, kind) >= n: + return count(name, kind) + time.sleep(1) + raise AssertionError("timeout waiting for " + str(n) + " " + kind + + " alerts on " + name + "; observed=" + repr(alerts(name)[-3:])) + +def must_count(name, kind, n): + actual = count(name, kind) + ensure(actual == n, name + " " + kind + ": expected " + str(n) + " got " + str(actual)) + return actual + +def make_volume(suffix): + name = PREFIX + "-" + suffix + run(["volume", "create", "--label", LABEL, name]) + volumes.append(name) + # The actual test process is UID 1000; test control files must be writable. + if suffix != "state": + run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", name + ":/control", "--entrypoint", "python3", + IMAGE, "-c", "import os;os.chown('/control',1000,1000)"]) + return name + +def write_control(volume, filename, present=True): + code = ("import pathlib; p=pathlib.Path('/control')/" + repr(filename) + + "; p.write_text('yes') if " + repr(present) + " else p.unlink(missing_ok=True)") + run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", volume + ":/control", "--entrypoint", "python3", + IMAGE, "-c", code], timeout=35) + +def wait_restarts(name, minimum, limit=120): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + n = int(inspect(name, "{{.RestartCount}}")) + if n >= minimum: + return n + time.sleep(2) + raise AssertionError("restart count did not reach " + str(minimum)) + +def require_syslog(): + raw = run(["exec", RECEIVER, "cat", "/var/log/syslog"], timeout=20) + (BASE/"syslog-udp-raw.txt").write_text(raw + "\n") + return [line for line in raw.splitlines() + if 'host="' + HOST + '"' in line and 'event="watchdog_alert"' in line] + + +def init(): + # verify_engine_is_test_only() has already established the trust boundary. + revision = run(["image", "inspect", "--format", + '{{index .Config.Labels "org.opencontainers.image.revision"}}', IMAGE]) + if EXPECTED_REV: + ensure(revision == EXPECTED_REV, "wrong image source revision " + revision) + image_id = run(["image", "inspect", "--format", "{{.Id}}", IMAGE]) + sources = Path(os.environ["SMOKE_SOURCE_DIR"]) + expected_sha = { + "/app/" + name: hashlib.sha256((sources/name).read_bytes()).hexdigest() + for name in ("watchdog.py", "lifecycle.py", "ha_monitor.py") + } + embedded = run(["run", "--rm", "--network", "none", "--entrypoint", "sha256sum", + IMAGE, *expected_sha.keys()]) + actual_sha = {line.split()[-1]: line.split()[0] for line in embedded.splitlines()} + ensure(actual_sha == expected_sha, "release image source SHA mismatch: " + repr(actual_sha)) + facts["embedded_source_sha256"] = actual_sha + existing = run(["ps", "-a", "--format", "{{.Names}}"]).splitlines() + ensure(not any(n.startswith(PREFIX) for n in existing), "smoke prefix collision") + ensure(not run(["volume", "ls", "--filter", "label=" + LABEL, "-q"]), "old test volumes") + collector = make_volume("syslogdata") + udp_code = """import socket +s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) +s.bind(("0.0.0.0",514)) +with open("/var/log/syslog","a",buffering=1) as output: + print("SMOKE_SYSLOG_RECEIVER_READY", flush=True) + while True: + packet,source=s.recvfrom(65535) + output.write(packet.decode(errors="replace")+"\\n") +""" + run(["run", "-d", "--name", RECEIVER, "--label", LABEL, "--network", "host", + "--restart", "no", "--memory", "64m", "--pids-limit", "32", + "--volume", collector + ":/var/log", "--user", "0:0", + "--entrypoint", "python3", IMAGE, "-u", "-c", udp_code]) + containers.append(RECEIVER) + receiver_logs = "" + for _ in range(18): + time.sleep(1) + receiver_logs = run(["logs", RECEIVER], check=False) + if "SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs: + break + if inspect(RECEIVER, "{{.State.Status}}") == "exited": + break + ensure("SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs, + "isolated UDP syslog collector failed; state=" + + inspect(RECEIVER, "{{.State.Status}}") + ", log=" + receiver_logs[-1000:]) + vol = make_volume("state") + excludes = sorted(set(existing + [OBS])) + config = { + "alert_channels": [], "log_level": "INFO", "log_file": None, + "check_interval_seconds": 2, "cooldown_minutes": 5, + "restart_threshold": 5, "restart_window_minutes": 10, + "restart_recovery_stability_seconds": 120, "unhealthy_cycles_threshold": 3, + "alert_on_recovery": True, "boot_grace_seconds": 0, + "lifecycle_detection": {"enabled": True, "correlation_window_seconds": 3}, + "ha_monitor": {"enabled": False}, + "auto_health_check": {"enabled": False}, + "excluded_containers": excludes, + "state_file": "/var/log/watchdog/state.json", + "syslog": {"enabled": True, "host": "127.0.0.1", "port": 514, + "protocol": "udp", "facility": "local0"} + } + setup = ("import os,json,pathlib; p=pathlib.Path('/data'); " + "c=json.loads(os.environ['SMOKE_CONFIG']); " + "(p/'watchdog-config.yaml').write_text(json.dumps(c)); " + "(p/'watchdog-config.yaml').chmod(0o644); os.chown(p,1000,1000)") + run(["run", "--rm", "--network", "none", "--user", "0:0", + "-e", "SMOKE_CONFIG=" + json.dumps(config), + "--volume", vol + ":/data", "--entrypoint", "python3", + IMAGE, "-c", setup], timeout=30) + gid_value = os.environ.get("SMOKE_DOCKER_SOCKET_GID") + gid = int(gid_value) if gid_value is not None else os.stat("/var/run/docker.sock").st_gid + arm_events() + args = ["run", "-d", "--name", OBS, "--label", LABEL, "--restart", "no", + "--label", "com.docker.compose.project=release-smoke", + "--label", "com.docker.compose.service=observer", + "--network", "host", "--cpus", "0.35", "--memory", "256m", + "--pids-limit", "96", "--read-only", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", + "--security-opt", "no-new-privileges", "--cap-drop", "ALL", + "--user", "1000:1000", "--group-add", str(gid), + "--log-driver", "local", "--log-opt", "max-size=5m", + "--log-opt", "max-file=2", + "-e", "WATCHDOG_HOST=" + HOST, + "-e", "WATCHDOG_CONFIG=/etc/watchdog/watchdog-config.yaml", + "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", + "-v", vol + ":/etc/watchdog:ro", + "-v", vol + ":/var/log/watchdog:rw", + IMAGE] + run(args) + containers.append(OBS) + time.sleep(3) + lines = run(["logs", OBS]) + ensure("Docker event listener started" in lines and "Poll loop started" in lines, + "observer did not initialize: " + lines[-700:]) + ensure(inspect(OBS, "{{.State.Status}}") == "running", "observer stopped") + facts.update({"host": run(["info", "--format", "{{.Name}}"]), + "revision": revision, "image_id": image_id, "observer": OBS, + "syslog_source_host": HOST, "observer_poll_seconds": 2, + "restart_threshold": 5, "recovery_stability_seconds": 120, + "excluded_existing_containers": len(existing), + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": os.environ.get("DOCKER_HOST", "default")}) + log("REAL_DOCKER_TEST_READY " + HOST + " image=" + image_id[:24]) + +def t01_clean(): + name = fixture("clean", "import sys;sys.exit(0)") + start(name) + ensure(wait_exit(name) == 0, "clean exit not 0") + time.sleep(7) + ensure(not alerts(name), "unexpected alert on clean exit") + return "real exit=0, no alerts" + +def t02_crash(): + name = fixture("crash", "raise SystemExit(2)") + start(name) + ensure(wait_exit(name) == 2, "unexpected exit code") + wait_alert(name, "crashed", limit=25) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure('severity="CRITICAL"' in alerts(name, "crashed")[0], "wrong severity") + return "Docker exit=2, exactly one CRITICAL crashed alert" + +def t03_planned_stop(): + name = fixture("planned", "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(300)") + start(name) + time.sleep(2) + run(["stop", "-t", "1", name], timeout=20) + ensure(wait_exit(name, 20) == 137, "planned stop not exit=137") + ensure(state(name)["OOMKilled"] is False, "planned stop unexpectedly OOM") + time.sleep(7) + ensure(not alerts(name), "planned stop misclassified: " + str(alerts(name))) + ensure(state(name)["Status"] == "exited", "unexpected planned fixture restart") + # The HA lab has an external restart manager; remove the finished fixture + # before that independent service can start a new Docker run. + run(["rm", "-f", name]) + return "real docker stop -> SIGKILL/137; no crash or OOM alert" + +def t04_direct_kill(): + name = fixture("sigkill", "import time;time.sleep(300)") + start(name) + time.sleep(1) + run(["kill", "--signal", "SIGKILL", name]) + ensure(wait_exit(name) == 137, "SIGKILL exit not 137") + ensure(state(name)["OOMKilled"] is False, "SIGKILL incorrectly marked OOM") + wait_alert(name, "crashed", limit=25) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure(must_count(name, "oom", 0) == 0, "false OOM") + facts["sigkill_state"] = {"exit": 137, "OOMKilled": False} + run(["rm", "-f", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "false recovery after removed SIGKILL") + ensure(must_count(name, "crashed", 1) == 1, "repeated crash") + return "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash" + +def t05_oom(): + name = fixture("oom", "import time;x=[]\nwhile True:\n x.append(bytearray(8*1024*1024));time.sleep(.02)", + memory="64m") + start(name) + ensure(wait_exit(name, limit=35) == 137, "OOM did not exit 137") + ensure(state(name)["OOMKilled"] is True, "Docker cgroup did not confirm OOMKilled") + wait_alert(name, "oom", limit=30) + facts["oom_state"] = {"exit": 137, "OOMKilled": True} + run(["rm", "-f", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "false recovery after removed OOM") + ensure(must_count(name, "oom", 1) == 1, "duplicate OOM") + ensure(must_count(name, "crashed", 0) == 0, "false crash after confirmed OOM") + line = alerts(name, "oom")[0] + ensure('severity="CRITICAL"' in line, "incorrect OOM severity") + return "kernel cgroup OOMKilled=true, exit=137, one CRITICAL OOM, zero crash" + +def t06_t08_loop(): + vol = make_volume("loopcontrol") + program = """import os,sys,time +if not os.path.exists('/control/recover'): + time.sleep(0.7) + sys.exit(2) +while True: + if os.path.exists('/control/interrupt'): + try: os.unlink('/control/interrupt') + except OSError: pass + sys.exit(2) + time.sleep(0.4) +""" + name = fixture("loop", program, options=("--restart", "always"), control=vol) + start(name) + wait_alert(name, "crashed", limit=50) + wait_alert(name, "restart-loop", limit=135) + n = int(inspect(name, "{{.RestartCount}}")) + ensure(n >= 5, "loop threshold crossed without 5 real Docker restarts, saw " + str(n)) + ensure(must_count(name, "crashed", 1) == 1, "repeated crashes") + ensure(must_count(name, "restart-loop", 1) == 1, "repeat loop") + mark("T06_RESTART_LOOP", "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "real Docker RestartCount=" + str(n) + "; crash=1; restart-loop=1") + write_control(vol, "recover") + running_at = time.monotonic() + time.sleep(11) + ensure(inspect(name, "{{.State.Status}}") == "running", "not recovering") + ensure(must_count(name, "recovered", 0) == 0, "premature recovered after 11s") + mark("T07_BRIEF_RECOVERY", "no recovery before 120s stability", "running after 11s, RECOVERED=0") + # A new real process failure during the stability period resets the recovery clock. + before = int(inspect(name, "{{.RestartCount}}")) + write_control(vol, "interrupt") + after = wait_restarts(name, before + 1, limit=45) + time.sleep(14) + ensure(must_count(name, "recovered", 0) == 0, + "premature recovered 14s after interrupted recovery") + mark("T08_INTERRUPTED_RECOVERY", "new actual die/reset prevents premature recovery", + "Docker RestartCount " + str(before) + " -> " + str(after) + + "; RECOVERED=0 after 14s") + wait_alert(name, "recovered", limit=165) + time.sleep(6) + ensure(must_count(name, "crashed", 1) == 1, "crash storm") + ensure(must_count(name, "restart-loop", 1) == 1, "loop repeated") + ensure(must_count(name, "recovered", 1) == 1, "recovered not exactly once") + ensure('severity="INFO"' in alerts(name, "recovered")[0], "recovery level not INFO") + mark("T09_STABLE_RECOVERY", "one INFO RECOVERED after 120s sustained uptime", + "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery") + facts["loop_restart_count_after"] = int(inspect(name, "{{.RestartCount}}")) + return "one CRASH + one LOOP + one RECOVERED after real interrupted recovery" + +def t14_external_restart(): + # A Docker policy=no container is intentionally started by an external actor. + # RECOVERED must wait for sustained uptime, just like auto-restart containers. + vol = make_volume("externalcontrol") + script = """import os,sys,time +if not os.path.exists("/control/recover"): sys.exit(2) +time.sleep(300) +""" + name = fixture("external", script, control=vol) + start(name) + ensure(wait_exit(name) == 2, "initial actual crash not exit 2") + wait_alert(name, "crashed", limit=35) + write_control(vol, "recover") + ensure(inspect(name, "{{.HostConfig.RestartPolicy.Name}}") == "no", + "external fixture should not auto-restart") + run(["start", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, + "premature RECOVERED after external Docker start, before 120s uptime") + wait_alert(name, "recovered", limit=150) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure(must_count(name, "recovered", 1) == 1, "duplicate recovery") + stable = confirmed_stable_uptime(name) + return "real manual Docker start after crash; RECOVERED after " + str(stable) + "s uptime" + +def confirmed_stable_uptime(name): + line = alerts(name, "recovered")[0] + stamp = re.search(r'time_utc="([^"]+)"', line) + ensure(stamp is not None, "RECOVERED alert missing timestamp") + started = dt.datetime.fromisoformat(inspect(name, "{{.State.StartedAt}}").replace("Z","+00:00")) + recovered = dt.datetime.fromisoformat(stamp.group(1).replace("Z","+00:00")) + elapsed = (recovered-started).total_seconds() + ensure(elapsed >= 119, "RECOVERED was emitted after only " + str(elapsed) + " seconds") + return round(elapsed,1) + +def t15_external_oom_restart(): + vol = make_volume("externaloomcontrol") + script = """import os,time +if os.path.exists("/control/recover"): + while True: time.sleep(2) +else: + chunks=[] + while True: + chunks.append(bytearray(8*1024*1024)) + time.sleep(.02) +""" + name = fixture("externaloom", script, memory="64m", control=vol) + ensure(inspect(name, "{{.HostConfig.RestartPolicy.Name}}") == "no", + "external OOM fixture must have restart policy no") + start(name) + ensure(wait_exit(name, 40) == 137, "kernel OOM did not exit 137") + ensure(state(name)["OOMKilled"] is True, "Docker did not confirm real OOM") + wait_alert(name, "oom", limit=35) + ensure(must_count(name, "crashed", 0) == 0, "OOM produced CRASH") + write_control(vol, "recover") + run(["start", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "early recovered after external OOM start") + wait_alert(name, "recovered", limit=150) + ensure(must_count(name, "oom", 1) == 1, "repeated OOM notification") + ensure(must_count(name, "recovered", 1) == 1, "repeated RECOVERED") + stable = confirmed_stable_uptime(name) + return "real kernel OOM -> external Docker start -> INFO RECOVERED after " + str(stable) + "s" + +def t10_name_reuse(): + name = PREFIX + "-reuse" + ids = [] + for code in (2, 42): + label = "reuse-" + str(code) + cid = fixture("reuse", "raise SystemExit(" + str(code) + ")") if code == 2 else None + if code != 2: + # Reuse the exact same name after deleting the old container. + ensure(not run(["ps", "-aq", "--filter", "name=^/" + name + "$"]), + "old ID still owns name") + cid = fixture("reuse", "raise SystemExit(42)") + ids.append(inspect(name, "{{.Id}}")) + start(name) + ensure(wait_exit(name) == code, "unexpected " + label + " exit") + wait_alert(name, "crashed", n=len(ids), limit=30) + if code == 2: + run(["rm", name]) + time.sleep(2) + ensure(ids[0] != ids[1], "same Docker instance reused") + ensure(must_count(name, "crashed", 2) == 2, "new name inherited old alert") + return "two unique container IDs, same name, each gets one CRITICAL crash" + +def t11_fast_removal(): + names = [] + for k in range(3): + name = PREFIX + "-rm" + str(k) + argv = ["docker", "run", "--rm", "--name", name, "--label", LABEL, + "--network", "none", "--memory", "128m", "--memory-swap", "128m", + "--pids-limit", "64", "--cpus", "0.2", "--cap-drop", "ALL", + "--entrypoint", "python3", IMAGE, "-c", "raise SystemExit(2)"] + result = subprocess.run(argv, capture_output=True, text=True, timeout=30) + ensure(result.returncode == 2, + "Fast removal fixture did not exit 2: " + name + + ", actual=" + str(result.returncode) + ", stderr=" + result.stderr[-500:]) + names.append(name) + time.sleep(8) + observed = [json.loads(line) for line in + (BASE/"docker-events-live.jsonl").read_text().splitlines() if line.strip()] + for name in names: + ensure(any(ev.get("Action") == "die" and + ev.get("Actor", {}).get("Attributes", {}).get("name") == name and + ev.get("Actor", {}).get("Attributes", {}).get("exitCode") == "2" + for ev in observed), + "Missing genuine Docker die event / exit 2: " + name) + wd_logs = run(["logs", "--since", START, OBS], timeout=20) + misses = [x for x in wd_logs.splitlines() if "already removed before" in x and + any(n in x for n in names)] + ensure(len(misses) <= 6, "missing-container log flood: " + str(len(misses))) + ensure(inspect(OBS, "{{.State.Status}}") == "running", "event watcher died") + facts["fast_remove_warning_count"] = len(misses) + return "3 real --rm containers, " + str(len(misses)) + " bounded missing-ID warnings, observer alive" + +def t12_healthcheck(): + vol = make_volume("healthcontrol") + name = fixture("health", "import time;time.sleep(300)", control=vol, + options=("--health-cmd", + "python3 -c 'import os,sys;sys.exit(0 if os.path.exists(\"/control/healthy\") else 1)'", + "--health-interval", "2s", "--health-timeout", "2s", + "--health-retries", "2", "--health-start-period", "0s")) + start(name) + wait_alert(name, "unhealthy", limit=55) + ensure(must_count(name, "unhealthy", 1) == 1, "health alert duplicated") + ensure('severity="HIGH"' in alerts(name, "unhealthy")[0], "unhealthy severity") + write_control(vol, "healthy") + wait_alert(name, "recovered", limit=40) + ensure(must_count(name, "recovered", 1) == 1, "health recovery duplicated") + ensure('severity="INFO"' in alerts(name, "recovered")[0], "recovery priority") + ensure(inspect(name, "{{.State.Health.Status}}") == "healthy", "Docker health not healthy") + return "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery" + +def t13_dfc(): + sent = [x for x in run(["logs", "--since", START, OBS]).splitlines() + if 'event="watchdog_alert"' in x] + received = require_syslog() + (BASE/"dfc-syslog.txt").write_text("\n".join(received) + "\n") + facts["watchdog_alert_count"] = len(sent) + facts["dfc_received_count"] = len(received) + ensure(sent, "watchdog generated no alert records") + for name in ("crash", "sigkill", "oom", "loop", "reuse", "health", "external"): + actual = PREFIX + "-" + name + local = [x for x in sent if 'container="' + actual + '"' in x] + remote = [x for x in received if 'container="' + actual + '"' in x] + ensure(len(remote) == len(local), + name + " alert count differs watchdog=" + str(len(local)) + + " DFC=" + str(len(remote))) + ensure(not [x for x in received if 'container="' + PREFIX + '-planned"' in x], + "planned stop sent to DFC") + ensure(not [x for x in received if 'container="' + PREFIX + '-clean"' in x], + "clean exit sent to DFC") + for kind,severity in (("crashed","CRITICAL"),("oom","CRITICAL"), + ("restart-loop","HIGH"),("unhealthy","HIGH"), + ("recovered","INFO")): + ensure(any('failure_type="' + kind + '"' in x and + 'severity="' + severity + '"' in x for x in received), + "DFC missing " + kind + "/" + severity) + return str(len(sent)) + " watchdog alerts delivered as " + str(len(received)) + \ + " isolated UDP syslog messages; CRITICAL/HIGH/INFO fields correct" + +def collect(): + stop_events() + if OBS in containers: + raw = run(["logs", "--since", START, OBS], timeout=30, check=False) + (BASE/"watchdog.log").write_text(raw + "\n") + try: + vol = next(v for v in volumes if v.endswith("-state")) + d = run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", vol + ":/data", "--entrypoint", "python3", + IMAGE, "-c", + "import json;print(json.dumps(json.load(open('/data/state.json')),sort_keys=True))"], + timeout=30) + (BASE/"observer-state.json").write_text(d + "\n") + except Exception as e: + (BASE/"state-read-error.txt").write_text(str(e)) + try: + before = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00","Z") + ev = run(["events", "--since", START, "--until", before, + "--filter", "type=container", + "--filter", "label=" + LABEL], timeout=40, check=False) + (BASE/"docker-events.log").write_text(ev + "\n") + facts["docker_event_lines"] = len([x for x in ev.splitlines() if x.strip()]) + except Exception as e: + facts["docker_events_error"] = str(e) + try: + if not (BASE/"dfc-syslog.txt").exists(): + (BASE/"dfc-syslog.txt").write_text("\n".join(require_syslog()) + "\n") + except Exception as e: + facts["syslog_read_error"] = str(e) + for n in list(containers): + if n == OBS: + continue + try: + state_json = run(["inspect", "--format", "{{json .State}}", n], check=False) + if state_json: + facts.setdefault("final_fixture_states", {})[n] = json.loads(state_json) + except Exception: + pass + +def cleanup(): + """Remove only resources created by this run and verify none remain.""" + errors = facts.setdefault("cleanup_errors", []) + for n in reversed(containers): + if not n.startswith(PREFIX): + errors.append("Refusing to remove non-test container " + n) + continue + try: + label = run(["inspect", "--format", + '{{index .Config.Labels "container-alert.smoke"}}', + n], check=False) + if not label: + continue # Already removed by the test. + if label != TAG: + errors.append("Refusing mismatched test label on " + n + ": " + label) + continue + run(["rm", "-f", n], timeout=30) + except Exception as exc: + errors.append(n + ": " + str(exc)) + for vol in reversed(volumes): + if not vol.startswith(PREFIX): + errors.append("Refusing to remove non-test volume " + vol) + continue + try: + label = run(["volume", "inspect", "--format", + '{{index .Labels "container-alert.smoke"}}', + vol], check=False) + if not label: + continue + if label != TAG: + errors.append("Refusing mismatched test volume label " + vol) + continue + run(["volume", "rm", vol], timeout=30) + except Exception as exc: + errors.append(vol + ": " + str(exc)) + for kind, command in ( + ("containers", ["ps", "-a", "--filter", "label=" + LABEL, "-q"]), + ("volumes", ["volume", "ls", "--filter", "label=" + LABEL, "-q"]), + ): + try: + leftover = run(command) + if leftover: + errors.append("Leftover test " + kind + ": " + leftover) + except Exception as exc: + errors.append("Cannot verify cleanup of " + kind + ": " + str(exc)) + +try: + verify_engine_is_test_only() + init() + if os.environ.get("SMOKE_ONLY") == "T13_SYSLOG_QUICK": + ensure(ISOLATED, "SMOKE_ONLY quick test requires isolated engine") + checked("T02_UNEXPECTED_CRASH", "real CRITICAL crash", t02_crash) + def quick_syslog(): + lines = require_syslog() + ensure(any('container="' + PREFIX + '-crash"' in x and + 'severity="CRITICAL"' in x for x in lines), + "no matching CRITICAL Syslog packet; raw=" + + (BASE/"syslog-udp-raw.txt").read_text()[:700]) + return "CRITICAL Syslog datagram collected for real Docker crash" + checked("T13_SYSLOG_QUICK", "UDP receiver captures matching CRITICAL record", quick_syslog) + elif os.environ.get("SMOKE_ONLY") in ("T14_EXTERNAL_RESTART", "T15_EXTERNAL_OOM"): + ensure(ISOLATED, "SMOKE_ONLY external case must run on isolated engine") + if os.environ["SMOKE_ONLY"] == "T14_EXTERNAL_RESTART": + checked("T14_EXTERNAL_RESTART", "no recovery until 120 seconds after external start", + t14_external_restart) + else: + checked("T15_EXTERNAL_OOM", "real OOM then manual start requires 120s stability", + t15_external_oom_restart) + else: + checked("T01_CLEAN_EXIT", "exit=0 means no alert", t01_clean) + checked("T02_UNEXPECTED_CRASH", "exit=2 sends one CRITICAL crash", t02_crash) + checked("T03_PLANNED_STOP_137", "Docker stop exit 137 is suppressed", t03_planned_stop) + checked("T04_DIRECT_SIGKILL", "SIGKILL 137 is crash, not OOM", t04_direct_kill) + checked("T05_REAL_OOM", "OOMKilled=true sends only CRITICAL OOM", t05_oom) + checked("T06_TO_T09_RESTART_LIFECYCLE", + "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", t06_t08_loop) + checked("T14_EXTERNAL_RESTART", "manual start after CRASH waits 120s before RECOVERED", + t14_external_restart) + checked("T10_NAME_REUSE", "replacement with same name gets new incident", t10_name_reuse) + checked("T11_FAST_REMOVAL", "no event flood, event listener survives", t11_fast_removal) + checked("T12_NATIVE_HEALTHCHECK", "unhealthy -> recovered", t12_healthcheck) + checked("T13_SYSLOG", "all expected structured alerts reach test UDP collector", + t13_dfc) +except Exception as e: + issue("PREFLIGHT_OR_FATAL", "observer initializes and tests run", repr(e)) + facts["fatal_traceback"] = traceback.format_exc() +finally: + if ENGINE_VERIFIED: + try: + collect() + except Exception as e: + facts["collect_error"] = repr(e) + finally: + stop_events() + cleanup() + finished = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00","Z") + facts["total_seconds"] = round(time.monotonic() - start_mono, 1) + data = {"run_id": TAG, "started_utc": START, "finished_utc": finished, + "status": ("FAIL" if any(x["status"] == "FAIL" for x in results) + or facts.get("cleanup_errors") + or facts.get("collect_error") + or facts.get("syslog_read_error") + or not ENGINE_VERIFIED else "PASS"), + "image": IMAGE, "results": results, "facts": facts} + (BASE/"report.json").write_text(json.dumps(data, indent=2, default=str) + "\n") + lines = ["# Container Alert v1.6.0 real Docker smoke test", + "", "**Run:** " + TAG + " | **Result:** " + data["status"], + "**Started:** " + START + " | **Finished:** " + finished, + "**Image:** " + IMAGE + " | **Revision:** " + str(facts.get("revision","unknown")), + "**Host:** " + str(facts.get("host","unknown")) + " | **Observer host ID:** " + HOST, + "", "## Acceptance results", "", + "| Test | Status | Actual evidence |", "|---|---|---|"] + for item in results: + actual = str(item["observed"]).replace("|","/").replace("\n"," ") + lines.append("| " + item["id"] + " | " + item["status"] + " | " + actual[:290] + " |") + lines += ["", "## Evidence", + "Actual Docker container failures and events; exact release code in an isolated observer.", + "Observer has SMTP disabled; Syslog messages go to " + facts.get("syslog_backend", "receiver") + " UDP/514.", + "No synthetic Docker events, mocked exporter metrics, host packages, or Pacemaker changes.", + "Other watchdogs and unrelated existing containers are not modified.", + "Output files: watchdog.log, docker-events.log, dfc-syslog.txt, observer-state.json, report.json.", + "","## Facts",""] + for k,v in sorted(facts.items()): + if k == "final_fixture_states": + continue + lines.append("- **" + k + ":** " + str(v)[:400]) + lines += ["","**Cleanup:** only test-labelled disposable containers and volumes, " + + ("PASS" if not facts.get("cleanup_errors") else "ERRORS"), ""] + (BASE/"report.md").write_text("\n".join(lines)) + log("REPORT_DIR=" + str(BASE)) + log("SMOKE_RESULT=" + data["status"] + " checks=" + str(len(results)) + + " fails=" + str(sum(x["status"] == "FAIL" for x in results)) + + " runtime_seconds=" + str(facts["total_seconds"])) + sys.exit(0 if data["status"] == "PASS" else 1) +PY diff --git a/tests/test_customer_alert_text.py b/tests/test_customer_alert_text.py new file mode 100644 index 0000000..e9ee997 --- /dev/null +++ b/tests/test_customer_alert_text.py @@ -0,0 +1,88 @@ +"""Customer alert wording and channel consistency for v1.6.0 (no Docker/network).""" +import unittest +from unittest.mock import patch +from email.parser import BytesParser +from email.policy import default + +import watchdog + + +class CustomerAlertTextTests(unittest.TestCase): + EXPECTED = { + "crashed": "Docker reported an unexpected container termination. Run: docker logs --tail 50. The exit code and detection details provide additional context.", + "oom": "Docker reported that the container was terminated due to an out-of-memory condition.", + "unhealthy": "The container is running, but a monitoring health check is failing.", + "restart-loop": "The container has restarted repeatedly within the configured monitoring window. This pattern may be associated with application errors, resource pressure, or unavailable dependencies. The underlying cause has not been determined.", + "recovered": "The previously reported alert condition has cleared according to the available monitoring checks.", + "reboot-summary": "Monitoring has resumed following a host reboot. The reported summary includes container status, recovered incidents, and any conditions that remain unresolved.", + } + + def test_all_approved_messages_exact_and_informational(self): + self.assertEqual(set(watchdog.ACTIONS), set(self.EXPECTED)) + for reason, message in self.EXPECTED.items(): + with self.subTest(reason=reason): + self.assertEqual(watchdog.ACTIONS[reason][1], message) + self.assertNotIn("docker-compose.yaml", message) + self.assertNotIn("Increase memory limit", message) + self.assertNotIn("fix application crash", message) + self.assertIn("Run: docker logs --tail 50", self.EXPECTED["crashed"]) + + def test_fallback_is_informative_and_preserves_warning(self): + container = type("Container", (), { + "name": "dummy", "id": "abc123abc123", + "logs": lambda self, tail, timestamps: b"", + })() + payload = watchdog.build_payload(container, "new-unknown-type", + "test-host", "https://example.invalid") + self.assertEqual(payload.severity, "WARNING") + self.assertEqual( + payload.recommended_action, + "A container monitoring event was reported. The failure type and available detection details provide additional context.") + + def test_crash_replaces_name_and_syslog_preserves_action_key(self): + container = type("Container", (), { + "name": "dummy", "id": "abc123abc123", + "logs": lambda self, tail, timestamps: b"", + })() + payload = watchdog.build_payload( + container, "crashed", "test-host", "https://example.invalid", + exit_code=2, probe_detail="Crash probe failed - exit code 2") + self.assertIn("docker logs dummy --tail 50", payload.recommended_action) + record = watchdog.format_syslog_alert(payload) + self.assertIn('action="Docker reported an unexpected container termination.', record) + self.assertIn('failure_type="crashed"', record) + self.assertIn('exit_code="2"', record) + self.assertIn('detection="Crash probe failed - exit code 2"', record) + + def test_email_and_slack_use_context_title(self): + payload = watchdog.AlertPayload( + "CRITICAL", "dummy", "id", "host", "oom", 137, "", + self.EXPECTED["oom"], "", "OOM probe failed") + cfg = {"smtp": {"enabled": True, "auth": False, "tls": False, + "host": "example.invalid", "port": 25, + "sender": "test@example.invalid", + "recipients": ["recipient@example.invalid"]}} + with patch("smtplib.SMTP") as smtp: + watchdog.send_smtp(payload, cfg) + raw = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] + body = BytesParser(policy=default).parsebytes(raw).get_body( + preferencelist=("plain",)).get_content() + self.assertIn("Context: " + self.EXPECTED["oom"], body) + self.assertNotIn("Action:", body) + with patch.dict("watchdog.os.environ", {"SLACK_WEBHOOK_URL": "https://example.invalid"}): + with patch("watchdog.requests.post") as post: + watchdog.send_slack(payload, {"slack": {"enabled": True}}) + fields = post.call_args.kwargs["json"]["attachments"][0]["fields"] + mapping = {field["title"]: field["value"] for field in fields} + self.assertEqual(mapping["Context"], self.EXPECTED["oom"]) + self.assertNotIn("Action", mapping) + + def test_ha_guidance_remains_unchanged(self): + from inspect import getsource + self.assertIn( + "Verify whether the HA switchover was expected and check service health.", + getsource(watchdog.Watchdog)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_documentation.py b/tests/test_documentation.py new file mode 100644 index 0000000..c6dc86c --- /dev/null +++ b/tests/test_documentation.py @@ -0,0 +1,110 @@ +"""Read-only documentation consistency checks; no Docker or network access.""" + +import re +from pathlib import Path +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +DOCS = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] +LINK_RE = re.compile(r"\]\(([^)#]+)?(?:#([a-zA-Z0-9\-]+))?\)") + + +def heading_anchors(markdown): + counts = {} + anchors = set() + fence = None + for line in markdown.splitlines(): + marker = line.strip()[:3] + if marker in ("~~~", chr(96) * 3): + fence = None if fence == marker else marker + continue + if fence: + continue + matched = re.match(r"^#{1,6}\s+(.+?)\s*#*\s*$", line) + if not matched: + continue + slug = re.sub(r"[^\w\- ]", "", matched.group(1).lower()).replace(" ", "-") + count = counts.get(slug, 0) + counts[slug] = count + 1 + anchors.add(slug + (f"-{count}" if count else "")) + if fence: + raise AssertionError("Unclosed Markdown code fence") + return anchors + + +class DocumentationTests(unittest.TestCase): + def test_all_relative_links_and_headings_resolve(self): + for doc in DOCS: + source = doc.read_text() + heading_anchors(source) + for match in LINK_RE.finditer(source): + relative, anchor = match.groups() + if relative and relative.startswith(("http:", "https:", "mailto:")): + continue + target = (doc.parent / (relative or "")).resolve() if relative else doc + with self.subTest(document=str(doc.relative_to(ROOT)), link=match.group(0)): + self.assertTrue(target.exists(), f"Broken link to {relative}") + if anchor and target.suffix == ".md": + self.assertIn(anchor, heading_anchors(target.read_text())) + + def test_customer_docs_are_self_contained(self): + docs = [ROOT / "README.md", *(ROOT / "docs/operations").glob("*.md")] + for doc in docs: + for match in LINK_RE.finditer(doc.read_text()): + relative, _ = match.groups() + if not relative or relative.startswith(("http:", "https:", "mailto:")): + continue + target = (doc.parent / relative).resolve() + self.assertNotIn( + ROOT / "docs/dev", + target.parents, + f"Customer bundle refers to internal engineering file: {doc}", + ) + self.assertFalse((ROOT / "DEPLOYMENT.md").exists()) + self.assertFalse((ROOT / "docs/DEPLOYMENT_ADVANCED.md").exists()) + + def test_release_notes_links_work_when_published_as_github_release(self): + # GitHub Release notes are rendered at the release URL, not at + # docs/operations/, so links to other files must be absolute. + for notes in (ROOT / "docs/operations").glob("RELEASE_NOTES_V*.md"): + for match in LINK_RE.finditer(notes.read_text()): + destination, _ = match.groups() + if not destination or destination.startswith("#"): + continue + with self.subTest(release_notes=notes.name, link=destination): + self.assertTrue( + destination.startswith(("https:", "http:", "mailto:")), + "Relative links are broken when copied into GitHub Releases", + ) + + def test_operator_guide_covers_configuration_template_fields(self): + config = yaml.safe_load((ROOT / "watchdog-config.yaml.example").read_text()) + guide = (ROOT / "docs/operations/INSTALLATION_AND_OPERATIONS.md").read_text() + + def keys(item, prefix=""): + if isinstance(item, dict): + for key, value in item.items(): + field = prefix + str(key) + yield field + yield from keys(value, field + ".") + + for key in keys(config): + # Detailed event matching and individual custom-probe entries are + # supported by the operator guide without documenting test fixtures. + if key.startswith(("ignored_container_events", "container_health_checks")): + continue + with self.subTest(yaml_setting=key): + self.assertIn(key, guide) + + for line in (ROOT / ".env.example").read_text().splitlines(): + matched = re.match(r"^\s*#?\s*([A-Z][A-Z0-9_]+)=", line) + if matched: + with self.subTest(env_setting=matched.group(1)): + self.assertIn(matched.group(1), guide) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ha_monitor.py b/tests/test_ha_monitor.py index 93023fd..d84cce7 100644 --- a/tests/test_ha_monitor.py +++ b/tests/test_ha_monitor.py @@ -13,8 +13,8 @@ parse_ha_metrics, read_ha_observation, ) -A = "10.199.0.241" -B = "10.199.0.242" +A = "192.0.2.41" +B = "192.0.2.42" def metrics(owner=A, local=A, *, pacemaker=1, quorum=1, extras=""): @@ -84,7 +84,7 @@ def test_http_is_local_bounded_and_closed(self): "http://127.0.0.1:9664/metrics", timeout=3, stream=True, allow_redirects=False) def test_rejects_remote_or_wrong_path_endpoints(self): - for url in ("http://10.199.0.242:9664/metrics", + for url in ("http://192.0.2.42:9664/metrics", "http://localhost:9664/admin", "https://localhost:9664/metrics", "http://localhost:9664/metrics?x=1"): with self.subTest(url=url), self.assertRaises(ValueError): @@ -183,6 +183,19 @@ def test_real_watchdog_event_and_persistence(self): restarted._poll_ha(restarted._session) restarted._dispatch.assert_not_called() + def test_ha_exporter_failure_is_error_but_throttled(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + with patch("watchdog.read_ha_observation", side_effect=TimeoutError("offline")), \ + patch("watchdog.time.monotonic", side_effect=[1000, 1000, 1100, 1301, 1301]), \ + patch("watchdog.log.error") as err: + w._poll_ha(w._session) + w._poll_ha(w._session) + w._poll_ha(w._session) + self.assertEqual(err.call_count, 2) + self.assertIn("HA check unavailable", err.call_args.args[0]) + w._dispatch.assert_not_called() + def test_failed_exporter_does_not_overwrite_state_or_alert(self): with tempfile.TemporaryDirectory() as td: w = self.create_shell(f"{td}/state.json") @@ -351,9 +364,12 @@ def test_slack_ha_fields_and_no_false_container(self): def test_syslog_logging_includes_transition_and_dispatches_snmp(self): alert = self.make_alert() - with patch("watchdog.log.warning") as warn, patch("watchdog.send_snmp_trap") as trap: + with patch("watchdog.log.log") as logged, patch("watchdog.send_snmp_trap") as trap: watchdog.dispatch_alert(alert, {"alert_channels": ["snmp_trap", "syslog"]}) - self.assertIn(f"{A} -> {B}", warn.call_args.args[1]) + self.assertEqual(logged.call_args.args[0], watchdog.logging.WARNING) + self.assertIn(f'previous_active="{A}"', logged.call_args.args[1]) + self.assertIn(f'new_active="{B}"', logged.call_args.args[1]) + self.assertTrue(logged.call_args.kwargs["extra"]["watchdog_alert"]) trap.assert_called_once() def test_smtp_ha_notification_identifies_role_change(self): diff --git a/tests/test_installer_image.py b/tests/test_installer_image.py new file mode 100644 index 0000000..2ed2821 --- /dev/null +++ b/tests/test_installer_image.py @@ -0,0 +1,157 @@ +"""Validate the single bundled installer using fake Docker only.""" + +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +VERSION = (ROOT / "VERSION").read_text().strip() + + +class InstallerImageTests(unittest.TestCase): + def setUp(self): + self.workspace = tempfile.TemporaryDirectory(prefix="watchdog-installer-fake-") + self.addCleanup(self.workspace.cleanup) + self.root = Path(self.workspace.name) + self.bin = self.root / "fake-bin" + self.bin.mkdir() + self.state = self.root / "fake-state" + self.state.mkdir() + shutil.copy2(ROOT / "install.sh", self.root / "install.sh") + (self.root / "VERSION").write_text(VERSION + "\n") + self.log = self.root / "fake-docker-calls.log" + self.log.write_text("") + self.archive = b"fake image bytes, not a real Docker image" + self.env = os.environ.copy() + self.env.update( + PATH=str(self.bin) + os.pathsep + os.environ["PATH"], + FAKE_DOCKER_LOG=str(self.log), + FAKE_STATE_DIR=str(self.state), + FAKE_VERSION=VERSION, + FAKE_IMAGE_PRESENT="0", + ) + docker = self.bin / "docker" + docker.write_text("""#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + image) + if [ "$2" = inspect ]; then + if [ "$FAKE_IMAGE_PRESENT" = 1 ] || [ -f "$FAKE_STATE_DIR/loaded" ]; then + case "$*" in + *org.opencontainers.image.version*) echo "$FAKE_VERSION" ;; + *'{{.Id}}'*) echo 'sha256:fake-image-id' ;; + esac + exit 0 + fi + exit 1 + fi ;; + load) + touch "$FAKE_STATE_DIR/loaded" + echo "Loaded image: watchdog:$FAKE_TAG_VERSION" + exit 0 ;; + config|up) exit 0 ;; +esac +echo "Unexpected fake Docker arguments: $*" >&2 +exit 97 +""") + docker.chmod(0o755) + self.env["FAKE_TAG_VERSION"] = VERSION + + def write_archive(self, with_checksum=True, valid_checksum=True): + (self.root / "watchdog.tar").write_bytes(self.archive) + if with_checksum: + digest = hashlib.sha256(self.archive).hexdigest() + if not valid_checksum: + digest = "0" * 64 + (self.root / "SHA256SUMS").write_text( + f"{digest} watchdog.tar\n" + ) + + def run_function(self, body="load_docker_image"): + result = subprocess.run( + ["bash", "-c", "source ./install.sh; " + body], + cwd=self.root, env=self.env, + text=True, capture_output=True, timeout=10, + ) + result.output = result.stdout + result.stderr + return result + + def calls(self): + return self.log.read_text().splitlines() + + def test_valid_package_loads_image(self): + self.write_archive() + out = self.run_function() + self.assertEqual(out.returncode, 0, out.output) + self.assertTrue(any(c.startswith("load ") for c in self.calls())) + + def test_existing_local_image_is_not_silently_trusted(self): + self.write_archive() + self.env["FAKE_IMAGE_PRESENT"] = "1" + out = self.run_function() + self.assertEqual(out.returncode, 0, out.output) + self.assertTrue(any(c.startswith("load ") for c in self.calls())) + + def test_missing_package_fails_without_docker_mutations(self): + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("Incomplete release package", out.output) + self.assertEqual(self.calls(), []) + + def test_missing_checksum_rejected_before_loading(self): + self.write_archive(with_checksum=False) + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_invalid_checksum_rejected_before_loading(self): + self.write_archive(valid_checksum=False) + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("checksum mismatch", out.output) + self.assertEqual(self.calls(), []) + + def test_wrong_image_version_label_rejected(self): + self.write_archive() + self.env["FAKE_VERSION"] = "unexpected-version" + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("does not match", out.output) + + def test_wrong_image_tag_rejected(self): + self.write_archive() + self.env["FAKE_TAG_VERSION"] = "old" + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("expected tag", out.output) + + def test_existing_configuration_is_not_regenerated(self): + (self.root / ".env").write_text("SECRET=keep\n") + out = self.run_function("configure_all") + self.assertNotEqual(out.returncode, 0) + self.assertEqual((self.root / ".env").read_text(), "SECRET=keep\n") + self.assertFalse((self.root / "watchdog-config.yaml").exists()) + self.assertEqual(self.calls(), []) + + def test_recreate_targets_watchdog_only(self): + out = self.run_function("DC=docker; start_watchdog") + self.assertEqual(out.returncode, 0, out.output) + self.assertIn("config -q", self.calls()) + self.assertIn("up -d --no-deps --force-recreate docker-container-watchdog", + self.calls()) + self.assertFalse(any("down" in c for c in self.calls())) + + def test_no_network_image_download_path(self): + source = (ROOT / "install.sh").read_text() + self.assertNotIn("download_release_image()", source) + self.assertNotIn("curl --fail", source) + self.assertNotIn('docker build', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_notifications.py b/tests/test_notifications.py new file mode 100644 index 0000000..b45bd8a --- /dev/null +++ b/tests/test_notifications.py @@ -0,0 +1,192 @@ +"""Notification consistency and syslog regression tests (no network).""" +import logging +import socket +import tempfile +import unittest +from unittest.mock import patch +from email.parser import BytesParser +from email.policy import default +from pathlib import Path + +import watchdog + + +def alert(severity="CRITICAL", failure="crashed"): + return watchdog.AlertPayload( + severity, "demo-service", "abcdef123456789", "test-host", failure, 137, + "", "Inspect Docker events and logs.", "https://example.invalid/runbook", + 'Crash probe failed - SIGKILL; OOM not confirmed', + ) + + +class NotificationTests(unittest.TestCase): + def test_oom_evidence_only_when_confirmed(self): + p = alert("CRITICAL", "oom") + p.probe_detail = ("OOM probe failed - container was OOM-killed by the kernel; " + "post-event memory snapshot: usage=63.0 MB, limit=64.0 MB") + data = watchdog.format_syslog_alert(p) + self.assertIn('failure_type="oom"', data) + self.assertIn('post-event memory snapshot: usage=63.0 MB', data) + self.assertNotIn('failure_type="crashed"', data) + + def test_real_syslog_keeps_recovery_and_ha_at_local_warning_or_error(self): + for level in ("WARNING", "ERROR"): + with self.subTest(level=level), tempfile.TemporaryDirectory() as td: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as receiver: + receiver.bind(("127.0.0.1", 0)) + receiver.settimeout(0.6) + watchdog.setup_logging( + level, f"{td}/watchdog.log", + {"enabled": True, "host": "127.0.0.1", + "port": receiver.getsockname()[1], "protocol": "udp"} + ) + try: + watchdog.log.info("routine polling must stay local") + recovery = alert("INFO", "recovered") + recovery.exit_code = None + watchdog.dispatch_alert(recovery, {"alert_channels": []}) + ha = watchdog.AlertPayload( + "WARNING", "(HA cluster)", "", "test-host", "ha-failover", + None, "", "Check cluster health", "", + "HA Operator promoted to new node", + ha_previous_node="node-a", ha_new_node="node-b" + ) + watchdog.dispatch_alert(ha, {"alert_channels": []}) + watchdog.log.error("HA check unavailable (TimeoutError): offline") + received = [receiver.recvfrom(8192)[0].decode() + for _ in range(3)] + with self.assertRaises(socket.timeout): + receiver.recvfrom(8192) + self.assertEqual( + [int(msg.split(">", 1)[0][1:]) % 8 for msg in received], + [6, 4, 3]) + self.assertIn('failure_type="recovered"', received[0]) + self.assertIn('failure_type="ha-failover"', received[1]) + self.assertIn("HA check unavailable", received[2]) + self.assertFalse(any("routine polling" in msg for msg in received)) + text = Path(f"{td}/watchdog.log").read_text() + self.assertNotIn("routine polling", text) + self.assertNotIn('failure_type="recovered"', text) + if level == "ERROR": + self.assertNotIn('failure_type="ha-failover"', text) + else: + self.assertIn('failure_type="ha-failover"', text) + finally: + logging.basicConfig( + handlers=[logging.NullHandler()], + level=logging.WARNING, force=True) + + def test_pseudo_container_fields_absent_in_ha_and_reboot_events(self): + for failure, container_name in ( + ("ha-failover", "(HA cluster)"), + ("reboot-summary", "(post-reboot summary)") + ): + with self.subTest(failure=failure): + p = watchdog.AlertPayload( + "INFO", container_name, "", "test-host", failure, None, "", + "Check cluster", "", "Event details", + ha_previous_node="node-a", ha_new_node="node-b") + record = watchdog.format_syslog_alert(p) + self.assertIn('event="watchdog_alert"', record) + self.assertIn(f'failure_type="{failure}"', record) + self.assertNotIn('container=', record) + self.assertNotIn('container_id=', record) + self.assertIn('host="test-host"', record) + if failure == "ha-failover": + self.assertIn('previous_active="node-a"', record) + self.assertIn('new_active="node-b"', record) + + def test_memory_snapshot_omits_missing_zero_and_unavailable_stats(self): + from unittest.mock import Mock + container = Mock() + container.stats.return_value = { + "memory_stats": {"usage": 0, "limit": 67108864, "max_usage": 12582912} + } + sample = watchdog.get_memory_stats(container) + self.assertIn("post-event memory snapshot:", sample) + self.assertIn("limit=64.0 MB", sample) + self.assertIn("peak=12.0 MB", sample) + self.assertNotIn("usage=", sample) + container.stats.return_value = {"memory_stats": {}} + self.assertEqual(watchdog.get_memory_stats(container), "") + container.stats.side_effect = RuntimeError("unavailable after OOM") + self.assertEqual(watchdog.get_memory_stats(container), "") + + def test_native_levels_match_incident_severity(self): + for severity, expected in (("CRITICAL", logging.CRITICAL), + ("HIGH", logging.ERROR), + ("WARNING", logging.WARNING), + ("INFO", logging.INFO)): + with self.subTest(severity=severity): + p = alert(severity) + with patch("watchdog.log.log") as log: + watchdog.dispatch_alert(p, {"alert_channels": []}) + self.assertEqual(log.call_args.args[0], expected) + self.assertIn(f'severity="{severity}"', log.call_args.args[1]) + self.assertTrue(log.call_args.kwargs["extra"]["watchdog_alert"]) + + def test_syslog_filter_rejects_poll_noise_and_keeps_recovery_and_errors(self): + f = watchdog.AlertSyslogFilter() + def record(level, is_alert=False): + r = logging.LogRecord("watchdog", level, __file__, 1, "message", (), None) + if is_alert: + r.watchdog_alert = True + return r + self.assertFalse(f.filter(record(logging.INFO))) + self.assertFalse(f.filter(record(logging.WARNING))) + self.assertTrue(f.filter(record(logging.INFO, True))) + self.assertTrue(f.filter(record(logging.WARNING, True))) + self.assertTrue(f.filter(record(logging.ERROR))) + + def test_syslog_has_evidence_without_false_oom_and_escapes_content(self): + p = alert() + p.probe_detail = 'line one\nline "two"' + data = watchdog.format_syslog_alert(p) + for fragment in ('event="watchdog_alert"', 'severity="CRITICAL"', + 'failure_type="crashed"', 'exit_code="137"', + 'host="test-host"', 'container="demo-service"', + 'container_id="abcdef123456"', + 'probe_type="Docker event (crash)"', + 'action="Inspect Docker events and logs."'): + self.assertIn(fragment, data) + self.assertIn(r'line one\nline \"two\"', data) + self.assertNotIn('failure_type="oom"', data) + self.assertNotIn("\n", data) + + def test_long_untrusted_detection_is_bounded(self): + p = alert() + p.probe_detail = "x" * 20000 + data = watchdog.format_syslog_alert(p) + self.assertLess(len(data), 2000) + self.assertIn('...', data) + + def test_email_includes_detection_without_assuming_oom(self): + p = alert() + cfg = {"smtp": {"enabled": True, "auth": False, "tls": False, + "host": "mail.invalid", "port": 25, + "sender": "alerts@example.invalid", + "recipients": ["ops@example.invalid"]}} + with patch("smtplib.SMTP") as smtp: + watchdog.send_smtp(p, cfg) + sent = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] + body = BytesParser(policy=default).parsebytes(sent).get_body( + preferencelist=("plain",)).get_content() + for item in ("Container ID: abcdef123456", "Severity: CRITICAL", + "Exit Code: 137", "Probe Type: Docker event (crash)", + "OOM not confirmed", "Runbook: https://example.invalid/runbook"): + self.assertIn(item, body) + + def test_slack_fields_match_alert_payload(self): + p = alert() + with patch.dict("os.environ", {"SLACK_WEBHOOK_URL": "https://example.invalid/slack"}): + with patch("watchdog.requests.post") as post: + watchdog.send_slack(p, {"slack": {"enabled": True}}) + fields = post.call_args.kwargs["json"]["attachments"][0]["fields"] + values = {field["title"]: field["value"] for field in fields} + self.assertEqual(values["Severity"], "CRITICAL") + self.assertEqual(values["Exit Code"], "137") + self.assertEqual(values["Detection"], p.probe_detail) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_package.py b/tests/test_release_package.py new file mode 100644 index 0000000..4395156 --- /dev/null +++ b/tests/test_release_package.py @@ -0,0 +1,96 @@ +"""Validate single customer release package with a fake image; never use Docker.""" + +import hashlib +import posixpath +import re +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "package_release.sh" +VERSION = (ROOT / "VERSION").read_text().strip() +PREFIX = "CyberController-Container-Alert/" + + +class ReleasePackageTests(unittest.TestCase): + def setUp(self): + self.workspace = tempfile.TemporaryDirectory(prefix="container-alert-package-fake-") + self.addCleanup(self.workspace.cleanup) + self.path = Path(self.workspace.name) + self.fake_image = self.path / "fake-watchdog.tar" + self.fake_image.write_bytes(b"FAKE BINARY, NOT A DOCKER IMAGE") + self.dist = self.path / "dist" + result = subprocess.run( + ["bash", str(SCRIPT), str(self.fake_image), str(self.dist)], + capture_output=True, text=True, timeout=15, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.archive = self.dist / f"container-alert-v{VERSION}.tar.gz" + + def package_files(self): + with tarfile.open(self.archive, "r:gz") as tf: + return { + item.name: tf.extractfile(item).read() + for item in tf if item.isfile() + } + + def test_one_complete_release_package(self): + files = self.package_files() + required = ("install.sh", "uninstall.sh", "docker-compose.yaml", "VERSION", + "README.md", ".env.example", + "watchdog-config.yaml.example", "watchdog.tar", "SHA256SUMS", + "docs/operations/INSTALLATION_AND_OPERATIONS.md", + f"docs/operations/RELEASE_NOTES_V{VERSION}.md") + for name in required: + self.assertIn(PREFIX + name, files) + self.assertEqual(files[PREFIX + "VERSION"].decode().strip(), VERSION) + + def test_bundle_contains_image_not_python_source(self): + files = self.package_files() + self.assertEqual(files[PREFIX + "watchdog.tar"], self.fake_image.read_bytes()) + self.assertNotIn(PREFIX + "watchdog.py", files) + self.assertNotIn(PREFIX + "lifecycle.py", files) + self.assertNotIn(PREFIX + "Dockerfile", files) + self.assertNotIn(PREFIX + ".env", files) + self.assertNotIn(PREFIX + "watchdog-config.yaml", files) + self.assertFalse(any(path.startswith(PREFIX + "docs/dev/") for path in files)) + self.assertNotIn(PREFIX + "docs/DEVELOPMENT.md", files) + + def test_every_local_customer_link_is_packaged(self): + files = self.package_files() + links = re.compile(r"\\]\\(([^)#]+)?(?:#[a-zA-Z0-9\\-]+)?\\)") + for name, content in files.items(): + if not name.endswith(".md"): + continue + for match in links.finditer(content.decode("utf-8")): + relative = match.group(1) + if not relative or relative.startswith(("http:", "https:", "mailto:")): + continue + target = posixpath.normpath( + posixpath.join(posixpath.dirname(name), relative) + ) + with self.subTest(markdown=name, target=target): + self.assertIn(target, files) + + def test_image_checksum_inside_package(self): + files = self.package_files() + expected = hashlib.sha256(self.fake_image.read_bytes()).hexdigest() + self.assertEqual(files[PREFIX + "SHA256SUMS"].decode().strip(), + f"{expected} watchdog.tar") + + def test_release_checksum_covers_only_package(self): + self.assertTrue(self.archive.is_file()) + lines = (self.dist / "SHA256SUMS").read_text().splitlines() + self.assertEqual(lines, [ + f"{hashlib.sha256(self.archive.read_bytes()).hexdigest()} {self.archive.name}" + ]) + self.assertEqual(sorted(p.name for p in self.dist.iterdir()), + ["SHA256SUMS", self.archive.name]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_smoke_safety.py b/tests/test_release_smoke_safety.py new file mode 100644 index 0000000..8a403b7 --- /dev/null +++ b/tests/test_release_smoke_safety.py @@ -0,0 +1,90 @@ +"""Fail-closed release smoke preflight tests; never connect to real Docker.""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "tests" / "release-smoke.sh" + + +class SmokeSafetyTests(unittest.TestCase): + def try_runner(self, overrides, docker_info=None): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + marker = root / "docker-invocations.txt" + stub = root / "docker" + # Fake Docker deliberately records any invocation, including read-only. + stub.write_text( + "#!/bin/sh\n" + "echo \"$*\" >> \"" + str(marker) + "\"\n" + "printf '%s\\n' '" + (docker_info or "{}") + "'\n") + stub.chmod(0o755) + env = dict(os.environ) + for key in ("GITHUB_ACTIONS", "SMOKE_RUNNER_KIND", "SMOKE_ENGINE_MARKER", + "DOCKER_HOST", "DOCKER_CONTEXT", "SMOKE_ONLY"): + env.pop(key, None) + env.update(overrides) + env["PATH"] = td + os.pathsep + env.get("PATH", "") + env["SMOKE_OUTPUT_DIR"] = td + done = subprocess.run(["bash", str(SCRIPT)], env=env, cwd=REPO, + capture_output=True, text=True, timeout=15) + calls = marker.read_text().splitlines() if marker.exists() else [] + return done.returncode, calls + + def test_default_invocation_fails_without_any_docker_call(self): + rc, calls = self.try_runner({}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_live_docker_host_rejected_without_any_docker_call(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "unix:///var/run/docker.sock"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://192.0.2.41:2375"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_context_override_rejected(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_CONTEXT": "production", + "DOCKER_HOST": "tcp://127.0.0.1:32768"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_forged_or_nonempty_engine_fails_before_creating_resources(self): + base = {"GITHUB_ACTIONS": "true", + "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://127.0.0.1:32768"} + for info in ( + '{"ID":"unexpected","Labels":[],"Containers":0}', + '{"ID":"test","Labels":["com.radware.container-alert.smoke-engine=ci-123-1"],"Containers":1}', + ): + with self.subTest(info=info): + rc, calls = self.try_runner(base, docker_info=info) + self.assertNotEqual(rc, 0) + self.assertEqual(len(calls), 1) + self.assertTrue(calls[0].startswith("info --format "), calls) + + def test_self_hosted_runner_rejected(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "self-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://127.0.0.1:32768"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_restart_loop.py b/tests/test_restart_loop.py index f39d8e6..e433d32 100644 --- a/tests/test_restart_loop.py +++ b/tests/test_restart_loop.py @@ -144,13 +144,30 @@ def test_single_auto_restart_recovers_only_after_stability(self): wd._poll_all_containers() self.assertEqual([a.failure_type for a in alerts], ["crashed", "recovered"]) - def test_no_restart_policy_keeps_existing_immediate_recovery(self): + def test_no_restart_policy_crash_requires_stable_uptime(self): + # Real lab: an outside actor restarts Docker policy=no containers. container = FakeContainer(policy="no", uptime=3) wd, alerts = make_watchdog(container) die(wd) wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["crashed"]) + container.attrs["State"]["StartedAt"] = started_ago(130) + wd.state.get(container.name).last_failure_time = time.time() - 130 + wd._poll_all_containers() + wd._poll_all_containers() self.assertEqual([a.failure_type for a in alerts], ["crashed", "recovered"]) + def test_no_restart_policy_oom_requires_stable_uptime(self): + container = FakeContainer(policy="no", uptime=4) + wd, alerts = make_watchdog(container) + wd._handle_event({"Action": "oom", "Actor": {"Attributes": {"name": container.name}}}) + wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["oom"]) + container.attrs["State"]["StartedAt"] = started_ago(130) + wd.state.get(container.name).last_failure_time = time.time() - 130 + wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["oom", "recovered"]) + def test_poll_detects_lost_events_from_docker_restart_count(self): container = FakeContainer(count=0) wd, alerts = make_watchdog(container) diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py index 1cf26bd..d6be9c7 100644 --- a/tests/test_watchdog.py +++ b/tests/test_watchdog.py @@ -57,9 +57,14 @@ def test_recovery_disabled_still_clears_open_state(self): w.cfg = {"alert_on_recovery": False} w.state = watchdog.WatchdogState("") w._boot_time = 123 + w._restart_recovery_stability_secs = 120 state = w.state.get("svc") state.alerted_for = "crashed" - w._maybe_alert_recovery(SimpleNamespace(name="svc"), state) + state.last_failure_time = time.time() - 130 + stable_since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 130)) + container = SimpleNamespace(name="svc", attrs={ + "State": {"StartedAt": stable_since}, "RestartCount": 0}) + w._maybe_alert_recovery(container, state) self.assertEqual(state.alerted_for, "") def test_reboot_summary_uses_current_state_not_folded_failures(self): w = object.__new__(watchdog.Watchdog) diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index bcd6b6c..1398a02 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -4,8 +4,8 @@ # SECRETS: All passwords and webhook URLs are read from environment # variables, never hardcoded here. Set them in your .env file. # -# Copy this file to /opt/radware/storage/scripts/Alert_Container/watchdog-config.yaml -# and update the values for your environment. +# Reference template only: install.sh creates watchdog-config.yaml in your +# chosen installation directory. Edit that active file after installation. # ── Alert channels ──────────────────────────────────────────────────────────── # Empty for local testing — alerts are still logged to stdout/file. @@ -20,7 +20,7 @@ check_interval_seconds: 60 # how often to poll all containers cooldown_minutes: 5 # suppress duplicate alerts per container restart_threshold: 5 # Docker RestartCount increments within the window trigger alert restart_window_minutes: 10 # rolling window for restart counting -restart_recovery_stability_seconds: 120 # require 2 min continuous uptime before RECOVERED for auto-restarting containers +restart_recovery_stability_seconds: 120 # require stable Docker uptime before CRASH/OOM/RESTART-LOOP recovery unhealthy_cycles_threshold: 3 # consecutive unhealthy cycles before alerting alert_on_recovery: true # send an INFO "recovered" alert when a container returns to normal @@ -35,9 +35,16 @@ lifecycle_detection: excluded_containers: [] # - my-temp-container +# Narrow exception for labelled temporary syntax-check containers. +ignored_container_events: + - label: + com.radware.cybercontroller.role: mariadb-ha-syntax-check + events: + - die + # ── Logging ─────────────────────────────────────────────────────────────────── -log_level: DEBUG # DEBUG | INFO | WARNING | ERROR +log_level: INFO # DEBUG | INFO | WARNING | ERROR (default INFO) log_file: /var/log/watchdog/watchdog.log # bind-mounted to ./watchdog/watchdog.log on host # Alert state is persisted here so recovery alerts still fire after the host or @@ -52,7 +59,7 @@ boot_grace_seconds: 180 # ── Syslog (remote / central log server) ───────────────────────────────────── syslog: enabled: true # set to true to enable - host: 155.1.1.4 # IP or hostname of your syslog server + host: syslog.example.com # replace with your syslog server port: 514 # 514 = standard syslog (UDP); 601 = reliable syslog (TCP) protocol: udp # udp (fire-and-forget) or tcp (reliable) facility: local0 # syslog facility: local0–local7, daemon, user, etc. @@ -62,23 +69,23 @@ syslog: smtp: enabled: true # set to true to enable auth: false - host: smtp.radware.com + host: smtp.example.com port: 587 - sender: noc-alerts@radware.com + sender: alerts@example.com username_env: SMTP_USERNAME # env var: SMTP username or app token (Mailtrap, Gmail app password, etc.) recipients: - - ops-team@radware.com - - oncall@radware.com + - ops@example.com + - oncall@example.com tls: true password_env: SMTP_PASSWORD # env var: SMTP password or app token # ── SNMP Traps ──────────────────────────────────────────────────────────────── # Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. # Add "snmp_trap" to alert_channels above to enable. -# Requires: pip install pysnmp (already in requirements-watchdog.txt) +# Dependencies are included in the release image; no host pip install. snmp_trap: enabled: false # set to true to enable - host: 155.1.1.1 # IP or hostname of your SNMP trap receiver (NMS) + host: nms.example.com # replace with your NMS receiver port: 162 # UDP port — 162 is the standard SNMP trap port version: v2c # SNMP version: v1, v2c, or v3 community: public # SNMPv1/v2c community string (ignored for v3) diff --git a/watchdog.py b/watchdog.py index 61c1b0e..9922d84 100644 --- a/watchdog.py +++ b/watchdog.py @@ -31,7 +31,15 @@ # ── Logging setup ───────────────────────────────────────────────────────────── LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" -SYSLOG_FORMAT = "watchdog[%(process)d]: %(levelname)s %(name)s: %(message)s" +SYSLOG_FORMAT = "watchdog[%(process)d]: %(message)s" + + +class AlertSyslogFilter(logging.Filter): + """Forward alert records and actual errors, not routine poll diagnostics.""" + + def filter(self, record: logging.LogRecord) -> bool: + return bool(getattr(record, "watchdog_alert", False) or record.levelno >= logging.ERROR) + log = logging.getLogger("watchdog") @@ -39,6 +47,7 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = None) -> None: numeric = getattr(logging, level.upper(), logging.INFO) handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + syslog_active = False if log_file: log_dir = os.path.dirname(log_file) if log_dir: @@ -48,6 +57,8 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = No log_file, maxBytes=10 * 1024 * 1024, backupCount=5 ) ) + for handler in handlers: + handler.setLevel(numeric) if syslog_cfg and syslog_cfg.get("enabled"): host = syslog_cfg.get("host", "localhost") port = int(syslog_cfg.get("port", 514)) @@ -70,10 +81,15 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = No log.warning("Syslog disabled — cannot reach %s:%d (%s): %s", host, port, proto, exc) else: syslog_handler.setFormatter(logging.Formatter(SYSLOG_FORMAT)) + syslog_handler.setLevel(logging.INFO) + syslog_handler.addFilter(AlertSyslogFilter()) handlers.append(syslog_handler) + syslog_active = True # Use a plain formatter for other handlers, syslog gets its own log.info("Syslog handler added: %s:%d (%s)", host, port, proto) - logging.basicConfig(level=numeric, format=LOG_FORMAT, handlers=handlers, force=True) + # Let INFO recovery alerts reach Syslog even when local logs are WARNING/ERROR. + root_level = min(numeric, logging.INFO) if syslog_active else numeric + logging.basicConfig(level=root_level, format=LOG_FORMAT, handlers=handlers, force=True) # Suppress noisy third-party debug chatter (urllib3 Docker socket calls, etc.) for noisy in ("urllib3", "urllib3.connectionpool", "docker", "requests"): logging.getLogger(noisy).setLevel(logging.WARNING) @@ -402,12 +418,14 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: ha_event = payload.failure_type == "ha-failover" ha_fields = [ {"title": "Event", "value": "HA active-node change", "short": True}, + {"title": "Severity", "value": payload.severity, "short": True}, {"title": "Previous active", "value": payload.ha_previous_node, "short": True}, {"title": "New active", "value": payload.ha_new_node, "short": True}, {"title": "Host", "value": payload.host, "short": True}, {"title": "Time (UTC)", "value": payload.timestamp, "short": True}, *probe_field, - {"title": "Action", "value": payload.recommended_action, "short": False}, + {"title": "Context", "value": payload.recommended_action, "short": False}, + {"title": "Runbook", "value": payload.runbook_url, "short": False}, ] body = { "attachments": [{ @@ -422,7 +440,7 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: *exit_field, *probe_type_field, *probe_field, - {"title": "Action", "value": payload.recommended_action, "short": False}, + {"title": "Context", "value": payload.recommended_action, "short": False}, {"title": "Runbook", "value": payload.runbook_url, "short": False}, ], "footer": ("Source: HA cluster exporter" if ha_event @@ -439,19 +457,48 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: +def _syslog_field(value: object, limit: int = 512) -> str: + """Single-line, bounded, quote-safe key/value field for SIEM parsing.""" + value = str(value).replace("\\", "\\\\").replace("\r", "\\r").replace("\n", "\\n") + value = value[:limit] + ("..." if len(value) > limit else "") + return '"' + value.replace('"', '\\"') + '"' + + +def format_syslog_alert(payload: AlertPayload) -> str: + """Same incident facts as human-facing notifications; bounded for UDP syslog.""" + details = { + "event": "watchdog_alert", + "severity": payload.severity, + "failure_type": payload.failure_type, + "time_utc": payload.timestamp, + "host": payload.host, + "probe_type": payload.probe_type, + "detection": payload.probe_detail, + "action": payload.recommended_action, + "runbook": payload.runbook_url, + } + if payload.failure_type not in ("ha-failover", "reboot-summary"): + details["container"] = payload.container_name + details["container_id"] = payload.container_id + if payload.exit_code is not None: + details["exit_code"] = payload.exit_code + if payload.failure_type == "ha-failover": + details["previous_active"] = payload.ha_previous_node + details["new_active"] = payload.ha_new_node + return " ".join(f"{key}={_syslog_field(value, 250 if key in ('detection', 'action') else 120)}" + for key, value in details.items() if value not in ("", "N/A")) + + +def _alert_log_level(severity: str) -> int: + return {"CRITICAL": logging.CRITICAL, "HIGH": logging.ERROR, + "WARNING": logging.WARNING, "INFO": logging.INFO}.get(severity, logging.WARNING) + + def dispatch_alert(payload: AlertPayload, cfg: dict) -> None: """Send alert to all configured channels.""" channels = cfg.get("alert_channels", ["slack"]) - log_fn = log.info if payload.failure_type == "recovered" else log.warning - if payload.failure_type == "ha-failover": - log_fn("ALERT %s (channels: %s) — %s", - payload.subject(), channels, payload.probe_detail) - else: - log_fn( - "ALERT [%s] %s — %s (channels: %s)", - payload.severity, payload.container_name, - payload.failure_type, channels, - ) + log.log(_alert_log_level(payload.severity), format_syslog_alert(payload), + extra={"watchdog_alert": True}) if "slack" in channels: send_slack(payload, cfg) if "smtp" in channels: send_smtp(payload, cfg) if "snmp_trap" in channels: send_snmp_trap(payload, cfg) @@ -520,18 +567,23 @@ def send_smtp(payload: AlertPayload, cfg: dict) -> None: f"Previous active: {payload.ha_previous_node}\n" f"New active: {payload.ha_new_node}\n" f"Host: {payload.host}\n" + f"Severity: {payload.severity}\n" f"Observed (UTC): {payload.timestamp}\n" - f"Evidence: {payload.probe_detail}\n\n" - f"Action: {payload.recommended_action}") + f"Evidence: {payload.probe_detail}\n" + f"Context: {payload.recommended_action}\n" + f"Runbook: {payload.runbook_url}") else: body = f"""Container: {payload.container_name} Host: {payload.host} Status: {payload.failure_type} Severity: {payload.severity} Time: {payload.timestamp} +Container ID: {payload.container_id} Exit Code: {payload.exit_code if payload.exit_code is not None else 'N/A'} - -{payload.recommended_action}""" +Probe Type: {payload.probe_type or 'N/A'} +Detection: {payload.probe_detail or 'N/A'} +Context: {payload.recommended_action} +Runbook: {payload.runbook_url or 'N/A'}""" # Build an RFC-compliant MIME message. smtplib.sendmail(str) tries to # encode the whole message as ASCII, which breaks on Unicode text such as @@ -1030,31 +1082,32 @@ def _suppress_invalid_state(loop, context): ACTIONS = { "crashed": ( "CRITICAL", - "Check exit code and logs. Run: docker logs --tail 50. " - "Exit 137 means SIGKILL, not necessarily OOM. Check Docker stop/oom events and OOMKilled.", + "Docker reported an unexpected container termination. " + "Run: docker logs --tail 50. " + "The exit code and detection details provide additional context.", ), "oom": ( "CRITICAL", - "Container was OOM-killed. Increase memory limit in docker-compose.yaml " - "or investigate memory leak. Run: docker stats .", + "Docker reported that the container was terminated due to an out-of-memory condition.", ), "unhealthy": ( "HIGH", - "Container is running but health probe is failing. " - "Check the application inside the container. Run: docker inspect .", + "The container is running, but a monitoring health check is failing.", ), "restart-loop": ( "HIGH", - "Container is repeatedly restarting. Inspect docker logs and the " - "restart policy; fix application crash, OOM, or dependency failure.", + "The container has restarted repeatedly within the configured monitoring window. " + "This pattern may be associated with application errors, resource pressure, " + "or unavailable dependencies. The underlying cause has not been determined.", ), "recovered": ( "INFO", - "Container is back to normal operation. No action required.", + "The previously reported alert condition has cleared according to the available monitoring checks.", ), "reboot-summary": ( "INFO", - "Post-reboot status summary. No action required unless containers are listed as still failing.", + "Monitoring has resumed following a host reboot. The reported summary includes " + "container status, recovered incidents, and any conditions that remain unresolved.", ), } @@ -1085,19 +1138,17 @@ def get_docker_health_log(container) -> str: def get_memory_stats(container) -> str: + """Best-effort post-event sample, not memory usage at the time of the OOM.""" try: - stats = container.stats(stream=False) - mem = stats.get("memory_stats", {}) - usage = mem.get("usage", 0) - limit = mem.get("limit", 0) - peak = mem.get("max_usage", 0) - def _mb(b: int) -> str: - return f"{b / 1024 / 1024:.1f} MB" - return ( - f"memory usage={_mb(usage)} limit={_mb(limit)} peak={_mb(peak)}" - ) - except Exception as exc: - return f"(memory stats unavailable: {exc})" + mem = container.stats(stream=False).get("memory_stats") or {} + parts = [] + for key, label in (("usage", "usage"), ("limit", "limit"), ("max_usage", "peak")): + value = mem.get(key) + if type(value) is int and value > 0: + parts.append(f"{label}={value / 1024 / 1024:.1f} MB") + return ("post-event memory snapshot: " + ", ".join(parts)) if parts else "" + except Exception: + return "" def get_container_id(container) -> str: @@ -1239,7 +1290,7 @@ def build_payload( extra_context: str = "", probe_detail: str = "", ) -> AlertPayload: - severity, action = ACTIONS.get(failure_type, ("WARNING", "Investigate container.")) + severity, action = ACTIONS.get(failure_type, ("WARNING", "A container monitoring event was reported. The failure type and available detection details provide additional context.")) log_tail = get_log_tail(container) if extra_context: log_tail = f"{extra_context}\n\n{log_tail}" @@ -1523,17 +1574,16 @@ def _sync_container_identity(self, state: ContainerState, container) -> None: # ── Recovery alert ───────────────────────────────────────────────────────── def _maybe_alert_recovery(self, container, state: ContainerState) -> None: - """Resolve once stable; a brief RUNNING state is not loop recovery.""" - automatic = has_automatic_restart_policy(container) + """Resolve real failures only after sustained Docker uptime.""" started = container_started_timestamp(container) now = time.time() with state.lock: previous = state.alerted_for if not previous or previous == "recovered": return - # Once a restart-loop was reported, changing Docker's restart - # policy must not bypass the stability period. - if previous == "restart-loop" or (automatic and previous in ("crashed", "oom")): + # Even a policy=no container may be restarted by an external actor. + # Never mark a crash, OOM, or loop recovered after a brief RUNNING poll. + if previous in ("restart-loop", "crashed", "oom"): stability = self._restart_recovery_stability_secs # If we cannot prove continuous uptime, don't clear a real incident. if started is None or now - started < stability: @@ -1626,7 +1676,7 @@ def _resolve_lifecycle_death(self, death) -> None: return if death.stopped: if death.exit_code == 137: - log.warning("%s: Docker-requested stop ended with SIGKILL (137), " + log.info("%s: Docker-requested stop ended with SIGKILL (137), " "not proof of OOM", death.name) else: log.info("%s: Docker-requested stop (exit %s)", death.name, death.exit_code) @@ -1789,10 +1839,12 @@ def _handle_event(self, event: dict) -> None: log.info("%s: OOM during active restart-loop — suppressing per-cycle OOM alert", name) return if self._claim_alert(state, failure_type): + memory = get_memory_stats(container) payload = build_payload(container, failure_type, self.host, self.runbook_base, - extra_context=get_memory_stats(container), - probe_detail="OOM probe failed - container was OOM-killed by the kernel") + extra_context=memory, + probe_detail=("OOM probe failed - container was OOM-killed by the kernel" + + (f"; {memory}" if memory else ""))) self._dispatch(payload) elif action == "health_status: unhealthy": @@ -1832,7 +1884,7 @@ def _poll_ha(self, session: requests.Session) -> None: self._ha_tracker.reset_candidate() if time.monotonic() - self._ha_error_log_at >= 300: self._ha_error_log_at = time.monotonic() - log.warning("HA check unavailable (%s): %s", type(exc).__name__, exc) + log.error("HA check unavailable (%s): %s", type(exc).__name__, exc) return changed, change = self._ha_tracker.observe(observation) @@ -1949,11 +2001,11 @@ def _poll_all_containers(self) -> None: self._dispatch(payload) if status == "restarting": - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) # Detect stuck containers missed by event stream elif status == "running" and health == "unhealthy": - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) with state.lock: state.unhealthy_cycles += 1 cycles = state.unhealthy_cycles @@ -1981,7 +2033,7 @@ def _poll_all_containers(self) -> None: # Log with watchdog-assessed health (replaces Docker's "none") display_health = health if health != "none" else ("healthy" if is_healthy else "unhealthy") - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, display_health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, display_health) if not is_healthy: with state.lock: @@ -2010,7 +2062,7 @@ def _poll_all_containers(self) -> None: self._maybe_alert_recovery(container, state) else: - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) # During the post-reboot window a restored container may not exist yet — # keep its open alert so the recovery still fires once it comes back.