From 730c64ba7970f754fa94d3176ef45bea06dca2d6 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Wed, 14 Oct 2026 19:07:33 +0000 Subject: [PATCH 01/18] chore(release): prepare v1.6.0 --- README.md | 4 ++-- VERSION | 2 +- install.sh | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67593ee..2574a8c 100644 --- a/README.md +++ b/README.md @@ -629,13 +629,13 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor ## 10. Version History -**Release status (2026-10-14):** `VERSION` remains `1.5.4`. The changes below are merged into `main` but **unreleased**: the running `watchdog:1.5.4` image does not acquire them from `git pull`. +**Release status (2026-10-14):** `VERSION` is set to `1.6.0` for the release candidate. The new image, Git tag, and GitHub Release are not yet published. The deployed `watchdog:1.5.4` remains unchanged until a validated new image is deployed. **Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| Unreleased (post-1.5.4) | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts. | +| 1.6.0 | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts. | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/VERSION b/VERSION index 94fe62c..dc1e644 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.4 +1.6.0 diff --git a/install.sh b/install.sh index 3228326..61eeaaa 100755 --- a/install.sh +++ b/install.sh @@ -371,6 +371,8 @@ load_docker_image() { trap "rm -rf '${tmpdir}'" EXIT cp "${INSTALL_DIR}/Dockerfile" "${tmpdir}/" cp "${INSTALL_DIR}/watchdog.py" "${tmpdir}/" + cp "${INSTALL_DIR}/lifecycle.py" "${tmpdir}/" + cp "${INSTALL_DIR}/ha_monitor.py" "${tmpdir}/" if ! docker build -t "${IMAGE_NAME}" "${tmpdir}"; then print_error "Build failed — cannot produce ${IMAGE_NAME}" exit 1 From 3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05 Mon Sep 17 00:00:00 2001 From: EgorE Date: Wed, 14 Oct 2026 16:31:44 -0400 Subject: [PATCH 02/18] fix: align watchdog notifications and preserve syslog alerts Review findings corrected; 108 local tests and GitHub CI run 38630917676 passed. --- .github/workflows/ci.yml | 1 + README.md | 19 ++++ tests/test_ha_monitor.py | 20 +++- tests/test_notifications.py | 192 ++++++++++++++++++++++++++++++++++++ watchdog.py | 124 ++++++++++++++++------- 5 files changed, 318 insertions(+), 38 deletions(-) create mode 100644 tests/test_notifications.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 950e08f..60f9293 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - "release/**" jobs: validate: diff --git a/README.md b/README.md index 2574a8c..f740e79 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,21 @@ syslog: facility: local0 ``` +Syslog is an **alert and error stream**, not a per-poll health feed. It sends +one structured, single-line event="watchdog_alert" record per emitted incident +(including INFO recoveries and reboot summaries), plus watchdog ERROR/CRITICAL +diagnostics. Routine INFO/WARNING polling messages stay local; per-container +poll output is available at log_level: DEBUG. The alert records include the +severity, UTC time, host, container, failure type, detection evidence and action, +with special previous/new active-node fields for HA. The syslog priority is +CRITICAL=2, HIGH=3 (error), WARNING=4 and INFO=6; the facility remains +configurable. Values are quoted, escaped and length-limited for ingestion. +Syslog is configured independently from alert_channels. +Local log_level controls only console/file verbosity; INFO recoveries and +WARNING HA alerts still reach Syslog when local log_level is WARNING or ERROR. +HA and reboot-summary records omit pseudo-container fields. + + ### SMTP (Email) Add `smtp` to `alert_channels` to enable. @@ -289,6 +304,10 @@ snmp_trap: # v3_local_engine_id: "" # hex engine ID pinned from first-run log; see DEPLOYMENT.md ``` +SNMP retains its five existing var-binds and OIDs for NMS compatibility. +The human-readable cwSummary contains severity; cwProbeDetail contains +detection evidence. No additional SNMP objects are introduced in v1.6.0. + Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3.6.1.4.1.89.110`): | OID | Object | Value | diff --git a/tests/test_ha_monitor.py b/tests/test_ha_monitor.py index 93023fd..1ff5c9a 100644 --- a/tests/test_ha_monitor.py +++ b/tests/test_ha_monitor.py @@ -183,6 +183,19 @@ def test_real_watchdog_event_and_persistence(self): restarted._poll_ha(restarted._session) restarted._dispatch.assert_not_called() + def test_ha_exporter_failure_is_error_but_throttled(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + with patch("watchdog.read_ha_observation", side_effect=TimeoutError("offline")), \ + patch("watchdog.time.monotonic", side_effect=[1000, 1000, 1100, 1301, 1301]), \ + patch("watchdog.log.error") as err: + w._poll_ha(w._session) + w._poll_ha(w._session) + w._poll_ha(w._session) + self.assertEqual(err.call_count, 2) + self.assertIn("HA check unavailable", err.call_args.args[0]) + w._dispatch.assert_not_called() + def test_failed_exporter_does_not_overwrite_state_or_alert(self): with tempfile.TemporaryDirectory() as td: w = self.create_shell(f"{td}/state.json") @@ -351,9 +364,12 @@ def test_slack_ha_fields_and_no_false_container(self): def test_syslog_logging_includes_transition_and_dispatches_snmp(self): alert = self.make_alert() - with patch("watchdog.log.warning") as warn, patch("watchdog.send_snmp_trap") as trap: + with patch("watchdog.log.log") as logged, patch("watchdog.send_snmp_trap") as trap: watchdog.dispatch_alert(alert, {"alert_channels": ["snmp_trap", "syslog"]}) - self.assertIn(f"{A} -> {B}", warn.call_args.args[1]) + self.assertEqual(logged.call_args.args[0], watchdog.logging.WARNING) + self.assertIn(f'previous_active="{A}"', logged.call_args.args[1]) + self.assertIn(f'new_active="{B}"', logged.call_args.args[1]) + self.assertTrue(logged.call_args.kwargs["extra"]["watchdog_alert"]) trap.assert_called_once() def test_smtp_ha_notification_identifies_role_change(self): diff --git a/tests/test_notifications.py b/tests/test_notifications.py new file mode 100644 index 0000000..604de8b --- /dev/null +++ b/tests/test_notifications.py @@ -0,0 +1,192 @@ +"""Notification consistency and syslog regression tests (no network).""" +import logging +import socket +import tempfile +import unittest +from unittest.mock import patch +from email.parser import BytesParser +from email.policy import default +from pathlib import Path + +import watchdog + + +def alert(severity="CRITICAL", failure="crashed"): + return watchdog.AlertPayload( + severity, "demo-service", "abcdef123456789", "vision", failure, 137, + "", "Inspect Docker events and logs.", "https://example.invalid/runbook", + 'Crash probe failed - SIGKILL; OOM not confirmed', + ) + + +class NotificationTests(unittest.TestCase): + def test_oom_evidence_only_when_confirmed(self): + p = alert("CRITICAL", "oom") + p.probe_detail = ("OOM probe failed - container was OOM-killed by the kernel; " + "post-event memory snapshot: usage=63.0 MB, limit=64.0 MB") + data = watchdog.format_syslog_alert(p) + self.assertIn('failure_type="oom"', data) + self.assertIn('post-event memory snapshot: usage=63.0 MB', data) + self.assertNotIn('failure_type="crashed"', data) + + def test_real_syslog_keeps_recovery_and_ha_at_local_warning_or_error(self): + for level in ("WARNING", "ERROR"): + with self.subTest(level=level), tempfile.TemporaryDirectory() as td: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as receiver: + receiver.bind(("127.0.0.1", 0)) + receiver.settimeout(0.6) + watchdog.setup_logging( + level, f"{td}/watchdog.log", + {"enabled": True, "host": "127.0.0.1", + "port": receiver.getsockname()[1], "protocol": "udp"} + ) + try: + watchdog.log.info("routine polling must stay local") + recovery = alert("INFO", "recovered") + recovery.exit_code = None + watchdog.dispatch_alert(recovery, {"alert_channels": []}) + ha = watchdog.AlertPayload( + "WARNING", "(HA cluster)", "", "vision", "ha-failover", + None, "", "Check cluster health", "", + "HA Operator promoted to new node", + ha_previous_node="node-a", ha_new_node="node-b" + ) + watchdog.dispatch_alert(ha, {"alert_channels": []}) + watchdog.log.error("HA check unavailable (TimeoutError): offline") + received = [receiver.recvfrom(8192)[0].decode() + for _ in range(3)] + with self.assertRaises(socket.timeout): + receiver.recvfrom(8192) + self.assertEqual( + [int(msg.split(">", 1)[0][1:]) % 8 for msg in received], + [6, 4, 3]) + self.assertIn('failure_type="recovered"', received[0]) + self.assertIn('failure_type="ha-failover"', received[1]) + self.assertIn("HA check unavailable", received[2]) + self.assertFalse(any("routine polling" in msg for msg in received)) + text = Path(f"{td}/watchdog.log").read_text() + self.assertNotIn("routine polling", text) + self.assertNotIn('failure_type="recovered"', text) + if level == "ERROR": + self.assertNotIn('failure_type="ha-failover"', text) + else: + self.assertIn('failure_type="ha-failover"', text) + finally: + logging.basicConfig( + handlers=[logging.NullHandler()], + level=logging.WARNING, force=True) + + def test_pseudo_container_fields_absent_in_ha_and_reboot_events(self): + for failure, container_name in ( + ("ha-failover", "(HA cluster)"), + ("reboot-summary", "(post-reboot summary)") + ): + with self.subTest(failure=failure): + p = watchdog.AlertPayload( + "INFO", container_name, "", "vision", failure, None, "", + "Check cluster", "", "Event details", + ha_previous_node="node-a", ha_new_node="node-b") + record = watchdog.format_syslog_alert(p) + self.assertIn('event="watchdog_alert"', record) + self.assertIn(f'failure_type="{failure}"', record) + self.assertNotIn('container=', record) + self.assertNotIn('container_id=', record) + self.assertIn('host="vision"', record) + if failure == "ha-failover": + self.assertIn('previous_active="node-a"', record) + self.assertIn('new_active="node-b"', record) + + def test_memory_snapshot_omits_missing_zero_and_unavailable_stats(self): + from unittest.mock import Mock + container = Mock() + container.stats.return_value = { + "memory_stats": {"usage": 0, "limit": 67108864, "max_usage": 12582912} + } + sample = watchdog.get_memory_stats(container) + self.assertIn("post-event memory snapshot:", sample) + self.assertIn("limit=64.0 MB", sample) + self.assertIn("peak=12.0 MB", sample) + self.assertNotIn("usage=", sample) + container.stats.return_value = {"memory_stats": {}} + self.assertEqual(watchdog.get_memory_stats(container), "") + container.stats.side_effect = RuntimeError("unavailable after OOM") + self.assertEqual(watchdog.get_memory_stats(container), "") + + def test_native_levels_match_incident_severity(self): + for severity, expected in (("CRITICAL", logging.CRITICAL), + ("HIGH", logging.ERROR), + ("WARNING", logging.WARNING), + ("INFO", logging.INFO)): + with self.subTest(severity=severity): + p = alert(severity) + with patch("watchdog.log.log") as log: + watchdog.dispatch_alert(p, {"alert_channels": []}) + self.assertEqual(log.call_args.args[0], expected) + self.assertIn(f'severity="{severity}"', log.call_args.args[1]) + self.assertTrue(log.call_args.kwargs["extra"]["watchdog_alert"]) + + def test_syslog_filter_rejects_poll_noise_and_keeps_recovery_and_errors(self): + f = watchdog.AlertSyslogFilter() + def record(level, is_alert=False): + r = logging.LogRecord("watchdog", level, __file__, 1, "message", (), None) + if is_alert: + r.watchdog_alert = True + return r + self.assertFalse(f.filter(record(logging.INFO))) + self.assertFalse(f.filter(record(logging.WARNING))) + self.assertTrue(f.filter(record(logging.INFO, True))) + self.assertTrue(f.filter(record(logging.WARNING, True))) + self.assertTrue(f.filter(record(logging.ERROR))) + + def test_syslog_has_evidence_without_false_oom_and_escapes_content(self): + p = alert() + p.probe_detail = 'line one\nline "two"' + data = watchdog.format_syslog_alert(p) + for fragment in ('event="watchdog_alert"', 'severity="CRITICAL"', + 'failure_type="crashed"', 'exit_code="137"', + 'host="vision"', 'container="demo-service"', + 'container_id="abcdef123456"', + 'probe_type="Docker event (crash)"', + 'action="Inspect Docker events and logs."'): + self.assertIn(fragment, data) + self.assertIn(r'line one\nline \"two\"', data) + self.assertNotIn('failure_type="oom"', data) + self.assertNotIn("\n", data) + + def test_long_untrusted_detection_is_bounded(self): + p = alert() + p.probe_detail = "x" * 20000 + data = watchdog.format_syslog_alert(p) + self.assertLess(len(data), 2000) + self.assertIn('...', data) + + def test_email_includes_detection_without_assuming_oom(self): + p = alert() + cfg = {"smtp": {"enabled": True, "auth": False, "tls": False, + "host": "mail.invalid", "port": 25, + "sender": "alerts@example.invalid", + "recipients": ["ops@example.invalid"]}} + with patch("smtplib.SMTP") as smtp: + watchdog.send_smtp(p, cfg) + sent = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] + body = BytesParser(policy=default).parsebytes(sent).get_body( + preferencelist=("plain",)).get_content() + for item in ("Container ID: abcdef123456", "Severity: CRITICAL", + "Exit Code: 137", "Probe Type: Docker event (crash)", + "OOM not confirmed", "Runbook: https://example.invalid/runbook"): + self.assertIn(item, body) + + def test_slack_fields_match_alert_payload(self): + p = alert() + with patch.dict("os.environ", {"SLACK_WEBHOOK_URL": "https://example.invalid/slack"}): + with patch("watchdog.requests.post") as post: + watchdog.send_slack(p, {"slack": {"enabled": True}}) + fields = post.call_args.kwargs["json"]["attachments"][0]["fields"] + values = {field["title"]: field["value"] for field in fields} + self.assertEqual(values["Severity"], "CRITICAL") + self.assertEqual(values["Exit Code"], "137") + self.assertEqual(values["Detection"], p.probe_detail) + + +if __name__ == "__main__": + unittest.main() diff --git a/watchdog.py b/watchdog.py index 61c1b0e..af69d1d 100644 --- a/watchdog.py +++ b/watchdog.py @@ -31,7 +31,15 @@ # ── Logging setup ───────────────────────────────────────────────────────────── LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" -SYSLOG_FORMAT = "watchdog[%(process)d]: %(levelname)s %(name)s: %(message)s" +SYSLOG_FORMAT = "watchdog[%(process)d]: %(message)s" + + +class AlertSyslogFilter(logging.Filter): + """Forward alert records and actual errors, not routine poll diagnostics.""" + + def filter(self, record: logging.LogRecord) -> bool: + return bool(getattr(record, "watchdog_alert", False) or record.levelno >= logging.ERROR) + log = logging.getLogger("watchdog") @@ -39,6 +47,7 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = None) -> None: numeric = getattr(logging, level.upper(), logging.INFO) handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + syslog_active = False if log_file: log_dir = os.path.dirname(log_file) if log_dir: @@ -48,6 +57,8 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = No log_file, maxBytes=10 * 1024 * 1024, backupCount=5 ) ) + for handler in handlers: + handler.setLevel(numeric) if syslog_cfg and syslog_cfg.get("enabled"): host = syslog_cfg.get("host", "localhost") port = int(syslog_cfg.get("port", 514)) @@ -70,10 +81,15 @@ def setup_logging(level: str, log_file: str | None, syslog_cfg: dict | None = No log.warning("Syslog disabled — cannot reach %s:%d (%s): %s", host, port, proto, exc) else: syslog_handler.setFormatter(logging.Formatter(SYSLOG_FORMAT)) + syslog_handler.setLevel(logging.INFO) + syslog_handler.addFilter(AlertSyslogFilter()) handlers.append(syslog_handler) + syslog_active = True # Use a plain formatter for other handlers, syslog gets its own log.info("Syslog handler added: %s:%d (%s)", host, port, proto) - logging.basicConfig(level=numeric, format=LOG_FORMAT, handlers=handlers, force=True) + # Let INFO recovery alerts reach Syslog even when local logs are WARNING/ERROR. + root_level = min(numeric, logging.INFO) if syslog_active else numeric + logging.basicConfig(level=root_level, format=LOG_FORMAT, handlers=handlers, force=True) # Suppress noisy third-party debug chatter (urllib3 Docker socket calls, etc.) for noisy in ("urllib3", "urllib3.connectionpool", "docker", "requests"): logging.getLogger(noisy).setLevel(logging.WARNING) @@ -402,12 +418,14 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: ha_event = payload.failure_type == "ha-failover" ha_fields = [ {"title": "Event", "value": "HA active-node change", "short": True}, + {"title": "Severity", "value": payload.severity, "short": True}, {"title": "Previous active", "value": payload.ha_previous_node, "short": True}, {"title": "New active", "value": payload.ha_new_node, "short": True}, {"title": "Host", "value": payload.host, "short": True}, {"title": "Time (UTC)", "value": payload.timestamp, "short": True}, *probe_field, {"title": "Action", "value": payload.recommended_action, "short": False}, + {"title": "Runbook", "value": payload.runbook_url, "short": False}, ] body = { "attachments": [{ @@ -439,19 +457,48 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: +def _syslog_field(value: object, limit: int = 512) -> str: + """Single-line, bounded, quote-safe key/value field for SIEM parsing.""" + value = str(value).replace("\\", "\\\\").replace("\r", "\\r").replace("\n", "\\n") + value = value[:limit] + ("..." if len(value) > limit else "") + return '"' + value.replace('"', '\\"') + '"' + + +def format_syslog_alert(payload: AlertPayload) -> str: + """Same incident facts as human-facing notifications; bounded for UDP syslog.""" + details = { + "event": "watchdog_alert", + "severity": payload.severity, + "failure_type": payload.failure_type, + "time_utc": payload.timestamp, + "host": payload.host, + "probe_type": payload.probe_type, + "detection": payload.probe_detail, + "action": payload.recommended_action, + "runbook": payload.runbook_url, + } + if payload.failure_type not in ("ha-failover", "reboot-summary"): + details["container"] = payload.container_name + details["container_id"] = payload.container_id + if payload.exit_code is not None: + details["exit_code"] = payload.exit_code + if payload.failure_type == "ha-failover": + details["previous_active"] = payload.ha_previous_node + details["new_active"] = payload.ha_new_node + return " ".join(f"{key}={_syslog_field(value, 250 if key in ('detection', 'action') else 120)}" + for key, value in details.items() if value not in ("", "N/A")) + + +def _alert_log_level(severity: str) -> int: + return {"CRITICAL": logging.CRITICAL, "HIGH": logging.ERROR, + "WARNING": logging.WARNING, "INFO": logging.INFO}.get(severity, logging.WARNING) + + def dispatch_alert(payload: AlertPayload, cfg: dict) -> None: """Send alert to all configured channels.""" channels = cfg.get("alert_channels", ["slack"]) - log_fn = log.info if payload.failure_type == "recovered" else log.warning - if payload.failure_type == "ha-failover": - log_fn("ALERT %s (channels: %s) — %s", - payload.subject(), channels, payload.probe_detail) - else: - log_fn( - "ALERT [%s] %s — %s (channels: %s)", - payload.severity, payload.container_name, - payload.failure_type, channels, - ) + log.log(_alert_log_level(payload.severity), format_syslog_alert(payload), + extra={"watchdog_alert": True}) if "slack" in channels: send_slack(payload, cfg) if "smtp" in channels: send_smtp(payload, cfg) if "snmp_trap" in channels: send_snmp_trap(payload, cfg) @@ -520,18 +567,23 @@ def send_smtp(payload: AlertPayload, cfg: dict) -> None: f"Previous active: {payload.ha_previous_node}\n" f"New active: {payload.ha_new_node}\n" f"Host: {payload.host}\n" + f"Severity: {payload.severity}\n" f"Observed (UTC): {payload.timestamp}\n" - f"Evidence: {payload.probe_detail}\n\n" - f"Action: {payload.recommended_action}") + f"Evidence: {payload.probe_detail}\n" + f"Action: {payload.recommended_action}\n" + f"Runbook: {payload.runbook_url}") else: body = f"""Container: {payload.container_name} Host: {payload.host} Status: {payload.failure_type} Severity: {payload.severity} Time: {payload.timestamp} +Container ID: {payload.container_id} Exit Code: {payload.exit_code if payload.exit_code is not None else 'N/A'} - -{payload.recommended_action}""" +Probe Type: {payload.probe_type or 'N/A'} +Detection: {payload.probe_detail or 'N/A'} +Action: {payload.recommended_action} +Runbook: {payload.runbook_url or 'N/A'}""" # Build an RFC-compliant MIME message. smtplib.sendmail(str) tries to # encode the whole message as ASCII, which breaks on Unicode text such as @@ -1085,19 +1137,17 @@ def get_docker_health_log(container) -> str: def get_memory_stats(container) -> str: + """Best-effort post-event sample, not memory usage at the time of the OOM.""" try: - stats = container.stats(stream=False) - mem = stats.get("memory_stats", {}) - usage = mem.get("usage", 0) - limit = mem.get("limit", 0) - peak = mem.get("max_usage", 0) - def _mb(b: int) -> str: - return f"{b / 1024 / 1024:.1f} MB" - return ( - f"memory usage={_mb(usage)} limit={_mb(limit)} peak={_mb(peak)}" - ) - except Exception as exc: - return f"(memory stats unavailable: {exc})" + mem = container.stats(stream=False).get("memory_stats") or {} + parts = [] + for key, label in (("usage", "usage"), ("limit", "limit"), ("max_usage", "peak")): + value = mem.get(key) + if type(value) is int and value > 0: + parts.append(f"{label}={value / 1024 / 1024:.1f} MB") + return ("post-event memory snapshot: " + ", ".join(parts)) if parts else "" + except Exception: + return "" def get_container_id(container) -> str: @@ -1626,7 +1676,7 @@ def _resolve_lifecycle_death(self, death) -> None: return if death.stopped: if death.exit_code == 137: - log.warning("%s: Docker-requested stop ended with SIGKILL (137), " + log.info("%s: Docker-requested stop ended with SIGKILL (137), " "not proof of OOM", death.name) else: log.info("%s: Docker-requested stop (exit %s)", death.name, death.exit_code) @@ -1789,10 +1839,12 @@ def _handle_event(self, event: dict) -> None: log.info("%s: OOM during active restart-loop — suppressing per-cycle OOM alert", name) return if self._claim_alert(state, failure_type): + memory = get_memory_stats(container) payload = build_payload(container, failure_type, self.host, self.runbook_base, - extra_context=get_memory_stats(container), - probe_detail="OOM probe failed - container was OOM-killed by the kernel") + extra_context=memory, + probe_detail=("OOM probe failed - container was OOM-killed by the kernel" + + (f"; {memory}" if memory else ""))) self._dispatch(payload) elif action == "health_status: unhealthy": @@ -1832,7 +1884,7 @@ def _poll_ha(self, session: requests.Session) -> None: self._ha_tracker.reset_candidate() if time.monotonic() - self._ha_error_log_at >= 300: self._ha_error_log_at = time.monotonic() - log.warning("HA check unavailable (%s): %s", type(exc).__name__, exc) + log.error("HA check unavailable (%s): %s", type(exc).__name__, exc) return changed, change = self._ha_tracker.observe(observation) @@ -1949,11 +2001,11 @@ def _poll_all_containers(self) -> None: self._dispatch(payload) if status == "restarting": - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) # Detect stuck containers missed by event stream elif status == "running" and health == "unhealthy": - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) with state.lock: state.unhealthy_cycles += 1 cycles = state.unhealthy_cycles @@ -1981,7 +2033,7 @@ def _poll_all_containers(self) -> None: # Log with watchdog-assessed health (replaces Docker's "none") display_health = health if health != "none" else ("healthy" if is_healthy else "unhealthy") - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, display_health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, display_health) if not is_healthy: with state.lock: @@ -2010,7 +2062,7 @@ def _poll_all_containers(self) -> None: self._maybe_alert_recovery(container, state) else: - log.info("CONTAINER %-30s status=%-12s health=%s", name, status, health) + log.debug("CONTAINER %-30s status=%-12s health=%s", name, status, health) # During the post-reboot window a restored container may not exist yet — # keep its open alert so the recovery still fires once it comes back. From 7afbc3df6a4afd57c94cb19d6f18f5b75253d178 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Wed, 14 Oct 2026 21:52:37 +0000 Subject: [PATCH 03/18] test: record real v1.6 Docker smoke acceptance evidence --- docs/RELEASE_SMOKE_V1.6.0_20261014.json | 276 +++++++++++ docs/RELEASE_VALIDATION_V1.6.0.md | 104 +++++ tests/release-smoke.sh | 584 ++++++++++++++++++++++++ 3 files changed, 964 insertions(+) create mode 100644 docs/RELEASE_SMOKE_V1.6.0_20261014.json create mode 100644 docs/RELEASE_VALIDATION_V1.6.0.md create mode 100755 tests/release-smoke.sh diff --git a/docs/RELEASE_SMOKE_V1.6.0_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_20261014.json new file mode 100644 index 0000000..7fb7e4e --- /dev/null +++ b/docs/RELEASE_SMOKE_V1.6.0_20261014.json @@ -0,0 +1,276 @@ +{ + "run_id": "261014214436754", + "started_utc": "2026-10-14T21:44:36.266768Z", + "finished_utc": "2026-10-14T21:48:30.206757Z", + "status": "FAIL", + "image": "watchdog:1.6.0", + "results": [ + { + "id": "T01_CLEAN_EXIT", + "status": "PASS", + "expected": "exit=0 means no alert", + "observed": "real exit=0, no alerts", + "elapsed_seconds": 12.7 + }, + { + "id": "T02_UNEXPECTED_CRASH", + "status": "PASS", + "expected": "exit=2 sends one CRITICAL crash", + "observed": "Docker exit=2, exactly one CRITICAL crashed alert", + "elapsed_seconds": 15.7 + }, + { + "id": "T03_PLANNED_STOP_137", + "status": "FAIL", + "expected": "Docker stop exit 137 is suppressed", + "observed": "unexpected planned fixture restart", + "elapsed_seconds": 26.6 + }, + { + "id": "T04_DIRECT_SIGKILL", + "status": "PASS", + "expected": "SIGKILL 137 is crash, not OOM", + "observed": "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash", + "elapsed_seconds": 43.6 + }, + { + "id": "T05_REAL_OOM", + "status": "FAIL", + "expected": "OOMKilled=true sends only CRITICAL OOM", + "observed": "Docker cgroup did not confirm OOMKilled", + "elapsed_seconds": 45.1 + }, + { + "id": "T06_RESTART_LOOP", + "status": "PASS", + "expected": "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "observed": "real Docker RestartCount=6; crash=1; restart-loop=1", + "elapsed_seconds": 57.8 + }, + { + "id": "T07_BRIEF_RECOVERY", + "status": "PASS", + "expected": "no recovery before 120s stability", + "observed": "running after 11s, RECOVERED=0", + "elapsed_seconds": 69.5 + }, + { + "id": "T08_INTERRUPTED_RECOVERY", + "status": "PASS", + "expected": "new actual die/reset prevents premature recovery", + "observed": "Docker RestartCount 6 -> 7; RECOVERED=0 after 14s", + "elapsed_seconds": 86.2 + }, + { + "id": "T09_STABLE_RECOVERY", + "status": "PASS", + "expected": "one INFO RECOVERED after 120s sustained uptime", + "observed": "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery", + "elapsed_seconds": 198 + }, + { + "id": "T06_TO_T09_RESTART_LIFECYCLE", + "status": "PASS", + "expected": "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", + "observed": "one CRASH + one LOOP + one RECOVERED after real interrupted recovery", + "elapsed_seconds": 198 + }, + { + "id": "T10_NAME_REUSE", + "status": "PASS", + "expected": "replacement with same name gets new incident", + "observed": "two unique container IDs, same name, each gets one CRITICAL crash", + "elapsed_seconds": 207.9 + }, + { + "id": "T11_FAST_REMOVAL", + "status": "PASS", + "expected": "no event flood, event listener survives", + "observed": "3 real --rm containers, 0 bounded missing-ID warnings, observer alive", + "elapsed_seconds": 218.2 + }, + { + "id": "T12_NATIVE_HEALTHCHECK", + "status": "PASS", + "expected": "unhealthy -> recovered", + "observed": "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery", + "elapsed_seconds": 231.5 + }, + { + "id": "T13_DFC_SYSLOG", + "status": "FAIL", + "expected": "all expected structured alerts reach actual DFC", + "observed": "planned stop sent to DFC", + "elapsed_seconds": 231.7 + } + ], + "facts": { + "host": "cc-ea-iad1", + "revision": "3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05", + "image_id": "sha256:e3d02f6ef56cde0bc7021919def9e240505434a15769aac3884cf15ac8f0ccff", + "observer": "ca-smoke-261014214436754-observer", + "syslog_source_host": "release-smoke-261014214436754", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 50, + "clean": { + "name": "ca-smoke-261014214436754-clean", + "container_id": "98f25a334562103075cfc01df542f3f55fd8356565bbc6533b1922f9f2f640a3" + }, + "crash": { + "name": "ca-smoke-261014214436754-crash", + "container_id": "748a19f96463f79fcba026b9fcbe7d11aec540b728f04fd9012175888b9057d1" + }, + "planned": { + "name": "ca-smoke-261014214436754-planned", + "container_id": "34ac9faaa74856caebfe6f3d1e85975e03eb38104837162da0a3fdbffe10936c" + }, + "sigkill": { + "name": "ca-smoke-261014214436754-sigkill", + "container_id": "19f291e6c4186fb0a3d0c24ebdb503fee846a30c325c810a69f827422956e4a6" + }, + "sigkill_state": { + "exit": 137, + "OOMKilled": false + }, + "oom": { + "name": "ca-smoke-261014214436754-oom", + "container_id": "ddb952d6842830181d32a458d4fcac3cf4509a1b6d10fa4739e2d53dfb334bd7" + }, + "loop": { + "name": "ca-smoke-261014214436754-loop", + "container_id": "40700dcc7322b59cf86ffb4894c8cb946e31d2fcfee1f0afa2456002d7da59ad" + }, + "loop_restart_count_after": 7, + "reuse": { + "name": "ca-smoke-261014214436754-reuse", + "container_id": "e346b9f8a2282179ed00b4d9cc3bd55300b3bf517f76f138c63a2d6d525fb3f7" + }, + "fast_remove_warning_count": 0, + "health": { + "name": "ca-smoke-261014214436754-health", + "container_id": "4713718efb4607ffca32afdbac8ec44e77ad91dda0dc69d55ba7a3890b3c4bb1" + }, + "watchdog_alert_count": 26, + "dfc_received_count": 26, + "live_docker_events": 206, + "live_docker_action_counts": { + "create": 13, + "start": 57, + "die": 54, + "oom": 12, + "stop": 1, + "kill": 5, + "restart": 37, + "health_status: unhealthy": 1, + "health_status: healthy": 1, + "destroy": 7 + }, + "docker_event_lines": 53, + "final_fixture_states": { + "ca-smoke-261014214436754-clean": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:44:41.368218838Z", + "FinishedAt": "2026-10-14T21:44:41.820033442Z" + }, + "ca-smoke-261014214436754-crash": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 2, + "Error": "", + "StartedAt": "2026-10-14T21:48:22.990852752Z", + "FinishedAt": "2026-10-14T21:48:23.424342384Z" + }, + "ca-smoke-261014214436754-loop": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 2784034, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:45:46.731517222Z", + "FinishedAt": "2026-10-14T21:45:46.633484834Z" + }, + "ca-smoke-261014214436754-reuse": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 42, + "Error": "", + "StartedAt": "2026-10-14T21:48:22.984997284Z", + "FinishedAt": "2026-10-14T21:48:23.418274429Z" + }, + "ca-smoke-261014214436754-health": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 2797095, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T21:48:15.175556816Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-14T21:48:19.632370905Z", + "End": "2026-10-14T21:48:19.915665384Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:21.916497231Z", + "End": "2026-10-14T21:48:22.134371817Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:24.135145923Z", + "End": "2026-10-14T21:48:24.330071676Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:26.331087306Z", + "End": "2026-10-14T21:48:26.61488817Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-14T21:48:28.615762457Z", + "End": "2026-10-14T21:48:28.827588657Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "total_seconds": 233.9 + } +} diff --git a/docs/RELEASE_VALIDATION_V1.6.0.md b/docs/RELEASE_VALIDATION_V1.6.0.md new file mode 100644 index 0000000..be9cdcb --- /dev/null +++ b/docs/RELEASE_VALIDATION_V1.6.0.md @@ -0,0 +1,104 @@ +# Container Alert v1.6.0 — Release Acceptance Validation + +**Date:** October 14, 2026 (UTC) +**Repository:** `Radware/CyberController-Container-Alert` +**Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) — open +**Notification fix:** [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10) — merged into release branch +**Source revision:** `3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05` +**Lab image:** `watchdog:1.6.0`, image ID `sha256:e3d02f6ef56cde0bc7021919def9e240505434a15769aac3884cf15ac8f0ccff` +**Verdict: CONDITIONAL / NOT FULLY ACCEPTED.** HA passes. Most real Docker cases pass; some checks are confounded by out-of-band Docker restarts and expose premature recovery / extra alert behavior. Do not represent the entire release as green. + +## Test methods and safeguards + +- Tested against the real Docker Engine on `ha-test-admin-ea-iad1` (`10.199.0.241`). No simulated Docker events. +- A separate resource-limited v1.6.0 observer used its own Docker volume/state, 2-second Docker polling, real 5-restart threshold, and **real 120-second** recovery requirement. SMTP disabled only on the observer; Syslog sent to the **existing** DFC UDP/514 receiver. Production-style main watchdog and cluster were not reconfigured. +- Used uniquely prefixed, labeled disposable containers, restricted to 0.2 CPU and 64–128 MiB each, with `--network none`. Verified actual Docker state, events, watchdog structured alert records, and DFC `/var/log/syslog` deliveries. +- No host packages, new Docker networks, Pacemaker operations, or host service modifications. Cleanup removed all smoke-owned containers and volumes. Original watchdog stayed healthy. +- Runner: [`tests/release-smoke.sh`](../tests/release-smoke.sh). It saves `report.json`, `report.md`, `watchdog.log`, `docker-events-live.jsonl`, and `dfc-syslog.txt` to `/tmp/container-alert-smoke-/` inside the authorized RDC client. + +## 1. Automated, packaging, and release checks + +| Check | Acceptance | Result | +|---|---|---| +| Automated regression suite | All tests pass against final built source | **PASS — 108/108** inside m920q Docker image | +| GitHub CI for PR #9 head | Green | **PASS** on `3b1de153` (before adding this documentation) | +| Release code/image integrity | Embedded `watchdog.py`, `lifecycle.py`, `ha_monitor.py` hashes match Git | **PASS** | +| `install.sh` / smoke script syntax | `bash -n`, smoke embedded Python compiles | **PASS** | +| Docker Compose build configuration | Valid config with `WATCHDOG_VERSION=1.6.0` | **PASS** | +| Installer source-build fallback | Dockerfile and three Python modules build/import | **PASS** with normal cached/online Docker build | +| Offline archive | `docker save` / SHA-256 / `docker load`; same release image ID | **PASS** | +| Source-build without network | All `pip` dependencies available offline | **NOT SUPPORTED**. No-network source build failed because pip must download dependencies; use the tested `watchdog.tar` for offline installations | + +## 2. Real two-node HA acceptance (completed before Docker smoke) + +| Scenario | Acceptance | Actual result | +|---|---|---| +| Baseline | Both nodes agree `.241` active, no startup failover | **PASS** | +| Failover `.241 → .242` | One WARNING on new owner, zero on previous | **PASS** — alert on `.242` at **20:56:16 UTC**; `.241` logged "new owner will notify" | +| HA Syslog on `.242` | Structured failover record delivered to DFC | **PASS** — stored in DFC `/var/log/syslog` | +| Failback `.242 → .241` | One WARNING on new owner, zero on previous | **PASS** — alert on `.241` at **21:09:47 UTC**; `.242` logged owner defer | +| HA Syslog on `.241` | Structured failback record delivered to DFC | **PASS** | +| Final cluster status | One promoted HA Operator, quorum, both watchdogs healthy | **PASS** — `.241` active; both watchdogs image `1.6.0`, healthy, zero restarts | +| Transient exporter ambiguity during transition | No fabricated failover | **PASS** — `.242` logged one invalid-promoted-owner observation at 20:54:16 UTC, then recovered and sent correct alert | +| SMTP delivery | Excluded by test owner | **EXCLUDED** — SMTP network timeouts are not part of this verdict | + +## 3. Real Docker acceptance — final run + +**Final run ID:** `261014214436754` +**Time:** 21:44:36–21:48:30 UTC (233.9 seconds) +**Final summary: 11 PASS / 3 FAIL** across 14 runner assertions (includes a grouped restart lifecycle check). + +| ID | Scenario / acceptance criteria | Result and evidence | +|---|---|---| +| T01 | Exit 0 yields no crash alert | **PASS** | +| T02 | Nonzero exit 2 yields exactly one CRITICAL crash | **PASS** | +| T03 | Planned `docker stop`, SIGKILL/137, yields no downstream false crash | **FAIL (interference)**. Initial Docker stop had matching `stop/die` evidence; the lab later restarted the stopped fixture, and the subsequent forced cleanup generated an additional CRASH | +| T04 | Direct SIGKILL/137 has `OOMKilled=false`, one CRITICAL crash, no false OOM | **PASS** in final run | +| T05 | Genuine cgroup OOM; Docker confirms `OOMKilled=true`; one CRITICAL OOM, no duplicate | **FAIL (unstable state)** in final run. Earlier run confirmed true `OOMKilled`; final live event capture proved **12 real OOM events** after unexpected restarts, so the subsequent Docker state was no longer the first killed instance | +| T06 | Restart loop escalates after 5 actual restarts, with one CRASH and one HIGH RESTART-LOOP | **PASS** — Docker `RestartCount=6` | +| T07 | Short period running does not prematurely recover restart loop | **PASS** — 11 seconds healthy, zero RECOVERED | +| T08 | New actual failure during recovery interrupts stability clock | **PASS** — Docker restart count incremented 6→7; no early RECOVERED | +| T09 | After at least 120 seconds sustained recovery, exactly one INFO RECOVERED | **PASS** — one CRASH, one RESTART-LOOP, one RECOVERED | +| T10 | Reuse exact container name with a different ID gets a fresh incident | **PASS** — 2 unique IDs, one crash per instance | +| T11 | Three `--rm` containers do not flood missing-container warnings or kill event listener | **PASS for no-flood** — 0 warnings observed; missing-ID race branch not independently forced | +| T12 | Native Docker health check transitions unhealthy→healthy | **PASS** — one HIGH UNHEALTHY, one INFO RECOVERED | +| T13 | All correct structured alert records arrive at DFC; no false planned-stop alert | **FAIL strict semantic criterion** — **26/26** actual watchdog records reached DFC, but one unwanted planned-fixture CRASH also reached it | +| Grouped T06–T09 | Correct restart-loop lifecycle as a whole | **PASS** | + +### Three attempts and reproducibility + +- **First run** (`261014212952944`): core real-event cases, name reuse, health, and DFC transport passed. A **test-runner formatting error** (missing grouped-test return value) prevented a clean report; fixed afterward. Unexpected out-of-band restarts began causing extra OOM/recovery alerts. +- **Second run** (`261014213819904`): **12 PASS / 2 FAIL**. Unexpected SIGKILL fixture restart and false recovery of an OOM fixture were observed. The runner was corrected to record these explicitly. +- **Final run** (above): **11 PASS / 3 FAIL**; includes live capture of **206 Docker events**. The failure conditions vary with the lab's externally initiated restarts, not a deterministic source-code change. + +### Investigation: out-of-band restarts + +The live event capture on `.241` included **37 `restart` actions, 57 starts, 54 dies, and 12 `oom` actions** for test-tagged containers. Importantly, some affected fixtures had `HostConfig.RestartPolicy.Name=no`, yet their Docker IDs were started/restarted repeatedly. The initiator of these restarts was **not established**; do not attribute them to Container Alert without additional evidence. + +The planned-stop fixture initially emitted real `kill(SIGTERM)`, `kill(SIGKILL)`, `stop`, and `die(137)` events. The unwanted CRASH appeared **later**, when that fixture had already been restarted and the test suite used force-remove to clean it up; that final kill had no `stop` evidence. This is not evidence that the original `docker stop` was misclassified. + +The OOM fixture produced genuine kernel `oom` and `die(137)` events. When another actor immediately restarts a container whose Docker policy is `no`, the current recovery rule can declare an OOM/CRASH incident recovered after only a brief running interval. Repeated OOM/recovery alert pairs were observed in the earlier run. **This is a real edge case worth addressing or accepting explicitly**, separate from the one-shot OOM detector. + +The DFC Syslog pipeline itself had **100% message-count correspondence in the final run (26/26)** and preserved structured severity/type fields. However, transport delivery is not the same as correct incident classification. + +## 4. Additional coverage and explicit exclusions + +- PR #10 automated tests check alert severity mapping, SMTP/Slack formatting, missing container fields on HA alerts, and mocked SNMP alert dispatch. The existing five-var-bind SNMP MIB mapping was kept unchanged, but it was **not verified by a live SNMP trap receiver**. Lab smoke verified real Syslog emission/receipt of CRITICAL, HIGH, and INFO severity fields; it did **not** independently capture raw UDP PRI for every severity. +- Live SNMP trap delivery, Slack delivery, and SMTP delivery were **not** exercised end-to-end; these require configured destinations (SMTP explicitly excluded by the owner). +- Real network partition, forced host reboot, Pacemaker failure injection, and blocked Docker daemon were intentionally **not** performed. HA independence and exporter failure handling are covered by automated tests; the transient exporter error was also observed during actual switchover. + +## 5. Cleanup, impact, and release decision + +- Both HA nodes remain on `watchdog:1.6.0` with their existing configs/state and healthy watchdogs; cluster role and quorum remain intact. +- All test-labelled temporary containers and named volumes were removed. No host packages, new networks, firewall rules, or HA resources were installed/changed. Report/evidence files reside in the RDC admin client's `/tmp` (ephemeral, not a host install). +- **Do not mark complete release acceptance yet.** HA, regression suite, packaging, and Syslog transport are verified. Real Docker repeated OOM/recovery and forced-cleanup false-alert behavior need a controlled-engine rerun or an explicit scoped acceptance. +- Minimal next investigation: isolate the lab's external Docker restarter, reproduce crash/OOM with exactly one failure, and decide whether all crash/OOM recoveries should require the already-existing 120-second continuous-uptime check (not just containers with Docker auto-restart). Avoid broad lifecycle redesign before validating that decision. +- PR #9 remains **unmerged**; do not tag or publish `v1.6.0` solely on these conditional results. + +## Test evidence pointers + +- Lab RDC container: `/tmp/container-alert-smoke-261014214436754/` +- Raw files: `report.json`, `report.md`, `docker-events-live.jsonl`, `watchdog.log`, `dfc-syslog.txt`, and `observer-state.json`. +- Source-controlled runner: [`tests/release-smoke.sh`](../tests/release-smoke.sh). Use `bash release-smoke.sh` from the authorized lab admin environment; its `/tmp` filesystem is mounted `noexec`. + +Machine-readable final run: [`RELEASE_SMOKE_V1.6.0_20261014.json`](RELEASE_SMOKE_V1.6.0_20261014.json). diff --git a/tests/release-smoke.sh b/tests/release-smoke.sh new file mode 100755 index 0000000..25d97f0 --- /dev/null +++ b/tests/release-smoke.sh @@ -0,0 +1,584 @@ +#!/usr/bin/env bash +# Container Alert v1.6.0: real Docker engine smoke tests, without host installations. +# Run on an authorized lab admin client with docker CLI, Python 3, a local DFC syslog receiver. +# Creates only clearly labelled disposable containers and Docker named volumes. +set -euo pipefail +exec python3 - "$@" <<'PY' +import datetime as dt +import json +import os +import pathlib +import re +import subprocess +import sys +import time +import traceback + +IMAGE = os.environ.get("SMOKE_IMAGE", "watchdog:1.6.0") +EXPECTED_REV = "3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05" +TAG = dt.datetime.now(dt.timezone.utc).strftime("%y%m%d%H%M%S") + str(os.getpid())[-3:] +PREFIX = "ca-smoke-" + TAG +LABEL = "container-alert.smoke=" + TAG +HOST = "release-smoke-" + TAG +OBS = PREFIX + "-observer" +STATE_VOL = PREFIX + "-state" +BASE = pathlib.Path(os.environ.get("SMOKE_OUTPUT_DIR", "/tmp")) / ("container-alert-smoke-" + TAG) +START = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") +containers, volumes, results, facts = [], [], [], {} +EVENT_PROC, EVENT_FILE = None, None +BASE.mkdir(parents=True, exist_ok=False) +start_mono = time.monotonic() + +def arm_events(): + # Capture live to avoid Docker's short retrospective event-history buffer. + global EVENT_PROC, EVENT_FILE + EVENT_FILE = (BASE/"docker-events-live.jsonl").open("w") + EVENT_PROC = subprocess.Popen( + ["docker", "events", "--format", "{{json .}}", "--filter", "type=container", + "--filter", "label=" + LABEL], + stdout=EVENT_FILE, stderr=subprocess.DEVNULL) + time.sleep(0.5) + ensure(EVENT_PROC.poll() is None, "real-time docker events capture failed") + +def stop_events(): + global EVENT_PROC, EVENT_FILE + if EVENT_PROC is not None: + EVENT_PROC.terminate() + try: + EVENT_PROC.wait(timeout=4) + except subprocess.TimeoutExpired: + EVENT_PROC.kill() + EVENT_PROC.wait(timeout=4) + EVENT_PROC = None + if EVENT_FILE is not None: + EVENT_FILE.close() + EVENT_FILE = None + p = BASE/"docker-events-live.jsonl" + if p.exists(): + events = [json.loads(x) for x in p.read_text().splitlines() if x.strip()] + facts["live_docker_events"] = len(events) + facts["live_docker_action_counts"] = { + a: sum(e.get("Action") == a for e in events) + for a in ("create","start","die","oom","stop","kill","restart","health_status: unhealthy", + "health_status: healthy","destroy") + } + +def run(argv, timeout=30, check=True): + args = ["docker"] + [str(v) for v in argv] + p = subprocess.run(args, text=True, capture_output=True, timeout=timeout) + if check and p.returncode: + raise RuntimeError("docker " + " ".join(args[1:5]) + + " returned " + str(p.returncode) + ": " + (p.stderr or p.stdout)[-1500:]) + return p.stdout.strip() + +def log(s): + print(s, flush=True) + +def mark(name, expected, evidence): + evidence = str(evidence) + elapsed = round(time.monotonic() - start_mono, 1) + results.append({"id": name, "status": "PASS", "expected": expected, + "observed": evidence, "elapsed_seconds": elapsed}) + log("PASS " + name + " - " + evidence[:190]) + +def issue(name, expected, err): + elapsed = round(time.monotonic() - start_mono, 1) + results.append({"id": name, "status": "FAIL", "expected": expected, + "observed": str(err), "elapsed_seconds": elapsed}) + log("FAIL " + name + " - " + str(err)[:350]) + +def checked(name, expected, fn): + try: + mark(name, expected, fn()) + return True + except Exception as exc: + issue(name, expected, exc) + return False + +def ensure(ok, message): + if not ok: + raise AssertionError(message) + +def fixture(suffix, program, options=(), memory="128m", control=None): + name = PREFIX + "-" + suffix + args = ["create", "--name", name, "--network", "none", "--label", LABEL, + # The image was built by Compose and carries service labels; do not + # let the lab reconciliation loop mistake test fixtures for watchdogs. + "--label", "com.docker.compose.project=release-smoke", + "--label", "com.docker.compose.service=fixture", + "--restart", "no", + "--cpus", "0.20", "--memory", memory, "--memory-swap", memory, + "--pids-limit", "64", "--security-opt", "no-new-privileges", + "--cap-drop", "ALL"] + if control: + args += ["--mount", "type=volume,source=" + control + ",target=/control"] + args += list(options) + ["--entrypoint", "python3", IMAGE, "-u", "-c", program] + cid = run(args) + containers.append(name) + facts[suffix] = {"name": name, "container_id": cid} + return name + +def start(name): + run(["start", name]) + +def inspect(name, value): + return run(["inspect", "--format", value, name]) + +def state(name): + return json.loads(inspect(name, "{{json .State}}")) + +def wait_exit(name, limit=30): + return int(run(["wait", name], timeout=limit).splitlines()[-1]) + +def count(name, kind): + a = alerts(name, kind) + return len(a) + +def alerts(name, kind=None): + raw = run(["logs", "--since", START, OBS], timeout=20, check=False) + lines = [x for x in raw.splitlines() if 'event="watchdog_alert"' in x + and ('container="' + name + '"') in x] + if kind is not None: + lines = [x for x in lines if ('failure_type="' + kind + '"') in x] + return lines + +def wait_alert(name, kind, n=1, limit=45): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + if count(name, kind) >= n: + return count(name, kind) + time.sleep(1) + raise AssertionError("timeout waiting for " + str(n) + " " + kind + + " alerts on " + name + "; observed=" + repr(alerts(name)[-3:])) + +def must_count(name, kind, n): + actual = count(name, kind) + ensure(actual == n, name + " " + kind + ": expected " + str(n) + " got " + str(actual)) + return actual + +def make_volume(suffix): + name = PREFIX + "-" + suffix + run(["volume", "create", "--label", LABEL, name]) + volumes.append(name) + # The actual test process is UID 1000; test control files must be writable. + if suffix != "state": + run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", name + ":/control", "--entrypoint", "python3", + IMAGE, "-c", "import os;os.chown('/control',1000,1000)"]) + return name + +def write_control(volume, filename, present=True): + code = ("import pathlib; p=pathlib.Path('/control')/" + repr(filename) + + "; p.write_text('yes') if " + repr(present) + " else p.unlink(missing_ok=True)") + run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", volume + ":/control", "--entrypoint", "python3", + IMAGE, "-c", code], timeout=35) + +def wait_restarts(name, minimum, limit=120): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + n = int(inspect(name, "{{.RestartCount}}")) + if n >= minimum: + return n + time.sleep(2) + raise AssertionError("restart count did not reach " + str(minimum)) + +def require_syslog(): + # This file is in the EXISTING DFC container, not created on the host. + data = run(["exec", "config_dfc_1", "sh", "-c", + "grep -F " + "'host=" + '"' + HOST + '"' + "'" + + " /var/log/syslog || true"], timeout=20, check=False) + return [x for x in data.splitlines() + if 'host="' + HOST + '"' in x and 'event="watchdog_alert"' in x] + +def init(): + ensure(run(["inspect", "--format", "{{.State.Health.Status}}", + "docker-container-watchdog"]) == "healthy", "existing watchdog not healthy") + revision = run(["image", "inspect", "--format", + '{{index .Config.Labels "org.opencontainers.image.revision"}}', IMAGE]) + ensure(revision == EXPECTED_REV, "wrong image source revision " + revision) + image_id = run(["image", "inspect", "--format", "{{.Id}}", IMAGE]) + ensure("config_dfc_1" in run(["ps", "--format", "{{.Names}}"]).splitlines(), + "DFC syslog receiver not running") + existing = run(["ps", "-a", "--format", "{{.Names}}"]).splitlines() + ensure(not any(n.startswith(PREFIX) for n in existing), "smoke prefix collision") + ensure(not run(["volume", "ls", "--filter", "label=" + LABEL, "-q"]), "old test volumes") + vol = make_volume("state") + excludes = sorted(set(existing + [OBS])) + config = { + "alert_channels": [], "log_level": "INFO", "log_file": None, + "check_interval_seconds": 2, "cooldown_minutes": 5, + "restart_threshold": 5, "restart_window_minutes": 10, + "restart_recovery_stability_seconds": 120, "unhealthy_cycles_threshold": 3, + "alert_on_recovery": True, "boot_grace_seconds": 0, + "lifecycle_detection": {"enabled": True, "correlation_window_seconds": 3}, + "ha_monitor": {"enabled": False}, + "auto_health_check": {"enabled": False}, + "excluded_containers": excludes, + "state_file": "/var/log/watchdog/state.json", + "syslog": {"enabled": True, "host": "127.0.0.1", "port": 514, + "protocol": "udp", "facility": "local0"} + } + setup = ("import os,json,pathlib; p=pathlib.Path('/data'); " + "c=json.loads(os.environ['SMOKE_CONFIG']); " + "(p/'watchdog-config.yaml').write_text(json.dumps(c)); " + "(p/'watchdog-config.yaml').chmod(0o644); os.chown(p,1000,1000)") + run(["run", "--rm", "--network", "none", "--user", "0:0", + "-e", "SMOKE_CONFIG=" + json.dumps(config), + "--volume", vol + ":/data", "--entrypoint", "python3", + IMAGE, "-c", setup], timeout=30) + gid = os.stat("/var/run/docker.sock").st_gid + arm_events() + args = ["run", "-d", "--name", OBS, "--label", LABEL, "--restart", "no", + "--label", "com.docker.compose.project=release-smoke", + "--label", "com.docker.compose.service=observer", + "--network", "host", "--cpus", "0.35", "--memory", "256m", + "--pids-limit", "96", "--read-only", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", + "--security-opt", "no-new-privileges", "--cap-drop", "ALL", + "--user", "1000:1000", "--group-add", str(gid), + "--log-driver", "local", "--log-opt", "max-size=5m", + "--log-opt", "max-file=2", + "-e", "WATCHDOG_HOST=" + HOST, + "-e", "WATCHDOG_CONFIG=/etc/watchdog/watchdog-config.yaml", + "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", + "-v", vol + ":/etc/watchdog:ro", + "-v", vol + ":/var/log/watchdog:rw", + IMAGE] + run(args) + containers.append(OBS) + time.sleep(3) + lines = run(["logs", OBS]) + ensure("Docker event listener started" in lines and "Poll loop started" in lines, + "observer did not initialize: " + lines[-700:]) + ensure(inspect(OBS, "{{.State.Status}}") == "running", "observer stopped") + facts.update({"host": run(["info", "--format", "{{.Name}}"]), + "revision": revision, "image_id": image_id, "observer": OBS, + "syslog_source_host": HOST, "observer_poll_seconds": 2, + "restart_threshold": 5, "recovery_stability_seconds": 120, + "excluded_existing_containers": len(existing)}) + log("REAL_DOCKER_TEST_READY " + HOST + " image=" + image_id[:24]) + +def t01_clean(): + name = fixture("clean", "import sys;sys.exit(0)") + start(name) + ensure(wait_exit(name) == 0, "clean exit not 0") + time.sleep(7) + ensure(not alerts(name), "unexpected alert on clean exit") + return "real exit=0, no alerts" + +def t02_crash(): + name = fixture("crash", "raise SystemExit(2)") + start(name) + ensure(wait_exit(name) == 2, "unexpected exit code") + wait_alert(name, "crashed", limit=25) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure('severity="CRITICAL"' in alerts(name, "crashed")[0], "wrong severity") + return "Docker exit=2, exactly one CRITICAL crashed alert" + +def t03_planned_stop(): + name = fixture("planned", "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(300)") + start(name) + time.sleep(2) + run(["stop", "-t", "1", name], timeout=20) + ensure(wait_exit(name, 20) == 137, "planned stop not exit=137") + ensure(state(name)["OOMKilled"] is False, "planned stop unexpectedly OOM") + time.sleep(7) + ensure(not alerts(name), "planned stop misclassified: " + str(alerts(name))) + ensure(state(name)["Status"] == "exited", "unexpected planned fixture restart") + # The HA lab has an external restart manager; remove the finished fixture + # before that independent service can start a new Docker run. + run(["rm", "-f", name]) + return "real docker stop -> SIGKILL/137; no crash or OOM alert" + +def t04_direct_kill(): + name = fixture("sigkill", "import time;time.sleep(300)") + start(name) + time.sleep(1) + run(["kill", "--signal", "SIGKILL", name]) + ensure(wait_exit(name) == 137, "SIGKILL exit not 137") + ensure(state(name)["OOMKilled"] is False, "SIGKILL incorrectly marked OOM") + wait_alert(name, "crashed", limit=25) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure(must_count(name, "oom", 0) == 0, "false OOM") + facts["sigkill_state"] = {"exit": 137, "OOMKilled": False} + run(["rm", "-f", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "false recovery after removed SIGKILL") + ensure(must_count(name, "crashed", 1) == 1, "repeated crash") + return "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash" + +def t05_oom(): + name = fixture("oom", "import time;x=[]\nwhile True:\n x.append(bytearray(8*1024*1024));time.sleep(.02)", + memory="64m") + start(name) + ensure(wait_exit(name, limit=35) == 137, "OOM did not exit 137") + ensure(state(name)["OOMKilled"] is True, "Docker cgroup did not confirm OOMKilled") + wait_alert(name, "oom", limit=30) + facts["oom_state"] = {"exit": 137, "OOMKilled": True} + run(["rm", "-f", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "false recovery after removed OOM") + ensure(must_count(name, "oom", 1) == 1, "duplicate OOM") + ensure(must_count(name, "crashed", 0) == 0, "false crash after confirmed OOM") + line = alerts(name, "oom")[0] + ensure('severity="CRITICAL"' in line, "incorrect OOM severity") + return "kernel cgroup OOMKilled=true, exit=137, one CRITICAL OOM, zero crash" + +def t06_t08_loop(): + vol = make_volume("loopcontrol") + program = """import os,sys,time +if not os.path.exists('/control/recover'): + time.sleep(0.7) + sys.exit(2) +while True: + if os.path.exists('/control/interrupt'): + try: os.unlink('/control/interrupt') + except OSError: pass + sys.exit(2) + time.sleep(0.4) +""" + name = fixture("loop", program, options=("--restart", "always"), control=vol) + start(name) + wait_alert(name, "crashed", limit=50) + wait_alert(name, "restart-loop", limit=135) + n = int(inspect(name, "{{.RestartCount}}")) + ensure(n >= 5, "loop threshold crossed without 5 real Docker restarts, saw " + str(n)) + ensure(must_count(name, "crashed", 1) == 1, "repeated crashes") + ensure(must_count(name, "restart-loop", 1) == 1, "repeat loop") + mark("T06_RESTART_LOOP", "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "real Docker RestartCount=" + str(n) + "; crash=1; restart-loop=1") + write_control(vol, "recover") + running_at = time.monotonic() + time.sleep(11) + ensure(inspect(name, "{{.State.Status}}") == "running", "not recovering") + ensure(must_count(name, "recovered", 0) == 0, "premature recovered after 11s") + mark("T07_BRIEF_RECOVERY", "no recovery before 120s stability", "running after 11s, RECOVERED=0") + # A new real process failure during the stability period resets the recovery clock. + before = int(inspect(name, "{{.RestartCount}}")) + write_control(vol, "interrupt") + after = wait_restarts(name, before + 1, limit=45) + time.sleep(14) + ensure(must_count(name, "recovered", 0) == 0, + "premature recovered 14s after interrupted recovery") + mark("T08_INTERRUPTED_RECOVERY", "new actual die/reset prevents premature recovery", + "Docker RestartCount " + str(before) + " -> " + str(after) + + "; RECOVERED=0 after 14s") + wait_alert(name, "recovered", limit=165) + time.sleep(6) + ensure(must_count(name, "crashed", 1) == 1, "crash storm") + ensure(must_count(name, "restart-loop", 1) == 1, "loop repeated") + ensure(must_count(name, "recovered", 1) == 1, "recovered not exactly once") + ensure('severity="INFO"' in alerts(name, "recovered")[0], "recovery level not INFO") + mark("T09_STABLE_RECOVERY", "one INFO RECOVERED after 120s sustained uptime", + "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery") + facts["loop_restart_count_after"] = int(inspect(name, "{{.RestartCount}}")) + return "one CRASH + one LOOP + one RECOVERED after real interrupted recovery" + +def t10_name_reuse(): + name = PREFIX + "-reuse" + ids = [] + for code in (2, 42): + label = "reuse-" + str(code) + cid = fixture("reuse", "raise SystemExit(" + str(code) + ")") if code == 2 else None + if code != 2: + # Reuse the exact same name after deleting the old container. + ensure(not run(["ps", "-aq", "--filter", "name=^/" + name + "$"]), + "old ID still owns name") + cid = fixture("reuse", "raise SystemExit(42)") + ids.append(inspect(name, "{{.Id}}")) + start(name) + ensure(wait_exit(name) == code, "unexpected " + label + " exit") + wait_alert(name, "crashed", n=len(ids), limit=30) + if code == 2: + run(["rm", name]) + time.sleep(2) + ensure(ids[0] != ids[1], "same Docker instance reused") + ensure(must_count(name, "crashed", 2) == 2, "new name inherited old alert") + return "two unique container IDs, same name, each gets one CRITICAL crash" + +def t11_fast_removal(): + names = [] + for k in range(3): + name = PREFIX + "-rm" + str(k) + run(["run", "--rm", "--name", name, "--label", LABEL, + "--network", "none", "--memory", "128m", "--memory-swap", "128m", + "--pids-limit", "64", "--cpus", "0.2", "--cap-drop", "ALL", + "--entrypoint", "python3", IMAGE, "-c", "raise SystemExit(2)"], + timeout=30, check=False) + names.append(name) + time.sleep(8) + wd_logs = run(["logs", "--since", START, OBS], timeout=20) + misses = [x for x in wd_logs.splitlines() if "already removed before" in x and + any(n in x for n in names)] + ensure(len(misses) <= 6, "missing-container log flood: " + str(len(misses))) + ensure(inspect(OBS, "{{.State.Status}}") == "running", "event watcher died") + facts["fast_remove_warning_count"] = len(misses) + return "3 real --rm containers, " + str(len(misses)) + " bounded missing-ID warnings, observer alive" + +def t12_healthcheck(): + vol = make_volume("healthcontrol") + name = fixture("health", "import time;time.sleep(300)", control=vol, + options=("--health-cmd", + "python3 -c 'import os,sys;sys.exit(0 if os.path.exists(\"/control/healthy\") else 1)'", + "--health-interval", "2s", "--health-timeout", "2s", + "--health-retries", "2", "--health-start-period", "0s")) + start(name) + wait_alert(name, "unhealthy", limit=55) + ensure(must_count(name, "unhealthy", 1) == 1, "health alert duplicated") + ensure('severity="HIGH"' in alerts(name, "unhealthy")[0], "unhealthy severity") + write_control(vol, "healthy") + wait_alert(name, "recovered", limit=40) + ensure(must_count(name, "recovered", 1) == 1, "health recovery duplicated") + ensure('severity="INFO"' in alerts(name, "recovered")[0], "recovery priority") + ensure(inspect(name, "{{.State.Health.Status}}") == "healthy", "Docker health not healthy") + return "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery" + +def t13_dfc(): + sent = [x for x in run(["logs", "--since", START, OBS]).splitlines() + if 'event="watchdog_alert"' in x] + received = require_syslog() + (BASE/"dfc-syslog.txt").write_text("\n".join(received) + "\n") + facts["watchdog_alert_count"] = len(sent) + facts["dfc_received_count"] = len(received) + ensure(sent, "watchdog generated no alert records") + for name in ("crash", "sigkill", "oom", "loop", "reuse", "health"): + actual = PREFIX + "-" + name + local = [x for x in sent if 'container="' + actual + '"' in x] + remote = [x for x in received if 'container="' + actual + '"' in x] + ensure(len(remote) == len(local), + name + " alert count differs watchdog=" + str(len(local)) + + " DFC=" + str(len(remote))) + ensure(not [x for x in received if 'container="' + PREFIX + '-planned"' in x], + "planned stop sent to DFC") + ensure(not [x for x in received if 'container="' + PREFIX + '-clean"' in x], + "clean exit sent to DFC") + for kind,severity in (("crashed","CRITICAL"),("oom","CRITICAL"), + ("restart-loop","HIGH"),("unhealthy","HIGH"), + ("recovered","INFO")): + ensure(any('failure_type="' + kind + '"' in x and + 'severity="' + severity + '"' in x for x in received), + "DFC missing " + kind + "/" + severity) + return str(len(sent)) + " watchdog alerts delivered as " + str(len(received)) + \ + " DFC syslog messages; CRITICAL/HIGH/INFO fields correct" + +def collect(): + stop_events() + if OBS in containers: + raw = run(["logs", "--since", START, OBS], timeout=30, check=False) + (BASE/"watchdog.log").write_text(raw + "\n") + try: + vol = next(v for v in volumes if v.endswith("-state")) + d = run(["run", "--rm", "--network", "none", "--user", "0:0", + "--volume", vol + ":/data", "--entrypoint", "python3", + IMAGE, "-c", + "import json;print(json.dumps(json.load(open('/data/state.json')),sort_keys=True))"], + timeout=30) + (BASE/"observer-state.json").write_text(d + "\n") + except Exception as e: + (BASE/"state-read-error.txt").write_text(str(e)) + try: + before = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00","Z") + ev = run(["events", "--since", START, "--until", before, + "--filter", "type=container", + "--filter", "label=" + LABEL], timeout=40, check=False) + (BASE/"docker-events.log").write_text(ev + "\n") + facts["docker_event_lines"] = len([x for x in ev.splitlines() if x.strip()]) + except Exception as e: + facts["docker_events_error"] = str(e) + try: + if not (BASE/"dfc-syslog.txt").exists(): + (BASE/"dfc-syslog.txt").write_text("\n".join(require_syslog()) + "\n") + except Exception as e: + facts["syslog_read_error"] = str(e) + for n in list(containers): + if n == OBS: + continue + try: + state_json = run(["inspect", "--format", "{{json .State}}", n], check=False) + if state_json: + facts.setdefault("final_fixture_states", {})[n] = json.loads(state_json) + except Exception: + pass + +def cleanup(): + for n in reversed(containers): + if n == OBS: + continue + try: + run(["rm", "-f", n], timeout=30, check=False) + except Exception as e: + facts.setdefault("cleanup_errors", []).append(n + ": " + str(e)) + if OBS in containers: + try: + run(["stop", "-t", "12", OBS], timeout=25, check=False) + run(["rm", "-f", OBS], timeout=25, check=False) + except Exception as e: + facts.setdefault("cleanup_errors", []).append(OBS + ": " + str(e)) + for vol in reversed(volumes): + try: + run(["volume", "rm", vol], timeout=30) + except Exception as e: + facts.setdefault("cleanup_errors", []).append(vol + ": " + str(e)) + +try: + init() + checked("T01_CLEAN_EXIT", "exit=0 means no alert", t01_clean) + checked("T02_UNEXPECTED_CRASH", "exit=2 sends one CRITICAL crash", t02_crash) + checked("T03_PLANNED_STOP_137", "Docker stop exit 137 is suppressed", t03_planned_stop) + checked("T04_DIRECT_SIGKILL", "SIGKILL 137 is crash, not OOM", t04_direct_kill) + checked("T05_REAL_OOM", "OOMKilled=true sends only CRITICAL OOM", t05_oom) + checked("T06_TO_T09_RESTART_LIFECYCLE", + "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", t06_t08_loop) + checked("T10_NAME_REUSE", "replacement with same name gets new incident", t10_name_reuse) + checked("T11_FAST_REMOVAL", "no event flood, event listener survives", t11_fast_removal) + checked("T12_NATIVE_HEALTHCHECK", "unhealthy -> recovered", t12_healthcheck) + checked("T13_DFC_SYSLOG", "all expected structured alerts reach actual DFC", t13_dfc) +except Exception as e: + issue("PREFLIGHT_OR_FATAL", "observer initializes and tests run", repr(e)) + facts["fatal_traceback"] = traceback.format_exc() +finally: + try: + collect() + except Exception as e: + facts["collect_error"] = repr(e) + finally: + stop_events() + cleanup() + finished = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00","Z") + facts["total_seconds"] = round(time.monotonic() - start_mono, 1) + data = {"run_id": TAG, "started_utc": START, "finished_utc": finished, + "status": ("FAIL" if any(x["status"] == "FAIL" for x in results) + or facts.get("cleanup_errors") else "PASS"), + "image": IMAGE, "results": results, "facts": facts} + (BASE/"report.json").write_text(json.dumps(data, indent=2, default=str) + "\n") + lines = ["# Container Alert v1.6.0 real Docker smoke test", + "", "**Run:** " + TAG + " | **Result:** " + data["status"], + "**Started:** " + START + " | **Finished:** " + finished, + "**Image:** " + IMAGE + " | **Revision:** " + str(facts.get("revision","unknown")), + "**Host:** " + str(facts.get("host","unknown")) + " | **Observer host ID:** " + HOST, + "", "## Acceptance results", "", + "| Test | Status | Actual evidence |", "|---|---|---|"] + for item in results: + actual = str(item["observed"]).replace("|","/").replace("\n"," ") + lines.append("| " + item["id"] + " | " + item["status"] + " | " + actual[:290] + " |") + lines += ["", "## Evidence", + "Actual Docker container failures and events; exact release code in an isolated observer.", + "Observer has SMTP disabled; Syslog messages are sent to pre-existing DFC UDP/514.", + "No synthetic Docker events, mocked exporter metrics, host packages, or Pacemaker changes.", + "Other watchdogs and unrelated existing containers are not modified.", + "Output files: watchdog.log, docker-events.log, dfc-syslog.txt, observer-state.json, report.json.", + "","## Facts",""] + for k,v in sorted(facts.items()): + if k == "final_fixture_states": + continue + lines.append("- **" + k + ":** " + str(v)[:400]) + lines += ["","**Cleanup:** only test-labelled disposable containers and volumes, " + + ("PASS" if not facts.get("cleanup_errors") else "ERRORS"), ""] + (BASE/"report.md").write_text("\n".join(lines)) + log("REPORT_DIR=" + str(BASE)) + log("SMOKE_RESULT=" + data["status"] + " checks=" + str(len(results)) + + " fails=" + str(sum(x["status"] == "FAIL" for x in results)) + + " runtime_seconds=" + str(facts["total_seconds"])) + sys.exit(0 if data["status"] == "PASS" else 1) +PY From 0666a1260999dd034a3455c588d1b72baf52b835 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Wed, 14 Oct 2026 22:54:57 +0000 Subject: [PATCH 04/18] fix: require stable recovery after manual Docker restarts --- README.md | 4 +- docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md | 99 ++++++ ...ELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json | 324 ++++++++++++++++++ ...ASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json | 81 +++++ ...E_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json | 76 ++++ docs/RELEASE_VALIDATION_V1.6.0.md | 2 + tests/release-smoke.sh | 193 +++++++++-- tests/test_restart_loop.py | 19 +- tests/test_watchdog.py | 7 +- watchdog.py | 9 +- 10 files changed, 779 insertions(+), 35 deletions(-) create mode 100644 docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md create mode 100644 docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json create mode 100644 docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json create mode 100644 docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json diff --git a/README.md b/README.md index f740e79..c41e425 100644 --- a/README.md +++ b/README.md @@ -648,13 +648,13 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor ## 10. Version History -**Release status (2026-10-14):** `VERSION` is set to `1.6.0` for the release candidate. The new image, Git tag, and GitHub Release are not yet published. The deployed `watchdog:1.5.4` remains unchanged until a validated new image is deployed. +**Release status (2026-10-14):** `VERSION` is set to `1.6.0` for the release candidate. Real HA failover/failback and the recovery-fixed image passed release acceptance tests; see [final validation](docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). PR #9 remains open, and the final Git tag, GitHub Release, and offline package have not been published. The HA labs still run the earlier v1.6.0 candidate; m920q continues to run `watchdog:1.5.4`. **Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts. | +| 1.6.0 | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts; PR #10: corrected notification consistency and Syslog priorities; release validation: require stable uptime before CRASH/OOM recovery, including manual/external container restarts. | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md new file mode 100644 index 0000000..9e8369f --- /dev/null +++ b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md @@ -0,0 +1,99 @@ +# Container Alert v1.6.0 - Final release-candidate acceptance + +**Validation date:** October 14, 2026 (UTC) +**Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) +**Source basis:** release branch including merged [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10), plus the targeted recovery fix in this PR +**Verdict: PASS for the tested Docker, HA, Syslog, regression, and image-packaging acceptance criteria.** Live SMTP, SNMP, Slack, destructive HA injection, and host reboot are not included. + +## Executive results + +| Area | Result | Evidence | +|---|---|---| +| Automated Python regression tests | **PASS 109/109** | `python3 -m unittest discover -s tests -q` inside actual rebuilt `watchdog:1.6.0-rc-fix` image | +| Full real Docker integration suite | **PASS 15/15** | Isolated Docker Engine on m920q, run `261014224557488`; 22:45:57-22:52:41 UTC; 404.4 seconds | +| Real external restart after OOM | **PASS 1/1** | Separate targeted real OOM run `261014224323244`; RECOVERED after 120.6s | +| Regression reproduction before fix | **CONFIRMED FAIL** | Real crash followed by manual Docker start produced premature RECOVERED within about three seconds; run `261014223252035` | +| Raw UDP Syslog integration | **PASS 15/15** | 15 locally generated `watchdog_alert` records delivered as 15 actual UDP datagrams to isolated collector | +| Two-node HA failover and failback | **PASS** | Previous *live* lab testing: promoted HAOperator `.241 -> .242 -> .241`; one new-owner WARNING per transition, no duplicate, both DFC Syslog receivers confirmed | +| Installer source-build / offline image bundle | **PASS** | Source-build includes `watchdog.py`, `lifecycle.py`, `ha_monitor.py`; image save/load preserved ID | +| Disposable test cleanup | **PASS** | No labelled Docker test containers or volumes remained after each completed suite | +| m920q host watchdog impact | **NONE** | Existing `watchdog:1.5.4` remained healthy, zero restarts, and saw no isolated Engine fixtures | + +## Reproducible environment + +- On **m920q**, ran a short-lived Docker-in-Docker engine `docker:27.5-dind`, exposed only at `127.0.0.1` using an ephemeral TCP port; it had a distinct Engine ID from m920q's running Docker Engine, memory limit 2 GiB, CPU limit 2 cores, no bridge/network provisioning inside the test engine, and its state was disposable. +- Real test observer ran the revised application code from `watchdog:1.6.0-rc-fix` with its own state volume and a 2s polling interval. Original 5-restart threshold and **120s recovery stability window** were preserved. The observer monitored **real Docker Engine events and cgroup OOMs**, not synthetic test events. +- Disposable failure containers were scoped with unique `ca-smoke-...` names/labels, limited CPU/memory, and no network access. Syslog receiver was a temporary UDP/514 collector on the isolated Engine. No production SMTP or external Syslog destinations were used. +- The built test image contained verified source hashes: + - `watchdog.py`: `4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa` + - `lifecycle.py`: `55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9` + - `ha_monitor.py`: `0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd` +- Test image ID within the isolated Engine: `sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9`. This was a locally built test candidate, **not** a published release artifact or Git tag. +- Real-event run captured **92 Docker events**, including 18 `die`, 22 `start`, 1 `oom`, 1 `stop`, native unhealthy/healthy transitions, and **zero unexpected `restart` actions**. The regular loop's expected internal restart behavior is tracked via Docker's actual `RestartCount`. + +## Full real Docker acceptance - final clean run + +| Test | Expected | Observed | Status | +|---|---|---|---| +| T01 CLEAN_EXIT | Exit 0, no incident | No alert | **PASS** | +| T02 UNEXPECTED_CRASH | Exit 2, one CRITICAL crash | Exactly one CRITICAL CRASH | **PASS** | +| T03 PLANNED_STOP_137 | Forced Docker stop/137 not mistaken for crash or OOM | No false alert | **PASS** | +| T04 DIRECT_SIGKILL | Direct signal 9/137 is crash, not OOM | `OOMKilled=false`, one CRITICAL CRASH | **PASS** | +| T05 REAL_OOM | Real kernel OOM, 137, single OOM alert, no redundant CRASH | `OOMKilled=true`, one CRITICAL OOM | **PASS** | +| T06 RESTART_LOOP | Repeated failures escalate after at least five real restarts | `RestartCount=5`, one CRASH, one HIGH RESTART-LOOP | **PASS** | +| T07 BRIEF_RECOVERY | 11 seconds of RUNNING is insufficient | Zero RECOVERED after 11 seconds | **PASS** | +| T08 INTERRUPTED_RECOVERY | A new actual failure restarts the stability clock | `RestartCount` 6 to 7; no early RECOVERED | **PASS** | +| T09 STABLE_RECOVERY | One INFO RECOVERED after 120s sustained uptime | One CRASH, one LOOP, one RECOVERED | **PASS** | +| T06-09 LIFECYCLE | One complete incident, no duplicate notifications | Correct alert order and totals | **PASS** | +| T14 EXTERNAL_RESTART | Manual start of policy-`no` crashed container must wait 120s | One RECOVERED near 120s (119.9s by second-resolution alert timestamp) | **PASS** | +| T10 NAME_REUSE | New Docker ID with reused name must start a new incident | Two distinct IDs, one CRASH per ID | **PASS** | +| T11 FAST_REMOVAL | Rapid `--rm` events do not flood missing-ID warnings or kill listener | Three ephemeral containers, zero observed warnings, observer alive | **PASS** | +| T12 NATIVE_HEALTHCHECK | Native Docker unhealthy then healthy | One HIGH UNHEALTHY, one INFO RECOVERED | **PASS** | +| T13 SYSLOG | Every generated alert reaches receiver with correct typed severity, no planned-stop false alarm | **15/15** structured UDP datagrams; no planned-stop false alarm | **PASS** | + +The isolated collector captured actual raw Syslog priorities: **CRITICAL `<130>` (10), HIGH `<131>` (2), INFO `<134>` (3)** with local0 facility. This verifies PR #10's intended priority mapping for Docker event classes. Previous real HA testing captured and stored HA WARNING through DFC (local0.WARNING `<132>`). + +### Additional OOM-then-manual-start case + +The test `T15_EXTERNAL_OOM` deliberately exceeded a 64 MiB cgroup memory limit, confirmed exit 137 with Docker `OOMKilled=true`, received one CRITICAL OOM alert, then manually started the same policy-`no` container with a healthy process. No early recovery was emitted; precisely one INFO RECOVERED occurred **120.6 seconds** after actual Docker uptime began. **PASS.** + +## Defect, fix, and historical HA-lab failures + +The earlier HA-lab smoke run (`261014214436754`, originally **11 PASS / 3 FAIL**) observed containers with Docker restart policy `no` starting again after a failure despite not being controlled by the test suite. The captured events prove this external restart behavior; the initiator was **not positively identified**. Do not attribute it to a particular CyberController host service without direct evidence. + +We reproduced the **application bug independently** on the isolated m920q Docker Engine: the old code emitted RECOVERED just a few seconds after a manual `docker start` of a crashed policy-`no` container. Original `_maybe_alert_recovery` applied the 120-second stability gate to CRASH/OOM only when the Docker restart policy was automatic. That assumption is invalid when an external manager manually restarts failed containers. + +**Minimal code correction:** for any open `crashed`, `oom`, or `restart-loop` incident, require sustained Docker uptime and no recent failure for the existing recovery-stability interval. The existing state machine, Docker event correlation, HA monitoring, and notification APIs were not redesigned. Two policy-`no` crash/OOM regressions now protect this path. All **109** automated tests pass. + +The original lab's external Docker restarter remains **unattributed and unchanged**. Its behavior is an integration-environment consideration, not proof the new application fix failed. The isolated Engine recorded zero unrelated restart actions and its full suite passed. + +## HA acceptance carried forward (actual Pacemaker operations) + +The user manually performed real switchover and failback of the existing two-node CyberController HA cluster while both real watchdogs ran the PR #10 1.6.0 candidate. + +| HA case | Evidence and outcome | +|---|---| +| `10.199.0.241 -> 10.199.0.242` | **PASS**: promoted operator moved to .242; one WARNING alert on newly active .242 at 20:56:16 UTC, no duplicate on .241 | +| .242 DFC delivery | **PASS**: structured `ha-failover` record received/stored in `config_dfc_1:/var/log/syslog` | +| `10.199.0.242 -> 10.199.0.241` | **PASS**: one WARNING alert on .241 at 21:09:47 UTC; .242 deferred, no duplicate | +| .241 DFC delivery | **PASS**: structured reverse alert received/stored in its DFC Syslog receiver | +| Final HA health | **PASS**: quorum healthy, .241 promoted, both watchdogs healthy and baselines .241 | + +The new Docker recovery fix does not change the HA monitoring code. Nevertheless, **the currently running HA-node images still contain the previous pre-fix candidate**, so they are not proof that the final recovery-fixed image has been installed on those nodes. + +## Packaging, exclusions, and release decision + +Installer source-build and offline archive `docker save`/`docker load` procedures were validated earlier in this release cycle; CI covered shell syntax, Compose, code compilation, unit tests, and image builds. The test suite itself runs without installing packages on m920q or the HA nodes. + +**Not validated end to end:** SNMP traps to a physical NMS, Slack delivery, SMTP delivery (explicitly excluded by user), host reboot, network partition, STONITH, destructive exporter injection. These are not represented as passed. Failover/failback, DFC Syslog, ordinary Docker incidents, and the fixed recovery path have direct live evidence. + +**Release recommendation:** The previously blocking Docker recovery bug is fixed and independently validated. The release branch is ready for final PR review and fresh CI validation. Do **not** claim that the Git tag, GitHub Release, or final v1.6.0 package was published; do not deploy or merge without the user's chosen release step. + +## Evidence and replay + +- [Final full suite JSON](RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json) - 15/15 and cleanup evidence +- [Before-fix real failure reproduction](RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json) +- [Real OOM/manual-restart 120.6s proof](RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json) +- [Earlier HA-lab conditional report](RELEASE_VALIDATION_V1.6.0.md) +- [Reusable real Docker smoke runner](../tests/release-smoke.sh) +- Runtime evidence: `/tmp/container-alert-smoke-261014224557488/` and `/tmp/container-alert-smoke-261014224323244/` on m920q (contains captured watchdog events, actual Docker events, raw received Syslog messages, and observer-state JSON). diff --git a/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json new file mode 100644 index 0000000..9462fde --- /dev/null +++ b/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json @@ -0,0 +1,324 @@ +{ + "run_id": "261014224557488", + "started_utc": "2026-10-14T22:45:57.106131Z", + "finished_utc": "2026-10-14T22:52:41.488066Z", + "status": "PASS", + "image": "watchdog:1.6.0-rc-fix", + "results": [ + { + "id": "T01_CLEAN_EXIT", + "status": "PASS", + "expected": "exit=0 means no alert", + "observed": "real exit=0, no alerts", + "elapsed_seconds": 21.9 + }, + { + "id": "T02_UNEXPECTED_CRASH", + "status": "PASS", + "expected": "exit=2 sends one CRITICAL crash", + "observed": "Docker exit=2, exactly one CRITICAL crashed alert", + "elapsed_seconds": 26.8 + }, + { + "id": "T03_PLANNED_STOP_137", + "status": "PASS", + "expected": "Docker stop exit 137 is suppressed", + "observed": "real docker stop -> SIGKILL/137; no crash or OOM alert", + "elapsed_seconds": 39.0 + }, + { + "id": "T04_DIRECT_SIGKILL", + "status": "PASS", + "expected": "SIGKILL 137 is crash, not OOM", + "observed": "real SIGKILL/137, OOMKilled=false, exactly one CRITICAL crash", + "elapsed_seconds": 57.5 + }, + { + "id": "T05_REAL_OOM", + "status": "PASS", + "expected": "OOMKilled=true sends only CRITICAL OOM", + "observed": "kernel cgroup OOMKilled=true, exit=137, one CRITICAL OOM, zero crash", + "elapsed_seconds": 71.9 + }, + { + "id": "T06_RESTART_LOOP", + "status": "PASS", + "expected": "one CRASH, one HIGH RESTART-LOOP at >=5 Docker restarts", + "observed": "real Docker RestartCount=5; crash=1; restart-loop=1", + "elapsed_seconds": 83.7 + }, + { + "id": "T07_BRIEF_RECOVERY", + "status": "PASS", + "expected": "no recovery before 120s stability", + "observed": "running after 11s, RECOVERED=0", + "elapsed_seconds": 96.7 + }, + { + "id": "T08_INTERRUPTED_RECOVERY", + "status": "PASS", + "expected": "new actual die/reset prevents premature recovery", + "observed": "Docker RestartCount 6 -> 7; RECOVERED=0 after 14s", + "elapsed_seconds": 112.6 + }, + { + "id": "T09_STABLE_RECOVERY", + "status": "PASS", + "expected": "one INFO RECOVERED after 120s sustained uptime", + "observed": "1 crashed, 1 restart-loop, 1 recovered after interrupted recovery", + "elapsed_seconds": 225.2 + }, + { + "id": "T06_TO_T09_RESTART_LIFECYCLE", + "status": "PASS", + "expected": "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", + "observed": "one CRASH + one LOOP + one RECOVERED after real interrupted recovery", + "elapsed_seconds": 225.3 + }, + { + "id": "T14_EXTERNAL_RESTART", + "status": "PASS", + "expected": "manual start after CRASH waits 120s before RECOVERED", + "observed": "real manual Docker start after crash; RECOVERED after 119.9s uptime", + "elapsed_seconds": 355.0 + }, + { + "id": "T10_NAME_REUSE", + "status": "PASS", + "expected": "replacement with same name gets new incident", + "observed": "two unique container IDs, same name, each gets one CRITICAL crash", + "elapsed_seconds": 367.1 + }, + { + "id": "T11_FAST_REMOVAL", + "status": "PASS", + "expected": "no event flood, event listener survives", + "observed": "3 real --rm containers, 0 bounded missing-ID warnings, observer alive", + "elapsed_seconds": 381.3 + }, + { + "id": "T12_NATIVE_HEALTHCHECK", + "status": "PASS", + "expected": "unhealthy -> recovered", + "observed": "real native Docker HEALTHCHECK unhealthy HIGH then healthy INFO recovery", + "elapsed_seconds": 398.2 + }, + { + "id": "T13_SYSLOG", + "status": "PASS", + "expected": "all expected structured alerts reach test UDP collector", + "observed": "15 watchdog alerts delivered as 15 isolated UDP syslog messages; CRITICAL/HIGH/INFO fields correct", + "elapsed_seconds": 398.3 + } + ], + "facts": { + "embedded_source_sha256": { + "/app/watchdog.py": "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa", + "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", + "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" + }, + "host": "b6df39059ff7", + "revision": "", + "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", + "observer": "ca-smoke-261014224557488-observer", + "syslog_source_host": "release-smoke-261014224557488", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "clean": { + "name": "ca-smoke-261014224557488-clean", + "container_id": "93dc6ec3664fa2b472f7031cde519ecbd49549a82cce2ef1e85c8581657ed781" + }, + "crash": { + "name": "ca-smoke-261014224557488-crash", + "container_id": "59e4a2af2ecf9c225a09df23c6b59fc504ae4f2f0b147b13b9ca2b45c60120de" + }, + "planned": { + "name": "ca-smoke-261014224557488-planned", + "container_id": "ee156fae6fcd5bef6c53165886e04bf0988fe0061378fb496e76d9f488e3d4d9" + }, + "sigkill": { + "name": "ca-smoke-261014224557488-sigkill", + "container_id": "80e2215066163d64b2bd6a62ce5c6e43a57a7638d472539c53e8db9c7a539920" + }, + "sigkill_state": { + "exit": 137, + "OOMKilled": false + }, + "oom": { + "name": "ca-smoke-261014224557488-oom", + "container_id": "19482ebcd087d5d302ea6e707075e1a874524ef2a33e2f2db1c2b572de59e8a1" + }, + "oom_state": { + "exit": 137, + "OOMKilled": true + }, + "loop": { + "name": "ca-smoke-261014224557488-loop", + "container_id": "1016f94b5e91523c0358698cbe9b48edad0ac087ae4c332a5f55cc9690f9b02e" + }, + "loop_restart_count_after": 7, + "external": { + "name": "ca-smoke-261014224557488-external", + "container_id": "d30bdc108c3a194a9fe4d75a09e8ce313eca86a8c0c9f6af3d2f26812624f969" + }, + "reuse": { + "name": "ca-smoke-261014224557488-reuse", + "container_id": "c09027edaf62f85bb5babaf6aa38aee13d719866ba108f83a9cdeb9e52544c51" + }, + "fast_remove_warning_count": 0, + "health": { + "name": "ca-smoke-261014224557488-health", + "container_id": "23b9356951d88eb98fbd92d8bdd08be6ed29ffb293125338db78a8d35560a300" + }, + "watchdog_alert_count": 15, + "dfc_received_count": 15, + "live_docker_events": 92, + "live_docker_action_counts": { + "create": 14, + "start": 22, + "die": 18, + "oom": 1, + "stop": 1, + "kill": 3, + "restart": 0, + "health_status: unhealthy": 1, + "health_status: healthy": 1, + "destroy": 7 + }, + "docker_event_lines": 97, + "final_fixture_states": { + "ca-smoke-261014224557488-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 5484, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:46:02.126339022Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014224557488-clean": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:46:11.519112255Z", + "FinishedAt": "2026-10-14T22:46:11.888551005Z" + }, + "ca-smoke-261014224557488-crash": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 2, + "Error": "", + "StartedAt": "2026-10-14T22:46:20.387611646Z", + "FinishedAt": "2026-10-14T22:46:20.651109479Z" + }, + "ca-smoke-261014224557488-loop": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 6714, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:47:35.416361955Z", + "FinishedAt": "2026-10-14T22:47:35.299866533Z" + }, + "ca-smoke-261014224557488-external": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 6976, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:49:51.127967239Z", + "FinishedAt": "2026-10-14T22:49:45.881841779Z" + }, + "ca-smoke-261014224557488-reuse": { + "Status": "exited", + "Running": false, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 0, + "ExitCode": 42, + "Error": "", + "StartedAt": "2026-10-14T22:52:00.724185739Z", + "FinishedAt": "2026-10-14T22:52:00.970303463Z" + }, + "ca-smoke-261014224557488-health": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 7439, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:52:21.630416377Z", + "FinishedAt": "0001-01-01T00:00:00Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [ + { + "Start": "2026-10-14T22:52:28.219276407Z", + "End": "2026-10-14T22:52:28.31610092Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:30.316786865Z", + "End": "2026-10-14T22:52:30.503706796Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:32.504440652Z", + "End": "2026-10-14T22:52:32.698663166Z", + "ExitCode": 1, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:34.699078389Z", + "End": "2026-10-14T22:52:35.108639315Z", + "ExitCode": 0, + "Output": "" + }, + { + "Start": "2026-10-14T22:52:37.109706599Z", + "End": "2026-10-14T22:52:37.218394592Z", + "ExitCode": 0, + "Output": "" + } + ] + } + } + }, + "total_seconds": 404.4 + } +} diff --git a/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json new file mode 100644 index 0000000..985c190 --- /dev/null +++ b/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json @@ -0,0 +1,81 @@ +{ + "run_id": "261014224323244", + "started_utc": "2026-10-14T22:43:23.601963Z", + "finished_utc": "2026-10-14T22:45:47.277368Z", + "status": "PASS", + "image": "watchdog:1.6.0-rc-fix", + "results": [ + { + "id": "T15_EXTERNAL_OOM", + "status": "PASS", + "expected": "real OOM then manual start requires 120s stability", + "observed": "real kernel OOM -> external Docker start -> INFO RECOVERED after 120.6s", + "elapsed_seconds": 139.9 + } + ], + "facts": { + "embedded_source_sha256": { + "/app/watchdog.py": "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa", + "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", + "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" + }, + "host": "b6df39059ff7", + "revision": "", + "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", + "observer": "ca-smoke-261014224323244-observer", + "syslog_source_host": "release-smoke-261014224323244", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "externaloom": { + "name": "ca-smoke-261014224323244-externaloom", + "container_id": "8c4b398b1b98f0c13523ba16e67ddec1f44319f6cd5e10311c9a9215db8d5e03" + }, + "live_docker_events": 7, + "live_docker_action_counts": { + "create": 2, + "start": 3, + "die": 1, + "oom": 1, + "stop": 0, + "kill": 0, + "restart": 0, + "health_status: unhealthy": 0, + "health_status: healthy": 0, + "destroy": 0 + }, + "docker_event_lines": 9, + "final_fixture_states": { + "ca-smoke-261014224323244-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 4754, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:43:28.60988319Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014224323244-externaloom": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 5131, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:43:42.417354209Z", + "FinishedAt": "2026-10-14T22:43:40.28403892Z" + } + }, + "total_seconds": 143.7 + } +} diff --git a/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json new file mode 100644 index 0000000..6530397 --- /dev/null +++ b/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json @@ -0,0 +1,76 @@ +{ + "run_id": "261014223252035", + "started_utc": "2026-10-14T22:32:52.613776Z", + "finished_utc": "2026-10-14T22:33:29.913178Z", + "status": "FAIL", + "image": "watchdog:1.6.0", + "results": [ + { + "id": "T14_EXTERNAL_RESTART", + "status": "FAIL", + "expected": "no recovery until 120 seconds after external start", + "observed": "ca-smoke-261014223252035-external recovered: expected 0 got 1", + "elapsed_seconds": 33.8 + } + ], + "facts": { + "host": "b6df39059ff7", + "revision": "", + "image_id": "sha256:3a9942e74f643b0c2b38c82bcd0ef2b805cd13dd0f7c100031080f2cf06e494a", + "observer": "ca-smoke-261014223252035-observer", + "syslog_source_host": "release-smoke-261014223252035", + "observer_poll_seconds": 2, + "restart_threshold": 5, + "recovery_stability_seconds": 120, + "excluded_existing_containers": 0, + "syslog_backend": "isolated UDP test collector", + "docker_engine_endpoint": "tcp://127.0.0.1:32768", + "external": { + "name": "ca-smoke-261014223252035-external", + "container_id": "b78eb7be8c5464972808a9a3a85a2240a4b57aac347ca91b31a7c445015604af" + }, + "live_docker_events": 6, + "live_docker_action_counts": { + "create": 2, + "start": 3, + "die": 1, + "oom": 0, + "stop": 0, + "kill": 0, + "restart": 0, + "health_status: unhealthy": 0, + "health_status: healthy": 0, + "destroy": 0 + }, + "docker_event_lines": 8, + "final_fixture_states": { + "ca-smoke-261014223252035-syslog-receiver": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 790, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:32:57.598924455Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "ca-smoke-261014223252035-external": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 1174, + "ExitCode": 0, + "Error": "", + "StartedAt": "2026-10-14T22:33:14.250958418Z", + "FinishedAt": "2026-10-14T22:33:09.081072575Z" + } + }, + "total_seconds": 37.3 + } +} diff --git a/docs/RELEASE_VALIDATION_V1.6.0.md b/docs/RELEASE_VALIDATION_V1.6.0.md index be9cdcb..b85fb87 100644 --- a/docs/RELEASE_VALIDATION_V1.6.0.md +++ b/docs/RELEASE_VALIDATION_V1.6.0.md @@ -1,5 +1,7 @@ # Container Alert v1.6.0 — Release Acceptance Validation +> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on m920q, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). Retain this document as evidence of the original findings, not the current release verdict. + **Date:** October 14, 2026 (UTC) **Repository:** `Radware/CyberController-Container-Alert` **Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) — open diff --git a/tests/release-smoke.sh b/tests/release-smoke.sh index 25d97f0..0eb6893 100755 --- a/tests/release-smoke.sh +++ b/tests/release-smoke.sh @@ -15,10 +15,12 @@ import time import traceback IMAGE = os.environ.get("SMOKE_IMAGE", "watchdog:1.6.0") -EXPECTED_REV = "3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05" +EXPECTED_REV = os.environ.get("SMOKE_EXPECTED_REV", "") TAG = dt.datetime.now(dt.timezone.utc).strftime("%y%m%d%H%M%S") + str(os.getpid())[-3:] PREFIX = "ca-smoke-" + TAG LABEL = "container-alert.smoke=" + TAG +ISOLATED = os.environ.get("SMOKE_ISOLATED", "") == "1" +RECEIVER = PREFIX + "-syslog-receiver" if ISOLATED else "config_dfc_1" HOST = "release-smoke-" + TAG OBS = PREFIX + "-observer" STATE_VOL = PREFIX + "-state" @@ -184,25 +186,71 @@ def wait_restarts(name, minimum, limit=120): raise AssertionError("restart count did not reach " + str(minimum)) def require_syslog(): - # This file is in the EXISTING DFC container, not created on the host. - data = run(["exec", "config_dfc_1", "sh", "-c", - "grep -F " + "'host=" + '"' + HOST + '"' + "'" + - " /var/log/syslog || true"], timeout=20, check=False) + # Keep the isolated collector's raw received packets as separate evidence. + if ISOLATED: + data = run(["exec", RECEIVER, "cat", "/var/log/syslog"], timeout=20) + (BASE/"syslog-udp-raw.txt").write_text(data+"\n") + else: + data = run(["exec", RECEIVER, "sh", "-c", + "grep -F " + "'host=" + '"' + HOST + '"' + "'" + + " /var/log/syslog || true"], timeout=20, check=False) return [x for x in data.splitlines() if 'host="' + HOST + '"' in x and 'event="watchdog_alert"' in x] def init(): - ensure(run(["inspect", "--format", "{{.State.Health.Status}}", - "docker-container-watchdog"]) == "healthy", "existing watchdog not healthy") + if not ISOLATED: + ensure(run(["inspect", "--format", "{{.State.Health.Status}}", + "docker-container-watchdog"]) == "healthy", "existing watchdog not healthy") revision = run(["image", "inspect", "--format", '{{index .Config.Labels "org.opencontainers.image.revision"}}', IMAGE]) - ensure(revision == EXPECTED_REV, "wrong image source revision " + revision) + if EXPECTED_REV: + ensure(revision == EXPECTED_REV, "wrong image source revision " + revision) image_id = run(["image", "inspect", "--format", "{{.Id}}", IMAGE]) - ensure("config_dfc_1" in run(["ps", "--format", "{{.Names}}"]).splitlines(), - "DFC syslog receiver not running") + embedded = run(["run", "--rm", "--network", "none", "--entrypoint", "sha256sum", + IMAGE, "/app/watchdog.py", "/app/lifecycle.py", "/app/ha_monitor.py"]) + expected_sha = { + "/app/watchdog.py": os.environ.get( + "SMOKE_EXPECTED_WATCHDOG_SHA256", + "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa"), + "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", + "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd", + } + actual_sha = {line.split()[-1]: line.split()[0] for line in embedded.splitlines()} + ensure(actual_sha == expected_sha, "release image source SHA mismatch: " + repr(actual_sha)) + facts["embedded_source_sha256"] = actual_sha + if not ISOLATED: + ensure(RECEIVER in run(["ps", "--format", "{{.Names}}"]).splitlines(), + "DFC syslog receiver not running") existing = run(["ps", "-a", "--format", "{{.Names}}"]).splitlines() ensure(not any(n.startswith(PREFIX) for n in existing), "smoke prefix collision") ensure(not run(["volume", "ls", "--filter", "label=" + LABEL, "-q"]), "old test volumes") + if ISOLATED: + collector = make_volume("syslogdata") + udp_code = """import socket +s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) +s.bind(("0.0.0.0",514)) +with open("/var/log/syslog","a",buffering=1) as output: + print("SMOKE_SYSLOG_RECEIVER_READY", flush=True) + while True: + packet,source=s.recvfrom(65535) + output.write(packet.decode(errors="replace")+"\\n") +""" + run(["run", "-d", "--name", RECEIVER, "--label", LABEL, "--network", "host", + "--restart", "no", "--memory", "64m", "--pids-limit", "32", + "--volume", collector + ":/var/log", "--user", "0:0", + "--entrypoint", "python3", IMAGE, "-u", "-c", udp_code]) + containers.append(RECEIVER) + receiver_logs = "" + for _ in range(18): + time.sleep(1) + receiver_logs = run(["logs", RECEIVER], check=False) + if "SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs: + break + if inspect(RECEIVER, "{{.State.Status}}") == "exited": + break + ensure("SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs, + "isolated UDP syslog collector failed; state=" + + inspect(RECEIVER, "{{.State.Status}}") + ", log=" + receiver_logs[-1000:]) vol = make_volume("state") excludes = sorted(set(existing + [OBS])) config = { @@ -227,7 +275,7 @@ def init(): "-e", "SMOKE_CONFIG=" + json.dumps(config), "--volume", vol + ":/data", "--entrypoint", "python3", IMAGE, "-c", setup], timeout=30) - gid = os.stat("/var/run/docker.sock").st_gid + gid = int(os.environ.get("SMOKE_DOCKER_SOCKET_GID", os.stat("/var/run/docker.sock").st_gid)) arm_events() args = ["run", "-d", "--name", OBS, "--label", LABEL, "--restart", "no", "--label", "com.docker.compose.project=release-smoke", @@ -256,7 +304,9 @@ def init(): "revision": revision, "image_id": image_id, "observer": OBS, "syslog_source_host": HOST, "observer_poll_seconds": 2, "restart_threshold": 5, "recovery_stability_seconds": 120, - "excluded_existing_containers": len(existing)}) + "excluded_existing_containers": len(existing), + "syslog_backend": "isolated UDP test collector" if ISOLATED else "real DFC", + "docker_engine_endpoint": os.environ.get("DOCKER_HOST", "default")}) log("REAL_DOCKER_TEST_READY " + HOST + " image=" + image_id[:24]) def t01_clean(): @@ -375,6 +425,70 @@ while True: facts["loop_restart_count_after"] = int(inspect(name, "{{.RestartCount}}")) return "one CRASH + one LOOP + one RECOVERED after real interrupted recovery" +def t14_external_restart(): + # A Docker policy=no container is intentionally started by an external actor. + # RECOVERED must wait for sustained uptime, just like auto-restart containers. + vol = make_volume("externalcontrol") + script = """import os,sys,time +if not os.path.exists("/control/recover"): sys.exit(2) +time.sleep(300) +""" + name = fixture("external", script, control=vol) + start(name) + ensure(wait_exit(name) == 2, "initial actual crash not exit 2") + wait_alert(name, "crashed", limit=35) + write_control(vol, "recover") + ensure(inspect(name, "{{.HostConfig.RestartPolicy.Name}}") == "no", + "external fixture should not auto-restart") + run(["start", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, + "premature RECOVERED after external Docker start, before 120s uptime") + wait_alert(name, "recovered", limit=150) + ensure(must_count(name, "crashed", 1) == 1, "duplicate crash") + ensure(must_count(name, "recovered", 1) == 1, "duplicate recovery") + stable = confirmed_stable_uptime(name) + return "real manual Docker start after crash; RECOVERED after " + str(stable) + "s uptime" + +def confirmed_stable_uptime(name): + line = alerts(name, "recovered")[0] + stamp = re.search(r'time_utc="([^"]+)"', line) + ensure(stamp is not None, "RECOVERED alert missing timestamp") + started = dt.datetime.fromisoformat(inspect(name, "{{.State.StartedAt}}").replace("Z","+00:00")) + recovered = dt.datetime.fromisoformat(stamp.group(1).replace("Z","+00:00")) + elapsed = (recovered-started).total_seconds() + ensure(elapsed >= 119, "RECOVERED was emitted after only " + str(elapsed) + " seconds") + return round(elapsed,1) + +def t15_external_oom_restart(): + vol = make_volume("externaloomcontrol") + script = """import os,time +if os.path.exists("/control/recover"): + while True: time.sleep(2) +else: + chunks=[] + while True: + chunks.append(bytearray(8*1024*1024)) + time.sleep(.02) +""" + name = fixture("externaloom", script, memory="64m", control=vol) + ensure(inspect(name, "{{.HostConfig.RestartPolicy.Name}}") == "no", + "external OOM fixture must have restart policy no") + start(name) + ensure(wait_exit(name, 40) == 137, "kernel OOM did not exit 137") + ensure(state(name)["OOMKilled"] is True, "Docker did not confirm real OOM") + wait_alert(name, "oom", limit=35) + ensure(must_count(name, "crashed", 0) == 0, "OOM produced CRASH") + write_control(vol, "recover") + run(["start", name]) + time.sleep(12) + ensure(must_count(name, "recovered", 0) == 0, "early recovered after external OOM start") + wait_alert(name, "recovered", limit=150) + ensure(must_count(name, "oom", 1) == 1, "repeated OOM notification") + ensure(must_count(name, "recovered", 1) == 1, "repeated RECOVERED") + stable = confirmed_stable_uptime(name) + return "real kernel OOM -> external Docker start -> INFO RECOVERED after " + str(stable) + "s" + def t10_name_reuse(): name = PREFIX + "-reuse" ids = [] @@ -442,7 +556,8 @@ def t13_dfc(): facts["watchdog_alert_count"] = len(sent) facts["dfc_received_count"] = len(received) ensure(sent, "watchdog generated no alert records") - for name in ("crash", "sigkill", "oom", "loop", "reuse", "health"): + for name in (("crash", "sigkill", "oom", "loop", "reuse", "health", "external") + if ISOLATED else ("crash", "sigkill", "oom", "loop", "reuse", "health")): actual = PREFIX + "-" + name local = [x for x in sent if 'container="' + actual + '"' in x] remote = [x for x in received if 'container="' + actual + '"' in x] @@ -460,7 +575,8 @@ def t13_dfc(): 'severity="' + severity + '"' in x for x in received), "DFC missing " + kind + "/" + severity) return str(len(sent)) + " watchdog alerts delivered as " + str(len(received)) + \ - " DFC syslog messages; CRITICAL/HIGH/INFO fields correct" + (" isolated UDP syslog messages" if ISOLATED else " DFC syslog messages") + \ + "; CRITICAL/HIGH/INFO fields correct" def collect(): stop_events() @@ -523,17 +639,42 @@ def cleanup(): try: init() - checked("T01_CLEAN_EXIT", "exit=0 means no alert", t01_clean) - checked("T02_UNEXPECTED_CRASH", "exit=2 sends one CRITICAL crash", t02_crash) - checked("T03_PLANNED_STOP_137", "Docker stop exit 137 is suppressed", t03_planned_stop) - checked("T04_DIRECT_SIGKILL", "SIGKILL 137 is crash, not OOM", t04_direct_kill) - checked("T05_REAL_OOM", "OOMKilled=true sends only CRITICAL OOM", t05_oom) - checked("T06_TO_T09_RESTART_LIFECYCLE", - "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", t06_t08_loop) - checked("T10_NAME_REUSE", "replacement with same name gets new incident", t10_name_reuse) - checked("T11_FAST_REMOVAL", "no event flood, event listener survives", t11_fast_removal) - checked("T12_NATIVE_HEALTHCHECK", "unhealthy -> recovered", t12_healthcheck) - checked("T13_DFC_SYSLOG", "all expected structured alerts reach actual DFC", t13_dfc) + if os.environ.get("SMOKE_ONLY") == "T13_SYSLOG_QUICK": + ensure(ISOLATED, "SMOKE_ONLY quick test requires isolated engine") + checked("T02_UNEXPECTED_CRASH", "real CRITICAL crash", t02_crash) + def quick_syslog(): + lines = require_syslog() + ensure(any('container="' + PREFIX + '-crash"' in x and + 'severity="CRITICAL"' in x for x in lines), + "no matching CRITICAL Syslog packet; raw=" + + (BASE/"syslog-udp-raw.txt").read_text()[:700]) + return "CRITICAL Syslog datagram collected for real Docker crash" + checked("T13_SYSLOG_QUICK", "UDP receiver captures matching CRITICAL record", quick_syslog) + elif os.environ.get("SMOKE_ONLY") in ("T14_EXTERNAL_RESTART", "T15_EXTERNAL_OOM"): + ensure(ISOLATED, "SMOKE_ONLY external case must run on isolated engine") + if os.environ["SMOKE_ONLY"] == "T14_EXTERNAL_RESTART": + checked("T14_EXTERNAL_RESTART", "no recovery until 120 seconds after external start", + t14_external_restart) + else: + checked("T15_EXTERNAL_OOM", "real OOM then manual start requires 120s stability", + t15_external_oom_restart) + else: + checked("T01_CLEAN_EXIT", "exit=0 means no alert", t01_clean) + checked("T02_UNEXPECTED_CRASH", "exit=2 sends one CRITICAL crash", t02_crash) + checked("T03_PLANNED_STOP_137", "Docker stop exit 137 is suppressed", t03_planned_stop) + checked("T04_DIRECT_SIGKILL", "SIGKILL 137 is crash, not OOM", t04_direct_kill) + checked("T05_REAL_OOM", "OOMKilled=true sends only CRITICAL OOM", t05_oom) + checked("T06_TO_T09_RESTART_LIFECYCLE", + "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", t06_t08_loop) + if ISOLATED: + checked("T14_EXTERNAL_RESTART", "manual start after CRASH waits 120s before RECOVERED", + t14_external_restart) + checked("T10_NAME_REUSE", "replacement with same name gets new incident", t10_name_reuse) + checked("T11_FAST_REMOVAL", "no event flood, event listener survives", t11_fast_removal) + checked("T12_NATIVE_HEALTHCHECK", "unhealthy -> recovered", t12_healthcheck) + checked("T13_SYSLOG" if ISOLATED else "T13_DFC_SYSLOG", + "all expected structured alerts reach " + ("test UDP collector" if ISOLATED else "DFC"), + t13_dfc) except Exception as e: issue("PREFLIGHT_OR_FATAL", "observer initializes and tests run", repr(e)) facts["fatal_traceback"] = traceback.format_exc() @@ -564,7 +705,7 @@ finally: lines.append("| " + item["id"] + " | " + item["status"] + " | " + actual[:290] + " |") lines += ["", "## Evidence", "Actual Docker container failures and events; exact release code in an isolated observer.", - "Observer has SMTP disabled; Syslog messages are sent to pre-existing DFC UDP/514.", + "Observer has SMTP disabled; Syslog messages go to " + facts.get("syslog_backend", "receiver") + " UDP/514.", "No synthetic Docker events, mocked exporter metrics, host packages, or Pacemaker changes.", "Other watchdogs and unrelated existing containers are not modified.", "Output files: watchdog.log, docker-events.log, dfc-syslog.txt, observer-state.json, report.json.", diff --git a/tests/test_restart_loop.py b/tests/test_restart_loop.py index f39d8e6..e433d32 100644 --- a/tests/test_restart_loop.py +++ b/tests/test_restart_loop.py @@ -144,13 +144,30 @@ def test_single_auto_restart_recovers_only_after_stability(self): wd._poll_all_containers() self.assertEqual([a.failure_type for a in alerts], ["crashed", "recovered"]) - def test_no_restart_policy_keeps_existing_immediate_recovery(self): + def test_no_restart_policy_crash_requires_stable_uptime(self): + # Real lab: an outside actor restarts Docker policy=no containers. container = FakeContainer(policy="no", uptime=3) wd, alerts = make_watchdog(container) die(wd) wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["crashed"]) + container.attrs["State"]["StartedAt"] = started_ago(130) + wd.state.get(container.name).last_failure_time = time.time() - 130 + wd._poll_all_containers() + wd._poll_all_containers() self.assertEqual([a.failure_type for a in alerts], ["crashed", "recovered"]) + def test_no_restart_policy_oom_requires_stable_uptime(self): + container = FakeContainer(policy="no", uptime=4) + wd, alerts = make_watchdog(container) + wd._handle_event({"Action": "oom", "Actor": {"Attributes": {"name": container.name}}}) + wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["oom"]) + container.attrs["State"]["StartedAt"] = started_ago(130) + wd.state.get(container.name).last_failure_time = time.time() - 130 + wd._poll_all_containers() + self.assertEqual([a.failure_type for a in alerts], ["oom", "recovered"]) + def test_poll_detects_lost_events_from_docker_restart_count(self): container = FakeContainer(count=0) wd, alerts = make_watchdog(container) diff --git a/tests/test_watchdog.py b/tests/test_watchdog.py index 1cf26bd..d6be9c7 100644 --- a/tests/test_watchdog.py +++ b/tests/test_watchdog.py @@ -57,9 +57,14 @@ def test_recovery_disabled_still_clears_open_state(self): w.cfg = {"alert_on_recovery": False} w.state = watchdog.WatchdogState("") w._boot_time = 123 + w._restart_recovery_stability_secs = 120 state = w.state.get("svc") state.alerted_for = "crashed" - w._maybe_alert_recovery(SimpleNamespace(name="svc"), state) + state.last_failure_time = time.time() - 130 + stable_since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 130)) + container = SimpleNamespace(name="svc", attrs={ + "State": {"StartedAt": stable_since}, "RestartCount": 0}) + w._maybe_alert_recovery(container, state) self.assertEqual(state.alerted_for, "") def test_reboot_summary_uses_current_state_not_folded_failures(self): w = object.__new__(watchdog.Watchdog) diff --git a/watchdog.py b/watchdog.py index af69d1d..2779b9d 100644 --- a/watchdog.py +++ b/watchdog.py @@ -1573,17 +1573,16 @@ def _sync_container_identity(self, state: ContainerState, container) -> None: # ── Recovery alert ───────────────────────────────────────────────────────── def _maybe_alert_recovery(self, container, state: ContainerState) -> None: - """Resolve once stable; a brief RUNNING state is not loop recovery.""" - automatic = has_automatic_restart_policy(container) + """Resolve real failures only after sustained Docker uptime.""" started = container_started_timestamp(container) now = time.time() with state.lock: previous = state.alerted_for if not previous or previous == "recovered": return - # Once a restart-loop was reported, changing Docker's restart - # policy must not bypass the stability period. - if previous == "restart-loop" or (automatic and previous in ("crashed", "oom")): + # Even a policy=no container may be restarted by an external actor. + # Never mark a crash, OOM, or loop recovered after a brief RUNNING poll. + if previous in ("restart-loop", "crashed", "oom"): stability = self._restart_recovery_stability_secs # If we cannot prove continuous uptime, don't clear a real incident. if started is None or now - started < stability: From 220d573f84496ba3916dbc3f76f415ef81b4b0b8 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Thu, 15 Oct 2026 15:03:21 +0000 Subject: [PATCH 05/18] fix: isolate destructive release tests and clarify customer alerts --- .github/workflows/ci.yml | 7 + .github/workflows/release-smoke.yml | 88 +++++++++ DEPLOYMENT.md | 11 ++ README.md | 170 ++++-------------- docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md | 9 + docs/RELEASE_NOTES_V1.6.0.md | 54 ++++++ docs/RELEASE_SMOKE_TESTING.md | 67 +++++++ docs/RELEASE_VALIDATION_V1.6.0.md | 9 + tests/release-smoke.sh | 228 +++++++++++++++--------- tests/test_customer_alert_text.py | 88 +++++++++ tests/test_release_smoke_safety.py | 90 ++++++++++ watchdog-config.yaml.example | 2 +- watchdog.py | 31 ++-- 13 files changed, 619 insertions(+), 235 deletions(-) create mode 100644 .github/workflows/release-smoke.yml create mode 100644 docs/RELEASE_NOTES_V1.6.0.md create mode 100644 docs/RELEASE_SMOKE_TESTING.md create mode 100644 tests/test_customer_alert_text.py create mode 100644 tests/test_release_smoke_safety.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60f9293..2badf6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,17 @@ jobs: run: | bash -n install.sh bash -n uninstall.sh + bash -n tests/release-smoke.sh - name: Validate Python syntax run: | python3 -m py_compile watchdog.py + python3 - <<'PYCODE' + from pathlib import Path + script = Path("tests/release-smoke.sh").read_text() + code = script.split("<<'PY'", 1)[1].rsplit("\nPY", 1)[0] + compile(code, "release-smoke", "exec") + PYCODE - name: Install test dependencies run: | diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml new file mode 100644 index 0000000..a0be819 --- /dev/null +++ b/.github/workflows/release-smoke.yml @@ -0,0 +1,88 @@ +name: Release smoke (isolated) + +# Only an explicitly dispatched GitHub-hosted worker runs destructive tests. +# This workflow never connects to CyberController or any shared Docker Engine. +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + real-docker: + name: Real Docker (disposable GitHub runner) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v4 + + - name: Build release candidate on disposable runner + run: docker build --pull=false --tag "watchdog:$(cat VERSION)" . + + - name: Create empty, labelled isolated Docker Engine + shell: bash + run: | + set -euo pipefail + marker="ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "SMOKE_ENGINE_MARKER=${marker}" >> "${GITHUB_ENV}" + docker run -d --name ca-smoke-engine --restart no \ + --privileged --cpus 2 --memory 2g --pids-limit 768 \ + --network bridge -p 127.0.0.1::2375 \ + -e DOCKER_TLS_CERTDIR="" \ + docker:27.5-dind \ + --storage-driver=vfs --iptables=false --ip6tables=false \ + --bridge=none \ + --label "com.radware.container-alert.smoke-engine=${marker}" + port="$(docker port ca-smoke-engine 2375/tcp | tail -1 | awk -F: '{print $NF}')" + test -n "${port}" + echo "DOCKER_HOST=tcp://127.0.0.1:${port}" >> "${GITHUB_ENV}" + for i in $(seq 1 40); do + if DOCKER_HOST="tcp://127.0.0.1:${port}" docker info >/dev/null 2>&1; then + echo "SMOKE_DOCKER_SOCKET_GID=$(docker exec ca-smoke-engine stat -c '%g' /var/run/docker.sock)" >> "${GITHUB_ENV}" + echo "Disposable Engine ready." + exit 0 + fi + sleep 2 + done + echo "Disposable Engine failed to start" >&2 + exit 1 + + - name: Load exact built image into isolated Engine + shell: bash + run: | + set -euo pipefail + DOCKER_HOST=unix:///var/run/docker.sock docker save "watchdog:$(cat VERSION)" | DOCKER_HOST="${DOCKER_HOST}" docker load + DOCKER_HOST="${DOCKER_HOST}" docker image inspect "watchdog:$(cat VERSION)" >/dev/null + + - name: Real Docker acceptance (15 scenarios, no operational access) + shell: bash + env: + SMOKE_RUNNER_KIND: ${{ runner.environment }} + run: | + set -euo pipefail + SMOKE_IMAGE="watchdog:$(cat VERSION)" bash tests/release-smoke.sh + + - name: Real OOM then external restart (full 120-second recovery) + shell: bash + env: + SMOKE_RUNNER_KIND: ${{ runner.environment }} + SMOKE_ONLY: T15_EXTERNAL_OOM + run: | + set -euo pipefail + SMOKE_IMAGE="watchdog:$(cat VERSION)" bash tests/release-smoke.sh + + - name: Upload test reports (test-only synthetic data) + if: always() + uses: actions/upload-artifact@v4 + with: + name: container-alert-release-smoke-${{ github.run_id }}-${{ github.run_attempt }} + path: /tmp/container-alert-smoke-*/ + if-no-files-found: warn + retention-days: 30 + + - name: Remove disposable test Engine + if: always() + shell: bash + run: | + DOCKER_HOST=unix:///var/run/docker.sock docker rm -f ca-smoke-engine 2>/dev/null || true diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index f6588fc..dcf31b8 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -4,6 +4,17 @@ The **CyberController Container Watchdog** is an autonomous monitoring service t This guide covers initial deployment, alert channel configuration, and ongoing operational management of the watchdog service. + +> **v1.6.0 release safety / customer messaging:** Do **not** run destructive +> Docker failure simulations on CyberController, HA nodes or shared operational +> Docker Engines. See the [hosted-only release test procedure](docs/RELEASE_SMOKE_TESTING.md). +> Container Alert notification descriptions are informational and do not direct +> customers to modify Compose YAML, resource limits, restart policies or other +> production settings. Such remediation requires the applicable Radware R&D / +> authorized change process. The optional CRASH `docker logs` command is read-only +> and is intended for authorized diagnostic access. See +> [v1.6.0 changes since 1.5.4](docs/RELEASE_NOTES_V1.6.0.md). + --- ## Table of Contents diff --git a/README.md b/README.md index c41e425..03b8574 100644 --- a/README.md +++ b/README.md @@ -365,151 +365,47 @@ Every alert includes two probe fields that identify exactly how and why the fail ## 7. Testing -The sections below describe optional manual tests that validate the watchdog's detection and alerting paths. Each creates a temporary Docker container and cleans up after itself. +**Production / CyberController safety boundary:** Do **not** simulate crashes, OOM +events, restart loops, or recovery by running `docker kill`, `docker stop`, +`docker restart`, disposable stress containers, or other destructive experiments +on a CyberController node or another shared/operational Docker Engine. +Even supposedly disposable test containers can consume shared resources or +trigger real monitoring, and a nested Docker-in-Docker container remains visible +to the host's normal watchdog. The earlier host-based examples were removed +for this reason. -### OOM Simulation +### Safe verification on an operational host (read-only) -Two options — choose based on whether the host has internet access. - -#### Option A — `polinux/stress` *(requires internet access)* - -> Pulls `polinux/stress` from Docker Hub. - -```bash -# --memory-swap=32m disables swap so the kernel is forced to OOM-kill rather than swap out -# Note: the polinux/stress entrypoint IS the stress binary — do not repeat "stress" in the args -docker run -d --name oom-test --memory=32m --memory-swap=32m polinux/stress \ - --vm 1 --vm-bytes 64M --vm-keep -``` - -```bash -# Clean up -docker rm -f oom-test -docker rmi polinux/stress -``` - -#### Option B — `watchdog:latest` *(no internet required)* - -> Uses the Watchdog image that is already present on the host, avoiding the need to pull the image during deployment. +Authorized operators may review the existing watchdog's current health and +its own recent logs without altering any container: ```bash -docker run -d --name oom-test --memory=32m --memory-swap=32m watchdog:latest \ - python3 -c "x = b'\xff' * (64 * 1024 * 1024)" +docker inspect docker-container-watchdog --format 'state={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}}' +docker logs --tail 50 docker-container-watchdog ``` -```bash -# Clean up — container only (keep the watchdog image) -docker rm -f oom-test -``` - -Docker kills the container with exit code `137` and emits an `oom` event. The watchdog captures memory stats and the last 20 log lines, then dispatches a CRITICAL alert — all within seconds of the kill. - -> **If the container exits cleanly (exit code 0) instead of being OOM-killed**, cgroup memory accounting may be disabled on this host. Verify with: -> ```bash -> docker info 2>&1 | grep -i "memory limit" -> ``` -> `WARNING: No memory limit support` means the `--memory` flag is silently ignored. Enable cgroup memory accounting by adding `cgroup_enable=memory swapaccount=1` to `GRUB_CMDLINE_LINUX` in `/etc/default/grub`, then run `sudo update-grub` and reboot. - ---- - -### Probe Testing - -> **Optional — requires internet access.** These tests pull images from Docker Hub and create temporary containers on the host. -> Each section includes full cleanup instructions (container + image). - -#### HTTP Probe +To validate an alert without changing a running service, use the unit/format +regression tests on a development workstation or the normal pull-request CI. +No failure injection is required for message formatting or severity checks. -```bash -# Start a container where every HTTP path returns 503 -# Note: the full server { } block is required — a bare location { } directive is not valid nginx config -docker run -d --name http-probe-test \ - nginx:alpine \ - sh -c "printf 'server { listen 80; location / { return 503; } }' > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" - -# Verify the container is running: -docker ps | grep http-probe-test +### Destructive real Docker acceptance — disposable hosted runner ONLY -# Wait one poll cycle (60s), then check watchdog detected it: -grep "http-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f http-probe-test -docker rmi nginx:alpine -``` - -During auto-discovery the watchdog sends an HTTP GET to `/-/healthy` on port 80. Nginx returns 503, so the watchdog immediately caches that endpoint as failing and reports unhealthy. After `unhealthy_cycles_threshold` consecutive unhealthy cycles an alert fires with Detection value `HTTP probe failed — error response 503 Service Unavailable, URI http://:80/-/healthy`. - ---- +Run the **Release smoke (isolated)** GitHub Actions workflow on the intended +release branch (`workflow_dispatch`); it uses an ephemeral GitHub-hosted worker +with an empty, specially labelled Docker Engine, separate alert state, and a +temporary UDP Syslog receiver. The smoke runner aborts before creating any +test resource unless it verifies the dedicated hosted worker, exact localhost +test endpoint, unique engine label, and zero existing containers. **There is no +live-host override.** -#### TCP Connect Probe +The [smoke-test procedure](docs/RELEASE_SMOKE_TESTING.md) details acceptance +criteria, evidence capture, cleanup, and use of the workflow. The normal PR CI +runs only non-destructive Python, shell and build validations; destructive Docker +scenarios require manual dispatch of the separate hosted workflow. -```bash -# Expose port 9999 but run nothing on it — TCP connect will be refused -docker run -d --name tcp-probe-test \ - --expose 9999 \ - alpine \ - sh -c "sleep infinity" - -# Watch the watchdog fall through HTTP → TCP and alert: -grep "tcp-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f tcp-probe-test -docker rmi alpine -``` - -After `unhealthy_cycles_threshold` consecutive failed cycles an alert fires with Detection value `TCP probe failed — connection refused or timed out on :9999`. - ---- - -#### `/proc` Alive Check - -```bash -# No EXPOSE — watchdog falls straight to /proc alive check -docker run -d --name proc-probe-test \ - alpine \ - sleep infinity - -# Verify the container is running: -docker ps | grep proc-probe-test - -# After one poll cycle, confirm the watchdog selected /proc as the probe type: -grep "proc-probe-test" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log - -# Clean up -docker rm -f proc-probe-test -docker rmi alpine -``` - -With no exposed ports, auto-discovery skips HTTP and TCP and falls back to reading `/proc/1/status` inside the container via `exec_run`. Since `sleep` is PID 1 in a live `S` (sleeping) state, no alert fires. The log will show `falling back to /proc alive-check` confirming the probe type was selected. In production, this probe fires when PID 1 enters zombie (`Z`) or stopped (`T`) state. - ---- - -### Recovery Alert Test - -Validates the INFO `recovered` alert fires once a previously-alarmed container is healthy again. Use any real, already-monitored container — no throwaway container needed. - -```bash -# 1. Force a failure (SIGKILL → non-zero exit code → CRITICAL "crashed" alert) -docker kill - -# 2. Confirm the crashed alert was logged/dispatched -grep "" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log | grep CRITICAL - -# 3. Bring it back -docker start - -# 4. Wait for it to report healthy again, then confirm the recovery alert: -grep "" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log | grep -i recovered -``` - -Expected log line: `ALERT [INFO] — recovered (channels: [...])`. - -- Containers with a Docker `HEALTHCHECK` recover via the `health_status: healthy` **event** — near-instant once Docker reports healthy (subject to the image's own `start_period`/`interval`). -- Containers on HTTP/TCP/`/proc` active probes recover on the **next poll cycle** (`check_interval_seconds`). -- No alert fires if `alert_on_recovery: false` is set, or if the container was never in an alarmed state (`state.alerted_for` empty). - ---- +Do not start a test Engine on CyberController or m920q. The previous 2026-10-14 +m920q Docker-in-Docker test inadvertently generated real host watchdog alerts; +that approach must not be repeated. ## 8. Troubleshooting @@ -648,13 +544,13 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor ## 10. Version History -**Release status (2026-10-14):** `VERSION` is set to `1.6.0` for the release candidate. Real HA failover/failback and the recovery-fixed image passed release acceptance tests; see [final validation](docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). PR #9 remains open, and the final Git tag, GitHub Release, and offline package have not been published. The HA labs still run the earlier v1.6.0 candidate; m920q continues to run `watchdog:1.5.4`. +**Release status (2026-10-15):** `VERSION` remains `1.6.0` for the **upcoming, untagged** release candidate. PR #9 is open. The complete changes since 1.5.4, approved customer-facing alert wording, release acceptance and destructive-test isolation are documented in [v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). Historical HA and isolated Docker tests passed on earlier candidates; the final hosted-runner safety workflow must pass against the latest commit before publication. No v1.6.0 tag or final offline package is published; deployed lab images are not automatically updated by this branch. **Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-14 | Radware engineering | [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6): Docker event ID resolution and bounded missing-container warnings; [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5): authoritative `RestartCount`, deduplicated restart-loop incidents, stable recovery; [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7): planned Docker stop correlation and corrected exit-137 handling; [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4): optional independent HA active-node change alerts; PR #10: corrected notification consistency and Syslog priorities; release validation: require stable uptime before CRASH/OOM recovery, including manual/external container restarts. | +| 1.6.0 | 2026-10-15 | Radware engineering | Upcoming since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md index 9e8369f..d473396 100644 --- a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md +++ b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md @@ -1,5 +1,14 @@ # Container Alert v1.6.0 - Final release-candidate acceptance + +> **Historical acceptance evidence (October 14, 2026), not current test instructions.** +> These records predate the v1.6.0 hosted-only smoke-test safety guard and final +> customer-facing message updates. Do **not** rerun their Docker commands on +> m920q, CyberController or another operational Engine. The prior outer +> Docker-in-Docker test container triggered genuine m920q watchdog alerts. +> See [current release notes](RELEASE_NOTES_V1.6.0.md) and +> [safe smoke workflow](RELEASE_SMOKE_TESTING.md). + **Validation date:** October 14, 2026 (UTC) **Release PR:** [#9](https://github.com/Radware/CyberController-Container-Alert/pull/9) **Source basis:** release branch including merged [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10), plus the targeted recovery fix in this PR diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md new file mode 100644 index 0000000..b6a517c --- /dev/null +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -0,0 +1,54 @@ +# v1.6.0 Release — Changes Since 1.5.4 + +**Status:** Release candidate, not yet tagged or published. Version remains `1.6.0` in `VERSION`. The [release PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) targets `main`. This file documents everything included in the upcoming release relative to v1.5.4, not a claim that an image or deployment has been published. + +## Feature and fix inventory + +| Area | Change since 1.5.4 | Change record | +|---|---|---| +| Baseline reliability and installation hardening | Serialize state persistence under concurrent event/poll writers; make manual exec probes bounded; consolidate post-reboot alert summaries; secure runtime config file as root-owned and group-readable; verify post-install health; improve image-archive validation and uninstall so unrelated tags are preserved | [PR #3](https://github.com/Radware/CyberController-Container-Alert/pull/3) | +| Docker event consistency | Resolve event identity by Docker ID; prevent historical names from corrupting a new container with the same name; rate-limit removed-container warnings; protect event processing from removed IDs | [PR #6](https://github.com/Radware/CyberController-Container-Alert/pull/6) | +| Restart-loop lifecycle | Count restarts using Docker's authoritative `RestartCount`; deduplicate CRASH / RESTART-LOOP notifications, prevent health checks from replacing active loop incidents; require sustained uptime before RECOVERED | [PR #5](https://github.com/Radware/CyberController-Container-Alert/pull/5) | +| Planned container shutdown | Correlate Docker stop intent with late / reversed `die` events, suppress false CRASH for legitimate Docker stops; distinguish SIGKILL exit 137 from confirmed kernel OOM; protect stop evidence across container starts | [PR #7](https://github.com/Radware/CyberController-Container-Alert/pull/7) | +| Optional CyberController HA change alert | Detect promoted/active HA Operator from existing Pacemaker exporter, not the Pacemaker DC. Separate HA polling from Docker; quiet baseline/rebaseline; only new active owner sends one WARNING on real failover/failback. Reuses existing notifications, state persistence and SNMP MIB | [PR #4](https://github.com/Radware/CyberController-Container-Alert/pull/4) | +| Release documentation | Clarify merged vs unreleased features and explicit image version/tag process | [PR #8](https://github.com/Radware/CyberController-Container-Alert/pull/8) | +| Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | +| Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Installer source fallback | Source-build mode copies both `lifecycle.py` and `ha_monitor.py` alongside `watchdog.py` and Dockerfile; normal versioned offline archive flow retained | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | + +## Customer-facing message catalogue (v1.6.0) + +These strings are informational, not instructions to change production configuration. The existing incident severity and detection evidence remain intact. + +| Event | Severity | Context text | +|---|---|---| +| `crashed` | CRITICAL | Docker reported an unexpected container termination. Run: `docker logs --tail 50`. The exit code and detection details provide additional context. | +| `oom` | CRITICAL | Docker reported that the container was terminated due to an out-of-memory condition. | +| `unhealthy` | HIGH | The container is running, but a monitoring health check is failing. | +| `restart-loop` | HIGH | The container has restarted repeatedly within the configured monitoring window. This pattern may be associated with application errors, resource pressure, or unavailable dependencies. The underlying cause has not been determined. | +| `recovered` | INFO | The previously reported alert condition has cleared according to the available monitoring checks. | +| `reboot-summary` | INFO or HIGH | Monitoring has resumed following a host reboot. The reported summary includes container status, recovered incidents, and any conditions that remain unresolved. | +| `ha-failover` | WARNING | Verify whether the HA switchover was expected and check service health. **Unchanged.** | +| Unknown type | WARNING | A container monitoring event was reported. The failure type and available detection details provide additional context. | + +The CRASH read-only logs command is intended for authorized diagnostic access. Customers must not modify Docker Compose, memory limits, restart policies, or production resources based solely on an alert. Configuration remediation is determined by Radware R&D or its authorized operational process. + +**Compatibility:** No change to machine event identifiers, severities, HA identities, container IDs, OIDs/var-binds, probe details, or Syslog key `action`. Only human-facing text and email/Slack label `Context` change. No new alert transport, background service or message queue. + +## Verification and safety boundaries + +- Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. +- Destructive real Docker failure testing is **only** through the manually dispatched [Release smoke (isolated)](../.github/workflows/release-smoke.yml) workflow on an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](RELEASE_SMOKE_TESTING.md). +- Historical test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md), [earlier live HA-lab findings](RELEASE_VALIDATION_V1.6.0.md), and JSON files in this directory. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. +- During previous m920q nested-engine testing, the outer test container was visible to the host's normal watchdog and caused genuine HIGH/CRITICAL notifications. This is why the nested Engine is now allowed **only on a disposable hosted CI machine** and never on m920q or CyberController. +- Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. + +## Exclusions, limitations and release gating + +**Not established by testing:** live customer SMTP, Slack and SNMP delivery end-to-end; arbitrary host reboot, network partition, STONITH, or production recovery policy. SMTP was explicitly excluded during lab acceptance. HA failover/failback was tested on real lab nodes with a prior source candidate, without any further failover required for text-only changes. + +The HA notification dispatch path has no persistent retry queue (intentional v1 limitation). Docker planned-stop correlation is not an availability policy; a deliberately stopped long-running service does not automatically produce a downtime alert. Unexpected external Docker restarts were seen on a lab; the initiator was not determined. These are documented operational limitations, not release regressions. + +**Before shipping:** PR #9 CI green, separate manual hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, package signed/verified artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. None of those final publish steps is automatic. diff --git a/docs/RELEASE_SMOKE_TESTING.md b/docs/RELEASE_SMOKE_TESTING.md new file mode 100644 index 0000000..59e0253 --- /dev/null +++ b/docs/RELEASE_SMOKE_TESTING.md @@ -0,0 +1,67 @@ +# Release Smoke Tests — Mandatory Isolation + +**Scope:** Container Alert v1.6.0 release validation. The 15 core cases and a separate true kernel-OOM/manual-restart test use real Docker events but must never access an operational or shared Docker Engine. + +## Safety rule + +**NEVER run `tests/release-smoke.sh` on CyberController, its HA nodes, m920q, a customer system, or another shared machine.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. + +Instead run the workflow [Release smoke (isolated)](../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to CyberController or m920q. All test-created resources are disposable and contained within that hosted worker. + +The workflow has no corporate customer credentials, no HA exporter dependency, no CyberController Docker socket, no external notification destinations, and no network access from the disposable failure fixtures. + +## Launch + +1. Review [release PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) and select its current `release/v1.6.0` branch. +2. In GitHub Actions select **Release smoke (isolated)** → **Run workflow** and choose that branch. Alternatively an authorized repository maintainer can run: + + ```bash + gh workflow run release-smoke.yml --ref release/v1.6.0 + ``` + +3. Confirm the workflow's commit SHA matches the release PR head. It builds the watchdog image on the **hosted runner only**, loads it into a freshly created nested Docker Engine, and checks that the image files match the checked-out source. +4. Download the attached structured evidence artifact (Docker events, alerts, raw UDP receiver packets, state snapshot, run reports). The workflow runs both the full acceptance suite and the additional kernel OOM/manual restart scenario. +5. All cases must PASS; the final cleanup must report zero test-labelled containers and volumes. Keep the actual workflow URL/commit in the release approval record. + +## Automatic fail-closed checks + +Before even inspecting an image or creating a test resource, the smoke runner requires all of the following: + +- GitHub Actions execution on a GitHub-hosted runner (`GITHUB_ACTIONS=true` and `SMOKE_RUNNER_KIND=github-hosted`). +- An explicit localhost-only TCP Docker Engine endpoint `DOCKER_HOST=tcp://127.0.0.1:`; no `DOCKER_CONTEXT` override. +- A unique run marker matching the Docker **daemon-side** label `com.radware.container-alert.smoke-engine`. +- **Zero existing containers** on the target Docker Engine. + +The runner has **no live-host override**. If preflight fails it must exit FAIL **without calling Docker to create/inspect any application containers, and without performing cleanup commands against the unverified Engine**. + +Within the verified Engine, each fixture uses unique names and a per-run ownership label. Cleanup re-verifies ownership before removal and fails if a command fails or a container/volume is left over. The fast-removal test verifies real exit codes and actual Docker `die` events, not only observer liveness. + +## Acceptance criteria + +| Scenario | Expected | +|---|---| +| Clean exit | Exit 0 → no alert | +| Nonzero crash | One CRITICAL CRASH | +| Planned forced stop | Docker-requested stop/137 → no false CRASH or OOM | +| Direct SIGKILL | CRITICAL CRASH, not kernel OOM | +| Genuine cgroup OOM | `OOMKilled=true`, exactly one CRITICAL OOM | +| Restart loop | One CRASH → one HIGH RESTART-LOOP; 5 actual restarts | +| Brief or interrupted recovery | No early RECOVERED; actual new failure resets stability | +| Stable recovery | Exactly one INFO RECOVERED after configured uptime | +| Manual restart after crash | Policy `no`, no premature RECOVERED | +| Manual restart after real OOM | Policy `no`, actual kernel OOM and 120-second stability gate | +| Container identity reuse | New Docker ID with old name does not inherit old state | +| Rapid auto-removal | All short-lived test fixtures actually exited, no listener crash or warning flood | +| Native Docker HEALTHCHECK | HIGH UNHEALTHY → INFO RECOVERED | +| Syslog | Locally generated structured alerts received exactly once as real UDP Syslog | +| Cleanup | No leftover test-labelled containers/volumes, no unauthorized modifications | + +## Important historical correction + +An earlier 2026-10-14 validation ran a nested Engine on **m920q**. Its outer `ca-v160-isolated-engine` container was monitored by the existing `watchdog:1.5.4` and generated HIGH UNHEALTHY and CRITICAL OOM alerts on the **normal SMTP channel**. That test Engine was subsequently removed, but its existence demonstrates that nesting alone is **not production isolation**. Do not repeat that method. + +Earlier acceptance JSON and reports remain in `docs/` as traceability, not authorization to rerun those procedures on operational Docker hosts. + +## Operator verification (read-only only) + +On an installed system, it is acceptable for authorized personnel to review the watchdog's own health, log records and configured alert destinations. It is **not** acceptable to intentionally kill/restart a service to "test notifications." Real failure injection occurs exclusively through the disposable hosted workflow. diff --git a/docs/RELEASE_VALIDATION_V1.6.0.md b/docs/RELEASE_VALIDATION_V1.6.0.md index b85fb87..cdd6d74 100644 --- a/docs/RELEASE_VALIDATION_V1.6.0.md +++ b/docs/RELEASE_VALIDATION_V1.6.0.md @@ -1,5 +1,14 @@ # Container Alert v1.6.0 — Release Acceptance Validation + +> **Historical acceptance evidence (October 14, 2026), not current test instructions.** +> These records predate the v1.6.0 hosted-only smoke-test safety guard and final +> customer-facing message updates. Do **not** rerun their Docker commands on +> m920q, CyberController or another operational Engine. The prior outer +> Docker-in-Docker test container triggered genuine m920q watchdog alerts. +> See [current release notes](RELEASE_NOTES_V1.6.0.md) and +> [safe smoke workflow](RELEASE_SMOKE_TESTING.md). + > **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on m920q, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). Retain this document as evidence of the original findings, not the current release verdict. **Date:** October 14, 2026 (UTC) diff --git a/tests/release-smoke.sh b/tests/release-smoke.sh index 0eb6893..7b4171a 100755 --- a/tests/release-smoke.sh +++ b/tests/release-smoke.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash # Container Alert v1.6.0: real Docker engine smoke tests, without host installations. -# Run on an authorized lab admin client with docker CLI, Python 3, a local DFC syslog receiver. -# Creates only clearly labelled disposable containers and Docker named volumes. +# Destructive tests are allowed ONLY on an empty, labelled Docker Engine launched +# on an ephemeral GitHub-hosted runner. Never use CyberController or shared hosts. +# Preflight fails closed before creating any containers/volumes. set -euo pipefail +SMOKE_SOURCE_DIR="$(cd "$(dirname "$0")/.." && pwd -P)" +export SMOKE_SOURCE_DIR exec python3 - "$@" <<'PY' import datetime as dt +import hashlib import json import os import pathlib +from pathlib import Path import re import subprocess import sys @@ -19,8 +24,8 @@ EXPECTED_REV = os.environ.get("SMOKE_EXPECTED_REV", "") TAG = dt.datetime.now(dt.timezone.utc).strftime("%y%m%d%H%M%S") + str(os.getpid())[-3:] PREFIX = "ca-smoke-" + TAG LABEL = "container-alert.smoke=" + TAG -ISOLATED = os.environ.get("SMOKE_ISOLATED", "") == "1" -RECEIVER = PREFIX + "-syslog-receiver" if ISOLATED else "config_dfc_1" +ISOLATED = True # no live-engine execution path exists +RECEIVER = PREFIX + "-syslog-receiver" HOST = "release-smoke-" + TAG OBS = PREFIX + "-observer" STATE_VOL = PREFIX + "-state" @@ -28,6 +33,7 @@ BASE = pathlib.Path(os.environ.get("SMOKE_OUTPUT_DIR", "/tmp")) / ("container-al START = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") containers, volumes, results, facts = [], [], [], {} EVENT_PROC, EVENT_FILE = None, None +ENGINE_VERIFIED = False BASE.mkdir(parents=True, exist_ok=False) start_mono = time.monotonic() @@ -76,6 +82,38 @@ def run(argv, timeout=30, check=True): def log(s): print(s, flush=True) +def verify_engine_is_test_only(): + """Never issue any mutating Docker command without independent safeguards. + + A non-default, localhost-only endpoint is necessary but NOT sufficient: + require an ephemeral GitHub-hosted worker, a unique daemon-side test label, + and a completely empty target Engine. Do not support live-host opt-in flags. + """ + global ENGINE_VERIFIED + ensure(os.environ.get("GITHUB_ACTIONS") == "true" and + os.environ.get("SMOKE_RUNNER_KIND") == "github-hosted", + "Refusing destructive tests outside an ephemeral GitHub-hosted runner") + ensure(not os.environ.get("DOCKER_CONTEXT"), "DOCKER_CONTEXT could override DOCKER_HOST") + endpoint = os.environ.get("DOCKER_HOST", "") + ensure(re.fullmatch(r"tcp://127\.0\.0\.1:[0-9]{2,5}", endpoint) is not None, + "DOCKER_HOST must be the isolated localhost Docker TCP endpoint") + marker = os.environ.get("SMOKE_ENGINE_MARKER", "") + ensure(re.fullmatch(r"ci-[0-9]+-[0-9]+", marker) is not None, + "Missing unique CI test-engine marker") + info = json.loads(run(["info", "--format", "{{json .}}"], timeout=12)) + expected_label = "com.radware.container-alert.smoke-engine=" + marker + ensure(expected_label in (info.get("Labels") or []), + "Target Docker Engine is not the labelled disposable test Engine") + ensure(info.get("Containers") == 0, + "Refusing Engine with pre-existing containers: " + str(info.get("Containers"))) + ENGINE_VERIFIED = True + facts["isolated_engine_id"] = info.get("ID") + facts["isolated_engine_label"] = expected_label + facts["runner_kind"] = "github-hosted" + log("ISOLATION_VERIFIED " + str(info.get("ID"))) + + + def mark(name, expected, evidence): evidence = str(evidence) elapsed = round(time.monotonic() - start_mono, 1) @@ -186,47 +224,34 @@ def wait_restarts(name, minimum, limit=120): raise AssertionError("restart count did not reach " + str(minimum)) def require_syslog(): - # Keep the isolated collector's raw received packets as separate evidence. - if ISOLATED: - data = run(["exec", RECEIVER, "cat", "/var/log/syslog"], timeout=20) - (BASE/"syslog-udp-raw.txt").write_text(data+"\n") - else: - data = run(["exec", RECEIVER, "sh", "-c", - "grep -F " + "'host=" + '"' + HOST + '"' + "'" + - " /var/log/syslog || true"], timeout=20, check=False) - return [x for x in data.splitlines() - if 'host="' + HOST + '"' in x and 'event="watchdog_alert"' in x] + raw = run(["exec", RECEIVER, "cat", "/var/log/syslog"], timeout=20) + (BASE/"syslog-udp-raw.txt").write_text(raw + "\n") + return [line for line in raw.splitlines() + if 'host="' + HOST + '"' in line and 'event="watchdog_alert"' in line] + def init(): - if not ISOLATED: - ensure(run(["inspect", "--format", "{{.State.Health.Status}}", - "docker-container-watchdog"]) == "healthy", "existing watchdog not healthy") + # verify_engine_is_test_only() has already established the trust boundary. revision = run(["image", "inspect", "--format", '{{index .Config.Labels "org.opencontainers.image.revision"}}', IMAGE]) if EXPECTED_REV: ensure(revision == EXPECTED_REV, "wrong image source revision " + revision) image_id = run(["image", "inspect", "--format", "{{.Id}}", IMAGE]) - embedded = run(["run", "--rm", "--network", "none", "--entrypoint", "sha256sum", - IMAGE, "/app/watchdog.py", "/app/lifecycle.py", "/app/ha_monitor.py"]) + sources = Path(os.environ["SMOKE_SOURCE_DIR"]) expected_sha = { - "/app/watchdog.py": os.environ.get( - "SMOKE_EXPECTED_WATCHDOG_SHA256", - "4df8451b496a2935eba17fd56cc87459c87b4f96d9c2ace5bee9161fdf2c60fa"), - "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", - "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd", + "/app/" + name: hashlib.sha256((sources/name).read_bytes()).hexdigest() + for name in ("watchdog.py", "lifecycle.py", "ha_monitor.py") } + embedded = run(["run", "--rm", "--network", "none", "--entrypoint", "sha256sum", + IMAGE, *expected_sha.keys()]) actual_sha = {line.split()[-1]: line.split()[0] for line in embedded.splitlines()} ensure(actual_sha == expected_sha, "release image source SHA mismatch: " + repr(actual_sha)) facts["embedded_source_sha256"] = actual_sha - if not ISOLATED: - ensure(RECEIVER in run(["ps", "--format", "{{.Names}}"]).splitlines(), - "DFC syslog receiver not running") existing = run(["ps", "-a", "--format", "{{.Names}}"]).splitlines() ensure(not any(n.startswith(PREFIX) for n in existing), "smoke prefix collision") ensure(not run(["volume", "ls", "--filter", "label=" + LABEL, "-q"]), "old test volumes") - if ISOLATED: - collector = make_volume("syslogdata") - udp_code = """import socket + collector = make_volume("syslogdata") + udp_code = """import socket s=socket.socket(socket.AF_INET,socket.SOCK_DGRAM) s.bind(("0.0.0.0",514)) with open("/var/log/syslog","a",buffering=1) as output: @@ -235,22 +260,22 @@ with open("/var/log/syslog","a",buffering=1) as output: packet,source=s.recvfrom(65535) output.write(packet.decode(errors="replace")+"\\n") """ - run(["run", "-d", "--name", RECEIVER, "--label", LABEL, "--network", "host", - "--restart", "no", "--memory", "64m", "--pids-limit", "32", - "--volume", collector + ":/var/log", "--user", "0:0", - "--entrypoint", "python3", IMAGE, "-u", "-c", udp_code]) - containers.append(RECEIVER) - receiver_logs = "" - for _ in range(18): - time.sleep(1) - receiver_logs = run(["logs", RECEIVER], check=False) - if "SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs: - break - if inspect(RECEIVER, "{{.State.Status}}") == "exited": - break - ensure("SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs, - "isolated UDP syslog collector failed; state=" + - inspect(RECEIVER, "{{.State.Status}}") + ", log=" + receiver_logs[-1000:]) + run(["run", "-d", "--name", RECEIVER, "--label", LABEL, "--network", "host", + "--restart", "no", "--memory", "64m", "--pids-limit", "32", + "--volume", collector + ":/var/log", "--user", "0:0", + "--entrypoint", "python3", IMAGE, "-u", "-c", udp_code]) + containers.append(RECEIVER) + receiver_logs = "" + for _ in range(18): + time.sleep(1) + receiver_logs = run(["logs", RECEIVER], check=False) + if "SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs: + break + if inspect(RECEIVER, "{{.State.Status}}") == "exited": + break + ensure("SMOKE_SYSLOG_RECEIVER_READY" in receiver_logs, + "isolated UDP syslog collector failed; state=" + + inspect(RECEIVER, "{{.State.Status}}") + ", log=" + receiver_logs[-1000:]) vol = make_volume("state") excludes = sorted(set(existing + [OBS])) config = { @@ -275,7 +300,8 @@ with open("/var/log/syslog","a",buffering=1) as output: "-e", "SMOKE_CONFIG=" + json.dumps(config), "--volume", vol + ":/data", "--entrypoint", "python3", IMAGE, "-c", setup], timeout=30) - gid = int(os.environ.get("SMOKE_DOCKER_SOCKET_GID", os.stat("/var/run/docker.sock").st_gid)) + gid_value = os.environ.get("SMOKE_DOCKER_SOCKET_GID") + gid = int(gid_value) if gid_value is not None else os.stat("/var/run/docker.sock").st_gid arm_events() args = ["run", "-d", "--name", OBS, "--label", LABEL, "--restart", "no", "--label", "com.docker.compose.project=release-smoke", @@ -305,7 +331,7 @@ with open("/var/log/syslog","a",buffering=1) as output: "syslog_source_host": HOST, "observer_poll_seconds": 2, "restart_threshold": 5, "recovery_stability_seconds": 120, "excluded_existing_containers": len(existing), - "syslog_backend": "isolated UDP test collector" if ISOLATED else "real DFC", + "syslog_backend": "isolated UDP test collector", "docker_engine_endpoint": os.environ.get("DOCKER_HOST", "default")}) log("REAL_DOCKER_TEST_READY " + HOST + " image=" + image_id[:24]) @@ -515,13 +541,24 @@ def t11_fast_removal(): names = [] for k in range(3): name = PREFIX + "-rm" + str(k) - run(["run", "--rm", "--name", name, "--label", LABEL, - "--network", "none", "--memory", "128m", "--memory-swap", "128m", - "--pids-limit", "64", "--cpus", "0.2", "--cap-drop", "ALL", - "--entrypoint", "python3", IMAGE, "-c", "raise SystemExit(2)"], - timeout=30, check=False) + argv = ["docker", "run", "--rm", "--name", name, "--label", LABEL, + "--network", "none", "--memory", "128m", "--memory-swap", "128m", + "--pids-limit", "64", "--cpus", "0.2", "--cap-drop", "ALL", + "--entrypoint", "python3", IMAGE, "-c", "raise SystemExit(2)"] + result = subprocess.run(argv, capture_output=True, text=True, timeout=30) + ensure(result.returncode == 2, + "Fast removal fixture did not exit 2: " + name + + ", actual=" + str(result.returncode) + ", stderr=" + result.stderr[-500:]) names.append(name) time.sleep(8) + observed = [json.loads(line) for line in + (BASE/"docker-events-live.jsonl").read_text().splitlines() if line.strip()] + for name in names: + ensure(any(ev.get("Action") == "die" and + ev.get("Actor", {}).get("Attributes", {}).get("name") == name and + ev.get("Actor", {}).get("Attributes", {}).get("exitCode") == "2" + for ev in observed), + "Missing genuine Docker die event / exit 2: " + name) wd_logs = run(["logs", "--since", START, OBS], timeout=20) misses = [x for x in wd_logs.splitlines() if "already removed before" in x and any(n in x for n in names)] @@ -556,8 +593,7 @@ def t13_dfc(): facts["watchdog_alert_count"] = len(sent) facts["dfc_received_count"] = len(received) ensure(sent, "watchdog generated no alert records") - for name in (("crash", "sigkill", "oom", "loop", "reuse", "health", "external") - if ISOLATED else ("crash", "sigkill", "oom", "loop", "reuse", "health")): + for name in ("crash", "sigkill", "oom", "loop", "reuse", "health", "external"): actual = PREFIX + "-" + name local = [x for x in sent if 'container="' + actual + '"' in x] remote = [x for x in received if 'container="' + actual + '"' in x] @@ -575,8 +611,7 @@ def t13_dfc(): 'severity="' + severity + '"' in x for x in received), "DFC missing " + kind + "/" + severity) return str(len(sent)) + " watchdog alerts delivered as " + str(len(received)) + \ - (" isolated UDP syslog messages" if ISOLATED else " DFC syslog messages") + \ - "; CRITICAL/HIGH/INFO fields correct" + " isolated UDP syslog messages; CRITICAL/HIGH/INFO fields correct" def collect(): stop_events() @@ -618,26 +653,53 @@ def collect(): pass def cleanup(): + """Remove only resources created by this run and verify none remain.""" + errors = facts.setdefault("cleanup_errors", []) for n in reversed(containers): - if n == OBS: + if not n.startswith(PREFIX): + errors.append("Refusing to remove non-test container " + n) continue try: - run(["rm", "-f", n], timeout=30, check=False) - except Exception as e: - facts.setdefault("cleanup_errors", []).append(n + ": " + str(e)) - if OBS in containers: - try: - run(["stop", "-t", "12", OBS], timeout=25, check=False) - run(["rm", "-f", OBS], timeout=25, check=False) - except Exception as e: - facts.setdefault("cleanup_errors", []).append(OBS + ": " + str(e)) + label = run(["inspect", "--format", + '{{index .Config.Labels "container-alert.smoke"}}', + n], check=False) + if not label: + continue # Already removed by the test. + if label != TAG: + errors.append("Refusing mismatched test label on " + n + ": " + label) + continue + run(["rm", "-f", n], timeout=30) + except Exception as exc: + errors.append(n + ": " + str(exc)) for vol in reversed(volumes): + if not vol.startswith(PREFIX): + errors.append("Refusing to remove non-test volume " + vol) + continue try: + label = run(["volume", "inspect", "--format", + '{{index .Labels "container-alert.smoke"}}', + vol], check=False) + if not label: + continue + if label != TAG: + errors.append("Refusing mismatched test volume label " + vol) + continue run(["volume", "rm", vol], timeout=30) - except Exception as e: - facts.setdefault("cleanup_errors", []).append(vol + ": " + str(e)) + except Exception as exc: + errors.append(vol + ": " + str(exc)) + for kind, command in ( + ("containers", ["ps", "-a", "--filter", "label=" + LABEL, "-q"]), + ("volumes", ["volume", "ls", "--filter", "label=" + LABEL, "-q"]), + ): + try: + leftover = run(command) + if leftover: + errors.append("Leftover test " + kind + ": " + leftover) + except Exception as exc: + errors.append("Cannot verify cleanup of " + kind + ": " + str(exc)) try: + verify_engine_is_test_only() init() if os.environ.get("SMOKE_ONLY") == "T13_SYSLOG_QUICK": ensure(ISOLATED, "SMOKE_ONLY quick test requires isolated engine") @@ -666,31 +728,33 @@ try: checked("T05_REAL_OOM", "OOMKilled=true sends only CRITICAL OOM", t05_oom) checked("T06_TO_T09_RESTART_LIFECYCLE", "CRASH -> RESTART-LOOP -> stability -> RECOVERED, interrupted recovery", t06_t08_loop) - if ISOLATED: - checked("T14_EXTERNAL_RESTART", "manual start after CRASH waits 120s before RECOVERED", - t14_external_restart) + checked("T14_EXTERNAL_RESTART", "manual start after CRASH waits 120s before RECOVERED", + t14_external_restart) checked("T10_NAME_REUSE", "replacement with same name gets new incident", t10_name_reuse) checked("T11_FAST_REMOVAL", "no event flood, event listener survives", t11_fast_removal) checked("T12_NATIVE_HEALTHCHECK", "unhealthy -> recovered", t12_healthcheck) - checked("T13_SYSLOG" if ISOLATED else "T13_DFC_SYSLOG", - "all expected structured alerts reach " + ("test UDP collector" if ISOLATED else "DFC"), + checked("T13_SYSLOG", "all expected structured alerts reach test UDP collector", t13_dfc) except Exception as e: issue("PREFLIGHT_OR_FATAL", "observer initializes and tests run", repr(e)) facts["fatal_traceback"] = traceback.format_exc() finally: - try: - collect() - except Exception as e: - facts["collect_error"] = repr(e) - finally: - stop_events() - cleanup() + if ENGINE_VERIFIED: + try: + collect() + except Exception as e: + facts["collect_error"] = repr(e) + finally: + stop_events() + cleanup() finished = dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00","Z") facts["total_seconds"] = round(time.monotonic() - start_mono, 1) data = {"run_id": TAG, "started_utc": START, "finished_utc": finished, "status": ("FAIL" if any(x["status"] == "FAIL" for x in results) - or facts.get("cleanup_errors") else "PASS"), + or facts.get("cleanup_errors") + or facts.get("collect_error") + or facts.get("syslog_read_error") + or not ENGINE_VERIFIED else "PASS"), "image": IMAGE, "results": results, "facts": facts} (BASE/"report.json").write_text(json.dumps(data, indent=2, default=str) + "\n") lines = ["# Container Alert v1.6.0 real Docker smoke test", diff --git a/tests/test_customer_alert_text.py b/tests/test_customer_alert_text.py new file mode 100644 index 0000000..8269924 --- /dev/null +++ b/tests/test_customer_alert_text.py @@ -0,0 +1,88 @@ +"""Customer alert wording and channel consistency for v1.6.0 (no Docker/network).""" +import unittest +from unittest.mock import patch +from email.parser import BytesParser +from email.policy import default + +import watchdog + + +class CustomerAlertTextTests(unittest.TestCase): + EXPECTED = { + "crashed": "Docker reported an unexpected container termination. Run: docker logs --tail 50. The exit code and detection details provide additional context.", + "oom": "Docker reported that the container was terminated due to an out-of-memory condition.", + "unhealthy": "The container is running, but a monitoring health check is failing.", + "restart-loop": "The container has restarted repeatedly within the configured monitoring window. This pattern may be associated with application errors, resource pressure, or unavailable dependencies. The underlying cause has not been determined.", + "recovered": "The previously reported alert condition has cleared according to the available monitoring checks.", + "reboot-summary": "Monitoring has resumed following a host reboot. The reported summary includes container status, recovered incidents, and any conditions that remain unresolved.", + } + + def test_all_approved_messages_exact_and_informational(self): + self.assertEqual(set(watchdog.ACTIONS), set(self.EXPECTED)) + for reason, message in self.EXPECTED.items(): + with self.subTest(reason=reason): + self.assertEqual(watchdog.ACTIONS[reason][1], message) + self.assertNotIn("docker-compose.yaml", message) + self.assertNotIn("Increase memory limit", message) + self.assertNotIn("fix application crash", message) + self.assertIn("Run: docker logs --tail 50", self.EXPECTED["crashed"]) + + def test_fallback_is_informative_and_preserves_warning(self): + container = type("Container", (), { + "name": "dummy", "id": "abc123abc123", + "logs": lambda self, tail, timestamps: b"", + })() + payload = watchdog.build_payload(container, "new-unknown-type", + "test-host", "https://example.invalid") + self.assertEqual(payload.severity, "WARNING") + self.assertEqual( + payload.recommended_action, + "A container monitoring event was reported. The failure type and available detection details provide additional context.") + + def test_crash_replaces_name_and_syslog_preserves_action_key(self): + container = type("Container", (), { + "name": "dummy", "id": "abc123abc123", + "logs": lambda self, tail, timestamps: b"", + })() + payload = watchdog.build_payload( + container, "crashed", "test-host", "https://example.invalid", + exit_code=2, probe_detail="Crash probe failed - exit code 2") + self.assertIn("docker logs dummy --tail 50", payload.recommended_action) + record = watchdog.format_syslog_alert(payload) + self.assertIn('action="Docker reported an unexpected container termination.', record) + self.assertIn('failure_type="crashed"', record) + self.assertIn('exit_code="2"', record) + self.assertIn('detection="Crash probe failed - exit code 2"', record) + + def test_email_and_slack_use_context_title(self): + payload = watchdog.AlertPayload( + "CRITICAL", "dummy", "id", "host", "oom", 137, "", + self.EXPECTED["oom"], "", "OOM probe failed") + cfg = {"smtp": {"enabled": True, "auth": False, "tls": False, + "host": "example.invalid", "port": 25, + "sender": "test@example.invalid", + "recipients": ["recipient@example.invalid"]}} + with patch("watchdog.smtplib.SMTP") as smtp: + watchdog.send_smtp(payload, cfg) + raw = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] + body = BytesParser(policy=default).parsebytes(raw).get_body( + preferencelist=("plain",)).get_content() + self.assertIn("Context: " + self.EXPECTED["oom"], body) + self.assertNotIn("Action:", body) + with patch.dict("watchdog.os.environ", {"SLACK_WEBHOOK_URL": "https://example.invalid"}): + with patch("watchdog.requests.post") as post: + watchdog.send_slack(payload, {"slack": {"enabled": True}}) + fields = post.call_args.kwargs["json"]["attachments"][0]["fields"] + mapping = {field["title"]: field["value"] for field in fields} + self.assertEqual(mapping["Context"], self.EXPECTED["oom"]) + self.assertNotIn("Action", mapping) + + def test_ha_guidance_remains_unchanged(self): + from inspect import getsource + self.assertIn( + "Verify whether the HA switchover was expected and check service health.", + getsource(watchdog.Watchdog)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_smoke_safety.py b/tests/test_release_smoke_safety.py new file mode 100644 index 0000000..7599677 --- /dev/null +++ b/tests/test_release_smoke_safety.py @@ -0,0 +1,90 @@ +"""Fail-closed release smoke preflight tests; never connect to real Docker.""" +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "tests" / "release-smoke.sh" + + +class SmokeSafetyTests(unittest.TestCase): + def try_runner(self, overrides, docker_info=None): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + marker = root / "docker-invocations.txt" + stub = root / "docker" + # Fake Docker deliberately records any invocation, including read-only. + stub.write_text( + "#!/bin/sh\n" + "echo \"$*\" >> \"" + str(marker) + "\"\n" + "printf '%s\\n' '" + (docker_info or "{}") + "'\n") + stub.chmod(0o755) + env = dict(os.environ) + for key in ("GITHUB_ACTIONS", "SMOKE_RUNNER_KIND", "SMOKE_ENGINE_MARKER", + "DOCKER_HOST", "DOCKER_CONTEXT", "SMOKE_ONLY"): + env.pop(key, None) + env.update(overrides) + env["PATH"] = td + os.pathsep + env.get("PATH", "") + env["SMOKE_OUTPUT_DIR"] = td + done = subprocess.run(["bash", str(SCRIPT)], env=env, cwd=REPO, + capture_output=True, text=True, timeout=15) + calls = marker.read_text().splitlines() if marker.exists() else [] + return done.returncode, calls + + def test_default_invocation_fails_without_any_docker_call(self): + rc, calls = self.try_runner({}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_live_docker_host_rejected_without_any_docker_call(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "unix:///var/run/docker.sock"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://10.199.0.241:2375"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_context_override_rejected(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_CONTEXT": "production", + "DOCKER_HOST": "tcp://127.0.0.1:32768"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + def test_forged_or_nonempty_engine_fails_before_creating_resources(self): + base = {"GITHUB_ACTIONS": "true", + "SMOKE_RUNNER_KIND": "github-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://127.0.0.1:32768"} + for info in ( + '{"ID":"unexpected","Labels":[],"Containers":0}', + '{"ID":"test","Labels":["com.radware.container-alert.smoke-engine=ci-123-1"],"Containers":1}', + ): + with self.subTest(info=info): + rc, calls = self.try_runner(base, docker_info=info) + self.assertNotEqual(rc, 0) + self.assertEqual(len(calls), 1) + self.assertTrue(calls[0].startswith("info --format "), calls) + + def test_self_hosted_runner_rejected(self): + rc, calls = self.try_runner({ + "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "self-hosted", + "SMOKE_ENGINE_MARKER": "ci-123-1", + "DOCKER_HOST": "tcp://127.0.0.1:32768"}) + self.assertNotEqual(rc, 0) + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index bcd6b6c..5598c61 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -20,7 +20,7 @@ check_interval_seconds: 60 # how often to poll all containers cooldown_minutes: 5 # suppress duplicate alerts per container restart_threshold: 5 # Docker RestartCount increments within the window trigger alert restart_window_minutes: 10 # rolling window for restart counting -restart_recovery_stability_seconds: 120 # require 2 min continuous uptime before RECOVERED for auto-restarting containers +restart_recovery_stability_seconds: 120 # require stable Docker uptime before CRASH/OOM/RESTART-LOOP recovery unhealthy_cycles_threshold: 3 # consecutive unhealthy cycles before alerting alert_on_recovery: true # send an INFO "recovered" alert when a container returns to normal diff --git a/watchdog.py b/watchdog.py index 2779b9d..9922d84 100644 --- a/watchdog.py +++ b/watchdog.py @@ -424,7 +424,7 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: {"title": "Host", "value": payload.host, "short": True}, {"title": "Time (UTC)", "value": payload.timestamp, "short": True}, *probe_field, - {"title": "Action", "value": payload.recommended_action, "short": False}, + {"title": "Context", "value": payload.recommended_action, "short": False}, {"title": "Runbook", "value": payload.runbook_url, "short": False}, ] body = { @@ -440,7 +440,7 @@ def send_slack(payload: AlertPayload, cfg: dict) -> None: *exit_field, *probe_type_field, *probe_field, - {"title": "Action", "value": payload.recommended_action, "short": False}, + {"title": "Context", "value": payload.recommended_action, "short": False}, {"title": "Runbook", "value": payload.runbook_url, "short": False}, ], "footer": ("Source: HA cluster exporter" if ha_event @@ -570,7 +570,7 @@ def send_smtp(payload: AlertPayload, cfg: dict) -> None: f"Severity: {payload.severity}\n" f"Observed (UTC): {payload.timestamp}\n" f"Evidence: {payload.probe_detail}\n" - f"Action: {payload.recommended_action}\n" + f"Context: {payload.recommended_action}\n" f"Runbook: {payload.runbook_url}") else: body = f"""Container: {payload.container_name} @@ -582,7 +582,7 @@ def send_smtp(payload: AlertPayload, cfg: dict) -> None: Exit Code: {payload.exit_code if payload.exit_code is not None else 'N/A'} Probe Type: {payload.probe_type or 'N/A'} Detection: {payload.probe_detail or 'N/A'} -Action: {payload.recommended_action} +Context: {payload.recommended_action} Runbook: {payload.runbook_url or 'N/A'}""" # Build an RFC-compliant MIME message. smtplib.sendmail(str) tries to @@ -1082,31 +1082,32 @@ def _suppress_invalid_state(loop, context): ACTIONS = { "crashed": ( "CRITICAL", - "Check exit code and logs. Run: docker logs --tail 50. " - "Exit 137 means SIGKILL, not necessarily OOM. Check Docker stop/oom events and OOMKilled.", + "Docker reported an unexpected container termination. " + "Run: docker logs --tail 50. " + "The exit code and detection details provide additional context.", ), "oom": ( "CRITICAL", - "Container was OOM-killed. Increase memory limit in docker-compose.yaml " - "or investigate memory leak. Run: docker stats .", + "Docker reported that the container was terminated due to an out-of-memory condition.", ), "unhealthy": ( "HIGH", - "Container is running but health probe is failing. " - "Check the application inside the container. Run: docker inspect .", + "The container is running, but a monitoring health check is failing.", ), "restart-loop": ( "HIGH", - "Container is repeatedly restarting. Inspect docker logs and the " - "restart policy; fix application crash, OOM, or dependency failure.", + "The container has restarted repeatedly within the configured monitoring window. " + "This pattern may be associated with application errors, resource pressure, " + "or unavailable dependencies. The underlying cause has not been determined.", ), "recovered": ( "INFO", - "Container is back to normal operation. No action required.", + "The previously reported alert condition has cleared according to the available monitoring checks.", ), "reboot-summary": ( "INFO", - "Post-reboot status summary. No action required unless containers are listed as still failing.", + "Monitoring has resumed following a host reboot. The reported summary includes " + "container status, recovered incidents, and any conditions that remain unresolved.", ), } @@ -1289,7 +1290,7 @@ def build_payload( extra_context: str = "", probe_detail: str = "", ) -> AlertPayload: - severity, action = ACTIONS.get(failure_type, ("WARNING", "Investigate container.")) + severity, action = ACTIONS.get(failure_type, ("WARNING", "A container monitoring event was reported. The failure type and available detection details provide additional context.")) log_tail = get_log_tail(container) if extra_context: log_tail = f"{extra_context}\n\n{log_tail}" From 136ce0a323370eac33aafddcaa98356ada866c7b Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Thu, 15 Oct 2026 15:05:56 +0000 Subject: [PATCH 06/18] ci: validate isolated smoke on release pull requests --- .github/workflows/release-smoke.yml | 13 +++++++++++-- README.md | 7 ++++--- docs/RELEASE_NOTES_V1.6.0.md | 2 +- docs/RELEASE_SMOKE_TESTING.md | 9 ++++++++- tests/test_customer_alert_text.py | 2 +- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index a0be819..bec02ee 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -1,8 +1,16 @@ name: Release smoke (isolated) -# Only an explicitly dispatched GitHub-hosted worker runs destructive tests. -# This workflow never connects to CyberController or any shared Docker Engine. +# Runs only on disposable GitHub-hosted workers; never on operational engines. +# PR trigger permits validation before release; manual reruns after first merge. on: + pull_request: + branches: + - main + paths: + - "watchdog.py" + - "tests/release-smoke.sh" + - ".github/workflows/release-smoke.yml" + - "Dockerfile" workflow_dispatch: permissions: @@ -12,6 +20,7 @@ jobs: real-docker: name: Real Docker (disposable GitHub runner) runs-on: ubuntu-24.04 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository timeout-minutes: 20 steps: diff --git a/README.md b/README.md index 03b8574..bd996f2 100644 --- a/README.md +++ b/README.md @@ -390,8 +390,9 @@ No failure injection is required for message formatting or severity checks. ### Destructive real Docker acceptance — disposable hosted runner ONLY -Run the **Release smoke (isolated)** GitHub Actions workflow on the intended -release branch (`workflow_dispatch`); it uses an ephemeral GitHub-hosted worker +Use the **Release smoke (isolated)** GitHub Actions workflow on the intended +release PR (automatically for in-repository PR changes, or manual rerun once the +workflow exists on the default branch); it uses an ephemeral GitHub-hosted worker with an empty, specially labelled Docker Engine, separate alert state, and a temporary UDP Syslog receiver. The smoke runner aborts before creating any test resource unless it verifies the dedicated hosted worker, exact localhost @@ -401,7 +402,7 @@ live-host override.** The [smoke-test procedure](docs/RELEASE_SMOKE_TESTING.md) details acceptance criteria, evidence capture, cleanup, and use of the workflow. The normal PR CI runs only non-destructive Python, shell and build validations; destructive Docker -scenarios require manual dispatch of the separate hosted workflow. +scenarios run only in the separate, GitHub-hosted release workflow. Do not start a test Engine on CyberController or m920q. The previous 2026-10-14 m920q Docker-in-Docker test inadvertently generated real host watchdog alerts; diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index b6a517c..ccb7bcc 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -51,4 +51,4 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C The HA notification dispatch path has no persistent retry queue (intentional v1 limitation). Docker planned-stop correlation is not an availability policy; a deliberately stopped long-running service does not automatically produce a downtime alert. Unexpected external Docker restarts were seen on a lab; the initiator was not determined. These are documented operational limitations, not release regressions. -**Before shipping:** PR #9 CI green, separate manual hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, package signed/verified artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. None of those final publish steps is automatic. +**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, package signed/verified artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. None of those final publish steps is automatic. diff --git a/docs/RELEASE_SMOKE_TESTING.md b/docs/RELEASE_SMOKE_TESTING.md index 59e0253..dc3495f 100644 --- a/docs/RELEASE_SMOKE_TESTING.md +++ b/docs/RELEASE_SMOKE_TESTING.md @@ -12,8 +12,15 @@ The workflow has no corporate customer credentials, no HA exporter dependency, n ## Launch +The hosted smoke workflow triggers on in-repository release PR changes targeting +`main`. The first run can occur before the workflow is merged into the +default branch. A manual `workflow_dispatch` rerun is available once the +workflow file exists on the default branch. + 1. Review [release PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) and select its current `release/v1.6.0` branch. -2. In GitHub Actions select **Release smoke (isolated)** → **Run workflow** and choose that branch. Alternatively an authorized repository maintainer can run: +2. For the current PR, verify the automatically triggered **Release smoke (isolated)** + check. After the workflow is on `main`, an authorized maintainer can + also manually rerun it from Actions or use: ```bash gh workflow run release-smoke.yml --ref release/v1.6.0 diff --git a/tests/test_customer_alert_text.py b/tests/test_customer_alert_text.py index 8269924..e9ee997 100644 --- a/tests/test_customer_alert_text.py +++ b/tests/test_customer_alert_text.py @@ -62,7 +62,7 @@ def test_email_and_slack_use_context_title(self): "host": "example.invalid", "port": 25, "sender": "test@example.invalid", "recipients": ["recipient@example.invalid"]}} - with patch("watchdog.smtplib.SMTP") as smtp: + with patch("smtplib.SMTP") as smtp: watchdog.send_smtp(payload, cfg) raw = smtp.return_value.__enter__.return_value.sendmail.call_args.args[2] body = BytesParser(policy=default).parsebytes(raw).get_body( From 7077dfead2d23e3047c87ce13d1bd3109c390fb6 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Thu, 15 Oct 2026 18:53:39 +0000 Subject: [PATCH 07/18] docs: anonymize lab evidence and simplify v1.6 operator guidance --- DEPLOYMENT.md | 16 +--- README.md | 87 ++++--------------- docs/HA_ACTIVE_NODE_ALERT_DESIGN.md | 24 ++--- docs/HA_VALIDATION.md | 18 ++-- docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md | 28 +++--- docs/RELEASE_NOTES_V1.6.0.md | 4 +- docs/RELEASE_SMOKE_TESTING.md | 10 +-- docs/RELEASE_SMOKE_V1.6.0_20261014.json | 2 +- ...ELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json | 2 +- ...ASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json | 2 +- ...E_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json | 2 +- docs/RELEASE_VALIDATION_V1.6.0.md | 26 +++--- tests/test_ha_monitor.py | 6 +- tests/test_notifications.py | 10 +-- tests/test_release_smoke_safety.py | 2 +- watchdog-config.yaml.example | 4 +- 16 files changed, 88 insertions(+), 155 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index dcf31b8..b35e763 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -5,15 +5,7 @@ The **CyberController Container Watchdog** is an autonomous monitoring service t This guide covers initial deployment, alert channel configuration, and ongoing operational management of the watchdog service. -> **v1.6.0 release safety / customer messaging:** Do **not** run destructive -> Docker failure simulations on CyberController, HA nodes or shared operational -> Docker Engines. See the [hosted-only release test procedure](docs/RELEASE_SMOKE_TESTING.md). -> Container Alert notification descriptions are informational and do not direct -> customers to modify Compose YAML, resource limits, restart policies or other -> production settings. Such remediation requires the applicable Radware R&D / -> authorized change process. The optional CRASH `docker logs` command is read-only -> and is intended for authorized diagnostic access. See -> [v1.6.0 changes since 1.5.4](docs/RELEASE_NOTES_V1.6.0.md). +> **Production safety:** Verify watchdog operation using read-only status and log checks. Do not simulate failures on operational hosts. Contact Radware support before changing production monitoring settings. --- @@ -209,7 +201,7 @@ Enable the syslog block in `watchdog-config.yaml`: # ── Syslog (remote / central log server) ───────────────────────────────────── syslog: enabled: true # set to true to enable - host: 155.1.1.4 # IP or hostname of your syslog server + host: syslog.example.com # IP or hostname of your syslog server port: 514 # 514 = standard syslog (UDP); 601 = reliable syslog (TCP) protocol: udp # udp (fire-and-forget) or tcp (reliable) facility: local0 # syslog facility: local0–local7, daemon, user, etc. @@ -288,7 +280,7 @@ Add `snmp_trap` to `alert_channels` in `watchdog-config.yaml` and configure the # Requires: pip install pysnmp (already in requirements-watchdog.txt) snmp_trap: enabled: false # set to true to enable - host: 155.1.1.1 # IP or hostname of your SNMP trap receiver (NMS) + host: nms.example.com # IP or hostname of your SNMP trap receiver (NMS) port: 162 # UDP port — 162 is the standard SNMP trap port version: v2c # SNMP version: v1, v2c, or v3 community: public # SNMPv1/v2c community string (ignored for v3) @@ -848,6 +840,4 @@ new trap OID is introduced. If you have trouble accessing the `watchdog.tar` download link, or need an updated offline image from the Radware RE team, contact: -- rahulku@radware.com -- Egore@radware.com - northamericare@radware.com diff --git a/README.md b/README.md index bd996f2..5e7d3e2 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ node has a local Pacemaker HA cluster exporter: ha_monitor: enabled: true exporter_url: http://127.0.0.1:9664/metrics -~~~ +``` The watchdog reads HA cluster resource metrics and validates local-node identity, cluster quorum, and Pacemaker scrape success. The first valid reading @@ -250,21 +250,6 @@ syslog: facility: local0 ``` -Syslog is an **alert and error stream**, not a per-poll health feed. It sends -one structured, single-line event="watchdog_alert" record per emitted incident -(including INFO recoveries and reboot summaries), plus watchdog ERROR/CRITICAL -diagnostics. Routine INFO/WARNING polling messages stay local; per-container -poll output is available at log_level: DEBUG. The alert records include the -severity, UTC time, host, container, failure type, detection evidence and action, -with special previous/new active-node fields for HA. The syslog priority is -CRITICAL=2, HIGH=3 (error), WARNING=4 and INFO=6; the facility remains -configurable. Values are quoted, escaped and length-limited for ingestion. -Syslog is configured independently from alert_channels. -Local log_level controls only console/file verbosity; INFO recoveries and -WARNING HA alerts still reach Syslog when local log_level is WARNING or ERROR. -HA and reboot-summary records omit pseudo-container fields. - - ### SMTP (Email) Add `smtp` to `alert_channels` to enable. @@ -290,7 +275,7 @@ Add `snmp_trap` to `alert_channels` to enable. Sends SNMP traps (v1, v2c, or v3) ```yaml snmp_trap: enabled: true - host: 155.1.1.4 # IP or hostname of your SNMP trap receiver + host: syslog.example.com # IP or hostname of your SNMP trap receiver port: 162 # Standard SNMP trap port version: v2c # SNMP version: v1, v2c, or v3 community: public # SNMPv1/v2c community string (ignored for v3) @@ -304,10 +289,6 @@ snmp_trap: # v3_local_engine_id: "" # hex engine ID pinned from first-run log; see DEPLOYMENT.md ``` -SNMP retains its five existing var-binds and OIDs for NMS compatibility. -The human-readable cwSummary contains severity; cwProbeDetail contains -detection evidence. No additional SNMP objects are introduced in v1.6.0. - Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3.6.1.4.1.89.110`): | OID | Object | Value | @@ -355,58 +336,27 @@ Every alert includes two probe fields that identify exactly how and why the fail | `crashed` | `Docker event (crash)` | `Crash probe failed — container exited with exit code 1` | | `oom` | `Docker event (OOM)` | `OOM probe failed — container was OOM-killed by the kernel` | | `unhealthy` (Docker native HEALTHCHECK) | `Docker HEALTHCHECK` | `Docker health probe failed — exit code 1, output: connection refused` | -| `unhealthy` (HTTP auto-probe) | `HTTP GET` | `HTTP probe failed — error response 503 Service Unavailable, URI http://172.18.0.5:8080/health` | -| `unhealthy` (TCP connect) | `TCP connect` | `TCP probe failed — connection refused or timed out on 172.18.0.5:6379 (Connection refused)` | +| `unhealthy` (HTTP auto-probe) | `HTTP GET` | `HTTP probe failed — error response 503 Service Unavailable, URI http://192.0.2.100:8080/health` | +| `unhealthy` (TCP connect) | `TCP connect` | `TCP probe failed — connection refused or timed out on 192.0.2.100:6379 (Connection refused)` | | `unhealthy` (/proc fallback) | `/proc alive check` | `/proc alive check failed — PID 1 is zombie/stopped (exit code 1)` | > **Restart-loop detection** uses Docker `RestartCount` from the polling loop as its sole restart count. The first poll establishes a baseline without interpreting historical restarts. Counter increases are timestamped when observed, so detection can take up to one polling interval; the rolling window is approximate to polling precision. After a polling gap longer than the restart window, unknown-age increments are ignored rather than misattributed to the current window. Delayed Docker events cannot double-count restarts; timestamps on events also prevent failures from before confirmed recovery from reopening an incident. --- -## 7. Testing - -**Production / CyberController safety boundary:** Do **not** simulate crashes, OOM -events, restart loops, or recovery by running `docker kill`, `docker stop`, -`docker restart`, disposable stress containers, or other destructive experiments -on a CyberController node or another shared/operational Docker Engine. -Even supposedly disposable test containers can consume shared resources or -trigger real monitoring, and a nested Docker-in-Docker container remains visible -to the host's normal watchdog. The earlier host-based examples were removed -for this reason. - -### Safe verification on an operational host (read-only) +## 7. Verify Watchdog Operation -Authorized operators may review the existing watchdog's current health and -its own recent logs without altering any container: +On an installed system, use these read-only checks: ```bash -docker inspect docker-container-watchdog --format 'state={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}}' +docker ps --filter name=docker-container-watchdog +docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' docker logs --tail 50 docker-container-watchdog ``` -To validate an alert without changing a running service, use the unit/format -regression tests on a development workstation or the normal pull-request CI. -No failure injection is required for message formatting or severity checks. +Expected: the watchdog is running/healthy and monitoring logs are current. Check the configured notification receiver for any existing alerts. A healthy system does not need to generate a new alert just for verification. -### Destructive real Docker acceptance — disposable hosted runner ONLY - -Use the **Release smoke (isolated)** GitHub Actions workflow on the intended -release PR (automatically for in-repository PR changes, or manual rerun once the -workflow exists on the default branch); it uses an ephemeral GitHub-hosted worker -with an empty, specially labelled Docker Engine, separate alert state, and a -temporary UDP Syslog receiver. The smoke runner aborts before creating any -test resource unless it verifies the dedicated hosted worker, exact localhost -test endpoint, unique engine label, and zero existing containers. **There is no -live-host override.** - -The [smoke-test procedure](docs/RELEASE_SMOKE_TESTING.md) details acceptance -criteria, evidence capture, cleanup, and use of the workflow. The normal PR CI -runs only non-destructive Python, shell and build validations; destructive Docker -scenarios run only in the separate, GitHub-hosted release workflow. - -Do not start a test Engine on CyberController or m920q. The previous 2026-10-14 -m920q Docker-in-Docker test inadvertently generated real host watchdog alerts; -that approach must not be repeated. +Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](docs/RELEASE_SMOKE_TESTING.md). ## 8. Troubleshooting @@ -439,18 +389,11 @@ Common causes: - Confirm `alert_on_recovery` is not set to `false` in the live `watchdog-config.yaml`. - Confirm the container was actually alarmed first — `state.alerted_for` must be non-empty; a container that was never flagged won't generate a recovery alert. - For containers with a Docker `HEALTHCHECK`, confirm Docker actually reports `healthy` — `docker inspect --format '{{json .State.Health}}'`. If it's stuck on `starting`, the watchdog is waiting on Docker, not the other way around. -- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; add a manual entry under `container_health_checks` instead. +- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; contact Radware support before changing the health-check configuration. ### Duplicate and Excessive Alert Notifications -Increase `cooldown_minutes` in `watchdog-config.yaml` (default: `5` minutes per container per failure type). To silence a noisy container entirely, add it to `excluded_containers`: - -```yaml -excluded_containers: - - debug-shell - - load-test - - my-noisy-container -``` +Review the affected container's alert history and probe details. Contact Radware support or the responsible engineering team before changing monitoring thresholds, health checks or exclusions. ### Probe Type Not as Expected @@ -545,13 +488,13 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor ## 10. Version History -**Release status (2026-10-15):** `VERSION` remains `1.6.0` for the **upcoming, untagged** release candidate. PR #9 is open. The complete changes since 1.5.4, approved customer-facing alert wording, release acceptance and destructive-test isolation are documented in [v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). Historical HA and isolated Docker tests passed on earlier candidates; the final hosted-runner safety workflow must pass against the latest commit before publication. No v1.6.0 tag or final offline package is published; deployed lab images are not automatically updated by this branch. +**Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. **Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Upcoming since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | @@ -560,7 +503,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor | 1.4.0 | 2026-08-17 | Rahul Kumar | Added INFO "recovered" alert | | 1.3.3 | 2026-08-05 | Rahul Kumar | Added Auth True/false | | 1.3.2 | 2026-07-28 | Rahul Kumar | Fixed SMTP Auth issue | -| 1.3.1 | 2026-07-21 | Egor Egorov | Updated Readme and Deployment guides | +| 1.3.1 | 2026-07-21 | Engineering | Updated Readme and Deployment guides | | 1.3.0 | 2026-07-16 | Rahul Kumar | Added Upgrade Guide | | 1.2.0 | 2026-07-16 | Rahul Kumar | Removed sudo and renamed the container | | 1.1.0 | 2026-07-09 | Rahul Kumar | Added SNMPv3 support | diff --git a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md b/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md index 8d3b261..513ca1f 100644 --- a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md +++ b/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md @@ -13,17 +13,19 @@ Customer request: alert when CyberController HA changes its active node, includi This is an **observed active-node change** alert, not a claim to identify why a failover happened or that all CyberController applications are healthy. -## 2. Verified appliance evidence (read-only inspection) +## 2. HA validation evidence (read-only inspection) -On the two-node HA environment ha-test-admin-ea-iad1, Pacemaker reports nodes 10.199.0.241 and 10.199.0.242 online. The HA Operator, OpenSearch, PostgreSQL, MariaDB, and Failback promotable resources are currently **Promoted on .241** and **Unpromoted on .242**. The Pacemaker DC is .242; **DC is not the CyberController active node**. +Node addresses below are anonymized using documentation-only IPs. -- HA exporter: **ha_cluster_exporter.service**, active on .241, serving port **9664** and reachable at **http://127.0.0.1:9664/metrics** (HTTP 200). -- Promoted role metric: **ha_cluster_pacemaker_resources** includes resource=HAOperatorRep-Res, role=promoted, status=active, node=10.199.0.241, value 1. -- Host identity metric: **ha_cluster_corosync_member_votes** includes local=true, node=10.199.0.241. +In the two-node HA test environment, Pacemaker reports nodes 192.0.2.41 and 192.0.2.42 online. The HA Operator, OpenSearch, PostgreSQL, MariaDB, and Failback promotable resources are currently **Promoted on .41** and **Unpromoted on .42**. The Pacemaker DC is .42; **DC is not the CyberController active node**. + +- HA exporter: **ha_cluster_exporter.service**, active on .41, serving port **9664** and reachable at **http://127.0.0.1:9664/metrics** (HTTP 200). +- Promoted role metric: **ha_cluster_pacemaker_resources** includes resource=HAOperatorRep-Res, role=promoted, status=active, node=192.0.2.41, value 1. +- Host identity metric: **ha_cluster_corosync_member_votes** includes local=true, node=192.0.2.41. - Data-quality signals: **ha_cluster_scrape_success{collector="pacemaker"} = 1** and **ha_cluster_corosync_quorate = 1**. -- Existing Watchdog v1.5.4 runs on .241 in **host network** mode, polls every 60 seconds, has persistent state, and currently delivers syslog. +- Existing Watchdog v1.5.4 runs on .41 in **host network** mode, polls every 60 seconds, has persistent state, and currently delivers syslog. - Prometheus's native HA target is misconfigured: it scrapes **localhost:9002**, while the exporter serves **9664**. The existing rule **ha_failover_occured** compares per-node location-constraint sum vs average and is not reliable for an actual role change. Correcting the native configuration is a **separate Radware/R&D action**, not part of this feature. -- Standby .242 did **not** accept cross-node connections to 9664/9099/9093. It has not been inspected locally, so its exporter/watchdog installation is **unverified**. Release testing must cover both nodes. +- Standby .42 did **not** accept cross-node connections to 9664/9099/9093. It has not been inspected locally, so its exporter/watchdog installation is **unverified**. Release testing must cover both nodes. ## 3. Small architecture @@ -104,10 +106,10 @@ Retain a **small optional ha_monitor section in the existing state.json** (backw ~~~text [WARNING] CyberController HA active node changed -Previous active: 10.199.0.241 -New active: 10.199.0.242 +Previous active: 192.0.2.41 +New active: 192.0.2.42 Observed at: -Evidence: HAOperatorRep-Res promoted on 10.199.0.242 (Pacemaker exporter) +Evidence: HAOperatorRep-Res promoted on 192.0.2.42 (Pacemaker exporter) Next step: Confirm expected switchover and application health. ~~~ @@ -164,7 +166,7 @@ The implementation is ready for review/release when all automated tests and CI p ## 7. Release gates, risks, and intentional exclusions -**Before release:** confirm both HA nodes can send notifications end-to-end. Both watchdogs and local exporters were confirmed running during the manual .242 -> .241 test. The standby's earlier *remote* exporter connection refusal reflected an inaccessible cross-node port, not a missing local service. No inter-node exporter access is required. +**Before release:** confirm both HA nodes can send notifications end-to-end. Both watchdogs and local exporters were confirmed running during the manual .42 -> .41 test. The standby's earlier *remote* exporter connection refusal reflected an inaccessible cross-node port, not a missing local service. No inter-node exporter access is required. **Known limitations:** If the surviving node has no running watchdog/exporter or has never recorded the original active node, this check cannot reconstruct the failover after the fact. Polling can miss a transition that changes twice between samples. Metrics cannot distinguish planned switchover vs unexpected failover, and promotion does not guarantee the entire application is healthy. diff --git a/docs/HA_VALIDATION.md b/docs/HA_VALIDATION.md index 1682f8f..846ba11 100644 --- a/docs/HA_VALIDATION.md +++ b/docs/HA_VALIDATION.md @@ -14,19 +14,19 @@ ## Real two-node environment — user-initiated test (2026-10-10 UTC) - **Both nodes have HA exporters and watchdog builds running** during the user test, not just the original active node. These installations were done by the user, not by this code review. -- At about 02:53:46 UTC the user initiated a manual switchover on .242, which moved the promoted HA Operator resource to .241. -- .242 logged at **02:55:17**: "HA role changed 10.199.0.242 -> 10.199.0.241; new owner will notify." -- .241 logged at **02:55:37**: one WARNING, "CyberController HA active node changed: 10.199.0.242 -> 10.199.0.241". -- Both nodes stayed online and .241's promoted HA resources became active. The actual switch was detected correctly, without duplicate HA events observed. -- **Actual notification delivery still needs verification.** .241 had syslog configured to 127.0.0.1:514; receiving NMS confirmation was not established. .242 used SMTP, but smtp.radware.com did not resolve, so its container failure messages were not emailed. -- During the planned demotion, the HA Operator explicitly removed vDirect, TED and Assist Service on .242. The watchdog generated separate CRITICAL crash alerts for vDirect and Assist Service. Narrowly scoped suppression is **out of scope** for PR #4 and should have a separate code review. -- A **reverse failback (.241 -> .242)**, end-to-end receiver confirmation, and a safe surviving-node test after failure of the original active node have not been confirmed. +- At about 02:53:46 UTC the user initiated a manual switchover on .42, which moved the promoted HA Operator resource to .41. +- .42 logged at **02:55:17**: "HA role changed 192.0.2.42 -> 192.0.2.41; new owner will notify." +- .41 logged at **02:55:37**: one WARNING, "CyberController HA active node changed: 192.0.2.42 -> 192.0.2.41". +- Both nodes stayed online and .41's promoted HA resources became active. The actual switch was detected correctly, without duplicate HA events observed. +- **Actual notification delivery still needs verification.** .41 had syslog configured to 127.0.0.1:514; receiving NMS confirmation was not established. .42 used SMTP, but smtp.radware.com did not resolve, so its container failure messages were not emailed. +- During the planned demotion, the HA Operator explicitly removed vDirect, TED and Assist Service on .42. The watchdog generated separate CRITICAL crash alerts for vDirect and Assist Service. Narrowly scoped suppression is **out of scope** for PR #4 and should have a separate code review. +- A **reverse failback (.41 -> .42)**, end-to-end receiver confirmation, and a safe surviving-node test after failure of the original active node have not been confirmed. ## Release gates still open 1. CI passes for the current head, with new concurrency/re-enable tests. -2. Test a controlled **failback** from .241 to .242, verifying a single reverse alert. -3. Verify the configured receiving systems, not just local watchdog logs; resolve .242 SMTP DNS or configure a working syslog route. +2. Test a controlled **failback** from .41 to .42, verifying a single reverse alert. +3. Verify the configured receiving systems, not just local watchdog logs; resolve .42 SMTP DNS or configure a working syslog route. 4. Validate a supported active-node-loss scenario without deliberately partitioning a cluster where STONITH is disabled. 5. Confirm targeted CC-version compatibility and ordinary watchdog restart/disabled option behavior. diff --git a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md index d473396..9ec3add 100644 --- a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md +++ b/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md @@ -4,8 +4,8 @@ > **Historical acceptance evidence (October 14, 2026), not current test instructions.** > These records predate the v1.6.0 hosted-only smoke-test safety guard and final > customer-facing message updates. Do **not** rerun their Docker commands on -> m920q, CyberController or another operational Engine. The prior outer -> Docker-in-Docker test container triggered genuine m920q watchdog alerts. +> any shared or operational Docker Engine. Earlier nested testing triggered +> real notifications on its parent host. > See [current release notes](RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). @@ -19,18 +19,18 @@ | Area | Result | Evidence | |---|---|---| | Automated Python regression tests | **PASS 109/109** | `python3 -m unittest discover -s tests -q` inside actual rebuilt `watchdog:1.6.0-rc-fix` image | -| Full real Docker integration suite | **PASS 15/15** | Isolated Docker Engine on m920q, run `261014224557488`; 22:45:57-22:52:41 UTC; 404.4 seconds | +| Full real Docker integration suite | **PASS 15/15** | Isolated Docker Engine on a dedicated development host, run `261014224557488`; 22:45:57-22:52:41 UTC; 404.4 seconds | | Real external restart after OOM | **PASS 1/1** | Separate targeted real OOM run `261014224323244`; RECOVERED after 120.6s | | Regression reproduction before fix | **CONFIRMED FAIL** | Real crash followed by manual Docker start produced premature RECOVERED within about three seconds; run `261014223252035` | | Raw UDP Syslog integration | **PASS 15/15** | 15 locally generated `watchdog_alert` records delivered as 15 actual UDP datagrams to isolated collector | -| Two-node HA failover and failback | **PASS** | Previous *live* lab testing: promoted HAOperator `.241 -> .242 -> .241`; one new-owner WARNING per transition, no duplicate, both DFC Syslog receivers confirmed | +| Two-node HA failover and failback | **PASS** | Previous *live* lab testing: promoted HAOperator `.41 -> .42 -> .41`; one new-owner WARNING per transition, no duplicate, both DFC Syslog receivers confirmed | | Installer source-build / offline image bundle | **PASS** | Source-build includes `watchdog.py`, `lifecycle.py`, `ha_monitor.py`; image save/load preserved ID | | Disposable test cleanup | **PASS** | No labelled Docker test containers or volumes remained after each completed suite | -| m920q host watchdog impact | **NONE** | Existing `watchdog:1.5.4` remained healthy, zero restarts, and saw no isolated Engine fixtures | +| Parent-host watchdog impact | **NONE** | Existing `watchdog:1.5.4` remained healthy, zero restarts, and saw no isolated Engine fixtures | ## Reproducible environment -- On **m920q**, ran a short-lived Docker-in-Docker engine `docker:27.5-dind`, exposed only at `127.0.0.1` using an ephemeral TCP port; it had a distinct Engine ID from m920q's running Docker Engine, memory limit 2 GiB, CPU limit 2 cores, no bridge/network provisioning inside the test engine, and its state was disposable. +- On a **dedicated development host**, ran a short-lived Docker-in-Docker engine `docker:27.5-dind`, exposed only at `127.0.0.1` using an ephemeral TCP port; it had a distinct Engine ID from the parent host's Docker Engine, memory limit 2 GiB, CPU limit 2 cores, no bridge/network provisioning inside the test engine, and its state was disposable. - Real test observer ran the revised application code from `watchdog:1.6.0-rc-fix` with its own state volume and a 2s polling interval. Original 5-restart threshold and **120s recovery stability window** were preserved. The observer monitored **real Docker Engine events and cgroup OOMs**, not synthetic test events. - Disposable failure containers were scoped with unique `ca-smoke-...` names/labels, limited CPU/memory, and no network access. Syslog receiver was a temporary UDP/514 collector on the isolated Engine. No production SMTP or external Syslog destinations were used. - The built test image contained verified source hashes: @@ -70,7 +70,7 @@ The test `T15_EXTERNAL_OOM` deliberately exceeded a 64 MiB cgroup memory limit, The earlier HA-lab smoke run (`261014214436754`, originally **11 PASS / 3 FAIL**) observed containers with Docker restart policy `no` starting again after a failure despite not being controlled by the test suite. The captured events prove this external restart behavior; the initiator was **not positively identified**. Do not attribute it to a particular CyberController host service without direct evidence. -We reproduced the **application bug independently** on the isolated m920q Docker Engine: the old code emitted RECOVERED just a few seconds after a manual `docker start` of a crashed policy-`no` container. Original `_maybe_alert_recovery` applied the 120-second stability gate to CRASH/OOM only when the Docker restart policy was automatic. That assumption is invalid when an external manager manually restarts failed containers. +We reproduced the **application bug independently** on the isolated test Docker Engine: the old code emitted RECOVERED just a few seconds after a manual `docker start` of a crashed policy-`no` container. Original `_maybe_alert_recovery` applied the 120-second stability gate to CRASH/OOM only when the Docker restart policy was automatic. That assumption is invalid when an external manager manually restarts failed containers. **Minimal code correction:** for any open `crashed`, `oom`, or `restart-loop` incident, require sustained Docker uptime and no recent failure for the existing recovery-stability interval. The existing state machine, Docker event correlation, HA monitoring, and notification APIs were not redesigned. Two policy-`no` crash/OOM regressions now protect this path. All **109** automated tests pass. @@ -82,17 +82,17 @@ The user manually performed real switchover and failback of the existing two-nod | HA case | Evidence and outcome | |---|---| -| `10.199.0.241 -> 10.199.0.242` | **PASS**: promoted operator moved to .242; one WARNING alert on newly active .242 at 20:56:16 UTC, no duplicate on .241 | -| .242 DFC delivery | **PASS**: structured `ha-failover` record received/stored in `config_dfc_1:/var/log/syslog` | -| `10.199.0.242 -> 10.199.0.241` | **PASS**: one WARNING alert on .241 at 21:09:47 UTC; .242 deferred, no duplicate | -| .241 DFC delivery | **PASS**: structured reverse alert received/stored in its DFC Syslog receiver | -| Final HA health | **PASS**: quorum healthy, .241 promoted, both watchdogs healthy and baselines .241 | +| `192.0.2.41 -> 192.0.2.42` | **PASS**: promoted operator moved to .42; one WARNING alert on newly active .42 at 20:56:16 UTC, no duplicate on .41 | +| .42 DFC delivery | **PASS**: structured `ha-failover` record received/stored in `config_dfc_1:/var/log/syslog` | +| `192.0.2.42 -> 192.0.2.41` | **PASS**: one WARNING alert on .41 at 21:09:47 UTC; .42 deferred, no duplicate | +| .41 DFC delivery | **PASS**: structured reverse alert received/stored in its DFC Syslog receiver | +| Final HA health | **PASS**: quorum healthy, .41 promoted, both watchdogs healthy and baselines .41 | The new Docker recovery fix does not change the HA monitoring code. Nevertheless, **the currently running HA-node images still contain the previous pre-fix candidate**, so they are not proof that the final recovery-fixed image has been installed on those nodes. ## Packaging, exclusions, and release decision -Installer source-build and offline archive `docker save`/`docker load` procedures were validated earlier in this release cycle; CI covered shell syntax, Compose, code compilation, unit tests, and image builds. The test suite itself runs without installing packages on m920q or the HA nodes. +Installer source-build and offline archive `docker save`/`docker load` procedures were validated earlier in this release cycle; CI covered shell syntax, Compose, code compilation, unit tests, and image builds. The test suite itself runs without installing packages on operational HA nodes. **Not validated end to end:** SNMP traps to a physical NMS, Slack delivery, SMTP delivery (explicitly excluded by user), host reboot, network partition, STONITH, destructive exporter injection. These are not represented as passed. Failover/failback, DFC Syslog, ordinary Docker incidents, and the fixed recovery path have direct live evidence. @@ -105,4 +105,4 @@ Installer source-build and offline archive `docker save`/`docker load` procedure - [Real OOM/manual-restart 120.6s proof](RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json) - [Earlier HA-lab conditional report](RELEASE_VALIDATION_V1.6.0.md) - [Reusable real Docker smoke runner](../tests/release-smoke.sh) -- Runtime evidence: `/tmp/container-alert-smoke-261014224557488/` and `/tmp/container-alert-smoke-261014224323244/` on m920q (contains captured watchdog events, actual Docker events, raw received Syslog messages, and observer-state JSON). +- Historical runtime evidence: temporary smoke-run directories containing test events, Syslog records and observer state were retained during validation. diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index ccb7bcc..be7e2b8 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -1,6 +1,6 @@ # v1.6.0 Release — Changes Since 1.5.4 -**Status:** Release candidate, not yet tagged or published. Version remains `1.6.0` in `VERSION`. The [release PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) targets `main`. This file documents everything included in the upcoming release relative to v1.5.4, not a claim that an image or deployment has been published. +This document describes version **1.6.0** and its changes since v1.5.4. Building, publishing, and deploying its image are separate release steps. ## Feature and fix inventory @@ -42,7 +42,7 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C - Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. - Destructive real Docker failure testing is **only** through the manually dispatched [Release smoke (isolated)](../.github/workflows/release-smoke.yml) workflow on an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](RELEASE_SMOKE_TESTING.md). - Historical test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md), [earlier live HA-lab findings](RELEASE_VALIDATION_V1.6.0.md), and JSON files in this directory. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. -- During previous m920q nested-engine testing, the outer test container was visible to the host's normal watchdog and caused genuine HIGH/CRITICAL notifications. This is why the nested Engine is now allowed **only on a disposable hosted CI machine** and never on m920q or CyberController. +- Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. - Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. ## Exclusions, limitations and release gating diff --git a/docs/RELEASE_SMOKE_TESTING.md b/docs/RELEASE_SMOKE_TESTING.md index dc3495f..145516a 100644 --- a/docs/RELEASE_SMOKE_TESTING.md +++ b/docs/RELEASE_SMOKE_TESTING.md @@ -4,9 +4,9 @@ ## Safety rule -**NEVER run `tests/release-smoke.sh` on CyberController, its HA nodes, m920q, a customer system, or another shared machine.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. +**NEVER run `tests/release-smoke.sh` on CyberController, customer systems, or any shared Docker host.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. -Instead run the workflow [Release smoke (isolated)](../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to CyberController or m920q. All test-created resources are disposable and contained within that hosted worker. +Instead run the workflow [Release smoke (isolated)](../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to any operational host. All test-created resources are disposable and contained within that hosted worker. The workflow has no corporate customer credentials, no HA exporter dependency, no CyberController Docker socket, no external notification destinations, and no network access from the disposable failure fixtures. @@ -63,11 +63,9 @@ Within the verified Engine, each fixture uses unique names and a per-run ownersh | Syslog | Locally generated structured alerts received exactly once as real UDP Syslog | | Cleanup | No leftover test-labelled containers/volumes, no unauthorized modifications | -## Important historical correction +## Isolation lesson -An earlier 2026-10-14 validation ran a nested Engine on **m920q**. Its outer `ca-v160-isolated-engine` container was monitored by the existing `watchdog:1.5.4` and generated HIGH UNHEALTHY and CRITICAL OOM alerts on the **normal SMTP channel**. That test Engine was subsequently removed, but its existence demonstrates that nesting alone is **not production isolation**. Do not repeat that method. - -Earlier acceptance JSON and reports remain in `docs/` as traceability, not authorization to rerun those procedures on operational Docker hosts. +Earlier development tests showed that a nested test Docker Engine remains visible to its parent host. Destructive acceptance tests must run on a disposable hosted runner, not a shared or operational host. ## Operator verification (read-only only) diff --git a/docs/RELEASE_SMOKE_V1.6.0_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_20261014.json index 7fb7e4e..c465df7 100644 --- a/docs/RELEASE_SMOKE_V1.6.0_20261014.json +++ b/docs/RELEASE_SMOKE_V1.6.0_20261014.json @@ -105,7 +105,7 @@ } ], "facts": { - "host": "cc-ea-iad1", + "host": "ha-validation-host", "revision": "3b1de153e4d6c1b5ef33ca90c0b6ef755727fb05", "image_id": "sha256:e3d02f6ef56cde0bc7021919def9e240505434a15769aac3884cf15ac8f0ccff", "observer": "ca-smoke-261014214436754-observer", diff --git a/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json index 9462fde..cc7a168 100644 --- a/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json +++ b/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json @@ -117,7 +117,7 @@ "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" }, - "host": "b6df39059ff7", + "host": "isolated-test-engine", "revision": "", "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", "observer": "ca-smoke-261014224557488-observer", diff --git a/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json index 985c190..620cf74 100644 --- a/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json +++ b/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json @@ -19,7 +19,7 @@ "/app/lifecycle.py": "55b6a06ebb69cb5a70f6c1566798087510228ef93eeceb810dcadb5da09d81f9", "/app/ha_monitor.py": "0fb3ba165035c9970f91854914709fd7492e21a7583c74f34174275c12083dbd" }, - "host": "b6df39059ff7", + "host": "isolated-test-engine", "revision": "", "image_id": "sha256:670cbdd3a24317aa39c04036d1c90c72420c97a0afe32ad5e4a77e06ef464be9", "observer": "ca-smoke-261014224323244-observer", diff --git a/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json b/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json index 6530397..032e142 100644 --- a/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json +++ b/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json @@ -14,7 +14,7 @@ } ], "facts": { - "host": "b6df39059ff7", + "host": "isolated-test-engine", "revision": "", "image_id": "sha256:3a9942e74f643b0c2b38c82bcd0ef2b805cd13dd0f7c100031080f2cf06e494a", "observer": "ca-smoke-261014223252035-observer", diff --git a/docs/RELEASE_VALIDATION_V1.6.0.md b/docs/RELEASE_VALIDATION_V1.6.0.md index cdd6d74..5ba66e3 100644 --- a/docs/RELEASE_VALIDATION_V1.6.0.md +++ b/docs/RELEASE_VALIDATION_V1.6.0.md @@ -4,12 +4,12 @@ > **Historical acceptance evidence (October 14, 2026), not current test instructions.** > These records predate the v1.6.0 hosted-only smoke-test safety guard and final > customer-facing message updates. Do **not** rerun their Docker commands on -> m920q, CyberController or another operational Engine. The prior outer -> Docker-in-Docker test container triggered genuine m920q watchdog alerts. +> any shared or operational Docker Engine. Earlier nested testing triggered +> real notifications on its parent host. > See [current release notes](RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). -> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on m920q, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). Retain this document as evidence of the original findings, not the current release verdict. +> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on a dedicated development host, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). Retain this document as evidence of the original findings, not the current release verdict. **Date:** October 14, 2026 (UTC) **Repository:** `Radware/CyberController-Container-Alert` @@ -21,7 +21,7 @@ ## Test methods and safeguards -- Tested against the real Docker Engine on `ha-test-admin-ea-iad1` (`10.199.0.241`). No simulated Docker events. +- Tested against a real Docker Engine in the HA test environment (`192.0.2.41`). No simulated Docker events. - A separate resource-limited v1.6.0 observer used its own Docker volume/state, 2-second Docker polling, real 5-restart threshold, and **real 120-second** recovery requirement. SMTP disabled only on the observer; Syslog sent to the **existing** DFC UDP/514 receiver. Production-style main watchdog and cluster were not reconfigured. - Used uniquely prefixed, labeled disposable containers, restricted to 0.2 CPU and 64–128 MiB each, with `--network none`. Verified actual Docker state, events, watchdog structured alert records, and DFC `/var/log/syslog` deliveries. - No host packages, new Docker networks, Pacemaker operations, or host service modifications. Cleanup removed all smoke-owned containers and volumes. Original watchdog stayed healthy. @@ -31,7 +31,7 @@ | Check | Acceptance | Result | |---|---|---| -| Automated regression suite | All tests pass against final built source | **PASS — 108/108** inside m920q Docker image | +| Automated regression suite | All tests pass against final built source | **PASS — 108/108** inside a test Docker image | | GitHub CI for PR #9 head | Green | **PASS** on `3b1de153` (before adding this documentation) | | Release code/image integrity | Embedded `watchdog.py`, `lifecycle.py`, `ha_monitor.py` hashes match Git | **PASS** | | `install.sh` / smoke script syntax | `bash -n`, smoke embedded Python compiles | **PASS** | @@ -44,13 +44,13 @@ | Scenario | Acceptance | Actual result | |---|---|---| -| Baseline | Both nodes agree `.241` active, no startup failover | **PASS** | -| Failover `.241 → .242` | One WARNING on new owner, zero on previous | **PASS** — alert on `.242` at **20:56:16 UTC**; `.241` logged "new owner will notify" | -| HA Syslog on `.242` | Structured failover record delivered to DFC | **PASS** — stored in DFC `/var/log/syslog` | -| Failback `.242 → .241` | One WARNING on new owner, zero on previous | **PASS** — alert on `.241` at **21:09:47 UTC**; `.242` logged owner defer | -| HA Syslog on `.241` | Structured failback record delivered to DFC | **PASS** | -| Final cluster status | One promoted HA Operator, quorum, both watchdogs healthy | **PASS** — `.241` active; both watchdogs image `1.6.0`, healthy, zero restarts | -| Transient exporter ambiguity during transition | No fabricated failover | **PASS** — `.242` logged one invalid-promoted-owner observation at 20:54:16 UTC, then recovered and sent correct alert | +| Baseline | Both nodes agree `.41` active, no startup failover | **PASS** | +| Failover `.41 → .42` | One WARNING on new owner, zero on previous | **PASS** — alert on `.42` at **20:56:16 UTC**; `.41` logged "new owner will notify" | +| HA Syslog on `.42` | Structured failover record delivered to DFC | **PASS** — stored in DFC `/var/log/syslog` | +| Failback `.42 → .41` | One WARNING on new owner, zero on previous | **PASS** — alert on `.41` at **21:09:47 UTC**; `.42` logged owner defer | +| HA Syslog on `.41` | Structured failback record delivered to DFC | **PASS** | +| Final cluster status | One promoted HA Operator, quorum, both watchdogs healthy | **PASS** — `.41` active; both watchdogs image `1.6.0`, healthy, zero restarts | +| Transient exporter ambiguity during transition | No fabricated failover | **PASS** — `.42` logged one invalid-promoted-owner observation at 20:54:16 UTC, then recovered and sent correct alert | | SMTP delivery | Excluded by test owner | **EXCLUDED** — SMTP network timeouts are not part of this verdict | ## 3. Real Docker acceptance — final run @@ -84,7 +84,7 @@ ### Investigation: out-of-band restarts -The live event capture on `.241` included **37 `restart` actions, 57 starts, 54 dies, and 12 `oom` actions** for test-tagged containers. Importantly, some affected fixtures had `HostConfig.RestartPolicy.Name=no`, yet their Docker IDs were started/restarted repeatedly. The initiator of these restarts was **not established**; do not attribute them to Container Alert without additional evidence. +The live event capture on `.41` included **37 `restart` actions, 57 starts, 54 dies, and 12 `oom` actions** for test-tagged containers. Importantly, some affected fixtures had `HostConfig.RestartPolicy.Name=no`, yet their Docker IDs were started/restarted repeatedly. The initiator of these restarts was **not established**; do not attribute them to Container Alert without additional evidence. The planned-stop fixture initially emitted real `kill(SIGTERM)`, `kill(SIGKILL)`, `stop`, and `die(137)` events. The unwanted CRASH appeared **later**, when that fixture had already been restarted and the test suite used force-remove to clean it up; that final kill had no `stop` evidence. This is not evidence that the original `docker stop` was misclassified. diff --git a/tests/test_ha_monitor.py b/tests/test_ha_monitor.py index 1ff5c9a..d84cce7 100644 --- a/tests/test_ha_monitor.py +++ b/tests/test_ha_monitor.py @@ -13,8 +13,8 @@ parse_ha_metrics, read_ha_observation, ) -A = "10.199.0.241" -B = "10.199.0.242" +A = "192.0.2.41" +B = "192.0.2.42" def metrics(owner=A, local=A, *, pacemaker=1, quorum=1, extras=""): @@ -84,7 +84,7 @@ def test_http_is_local_bounded_and_closed(self): "http://127.0.0.1:9664/metrics", timeout=3, stream=True, allow_redirects=False) def test_rejects_remote_or_wrong_path_endpoints(self): - for url in ("http://10.199.0.242:9664/metrics", + for url in ("http://192.0.2.42:9664/metrics", "http://localhost:9664/admin", "https://localhost:9664/metrics", "http://localhost:9664/metrics?x=1"): with self.subTest(url=url), self.assertRaises(ValueError): diff --git a/tests/test_notifications.py b/tests/test_notifications.py index 604de8b..b45bd8a 100644 --- a/tests/test_notifications.py +++ b/tests/test_notifications.py @@ -13,7 +13,7 @@ def alert(severity="CRITICAL", failure="crashed"): return watchdog.AlertPayload( - severity, "demo-service", "abcdef123456789", "vision", failure, 137, + severity, "demo-service", "abcdef123456789", "test-host", failure, 137, "", "Inspect Docker events and logs.", "https://example.invalid/runbook", 'Crash probe failed - SIGKILL; OOM not confirmed', ) @@ -46,7 +46,7 @@ def test_real_syslog_keeps_recovery_and_ha_at_local_warning_or_error(self): recovery.exit_code = None watchdog.dispatch_alert(recovery, {"alert_channels": []}) ha = watchdog.AlertPayload( - "WARNING", "(HA cluster)", "", "vision", "ha-failover", + "WARNING", "(HA cluster)", "", "test-host", "ha-failover", None, "", "Check cluster health", "", "HA Operator promoted to new node", ha_previous_node="node-a", ha_new_node="node-b" @@ -83,7 +83,7 @@ def test_pseudo_container_fields_absent_in_ha_and_reboot_events(self): ): with self.subTest(failure=failure): p = watchdog.AlertPayload( - "INFO", container_name, "", "vision", failure, None, "", + "INFO", container_name, "", "test-host", failure, None, "", "Check cluster", "", "Event details", ha_previous_node="node-a", ha_new_node="node-b") record = watchdog.format_syslog_alert(p) @@ -91,7 +91,7 @@ def test_pseudo_container_fields_absent_in_ha_and_reboot_events(self): self.assertIn(f'failure_type="{failure}"', record) self.assertNotIn('container=', record) self.assertNotIn('container_id=', record) - self.assertIn('host="vision"', record) + self.assertIn('host="test-host"', record) if failure == "ha-failover": self.assertIn('previous_active="node-a"', record) self.assertIn('new_active="node-b"', record) @@ -144,7 +144,7 @@ def test_syslog_has_evidence_without_false_oom_and_escapes_content(self): data = watchdog.format_syslog_alert(p) for fragment in ('event="watchdog_alert"', 'severity="CRITICAL"', 'failure_type="crashed"', 'exit_code="137"', - 'host="vision"', 'container="demo-service"', + 'host="test-host"', 'container="demo-service"', 'container_id="abcdef123456"', 'probe_type="Docker event (crash)"', 'action="Inspect Docker events and logs."'): diff --git a/tests/test_release_smoke_safety.py b/tests/test_release_smoke_safety.py index 7599677..8a403b7 100644 --- a/tests/test_release_smoke_safety.py +++ b/tests/test_release_smoke_safety.py @@ -49,7 +49,7 @@ def test_live_docker_host_rejected_without_any_docker_call(self): rc, calls = self.try_runner({ "GITHUB_ACTIONS": "true", "SMOKE_RUNNER_KIND": "github-hosted", "SMOKE_ENGINE_MARKER": "ci-123-1", - "DOCKER_HOST": "tcp://10.199.0.241:2375"}) + "DOCKER_HOST": "tcp://192.0.2.41:2375"}) self.assertNotEqual(rc, 0) self.assertEqual(calls, []) diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index 5598c61..33684a0 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -52,7 +52,7 @@ boot_grace_seconds: 180 # ── Syslog (remote / central log server) ───────────────────────────────────── syslog: enabled: true # set to true to enable - host: 155.1.1.4 # IP or hostname of your syslog server + host: syslog.example.com # replace with your syslog server port: 514 # 514 = standard syslog (UDP); 601 = reliable syslog (TCP) protocol: udp # udp (fire-and-forget) or tcp (reliable) facility: local0 # syslog facility: local0–local7, daemon, user, etc. @@ -78,7 +78,7 @@ smtp: # Requires: pip install pysnmp (already in requirements-watchdog.txt) snmp_trap: enabled: false # set to true to enable - host: 155.1.1.1 # IP or hostname of your SNMP trap receiver (NMS) + host: nms.example.com # replace with your NMS receiver port: 162 # UDP port — 162 is the standard SNMP trap port version: v2c # SNMP version: v1, v2c, or v3 community: public # SNMPv1/v2c community string (ignored for v3) From d01d7e945f1a0d4c35fbf1bca3b6681649e866ab Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Thu, 15 Oct 2026 21:02:09 +0000 Subject: [PATCH 08/18] feat: guided online-offline release installation and packaging --- .env.example | 2 +- .github/workflows/release-package.yml | 84 +++ .github/workflows/release-smoke.yml | 8 +- DEPLOYMENT.md | 844 ++------------------------ Dockerfile | 4 + README.md | 43 +- docker-compose.build.yaml | 19 +- docker-compose.yaml | 16 +- docs/DEPLOYMENT_ADVANCED.md | 69 +++ docs/RELEASE_NOTES_V1.6.0.md | 6 +- install.sh | 215 ++++--- tests/test_installer_image.py | 184 ++++++ 12 files changed, 573 insertions(+), 921 deletions(-) create mode 100644 .github/workflows/release-package.yml create mode 100644 docs/DEPLOYMENT_ADVANCED.md create mode 100644 tests/test_installer_image.py diff --git a/.env.example b/.env.example index c826a1a..8357b20 100644 --- a/.env.example +++ b/.env.example @@ -48,7 +48,7 @@ SMTP_PASSWORD=your-smtp-password/api key value # ── Image version ───────────────────────────────────────────────────────────── # Release tag the container runs (see the repository VERSION file). install.sh -# sets this to the version it loaded from watchdog.tar; unset means watchdog:latest. +# sets this to the verified release image version (downloaded or loaded locally). # WATCHDOG_VERSION= # ── Tuning ──────────────────────────────────────────────────────────────────── diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml new file mode 100644 index 0000000..a3a7661 --- /dev/null +++ b/.github/workflows/release-package.yml @@ -0,0 +1,84 @@ +name: Publish Container Alert release + +# Publish only a reviewed, tagged commit already merged into main. +on: + push: + tags: + - "v*.*.*" + +permissions: + contents: write + +jobs: + package: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify tag, version, and main ancestry + shell: bash + run: | + set -euo pipefail + version="$(tr -d '[:space:]' < VERSION)" + test "$GITHUB_REF_NAME" = "v$version" + git fetch origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + echo "WATCHDOG_VERSION=$version" >> "$GITHUB_ENV" + + - name: Validate release scripts and Python tests + shell: bash + run: | + set -euo pipefail + bash -n install.sh uninstall.sh + python3 -m pip install docker==7.1.0 requests==2.32.3 PyYAML==6.0.2 + python3 -m unittest discover -s tests -q + + - name: Build version-labelled release image + shell: bash + run: | + set -euo pipefail + docker build --build-arg "WATCHDOG_VERSION=$WATCHDOG_VERSION" \ + -t "watchdog:$WATCHDOG_VERSION" . + image_version="$(docker image inspect "watchdog:$WATCHDOG_VERSION" \ + --format '{{index .Config.Labels "org.opencontainers.image.version"}}')" + test "$image_version" = "$WATCHDOG_VERSION" + docker run --rm --network none --entrypoint python3 \ + "watchdog:$WATCHDOG_VERSION" -c \ + "import docker, requests, yaml, pysnmp, cryptography, watchdog, lifecycle, ha_monitor" + + - name: Package online and offline downloads + shell: bash + run: | + set -euo pipefail + docker save "watchdog:$WATCHDOG_VERSION" -o watchdog.tar + if [ "$(stat -c '%s' watchdog.tar)" -ge 2147483648 ]; then + echo "Release image exceeds GitHub's 2 GiB asset limit" >&2 + exit 1 + fi + sha256sum watchdog.tar > SHA256SUMS + + staging="$RUNNER_TEMP/offline/CyberController-Container-Alert" + mkdir -p "$staging/docs" + cp install.sh uninstall.sh docker-compose.yaml VERSION \ + .env.example watchdog-config.yaml.example \ + README.md DEPLOYMENT.md watchdog.tar SHA256SUMS "$staging/" + cp docs/DEPLOYMENT_ADVANCED.md "$staging/docs/" + tar -czf "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ + -C "$RUNNER_TEMP/offline" CyberController-Container-Alert + sha256sum "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" >> SHA256SUMS + + - name: Publish GitHub Release + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + gh release create "$GITHUB_REF_NAME" \ + watchdog.tar SHA256SUMS \ + "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ + --verify-tag --title "Container Alert $GITHUB_REF_NAME" \ + --generate-notes diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index bec02ee..b5a27a8 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -8,6 +8,12 @@ on: - main paths: - "watchdog.py" + - "lifecycle.py" + - "ha_monitor.py" + - "install.sh" + - "VERSION" + - "docker-compose.yaml" + - "docker-compose.build.yaml" - "tests/release-smoke.sh" - ".github/workflows/release-smoke.yml" - "Dockerfile" @@ -27,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - name: Build release candidate on disposable runner - run: docker build --pull=false --tag "watchdog:$(cat VERSION)" . + run: docker build --pull=false --build-arg "WATCHDOG_VERSION=$(cat VERSION)" --tag "watchdog:$(cat VERSION)" . - name: Create empty, labelled isolated Docker Engine shell: bash diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index b35e763..312734e 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,843 +1,107 @@ -# CyberController Container Watchdog — Deployment Guide - -The **CyberController Container Watchdog** is an autonomous monitoring service that provides continuous health visibility across all Docker containers running on the host. It detects failures — including crashes, out-of-memory kills, prolonged unhealthy states, and restart loops — and dispatches real-time alerts through one or more configurable channels (**Slack**, **SMTP**, **SNMP Traps**, or **Syslog**). It also sends a one-time INFO **recovered** alert once a previously-alarmed container returns to normal operation. - -This guide covers initial deployment, alert channel configuration, and ongoing operational management of the watchdog service. - - -> **Production safety:** Verify watchdog operation using read-only status and log checks. Do not simulate failures on operational hosts. Contact Radware support before changing production monitoring settings. - ---- - -## Table of Contents - -1. [Prerequisites](#1-prerequisites) - - [1.1 Pre-flight Checks](#11-pre-flight-checks) -2. [Environment Setup](#2-environment-setup) - - [2.1 Clone the Repository](#21-clone-the-repository) - - [2.2 Configure Environment Variables (.env)](#22-configure-environment-variables-env) - - [2.3 Configure Watchdog Global Settings](#23-configure-watchdog-global-settings) - - [2.4 Configure Alert Channels](#24-configure-alert-channels) -3. [Deployment Steps](#3-deployment-steps) - - [3.1 Option A: Automated Installation (install.sh) (Recommended)](#31-option-a-automated-installation-installsh-recommended) - - [3.2 Option B: Manual Installation](#32-option-b-manual-installation) -4. [Verification](#4-verification) - - [4.1 Confirm Container Status and Health](#41-confirm-container-status-and-health) - - [4.2 Reviewing Logs](#42-reviewing-logs) -5. [Rollback](#5-rollback) - - [5.1 Roll Back a Failed Deployment](#51-roll-back-a-failed-deployment) - - [5.2 Roll Back an Image Upgrade](#52-roll-back-an-image-upgrade) - - [5.3 Roll Back an Application Upgrade (Git)](#53-roll-back-an-application-upgrade-git) -6. [Operations and Maintenance](#6-operations-and-maintenance) - - [6.1 Lifecycle Management](#61-lifecycle-management) - - [6.2 Applying Configuration Changes](#62-applying-configuration-changes) - - [6.3 Upgrade Guide](#63-upgrade-guide) - - [6.4 Uninstalling](#64-uninstalling) -7. [Troubleshooting](#7-troubleshooting) -8. [Appendix and Reference](#8-appendix-and-reference) - - [8.1 Repository Structure](#81-repository-structure) - - [8.2 Configuration Reference](#82-configuration-reference) - - [8.3 SNMP Trap Var-Binds](#83-snmp-trap-var-binds) - - [8.4 Support Contacts](#84-support-contacts) - ---- - -## 1. Prerequisites - -| Requirement | Minimum Version / Notes | -|---|---| -| Docker Engine | 20.10+ | -| Docker Compose | v2 plugin (`docker compose`) or standalone `docker-compose` | -| Linux host | Ubuntu 20.04 / RHEL 8 or later (required for Docker socket access) | -| Git | Required to clone the repository | -| Host permissions | Root, or membership in the `docker` group (to read `/var/run/docker.sock`) | - -> By default the container runs as root so a manual install needs no host-specific group setup. The automated installer ([3.1 Option A](#31-option-a-automated-installation-installsh-recommended)) instead hardens it to run non-root (UID/GID 1000, joined to the docker.sock group) automatically, with no input required. Either way it's still contained by a read-only root filesystem, dropped Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID limits (see [Container Hardening in the README](README.md#container-hardening)). - -```bash -docker compose version -``` - -### 1.1 Pre-flight Checks - -Run these checks before starting the deployment. Do not proceed if any of them fail. - -```bash -# Docker daemon is running and reachable -docker info +# CyberController Container Watchdog — Deployment -# Current user can talk to the Docker socket without sudo -docker ps +**One installer, one wizard.** The same `install.sh` supports Internet-connected and offline hosts. It deploys only the `docker-container-watchdog` service and does not change CyberController application containers. -# Compose plugin is installed -docker compose version +## Before you begin -# Sufficient free disk space for the image and log files -df -h . -``` - -Image and container size is documented in [README.md § Image Size](README.md#image-size). - -If you plan to use an alert channel that requires outbound network access (Slack webhook, SMTP relay, or SNMP trap receiver), confirm the host can reach that endpoint before deployment. - -> Repository layout is documented in [8.1 Repository Structure](#81-repository-structure). - ---- +The host needs Docker, a working Docker daemon, Docker Compose, and an operator account with Docker access (and sudo where required). Git is needed only to obtain or update files from GitHub; curl is needed only if downloading the release image online. -## 2. Environment Setup +Install only a published, approved release. The version in `VERSION` selects the corresponding prebuilt Docker image. Customer installations never compile application code. -> If you plan to use the automated installer ([3.1 Option A: Automated Installation (install.sh)](#31-option-a-automated-installation-installsh-recommended), recommended), the configuration wizard creates `.env` and `watchdog-config.yaml` for you interactively — you may skip 2.2–2.4 and go directly to [3. Deployment Steps](#3-deployment-steps). Complete 2.2–2.4 only if you are installing manually ([3.2 Option B: Manual Installation](#32-option-b-manual-installation)) or want to pre-stage configuration before running the installer. +## Install -### 2.1 Clone the Repository +Get the installation files in either of these ways: -# Navigate to your target installation directory -cd /path/to/Container_Alert +**GitHub available** — clone the default `main` branch: ```bash -git clone https://github.com/Radware/CyberController-Container-Alert Container_Alert -cd Container_Alert +git clone https://github.com/Radware/CyberController-Container-Alert.git +cd CyberController-Container-Alert ``` -All remaining commands in this guide are run from inside the `Container_Alert/` directory. - ---- -*Note*: If you use the automated installation method in Section 2.1, skip Sections 2.2, 2.3 and 2.4 and proceed directly to Section 3. - -### 2.2 Configure Environment Variables (.env) +**GitHub unavailable** — obtain the approved offline package from Radware, copy it to the host, and extract it: ```bash -cp .env.example .env -chmod 600 .env # restrict to owner only — contains secrets +tar -xzf container-alert-v1.6.0-offline.tar.gz +cd CyberController-Container-Alert ``` -Open `.env` and fill in every value: - -```dotenv -# Slack -SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T.../B.../... - -# SMTP (only if SMTP channel is enabled) -SMTP_USERNAME=your-smtp-username/login email -SMTP_PASSWORD=your-smtp-password/api key value - -# Tuning (optional — defaults shown) -LOG_LEVEL=INFO -``` - -Only credentials and secrets belong in `.env`. Hosts, ports, recipients, and thresholds are configured in `watchdog-config.yaml`, never hardcoded in this guide or in scripts. - -The container runs as root by default for a zero-configuration manual install. If you want the non-root hardening that [Option A](#31-option-a-automated-installation-installsh-recommended) applies automatically, uncomment the `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` lines in `.env.example` and fill in `DOCKER_GID` (`stat -c '%g' /var/run/docker.sock`). - ---- - -### 2.3 Configure Watchdog Global Settings - -```yaml -# ── Watchdog global settings ──────────────────────────────────────────────────────── -check_interval_seconds: 60 # poll interval -cooldown_minutes: 5 # no duplicate alerts within this window -restart_threshold: 5 # restarts before "restart-loop" alert -restart_window_minutes: 10 # rolling window for restart counting -unhealthy_cycles_threshold: 3 # consecutive unhealthy polls before alert -alert_on_recovery: true # send an INFO "recovered" alert when a container returns to normal - -excluded_containers: - - debug-shell # add any containers you don't want monitored - -ignored_container_events: - - label: - com.radware.cybercontroller.role: mariadb-ha-syntax-check - events: - - die # ignore only R&D-designated temporary syntax-check dies -``` - -Radware R&D to add `--label com.radware.cybercontroller.role=mariadb-ha-syntax-check` when creating the temporary MariaDB HA syntax-check container. That explicit label is the preferred contract: normal service container dies alert; R&D-designated temporary helper dies are ignored for the configured event. - -Until that label exists, the watchdog also has a built-in fallback for dynamic MariaDB dump helper crashes. It suppresses the alert only when image `kvision_infra_mariadb`, container name is not `config_kvision-infra-mariadb_1`, Docker `AutoRemove` is `true`, a mount contains `/mnt/cli/tmp/mariadb_dump.sql.gz`, and the exit code is `137` all match together. - ---- - -### 2.4 Configure Alert Channels - -| Channel | `alert_channels` identifier | What you need | -|---|---|---| -| Slack Webhook | `slack` | Slack incoming webhook URL | -| Syslog | `syslog` | Syslog server reachable from the Docker host | -| Email (SMTP) | `smtp` | SMTP relay and credentials | -| SNMP Traps | `snmp_trap` | SNMP trap receiver (NMS / SIEM) | - -At least one communication channel must be configured before the watchdog can send alerts. Choose from Slack, SMTP, SNMP Trap, or Syslog, then obtain the required credentials or endpoint details for your chosen channel(s) and configure them in `watchdog-config.yaml` and `.env`. - -For each channel you want to use, follow the relevant option below. Ensure its identifier is listed under `alert_channels` in `watchdog-config.yaml`. - -#### 2.4.1 Slack Webhook - -1. Go to [https://api.slack.com/apps](https://api.slack.com/apps) → **Create New App** → **From scratch** -2. Name it `Container Watchdog`, select your workspace -3. In the left menu: **Incoming Webhooks** → toggle **On** -4. Click **Add New Webhook to Workspace** → choose an alert channel (e.g. `#container-alerts`) -5. Copy the webhook URL (starts with `https://hooks.slack.com/services/...`) -6. Paste it into `.env` as `SLACK_WEBHOOK_URL` - -Edit `watchdog-config.yaml` to match your environment: -```yaml -# ── Slack ───────────────────────────────────────────────────────────────────── -slack: - enabled: true # set to true to enable - webhook_url_env: SLACK_WEBHOOK_URL # env var: https://hooks.slack.com/services/... - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - slack -``` - - -#### 2.4.2 Syslog Forwarding - -Enable the syslog block in `watchdog-config.yaml`: - -```yaml -# ── Syslog (remote / central log server) ───────────────────────────────────── -syslog: - enabled: true # set to true to enable - host: syslog.example.com # IP or hostname of your syslog server - port: 514 # 514 = standard syslog (UDP); 601 = reliable syslog (TCP) - protocol: udp # udp (fire-and-forget) or tcp (reliable) - facility: local0 # syslog facility: local0–local7, daemon, user, etc. - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - syslog -``` - ---- - -#### 2.4.3 SMTP (Email) - -Add `smtp` to `alert_channels` in `watchdog-config.yaml` and configure the block: - -```yaml -# ── Email (SMTP) ───────────────────────────────────────────────────────────── -# Add "smtp" to alert_channels above to enable. -smtp: - enabled: true # set to true to enable - host: smtp.radware.com - port: 587 - sender: noc-alerts@radware.com - username_env: SMTP_USERNAME # env var: SMTP username or app token (Mailtrap, Gmail app password, etc.) - recipients: - - ops-team@radware.com - - oncall@radware.com - tls: true - password_env: SMTP_PASSWORD # env var: SMTP password or app token - -#### SMTP Relay (No Authentication) -# ── Email (SMTP Relay - No Authentication) ─────────────────────────────────── -# Use this configuration only when your SMTP relay accepts unauthenticated -# connections from trusted hosts. - -smtp: - enabled: true - - # SMTP server settings - host: your-relay.internal - port: 25 - tls: false # Enable only if your relay requires STARTTLS - - # Authentication - auth: false - - # Email settings - sender: watchdog@yourdomain.com - recipients: - - ops@yourdomain.com - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - smtp -``` - -Set SMTP credentials in `.env`: - -```dotenv -SMTP_USERNAME=your-smtp-username/ login email -SMTP_PASSWORD=your-smtp-password/ api key value -``` - ---- - -#### 2.4.4 SNMP Traps - -Add `snmp_trap` to `alert_channels` in `watchdog-config.yaml` and configure the block: - -```yaml -# ── SNMP Traps ──────────────────────────────────────────────────────────────── -# Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. -# Add "snmp_trap" to alert_channels above to enable. -# Requires: pip install pysnmp (already in requirements-watchdog.txt) -snmp_trap: - enabled: false # set to true to enable - host: nms.example.com # IP or hostname of your SNMP trap receiver (NMS) - port: 162 # UDP port — 162 is the standard SNMP trap port - version: v2c # SNMP version: v1, v2c, or v3 - community: public # SNMPv1/v2c community string (ignored for v3) - # trap_oid is not set here — code defaults to Radware enterprise OID 1.3.6.1.4.1.89.110.0.1 - # SNMPv3 settings (used only when version: v3 — add secrets to .env) - # v3_username: watchdog-user - # v3_auth_protocol: SHA # MD5 | SHA | SHA256 | SHA384 | SHA512 - # v3_auth_key_env: SNMP_V3_AUTH_KEY # env var in .env holding the auth passphrase - # v3_priv_protocol: AES # AES (AES-128, recommended) | AES192 | AES256 | DES (weak — logs warning) - # v3_priv_key_env: SNMP_V3_PRIV_KEY # env var in .env holding the priv passphrase - # v3_local_engine_id: "" # Pin the sender engine ID (hex, with or without 0x). - # The watchdog logs the auto-derived engine ID at startup (INFO level) — copy it here. - # Then set in snmptrapd.conf: createUser -e 0x watchdog-user SHA "..." AES "..." - # This prevents snmptrapd from needing reconfiguration after container restarts. - -# ── Alert channels ──────────────────────────────────────────────────────────── -# Empty for local testing — alerts are still logged to stdout/file. -alert_channels: - - snmp_trap -``` - -> **Security note:** `public` is the well-known default SNMPv1/v2c community string and provides no real access control — anyone who can reach the trap receiver on UDP/162 with that string can be trusted implicitly. Change it to a non-default value, restrict network access to the trap receiver, or use SNMPv3 with authentication and encryption for production deployments. - -Trap var-binds and SNMPv3 setup notes are documented in [8.3 SNMP Trap Var-Binds](#83-snmp-trap-var-binds). - ---- - -## 3. Deployment Steps - -### 3.1 Option A: Automated Installation (install.sh) (Recommended) - -Run the installer from the `Container_Alert/` directory created in [2.1 Clone the Repository](#21-clone-the-repository). It guides you through every step interactively — including environment and alert-channel configuration — and requires no manual file editing. - -> **Before running:** If internet access is available, no additional steps are required as the installer will automatically build the Docker image from source; otherwise, download the pre-built Docker image archive [watchdog.tar](https://radwareil.sharepoint.com/:f:/s/NAResidentEngineers/IgC4d3ALtPwvSrz4is6QWmnPATRCQ7g6BXbnbWO3Q0FFMSk?e=oj4WGS) and place it in the same directory as install.sh before starting the installation. -If you have trouble accessing the download link, see [8.4 Support Contacts](#84-support-contacts). +Then **run the same command** in either case: ```bash bash install.sh ``` -The script walks through the following stages in order: - -| Stage | What happens | -|---|---| -| **Prerequisites** | Verifies Docker and Docker Compose are installed and running | -| **Log directory** | Creates `./watchdog/` for persistent log storage | -| **Load image** | Loads `watchdog.tar` and requires it to contain `watchdog:` (from the root `VERSION` file); aborts on a version mismatch, and builds from source only when the archive is absent | -| **Host identification** | `WATCHDOG_HOST` is taken from `.env` (the installer prompts for it), defaulting to `CyberController-Server` | -| **Configuration wizard** | Selects channels and prompts for Slack/SMTP/SNMP/Syslog values interactively | -| **Credentials + config write** | Generates/updates `.env` (including `WATCHDOG_VERSION`) and `watchdog-config.yaml` from wizard answers | -| **Start** | Runs `docker compose up -d` in the background, aborting if Compose reports a failure | -| **Verify** | Fails the installation unless the container is running **and** uses `watchdog:` | - -> **Re-running `install.sh` on an existing installation is safe.** If both configuration files exist, the installer asks whether to reconfigure from scratch. Choose `N` to keep them unchanged — the packaged image is still loaded and the deployment is still moved to the new version. If only one of the two files is present, the wizard regenerates both, since a half-present configuration cannot start. - -Continue to [4. Verification](#4-verification). - ---- - -### 3.2 Option B: Manual Installation - -Use this path when you need full control over configuration files before starting. This procedure is optional — skip it if you completed Option A. It assumes you already completed [2.1 Clone the Repository](#21-clone-the-repository) and [2.2–2.4 Environment Setup](#2-environment-setup). - -#### Online Installation - -*Note* - If internet access is available, build the image and skip to [Start the container](#start-the-container) below. - -```bash -WATCHDOG_VERSION="$(cat VERSION)" docker compose -f docker-compose.build.yaml build -``` - -#### Offline Installation - -*Note* - If internet access is unavailable, load the pre-built image instead of building it. - -Download [`watchdog.tar` — pre-built Docker image for offline installation](https://radwareil.sharepoint.com/:u:/s/NAResidentEngineers/IQASPcHrYh6ATqPzBOvz_TgnAYi1knv7GbmwctN7m1JWtOM?e=YKrQP0). If you have trouble accessing the download link, see [8.4 Support Contacts](#84-support-contacts). -```bash -docker load -i watchdog.tar -``` - -#### Start the container - -```bash -docker compose up -d -``` - -Continue to [4. Verification](#4-verification). - ---- - -## 4. Verification - -### 4.1 Confirm Container Status and Health - -```bash -docker compose ps -docker compose logs -f docker-container-watchdog -``` - -Expected output from `docker compose ps`: - -``` -NAME STATUS -docker-container-watchdog Up (healthy) -``` - -If the status does not reach `Up (healthy)` within a couple of minutes, see [7. Troubleshooting](#7-troubleshooting), or follow [5. Rollback](#5-rollback) to back out the deployment. - ---- - -### 4.2 Reviewing Logs - -| Location | Description | -|---|---| -| `docker compose logs docker-container-watchdog` | Live stdout logs | -| `./watchdog/watchdog.log` | Bind-mounted log file on the host (inside the `Container_Alert/` directory) | -| `/var/log/watchdog/watchdog.log` | Inside the container (rotated, max 10 MB × 5 files) | - -To follow logs with timestamps: - -```bash -docker compose logs -f --timestamps docker-container-watchdog -``` - -To read the log file directly from the volume: - -```bash -docker exec docker-container-watchdog tail -f /var/log/watchdog/watchdog.log -``` - ---- - -### Optional HA role-change verification - -This check is only for **two-node CyberController HA** installations and is -disabled by default. On **each HA host**: - -1. Verify the local exporter is active: curl -fsS http://127.0.0.1:9664/metrics. -2. Confirm metrics for Pacemaker resources, quorum, collector scrape success, - and local node identity are present. Port 9664 is the verified lab setting; - read /etc/ha_cluster_exporter.yaml if different. -3. Enable ha_monitor.enabled: true in watchdog-config.yaml on **both** - nodes; restart only the watchdog service using the normal documented - configuration-change procedure. -4. Verify the baseline is stored under ha_monitor.active_node in the persistent - watchdog state.json; startup should send **no** HA event. -5. During an approved HA maintenance window, perform one controlled switchover - and failback using the supported CyberController operations procedure. - Confirm exactly one notification from the newly active node for each - transition; check docker logs docker-container-watchdog on both hosts. -6. Disable by setting ha_monitor.enabled: false and restarting the watchdog. - This clears its saved HA baseline; re-enabling starts with a fresh, quiet - baseline (no notification for transitions that occurred while disabled). - An ordinary restart while HA remains enabled keeps the confirmed baseline. - No HA service or Pacemaker configuration changes are necessary. - -Delivery is best-effort. Test the actual alert receiver on **both** nodes: -an HA alert in watchdog logs does not prove a syslog or SMTP destination -received it. This release has no persisted notification retry queue. - -Do not attempt a network-partition simulation on a cluster with STONITH -disabled: split-brain and database inconsistency are possible. The native -Prometheus port mismatch (9002 vs exporter 9664) is a separate appliance -configuration issue; it does not block the watchdog's direct localhost check. - ---- - -## 5. Rollback - -### 5.1 Roll Back a Failed Deployment - -If verification fails immediately after `docker compose up -d` — the container will not start, restarts in a loop, or the health check never turns healthy: - -```bash -docker compose down -``` - -Diagnose the cause using [4.2 Reviewing Logs](#42-reviewing-logs) and [7. Troubleshooting](#7-troubleshooting), correct the configuration, then redeploy: - -```bash -docker compose up -d -``` - -Re-run [4.1 Confirm Container Status and Health](#41-confirm-container-status-and-health) before considering the deployment complete. - -### 5.2 Roll Back an Image Upgrade - -Each release is tagged `watchdog:` and the deployed tag is pinned by `WATCHDOG_VERSION` in `.env`. As long as the previous image is still on the host, rolling back is a version change: - -```bash -docker images watchdog # list the versions available locally -sed -i 's/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=1.5.3/' .env -docker compose up -d -``` - -If the previous image was only ever tagged `watchdog:latest`, preserve it under a rollback tag before loading a new archive: - -```bash -docker tag watchdog:latest watchdog:rollback -``` - -### 5.3 Roll Back an Application Upgrade (Git) - -Before running `git pull` (see [6.3 Upgrade Guide](#63-upgrade-guide)), record the current commit so you can return to it: - -```bash -git rev-parse HEAD > backup/previous-commit.txt -``` - -To restore the prior source checkout: - -```bash -git checkout "$(cat backup/previous-commit.txt)" -``` - -This **does not roll back the running container**. To revert deployed behavior, restore the previous `WATCHDOG_VERSION` and redeploy the corresponding image as described in [5.2 Image Rollback](#52-roll-back-an-image-upgrade). +The wizard asks whether this host can download the official image from GitHub Releases. -Also restore the `.env` and `watchdog-config.yaml` files backed up before the upgrade — see [6.3 Upgrade Guide](#63-upgrade-guide). +- **Yes:** Download the Docker image and its checksum, verify the image, then continue. If the download fails, you can choose a local image or archive instead. +- **No:** Check for the matching local image and ask whether to use it. If not selected, load `watchdog.tar` from the installation directory. An offline package includes this archive and its checksum. +- **Neither available:** The installer explains what is missing and exits without changing the running watchdog. Obtain the release package and rerun `bash install.sh`. ---- +The installer then asks for the desired notification channels (Syslog, SMTP, SNMP, Slack), the necessary connection settings, and the hostname shown in alerts. Existing `.env` and `watchdog-config.yaml` files are kept by default during upgrades. -## 6. Operations and Maintenance +**No Docker image build, manual `docker load`, or Compose editing is required for a normal installation.** -### 6.1 Lifecycle Management - -#### Stopping the Service +## Verify the installation ```bash -# Pause the container — restartable with `docker compose start` -docker compose stop - -# Stop and remove the container, keep the image and log files -docker compose down +docker ps --filter name=docker-container-watchdog +docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' +docker logs --tail 50 docker-container-watchdog ``` -#### Restarting After Configuration Changes +Expected: the watchdog container is running, its Docker health check becomes healthy, and recent logs show normal monitoring. Check the configured receiver for previously emitted alerts if available. A working monitor does not need an artificial failure to prove it is alive. -```bash -# Pick up watchdog-config.yaml changes (no rebuild needed) -docker compose restart docker-container-watchdog +Read-only inspection is sufficient on operational hosts. **Do not stop, kill, restart, or stress application containers to test alerting.** -# Re-read .env credential changes -docker compose up -d -``` +## Upgrade with the same installer -#### Updating the Watchdog Image +**GitHub installation:** ```bash -# Rebuild the image (requires internet) -WATCHDOG_VERSION="$(cat VERSION)" docker compose -f docker-compose.build.yaml build - -# Redeploy using the production runtime file -docker compose up -d -``` - ---- - -### 6.2 Applying Configuration Changes - -No image rebuild is required for any configuration change. The type of change determines the required command: - -| Changed file | What to run | -|---|---| -| `watchdog-config.yaml` (thresholds, channels, exclusions) | `docker compose restart docker-container-watchdog` | -| `.env` (secrets, credentials) | `docker compose up -d` | -| `docker-compose.yaml` (host name, resource limits) | `docker compose up -d` | - -#### Apply watchdog-config.yaml changes - -```bash -docker compose restart docker-container-watchdog -``` - -#### Apply .env changes (credentials / secrets) - -```bash -docker compose up -d -``` - -#### Apply docker-compose.yaml changes (container settings) - -For example, change the hostname shown in alerts by editing `WATCHDOG_HOST` in `.env`: - -```bash -WATCHDOG_HOST=my-new-server-name -``` - -Then redeploy to take effect: - -```bash -docker compose up -d -``` - ---- - -### 6.3 Upgrade Guide - -Updating the source checkout and upgrading the running Docker image are different operations. Only deploying a new image changes the running watchdog code. - -#### Before You Begin (Recommended) - -Back up your deployment-specific configuration files before upgrading. These backups are also used for [5. Rollback](#5-rollback) if the upgrade fails. - -```bash -mkdir -p backup - -cp .env backup/.env.bak -cp watchdog-config.yaml backup/watchdog-config.yaml.bak -``` - -To restore the configuration later (if required): - -```bash -cp backup/.env.bak .env -cp backup/watchdog-config.yaml.bak watchdog-config.yaml -``` - ---- - -#### Update Source Code (Not a Running Image) - -The application (`watchdog.py`, `lifecycle.py`, `ha_monitor.py`) is **built into the Docker image**, not bind-mounted. Updating the Git checkout alone **does not deploy code**, and `docker compose up -d` with the same pinned image will not apply the latest changes. Do not stop the running watchdog just to update source files. - -To update the repository without changing the running container: - -```bash -mkdir -p backup -git rev-parse HEAD > backup/previous-commit.txt -git fetch origin -git pull --ff-only origin main -``` - -To deploy new code, first prepare a **new numbered release**: update `VERSION` and the README version history, build and test the matching `watchdog:` image using `docker-compose.build.yaml`, and verify it contains all required Python modules. Keep the previous versioned image for rollback; update `.env`'s `WATCHDOG_VERSION` only when deploying the verified new image. For offline deployments, distribute the new `VERSION` and matching `watchdog.tar`, then use `install.sh`. See [Deployment Steps](#3-deployment-steps), [Image Rollback](#52-roll-back-an-image-upgrade), and [README Version History](README.md#10-version-history). - ---- - -#### Upgrade Docker Image Only - -Automatic upgrades are **not enabled**. Preserve the existing versioned `watchdog:` image for [rollback](#52-roll-back-an-image-upgrade). Only if the old image exists solely as `watchdog:latest`, save that image as `watchdog:rollback` before loading the replacement: - -```bash -docker tag watchdog:latest watchdog:rollback -``` - -There are two ways to upgrade the image — online or offline. Follow only one of the options. - -##### Option A: Online Image Upgrade - -If a new versioned image has been published to the image registry configured for this deployment, pull that tag. A local-only `watchdog:` image is not available via `docker compose pull`; build it or load the release archive instead: - -```bash -docker compose pull -docker compose up -d -``` - -##### Option B: Offline Image Upgrade - -1. Request the Radware RE team to create an updated pre-built Docker image for offline installation. See [8.4 Support Contacts](#84-support-contacts). -2. Load the image. The archive carries the release tag `watchdog:` matching the package's `VERSION` file. - - ```bash - docker load -i watchdog.tar - sed -i "s/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=$(cat VERSION)/" .env - docker compose up -d - ``` - - Running `bash install.sh` performs all three steps (and verifies the result) automatically. - ---- - -#### Checking the Installed Version - -View the image used by the running container: - -```bash -docker inspect docker-container-watchdog --format='{{.Config.Image}}' -``` - -View locally available Watchdog images: - -```bash -docker images watchdog -``` - ---- - -### 6.4 Uninstalling - -Run `uninstall.sh` from the `Container_Alert/` directory. Choose the option that matches how much you want removed: - -| Scenario | Container removed | Image removed | Logs removed | Command | -|---|:---:|:---:|:---:|---| -| Standard (container removed; image + logs prompted) | ✓ | Prompted | Prompted | `bash uninstall.sh` | -| Remove everything without prompts | ✓ | ✓ | ✓ | `bash uninstall.sh --remove-all --force` | -| Remove container, keep image + logs | ✓ | ✗ | ✗ | `bash uninstall.sh --keep-image --keep-logs` | -| Remove container + image, keep logs | ✓ | ✓ | ✗ | `bash uninstall.sh --keep-logs --force` | - -#### Standard interactive uninstall - -```bash -bash uninstall.sh -``` - -Shows a removal plan (container name, image size, log directory size), asks for confirmation, then prompts whether to delete log files. Safe default for manual runs. - -#### Remove container and image, preserve logs - -```bash -bash uninstall.sh --keep-logs -``` - -Stops and removes the `watchdog` container and the `watchdog:` and `watchdog:latest` Docker image tags. The `./watchdog/` log directory is left intact so you can review historical logs later. - -#### Remove everything including logs - -```bash -bash uninstall.sh --remove-all -``` - -Removes the container, image, and the entire `./watchdog/` log directory. Prompts for confirmation before proceeding. - -#### Fully non-interactive - -```bash -bash uninstall.sh --remove-all --force -``` - -Skips all confirmation prompts. Use in automated pipelines or when scripting repeated clean installs. - -> `--keep-logs` and `--remove-all` are mutually exclusive and cannot be combined. - ---- - -## 7. Troubleshooting - -### Service Fails to Start - -```bash -docker compose logs docker-container-watchdog -``` - -Common causes: -- `/var/run/docker.sock` is not accessible — ensure the host socket exists and Docker is running. Manual installs run the container as root, so no host-side group configuration is required; if you opted into non-root hardening (`WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env`), verify `DOCKER_GID` matches `stat -c '%g' /var/run/docker.sock`. -- Missing `.env` file — run `cp .env.example .env` and fill in values - -### Alert Notifications Not Received - -1. Verify `SLACK_WEBHOOK_URL` is set correctly in `.env` -2. Test the webhook manually: - ```bash - curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"Watchdog test"}' \ - "$SLACK_WEBHOOK_URL" - ``` - Expected response: `ok` - -### Duplicate Alert Notifications - -Increase `cooldown_minutes` in `watchdog-config.yaml`. The default is 5 minutes per container per failure type. - -### Excessive Alert Volume - -Add noisy containers to `excluded_containers` in `watchdog-config.yaml`: - -```yaml -excluded_containers: - - debug-shell - - load-test - - my-noisy-container +cd CyberController-Container-Alert +git pull --ff-only +bash install.sh ``` -If none of the above resolves the issue, see [5. Rollback](#5-rollback) to back out a recent change and redeploy from a known-good state. - ---- +**Offline installation:** obtain the new offline package and update the installation files in the **existing installation directory**, preserving `.env`, `watchdog-config.yaml`, and `watchdog/`. Then run `bash install.sh` there. -## 8. Appendix and Reference +On upgrade, the installer retains existing alert settings unless you explicitly request reconfiguration. It verifies the new image before changing deployment state and recreates **only the watchdog service**, preserving unrelated Docker containers. The prior image should be retained for rollback. -### 8.1 Repository Structure +### Apply configuration changes -``` -Container_Alert/ -├── docker-compose.yaml # Production stack — uses pre-built image -├── docker-compose.build.yaml # Developer build stack — requires internet -├── Dockerfile # Watchdog image definition -├── requirements-watchdog.txt # Python deps: docker, requests, PyYAML -├── watchdog.py # Watchdog agent -├── watchdog-config.yaml # Watchdog behaviour & channel settings -├── install.sh # Install helper script -├── uninstall.sh # Uninstall helper script -├── README.md # Quick-start reference -└── .env.example # Template — copy to .env and fill in -``` - ---- - -### 8.2 Configuration Reference - -All settings are in `watchdog-config.yaml`. No image rebuild is required for configuration changes — restart the watchdog container to apply updates: +Changes to `watchdog-config.yaml` require a watchdog restart. Changes to `.env` or Compose settings require recreation to apply the new container environment: ```bash docker compose restart docker-container-watchdog +# For changed environment or Compose settings: +docker compose up -d --no-deps --force-recreate docker-container-watchdog ``` -| Setting | Default | Description | -|---|---|---| -| `check_interval_seconds` | `60` | How often to poll all containers | -| `cooldown_minutes` | `5` | Suppress duplicate alerts per container | -| `restart_threshold` | `5` | Restarts within the window to trigger restart-loop alert | -| `restart_window_minutes` | `10` | Rolling window for restart counting | -| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy polls before alerting | -| `alert_on_recovery` | `true` | Send an INFO "recovered" alert once a previously-alarmed container returns to normal | -| `excluded_containers` | `[]` | Containers that are never alerted on | -| `log_level` | `INFO` | Logging verbosity: `INFO` (recommended for production) or `DEBUG` | +Production monitoring configuration changes should follow the normal authorized change process. -#### Host Identification in Alerts +### Roll back an image -Set `WATCHDOG_HOST` in `.env` (written by `install.sh` from the hostname prompt), then run `docker compose up -d`: +If the previous image is still available, restore its version in `WATCHDOG_VERSION` in `.env` and recreate **only** the watchdog service: ```bash -WATCHDOG_HOST=my-server-name +docker image ls watchdog +docker compose up -d --no-deps --force-recreate docker-container-watchdog ``` -When it is unset, `docker-compose.yaml` falls back to `CyberController-Server`. - ---- +Do not overwrite existing version tags with new code. Obtain the previous approved package if its image was removed. -### 8.3 SNMP Trap Var-Binds +## Troubleshooting -Configuration for the SNMP Traps channel is in [2.4.4 SNMP Traps](#244-snmp-traps). - -**Trap var-binds (enterprise OID arc `1.3.6.1.4.1.89.110`):** - -| OID | Object | Value | -|-----|--------|-------| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Hostname of the alerting node | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Human-readable alert summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Probe failure detail | - -**HA trap mapping:** this release uses the existing Container Watchdog MIB. -For HA, cwContainerName is (HA cluster), cwFailureType is ha-failover, and the -summary/detail fields contain previous and new active nodes. NMS rules should -match cwFailureType instead of treating all traps as container failures. No -new trap OID is introduced. - -**SNMPv3 notes:** - -- **Engine ID:** On the first run the watchdog logs an auto-stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...` in the logs). Copy it into `v3_local_engine_id` and register it in `snmptrapd.conf` with `createUser -e 0x watchdog-user SHA "..." AES "..."`. This keeps the ID stable across container restarts. -- **Key length:** Both auth and priv passphrases must be at least 8 characters (RFC 3414 §11.2). The watchdog rejects shorter keys with a clear error. -- **No privacy-only mode:** Setting a priv key without an auth key is rejected — SNMPv3 has no `privNoAuth` security level. Both keys must be set together for encrypted traps. -- **Protocol names:** Auth: `MD5`, `SHA`, `SHA256`, `SHA384`, `SHA512`. Priv: `AES` (AES-128, recommended), `AES192`, `AES256`, `DES` (weak — logs a warning). An unrecognised name is rejected with an error listing the supported values. - ---- - -### 8.4 Support Contacts +| Symptom | Check | +|---|---| +| Online image download failed | GitHub Release for the version in `VERSION` must be published and accessible; otherwise use the offline package | +| Local image rejected | The image must have a matching release version label; supply the official `watchdog.tar` | +| Cannot load image | Check available disk space, image archive, and `SHA256SUMS` | +| Docker/Compose check fails | Confirm the daemon is running and the operator has Docker access | +| Configuration is incomplete | Restore missing `.env` or `watchdog-config.yaml` from backup; installer does not discard the remaining file | +| Watchdog is not healthy | Review `docker logs --tail 100 docker-container-watchdog` and the configured notification endpoints | -If you have trouble accessing the `watchdog.tar` download link, or need an updated offline image from the Radware RE team, contact: +For configuration examples, security considerations, image-building instructions for developers, release packaging, and advanced troubleshooting, see [Advanced Deployment Reference](docs/DEPLOYMENT_ADVANCED.md). Notification-channel examples are also in [README.md](README.md). -- northamericare@radware.com +To remove the watchdog, run `bash uninstall.sh` from its installation directory and choose the desired retention options. diff --git a/Dockerfile b/Dockerfile index d906b30..2d6ec32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,10 @@ FROM python:3.11-slim WORKDIR /app +# Version is set by the release build and checked by install.sh. +ARG WATCHDOG_VERSION=unknown +LABEL org.opencontainers.image.version="$WATCHDOG_VERSION" + # Install dependencies RUN pip install --no-cache-dir \ docker==7.1.0 \ diff --git a/README.md b/README.md index 5e7d3e2..0a25b2f 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,7 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitori - [SNMP Traps](#snmp-traps) 5. [Deployment Instructions](#5-deployment-instructions) 6. [Failure Types & Severities](#6-failure-types--severities) -7. [Testing](#7-testing) - - [OOM Simulation](#oom-simulation) - - [Probe Testing](#probe-testing) - - [Recovery Alert Test](#recovery-alert-test) +7. [Verify Watchdog Operation](#7-verify-watchdog-operation) 8. [Troubleshooting](#8-troubleshooting) - [Service Fails to Start](#service-fails-to-start) - [Alert Notifications Not Received](#alert-notifications-not-received) @@ -138,13 +135,13 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ├── lifecycle.py # Bounded Docker stop/die evidence correlation ├── watchdog-config.yaml # All configuration (no secrets) ├── docker-compose.yaml # Production / offline runtime (image only) -├── docker-compose.build.yaml # Developer / online build +├── docker-compose.build.yaml # Developer-only source build ├── Dockerfile # Image build definition (python:3.11-slim) ├── requirements-watchdog.txt # Python dependencies (reference; Dockerfile pip-installs inline) -├── install.sh # Offline install script +├── install.sh # Guided online/offline installation ├── uninstall.sh # Stop + remove script ├── VERSION # Release version — drives the image tag and installer -├── watchdog.tar # Pre-built Docker image (provided, no internet needed) +├── watchdog.tar # Optional: image supplied in offline release package └── .env # Secrets — NOT committed to git ``` @@ -158,11 +155,11 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). | Running container (memory) | ~50–80 MB baseline; capped at `mem_limit: 256m` in `docker-compose.yaml` | | `watchdog.tar` export | ~170 MB | -> **Note:** The values above are approximate and may vary depending on the host operating system, Docker version, and installed dependencies. If usage approaches the 256 MB limit (check with `docker stats docker-container-watchdog`), raise `mem_limit`/`mem_reservation` in `docker-compose.yaml` rather than removing the limit — and raise `memswap_limit` to at least the new `mem_limit` in **both** `docker-compose.yaml` and `docker-compose.build.yaml`, since Docker rejects a config where `memswap_limit` is lower than `mem_limit`. +> **Note:** Image size and memory usage vary with the installed release. If resource usage is abnormal, capture Docker health, logs, and usage observations for engineering review before making production resource-limit changes. ### Container Hardening -By default the container runs as **root** so `docker compose up` works with zero configuration — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [DEPLOYMENT.md](DEPLOYMENT.md#22-configure-environment-variables-env)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. +By default the container runs as **root** so `docker compose up` works with zero configuration — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md#configuration-files)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. ### Python Dependencies @@ -286,7 +283,7 @@ snmp_trap: # v3_auth_key_env: SNMP_V3_AUTH_KEY # min 8 chars # v3_priv_protocol: AES # AES (recommended) | AES192 | AES256 | DES (weak) # v3_priv_key_env: SNMP_V3_PRIV_KEY # min 8 chars; requires auth key also set - # v3_local_engine_id: "" # hex engine ID pinned from first-run log; see DEPLOYMENT.md + # v3_local_engine_id: "" # hex engine ID; see docs/DEPLOYMENT_ADVANCED.md ``` Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3.6.1.4.1.89.110`): @@ -303,14 +300,18 @@ Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3 ## 5. Deployment Instructions -For full deployment instructions — including alert channel setup, offline installation, developer builds, and troubleshooting — see [DEPLOYMENT.md](DEPLOYMENT.md). - -**Quick start (automated):** +Use the same installer for Internet-connected and offline hosts. With access to GitHub, clone the default **main** branch: ```bash +git clone https://github.com/Radware/CyberController-Container-Alert.git +cd CyberController-Container-Alert bash install.sh ``` +For a disconnected host, extract the approved offline release package and run `bash install.sh` inside the extracted installation directory. The wizard offers download or local-image selection, asks for the selected notification channels, and verifies the watchdog. Customer installations do not build Docker images. + +See [Deployment Guide](DEPLOYMENT.md) for the short procedure, and [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) for developer and release-maintainer details. + --- ## 6. Failure Types & Severities @@ -336,8 +337,8 @@ Every alert includes two probe fields that identify exactly how and why the fail | `crashed` | `Docker event (crash)` | `Crash probe failed — container exited with exit code 1` | | `oom` | `Docker event (OOM)` | `OOM probe failed — container was OOM-killed by the kernel` | | `unhealthy` (Docker native HEALTHCHECK) | `Docker HEALTHCHECK` | `Docker health probe failed — exit code 1, output: connection refused` | -| `unhealthy` (HTTP auto-probe) | `HTTP GET` | `HTTP probe failed — error response 503 Service Unavailable, URI http://192.0.2.100:8080/health` | -| `unhealthy` (TCP connect) | `TCP connect` | `TCP probe failed — connection refused or timed out on 192.0.2.100:6379 (Connection refused)` | +| `unhealthy` (HTTP auto-probe) | `HTTP GET` | `HTTP probe failed — error response 503 Service Unavailable, URI http://172.18.0.5:8080/health` | +| `unhealthy` (TCP connect) | `TCP connect` | `TCP probe failed — connection refused or timed out on 172.18.0.5:6379 (Connection refused)` | | `unhealthy` (/proc fallback) | `/proc alive check` | `/proc alive check failed — PID 1 is zombie/stopped (exit code 1)` | > **Restart-loop detection** uses Docker `RestartCount` from the polling loop as its sole restart count. The first poll establishes a baseline without interpreting historical restarts. Counter increases are timestamped when observed, so detection can take up to one polling interval; the rolling window is approximate to polling precision. After a polling gap longer than the restart window, unknown-age increments are ignored rather than misattributed to the current window. Delayed Docker events cannot double-count restarts; timestamps on events also prevent failures from before confirmed recovery from reopening an incident. @@ -367,9 +368,9 @@ docker compose logs docker-container-watchdog ``` Common causes: -- **Missing `.env` file** — run `cp .env.example .env` and fill in credentials +- **Missing `.env` file** — complete the initial installer wizard, or restore it from a previous-installation backup - **Docker socket not accessible** — ensure `/var/run/docker.sock` exists and the container has read access -- **Image not loaded** — run `docker images watchdog`; if empty, set `WATCHDOG_VERSION=$(cat VERSION)` and build with `WATCHDOG_VERSION="$WATCHDOG_VERSION" docker compose -f docker-compose.build.yaml build` (see [DEPLOYMENT.md](DEPLOYMENT.md) for details) +- **Release image unavailable** — rerun `bash install.sh` and select online download, or supply the approved offline `watchdog.tar` and choose the local-image option (see [Deployment Guide](DEPLOYMENT.md)) ### Alert Notifications Not Received @@ -389,11 +390,11 @@ Common causes: - Confirm `alert_on_recovery` is not set to `false` in the live `watchdog-config.yaml`. - Confirm the container was actually alarmed first — `state.alerted_for` must be non-empty; a container that was never flagged won't generate a recovery alert. - For containers with a Docker `HEALTHCHECK`, confirm Docker actually reports `healthy` — `docker inspect --format '{{json .State.Health}}'`. If it's stuck on `starting`, the watchdog is waiting on Docker, not the other way around. -- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; contact Radware support before changing the health-check configuration. +- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; ### Duplicate and Excessive Alert Notifications -Review the affected container's alert history and probe details. Contact Radware support or the responsible engineering team before changing monitoring thresholds, health checks or exclusions. +Review the affected container's alert history and probe details. ### Probe Type Not as Expected @@ -494,7 +495,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Egor Egorov | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | @@ -503,7 +504,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor | 1.4.0 | 2026-08-17 | Rahul Kumar | Added INFO "recovered" alert | | 1.3.3 | 2026-08-05 | Rahul Kumar | Added Auth True/false | | 1.3.2 | 2026-07-28 | Rahul Kumar | Fixed SMTP Auth issue | -| 1.3.1 | 2026-07-21 | Engineering | Updated Readme and Deployment guides | +| 1.3.1 | 2026-07-21 | Egor Egorov | Updated Readme and Deployment guides | | 1.3.0 | 2026-07-16 | Rahul Kumar | Added Upgrade Guide | | 1.2.0 | 2026-07-16 | Rahul Kumar | Removed sudo and renamed the container | | 1.1.0 | 2026-07-09 | Rahul Kumar | Added SNMPv3 support | diff --git a/docker-compose.build.yaml b/docker-compose.build.yaml index c2fe83e..a181e24 100644 --- a/docker-compose.build.yaml +++ b/docker-compose.build.yaml @@ -1,22 +1,15 @@ -# ── DEVELOPER / ONLINE BUILD ────────────────────────────────────────────────── -# Use this file when you have internet access and want to build a fresh image. -# -# Usage: -# export WATCHDOG_VERSION=$(cat VERSION) -# docker compose -f docker-compose.build.yaml build -# docker compose -f docker-compose.build.yaml up -d -# -# After a successful build, export the image for offline customers — always -# ship the versioned tag so install.sh can pin the deployment to it: -# docker tag watchdog:"$WATCHDOG_VERSION" watchdog:latest -# docker save watchdog:"$WATCHDOG_VERSION" watchdog:latest -o watchdog.tar -# ───────────────────────────────────────────────────────────────────────────── +# DEVELOPER-ONLY SOURCE BUILD (not used during customer installation). +# export WATCHDOG_VERSION="$(cat VERSION)" +# docker compose -f docker-compose.build.yaml build +# Official offline bundles and images are built by the tagged release workflow. services: docker-container-watchdog: build: context: . + args: + WATCHDOG_VERSION: "$WATCHDOG_VERSION" image: watchdog:${WATCHDOG_VERSION:?Set WATCHDOG_VERSION from the VERSION file before building} container_name: docker-container-watchdog restart: always diff --git a/docker-compose.yaml b/docker-compose.yaml index 9ac162d..66eab9f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,15 +1,7 @@ -# ── PRODUCTION / OFFLINE RUNTIME ───────────────────────────────────────────── -# Uses a pre-built image. No internet access required. -# The image must be loaded first: -# docker load -i watchdog.tar -# -# Usage: -# cp .env.example .env # fill in SLACK_WEBHOOK_URL and any other channel secrets -# docker compose up -d -# -# To build a new image (requires internet): -# docker compose -f docker-compose.build.yaml build -# ───────────────────────────────────────────────────────────────────────────── +# PRODUCTION RUNTIME — use one verified, prebuilt versioned Docker image. +# bash install.sh obtains it from GitHub Releases, a local image, or watchdog.tar. +# Normal customer installation does not build source or require docker load. +# For authorized manual operations: docker compose up -d --no-deps docker-container-watchdog services: diff --git a/docs/DEPLOYMENT_ADVANCED.md b/docs/DEPLOYMENT_ADVANCED.md new file mode 100644 index 0000000..b7bda29 --- /dev/null +++ b/docs/DEPLOYMENT_ADVANCED.md @@ -0,0 +1,69 @@ +# Container Alert — Advanced Deployment Reference + +This page supports engineers and release maintainers. For normal installation or upgrades, use the [Deployment Guide](../DEPLOYMENT.md): one interactive installer for online and offline hosts. + +## Image distribution and provenance + +The production installer reads the root `VERSION` file (for example `1.6.0`) and deploys only `watchdog:`. It never builds Docker images from source. + +Official image delivery uses **GitHub Releases** at: + +```text +https://github.com/Radware/CyberController-Container-Alert/releases/tag/v +``` + +Each approved release contains: + +- `watchdog.tar` — the prebuilt, version-tagged Docker image. +- `SHA256SUMS` — SHA-256 hashes for the image archive and offline bundle. +- `container-alert-v-offline.tar.gz` — an installation folder containing `install.sh`, `VERSION`, Compose, templates, documentation, checksum, and image. + +The installer verifies downloaded archive checksum and the image's `org.opencontainers.image.version` label. When an operator explicitly accepts an existing local versioned image, the installer verifies its version label. The operator remains responsible for ensuring the local image came from an approved source. An offline archive with `SHA256SUMS` is independently checksum-checked before load. + +Image tags must not be reused for different source code. Build new code under a new version so rollbacks are unambiguous. + +## Release publication (maintainers) + +1. Finish review and merge the release changes into `main`. Do not publish directly from an unmerged release branch. +2. Confirm the existing PR CI and isolated release smoke workflow passed for the reviewed candidate. +3. Tag the approved commit as `v` and push the tag. The `Publish Container Alert release` GitHub Actions workflow checks that the tag matches `VERSION` and that the commit is on `main`. +4. The hosted workflow runs non-destructive tests, builds one version-labelled image, exports `watchdog.tar`, creates the offline bundle/checksums, and publishes the GitHub Release. +5. Verify the release page assets and checksum before announcing customer availability. + +Publication requires repository permission for GitHub Actions to create releases (contents: write), and approval to distribute the image publicly. Until that is approved and the release workflow succeeds, use a controlled offline package instead. No release publication happens merely by opening or merging a PR. + +## Developer-only source build + +Production operators should never build source as part of installation. In a development environment only: + +```bash +export WATCHDOG_VERSION="$(cat VERSION)" +docker compose -f docker-compose.build.yaml build +``` + +The build file passes `WATCHDOG_VERSION` into the image metadata. This produces a local image; it does **not** publish a GitHub Release or update an existing running container. The Dockerfile packages `watchdog.py`, `lifecycle.py`, and `ha_monitor.py` inside the image; a container restart alone does not apply host-side Python edits. + +## Configuration files + +The wizard generates and retains two files: + +| File | Purpose | Apply changes | +|---|---|---| +| `watchdog-config.yaml` | Alert channels, endpoints, checks and thresholds | Restart only the watchdog | +| `.env` | Authentication secrets, image version, hostname and Docker group settings | Recreate only the watchdog | + +The installer detects the group ID of `/var/run/docker.sock` and configures its existing non-root runtime permissions. The production Compose file maintains a read-only root filesystem, capability restrictions, CPU/memory limits and persistent log volume. + +Review the channel examples and settings in [README](../README.md#4-supported-alert-channels). Slack and SMTP secrets belong in `.env`; Syslog/SNMP destinations and health-check policy belong in `watchdog-config.yaml`. Do not commit local secrets to Git. + +For SNMPv3, the watchdog logs a stable engine ID based on the hostname at first run. Register that engine ID and the USM user with the authorized trap receiver; if setting the optional fixed `v3_local_engine_id`, use the logged hexadecimal value. Authentication/privacy passphrases must each be at least 8 characters. The trap receiver must accept the selected auth and encryption algorithms. + +For HA active-node notifications, both nodes need the supported local HA exporter and watchdog. See [HA design and validation](HA_ACTIVE_NODE_ALERT_DESIGN.md) for exporter metrics, alert behavior, and limitations. The installer does not create a Pacemaker exporter or change cluster roles. + +## Failure handling + +Image acquisition occurs before changing configuration or running containers. A failed download or missing offline archive does not stop the watchdog. Re-running the installer keeps existing configuration unless the operator explicitly elects to reconfigure. + +If Compose reports an invalid configuration, installation stops before attempting deployment. If a container recreation fails after it starts, inspect its logs and restore the previous approved version and configuration under the normal operations process; do not assume an automatic rollback occurred. + +For operational verification, use read-only Docker status, inspect and log commands in [Deployment](../DEPLOYMENT.md#verify-the-installation). **Never run destructive Docker acceptance tests on customer, shared or operational Docker hosts.** The isolated hosted acceptance instructions are in [Release Smoke Testing](RELEASE_SMOKE_TESTING.md). diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index be7e2b8..b140b84 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -14,7 +14,7 @@ This document describes version **1.6.0** and its changes since v1.5.4. Building | Release documentation | Clarify merged vs unreleased features and explicit image version/tag process | [PR #8](https://github.com/Radware/CyberController-Container-Alert/pull/8) | | Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | | Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Installer source fallback | Source-build mode copies both `lifecycle.py` and `ha_monitor.py` alongside `watchdog.py` and Dockerfile; normal versioned offline archive flow retained | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Guided release installation | One installer wizard selects verified GitHub Release image download or an approved local image/archive, preserves existing configuration, and recreates only the watchdog. Source builds remain developer-only. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | @@ -45,6 +45,10 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C - Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. - Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. +## Image distribution and deployment + +The v1.6.0 release publication workflow prepares a version-labelled `watchdog.tar`, `SHA256SUMS`, and a complete offline installation bundle through GitHub Releases. Publishing is triggered only after the approved code is merged to `main` and a matching `v1.6.0` tag is pushed; opening or merging this PR does not publish artifacts. Operators obtain files from `main` or the offline bundle and run the same `bash install.sh` wizard. Production installations no longer build Docker images from source. See [Deployment](../DEPLOYMENT.md) and [Advanced Deployment](DEPLOYMENT_ADVANCED.md). + ## Exclusions, limitations and release gating **Not established by testing:** live customer SMTP, Slack and SNMP delivery end-to-end; arbitrary host reboot, network partition, STONITH, or production recovery policy. SMTP was explicitly excluded during lab acceptance. HA failover/failback was tested on real lab nodes with a prior source candidate, without any further failover required for text-only changes. diff --git a/install.sh b/install.sh index 61eeaaa..ac6df17 100755 --- a/install.sh +++ b/install.sh @@ -323,75 +323,117 @@ detect_docker_gid() { # Docker Image ################################################################################ -load_docker_image() { - print_section "Loading Docker Image" +# Confirm that an image carries the release version, not just an arbitrary tag. +verify_release_image() { + local reported_version + docker image inspect "$IMAGE_NAME" >/dev/null 2>&1 || return 1 + reported_version=$(docker image inspect "$IMAGE_NAME" \ + --format '{{index .Config.Labels "org.opencontainers.image.version"}}' 2>/dev/null) || return 1 + if [ "$reported_version" != "$VERSION" ]; then + print_warning "$IMAGE_NAME has version label '$reported_version', expected '$VERSION'" + return 1 + fi +} - # Always load the packaged archive first. The previous behaviour — skipping - # the load whenever any watchdog image already existed — meant an upgrade - # kept running the old code, because watchdog.py ships inside the image. - if [ -f "$IMAGE_ARCHIVE" ]; then - print_info "Loading image from archive: $(basename "$IMAGE_ARCHIVE")" - local load_output - if ! load_output=$(docker load -i "$IMAGE_ARCHIVE" 2>&1); then - print_error "Failed to load ${IMAGE_ARCHIVE}" - echo "$load_output" - exit 1 +# Verify archive checksum when supplied (mandatory for online downloads). +load_release_archive() { + local archive="$1" sums="$2" expected actual loaded + if [ -n "$sums" ]; then + if [ ! -r "$sums" ] || ! command -v sha256sum >/dev/null 2>&1; then + print_error "Cannot verify the release archive checksum" + return 1 fi - echo "$load_output" - - # The archive itself must carry the release tag. Merely inspecting the - # local tag after docker load is insufficient because a stale tag may - # have existed before this installer ran. - if ! printf '%s\n' "$load_output" | grep -Fxq "Loaded image: ${IMAGE_NAME}"; then - print_error "${IMAGE_ARCHIVE} did not load the required tag ${IMAGE_NAME}" - echo "" - echo " docker load reported:" - printf ' %s\n' "$load_output" - echo "" - echo " This package is version ${VERSION} (from ${VERSION_FILE})." - echo " Rebuild the archive with the versioned tag:" - echo " docker build -t ${IMAGE_NAME} ." - echo " docker tag ${IMAGE_NAME} ${IMAGE_LATEST}" - echo " docker save ${IMAGE_NAME} ${IMAGE_LATEST} -o watchdog.tar" - exit 1 + expected=$(awk '$2 == "watchdog.tar" || $2 == "*watchdog.tar" {print $1}' "$sums" | tail -n1) + if ! [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]]; then + print_error "SHA256SUMS has no valid digest for watchdog.tar" + return 1 fi - if ! docker image inspect "$IMAGE_NAME" &>/dev/null; then - print_error "Required image ${IMAGE_NAME} is not available after docker load" - exit 1 + actual=$(sha256sum "$archive" | awk '{print $1}') || return 1 + if [ "$actual" != "$expected" ]; then + print_error "Archive checksum mismatch; image not loaded" + return 1 fi - else - print_warning "Image archive not found: ${IMAGE_ARCHIVE}" - - # No archive — build from source if the Dockerfile shipped. - if [ -f "${INSTALL_DIR}/Dockerfile" ]; then - print_info "Building ${IMAGE_NAME} from source — this may take a few minutes..." - # Use a clean temp dir so any .dockerignore in INSTALL_DIR is bypassed - local tmpdir - tmpdir=$(mktemp -d) - trap "rm -rf '${tmpdir}'" EXIT - cp "${INSTALL_DIR}/Dockerfile" "${tmpdir}/" - cp "${INSTALL_DIR}/watchdog.py" "${tmpdir}/" - cp "${INSTALL_DIR}/lifecycle.py" "${tmpdir}/" - cp "${INSTALL_DIR}/ha_monitor.py" "${tmpdir}/" - if ! docker build -t "${IMAGE_NAME}" "${tmpdir}"; then - print_error "Build failed — cannot produce ${IMAGE_NAME}" - exit 1 + print_success "Archive checksum verified" + fi + if ! loaded=$(docker load -i "$archive" 2>&1); then + print_error "Docker could not load the image" + printf '%s\n' "$loaded" + return 1 + fi + if ! printf '%s\n' "$loaded" | grep -Fxq "Loaded image: $IMAGE_NAME"; then + print_error "Archive does not contain the expected tag $IMAGE_NAME" + return 1 + fi + if ! verify_release_image; then + print_error "Loaded image does not match release version $VERSION" + return 1 + fi + print_success "Loaded and verified $IMAGE_NAME" +} + +download_release_image() { + local tempdir base + if ! command -v curl >/dev/null 2>&1; then + print_warning "curl is not installed" + return 1 + fi + tempdir=$(mktemp -d) || return 1 + base="https://github.com/Radware/CyberController-Container-Alert/releases/download/v$VERSION" + print_info "Downloading official release image $IMAGE_NAME" + if ! curl --fail --location --silent --show-error --retry 2 --connect-timeout 15 \ + --output "$tempdir/SHA256SUMS" "$base/SHA256SUMS" || \ + ! curl --fail --location --silent --show-error --retry 2 --connect-timeout 15 \ + --output "$tempdir/watchdog.tar" "$base/watchdog.tar"; then + rm -rf "$tempdir" + print_warning "Release download failed" + return 1 + fi + if load_release_archive "$tempdir/watchdog.tar" "$tempdir/SHA256SUMS"; then + rm -rf "$tempdir" + return 0 + fi + rm -rf "$tempdir" + return 1 +} + +use_local_image() { + if docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then + if verify_release_image; then + if [ "$(_prompt_yn "Use existing local $IMAGE_NAME?" yes)" = true ]; then + print_success "Using local verified image $IMAGE_NAME" + return 0 fi - print_success "Image built: ${IMAGE_NAME}" else - print_error "Cannot proceed: ${IMAGE_NAME} is not available" - echo "" - echo " watchdog.tar not found and Dockerfile not found." - echo " Copy watchdog.tar into: $(dirname "$IMAGE_ARCHIVE") and re-run." - exit 1 + print_warning "Local image is not a verified release build" fi fi + if [ -f "$IMAGE_ARCHIVE" ]; then + local sums="" + [ ! -f "$INSTALL_DIR/SHA256SUMS" ] || sums="$INSTALL_DIR/SHA256SUMS" + print_info "Loading supplied watchdog.tar" + load_release_archive "$IMAGE_ARCHIVE" "$sums" && return 0 + return 1 + fi + print_error "No matching local $IMAGE_NAME or watchdog.tar available" + echo " Obtain the official v$VERSION offline package from Radware." + echo " Copy watchdog.tar to $INSTALL_DIR and rerun bash install.sh." + echo " No running containers were changed." + return 1 +} - # Convenience alias only — docker-compose.yaml deploys ${IMAGE_NAME}. - docker tag "$IMAGE_NAME" "$IMAGE_LATEST" &>/dev/null || \ - print_warning "Could not update the ${IMAGE_LATEST} convenience tag" - - print_success "Image ready: ${IMAGE_NAME}" +load_docker_image() { + print_section "Select Docker Image" + if [ "$(_prompt_yn "Can this host download an image from GitHub Releases?" yes)" = true ]; then + if download_release_image; then + return 0 + fi + print_warning "Image download unavailable; source builds are not used in production" + if [ "$(_prompt_yn "Try a local image or archive instead?" yes)" != true ]; then + print_error "Installation cancelled; running containers unchanged" + return 1 + fi + fi + use_local_image } ################################################################################ @@ -452,8 +494,9 @@ configure_all() { [ -f "$ENV_FILE" ] || missing+=" .env" [ -f "$CONFIG_FILE" ] || missing+=" watchdog-config.yaml" print_warning "Incomplete configuration — missing:${missing}" - print_warning "The wizard regenerates both files — re-enter your settings when prompted" - echo "" + print_error "Restore the missing configuration file from backup, then rerun." + print_info "Existing configuration was not changed." + return 1 # Offer to reconfigure if files already exist elif [ -f "$ENV_FILE" ] && [ -f "$CONFIG_FILE" ]; then print_info "Existing configuration detected: .env watchdog-config.yaml" @@ -490,6 +533,16 @@ configure_all() { harden_config_permissions "$CONFIG_FILE" || exit 1 return 0 fi + # Preserve both existing files before explicitly requested reconfiguration. + local backup_dir + backup_dir="$INSTALL_DIR/backup/$(date -u +%Y%m%dT%H%M%SZ)" + if ! _run_priv mkdir -p "$backup_dir" || ! _run_priv chmod 700 "$backup_dir" || \ + ! _run_priv cp -p "$ENV_FILE" "$backup_dir/.env" || \ + ! _run_priv cp -p "$CONFIG_FILE" "$backup_dir/watchdog-config.yaml"; then + print_error "Could not back up existing configuration; aborting reconfiguration" + return 1 + fi + print_success "Existing configuration backed up in: $backup_dir" echo "" fi @@ -749,21 +802,14 @@ start_watchdog() { exit 1 fi - # Stop and remove existing container if present - if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - print_info "Stopping existing container..." - if ! $DC down; then - print_error "Failed to stop the existing container — aborting before redeploy" - exit 1 - fi + # Validate Compose first and recreate only our service, never the project. + if ! $DC config -q; then + print_error "Compose configuration invalid; running containers unchanged" + exit 1 fi - - print_info "Starting container with docker compose..." - if ! $DC up -d; then - print_error "docker compose up failed — the watchdog was not deployed" - echo "" - echo " Inspect the failure with:" - echo " $DC -f ${INSTALL_DIR}/docker-compose.yaml logs" + print_info "Deploying watchdog container only..." + if ! $DC up -d --no-deps --force-recreate docker-container-watchdog; then + print_error "Watchdog deployment failed; check its Compose logs" exit 1 fi print_success "Watchdog container started" @@ -806,8 +852,11 @@ verify_deployment() { # image left over from a previous version would otherwise look healthy. local RUNNING_IMAGE RUNNING_IMAGE=$(docker inspect --format '{{.Config.Image}}' "$CONTAINER_NAME" 2>/dev/null || echo "") - if [ "$RUNNING_IMAGE" = "$IMAGE_NAME" ]; then - print_success "Container is running the expected image: ${IMAGE_NAME}" + local RUNNING_IMAGE_ID EXPECTED_IMAGE_ID + RUNNING_IMAGE_ID=$(docker inspect --format '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true) + EXPECTED_IMAGE_ID=$(docker image inspect --format '{{.Id}}' "$IMAGE_NAME" 2>/dev/null || true) + if [ "$RUNNING_IMAGE" = "$IMAGE_NAME" ] && [ -n "$EXPECTED_IMAGE_ID" ] && [ "$RUNNING_IMAGE_ID" = "$EXPECTED_IMAGE_ID" ]; then + print_success "Container runs the verified image: $IMAGE_NAME" else print_error "Container is running '${RUNNING_IMAGE:-unknown}', expected '${IMAGE_NAME}'" echo "" @@ -868,13 +917,15 @@ display_usage_instructions() { main() { print_header check_prerequisites + load_docker_image || exit 1 setup_log_directory - load_docker_image - configure_all + configure_all || exit 1 start_watchdog verify_deployment display_usage_instructions } -main "$@" - +# Allow fake-Docker unit tests to source functions without invoking main. +if [[ "$BASH_SOURCE" == "$0" ]]; then + main "$@" +fi diff --git a/tests/test_installer_image.py b/tests/test_installer_image.py new file mode 100644 index 0000000..1c2f434 --- /dev/null +++ b/tests/test_installer_image.py @@ -0,0 +1,184 @@ +"""Installer decisions use fake Docker and fake downloads; never a real daemon.""" + +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +SOURCE = Path(__file__).resolve().parents[1] / "install.sh" +VERSION = "1.6.0" + + +class InstallerImageTests(unittest.TestCase): + def setUp(self): + self.t = tempfile.TemporaryDirectory(prefix="watchdog-installer-fake-") + self.addCleanup(self.t.cleanup) + self.root = Path(self.t.name) + self.bin = self.root / "fake-bin" + self.bin.mkdir() + self.release = self.root / "fake-release" + self.release.mkdir() + self.state = self.root / "fake-state" + self.state.mkdir() + shutil.copy2(SOURCE, self.root / "install.sh") + (self.root / "VERSION").write_text(VERSION + "\n") + self.log = self.root / "fake-docker-calls.log" + self.log.write_text("") + self.archive = b"small fake release archive, never a real Docker image" + self.write_release_archive(self.release) + + docker = self.bin / "docker" + docker.write_text("""#!/usr/bin/env bash +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +case "$1" in + image) + if [ "$2" = inspect ]; then + if [ "$FAKE_IMAGE_PRESENT" = 1 ] || [ -f "$FAKE_STATE_DIR/loaded" ]; then + case "$*" in + *org.opencontainers.image.version*) printf '%s\\n' "$FAKE_VERSION" ;; + *'{{.Id}}'*) echo 'sha256:fake-image-id' ;; + esac + exit 0 + fi + exit 1 + fi ;; + load) + touch "$FAKE_STATE_DIR/loaded" + echo 'Loaded image: watchdog:1.6.0' + exit 0 ;; + config|up) + exit 0 ;; +esac +echo "Unexpected fake Docker arguments: $*" >&2 +exit 97 +""") + docker.chmod(0o755) + curl = self.bin / "curl" + curl.write_text("""#!/usr/bin/env bash +[ "$FAKE_CURL_FAIL" != 1 ] || exit 22 +out="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output) out="$2"; shift 2 ;; + *) url="$1"; shift ;; + esac +done +case "$url" in + */watchdog.tar) cp "$FAKE_RELEASE_DIR/watchdog.tar" "$out" ;; + */SHA256SUMS) cp "$FAKE_RELEASE_DIR/SHA256SUMS" "$out" ;; + *) echo "Unexpected download URL" >&2; exit 22 ;; +esac +""") + curl.chmod(0o755) + + self.env = os.environ.copy() + self.env.update( + PATH=str(self.bin) + os.pathsep + os.environ["PATH"], + FAKE_DOCKER_LOG=str(self.log), + FAKE_STATE_DIR=str(self.state), + FAKE_RELEASE_DIR=str(self.release), + FAKE_VERSION=VERSION, + FAKE_IMAGE_PRESENT="0", + FAKE_CURL_FAIL="0", + ) + + def write_release_archive(self, folder, valid=True): + (folder / "watchdog.tar").write_bytes(self.archive) + digest = hashlib.sha256(self.archive).hexdigest() + if not valid: + digest = "0" * 64 + (folder / "SHA256SUMS").write_text(f"{digest} watchdog.tar\n") + + def run_installer(self, task="load_docker_image", online=False): + # Source install.sh. Overriding prompts never invokes main or Docker. + prompt = ( + '_prompt_yn() { case "$1" in ' + '*"Can this host download"*) printf "' + + ("true" if online else "false") + + '";; *) printf "true";; esac; }; ' + ) + command = "source ./install.sh; " + prompt + task + return subprocess.run( + ["bash", "-c", command], + env=self.env, + cwd=self.root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=10, + ) + + def docker_calls(self): + return self.log.read_text().splitlines() + + def test_offline_uses_matching_local_image_without_loading(self): + self.env["FAKE_IMAGE_PRESENT"] = "1" + res = self.run_installer() + self.assertEqual(res.returncode, 0, res.stdout) + self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) + + def test_offline_loads_checksum_verified_archive(self): + shutil.copy2(self.release / "watchdog.tar", self.root / "watchdog.tar") + shutil.copy2(self.release / "SHA256SUMS", self.root / "SHA256SUMS") + res = self.run_installer() + self.assertEqual(res.returncode, 0, res.stdout) + self.assertTrue(any(s.startswith("load ") for s in self.docker_calls())) + + def test_bad_offline_checksum_is_rejected_before_docker_load(self): + self.write_release_archive(self.root, valid=False) + res = self.run_installer() + self.assertNotEqual(res.returncode, 0) + self.assertIn("checksum mismatch", res.stdout) + self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) + + def test_unverified_local_image_is_rejected(self): + self.env["FAKE_IMAGE_PRESENT"] = "1" + self.env["FAKE_VERSION"] = "old-version" + res = self.run_installer() + self.assertNotEqual(res.returncode, 0) + self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) + + def test_online_download_and_load_never_builds(self): + res = self.run_installer(online=True) + self.assertEqual(res.returncode, 0, res.stdout) + self.assertTrue(any(s.startswith("load ") for s in self.docker_calls())) + self.assertFalse(any("build" in s for s in self.docker_calls())) + + def test_online_failure_can_use_matching_local_image(self): + self.env["FAKE_CURL_FAIL"] = "1" + self.env["FAKE_IMAGE_PRESENT"] = "1" + res = self.run_installer(online=True) + self.assertEqual(res.returncode, 0, res.stdout) + self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) + + def test_missing_image_fails_without_docker_mutations(self): + res = self.run_installer() + self.assertNotEqual(res.returncode, 0) + self.assertIn("No running containers were changed", res.stdout) + self.assertFalse(any(s.startswith(("load ", "up ", "run ", "stop ", "build ")) for s in self.docker_calls())) + + def test_incomplete_existing_configuration_does_not_reset(self): + (self.root / ".env").write_text("SENSITIVE=keep\n") + res = self.run_installer(task="configure_all") + self.assertNotEqual(res.returncode, 0) + self.assertEqual((self.root / ".env").read_text(), "SENSITIVE=keep\n") + self.assertFalse((self.root / "watchdog-config.yaml").exists()) + self.assertEqual(self.docker_calls(), []) + + def test_start_recreates_watchdog_only(self): + # The service call below is a fake Docker executable, not a host daemon. + res = self.run_installer(task='DC=docker; start_watchdog') + self.assertEqual(res.returncode, 0, res.stdout) + calls = self.docker_calls() + self.assertIn("config -q", calls) + self.assertIn("up -d --no-deps --force-recreate docker-container-watchdog", calls) + self.assertFalse(any("down" in s for s in calls)) + + +if __name__ == "__main__": + unittest.main() From 329fe22b8901020a7e26b685e15eb821d21aff62 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Thu, 15 Oct 2026 21:45:17 +0000 Subject: [PATCH 09/18] docs: restore complete deployment and operations guide --- .env.example | 7 +- DEPLOYMENT.md | 438 +++++++++++++++++++++++++++++++---- README.md | 161 +++---------- docs/DEPLOYMENT_ADVANCED.md | 36 ++- docs/RELEASE_NOTES_V1.6.0.md | 1 + install.sh | 4 +- watchdog-config.yaml.example | 15 +- 7 files changed, 468 insertions(+), 194 deletions(-) diff --git a/.env.example b/.env.example index 8357b20..e08fb26 100644 --- a/.env.example +++ b/.env.example @@ -3,8 +3,8 @@ # cp .env.example .env # chmod 600 .env # secrets — restrict to owner only # -# Only credentials/secrets belong here. -# All other settings (hosts, ports, recipients, thresholds) go in watchdog-config.yaml. +# Credentials/secrets and installer-managed deployment values live here. +# Monitoring thresholds, channel hosts, ports, and recipients belong in watchdog-config.yaml. # ── Slack ───────────────────────────────────────────────────────────────────── # Required if alert_channels includes "slack" @@ -50,6 +50,3 @@ SMTP_PASSWORD=your-smtp-password/api key value # Release tag the container runs (see the repository VERSION file). install.sh # sets this to the verified release image version (downloaded or loaded locally). # WATCHDOG_VERSION= - -# ── Tuning ──────────────────────────────────────────────────────────────────── -LOG_LEVEL=INFO diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 312734e..b050407 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,107 +1,453 @@ -# CyberController Container Watchdog — Deployment +# CyberController Container Watchdog — Deployment Guide + +This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. The same `install.sh` wizard is used for online and offline installations. It deploys only the `docker-container-watchdog` service; it never restarts CyberController application containers. + +## Table of Contents + +1. [Prerequisites](#1-prerequisites) +2. [Installation](#2-installation) + - [Online installation](#21-online-installation) + - [Offline installation](#22-offline-installation) +3. [Verify the installation](#3-verify-the-installation) +4. [Configuration reference](#4-configuration-reference) + - [Configuration files and editing](#41-configuration-files-and-editing) + - [Watchdog monitoring settings](#42-watchdog-monitoring-settings) + - [Notification channels](#43-notification-channels) + - [Health checks, exclusions, and HA](#44-health-checks-exclusions-and-ha) + - [Environment variables](#45-environment-variables) + - [Apply configuration changes](#46-apply-configuration-changes) +5. [Operations and maintenance](#5-operations-and-maintenance) + - [Common commands and logs](#51-common-commands-and-logs) + - [Backup and restore](#52-backup-and-restore) + - [Upgrade the watchdog](#53-upgrade-the-watchdog) + - [Rollback](#54-rollback) + - [Uninstall](#55-uninstall) +6. [Troubleshooting](#6-troubleshooting) +7. [Advanced references and support](#7-advanced-references-and-support) + +## 1. Prerequisites + +| Requirement | What is needed | +|---|---| +| Operating system | Supported Linux host with Docker (Ubuntu 20.04 / RHEL 8 or later are the documented baseline) | +| Docker Engine | 20.10+ and a running Docker daemon | +| Docker Compose | Version 2 plugin (`docker compose`) recommended; `install.sh` also supports standalone `docker-compose` | +| Host permissions | Root or an operator with Docker access; `sudo` may be required to secure configuration files | +| Online installation | Git access to this repository **and** HTTPS access to its published GitHub Release image; `curl` is used for the image download | +| Offline installation | Approved release package copied to the host; Git and Internet are not required | +| Alert delivery | Network connectivity to the chosen Syslog, SMTP, SNMP, or Slack destination | + +Read-only prerequisite checks: -**One installer, one wizard.** The same `install.sh` supports Internet-connected and offline hosts. It deploys only the `docker-container-watchdog` service and does not change CyberController application containers. +```bash +docker info +docker ps +docker compose version # or: docker-compose --version +df -h . +``` -## Before you begin +Docker socket access is highly privileged even if a container runs as a non-root user. Restrict access to authorized operators. The installer automatically detects the socket group and configures the watchdog's existing non-root execution mode. -The host needs Docker, a working Docker daemon, Docker Compose, and an operator account with Docker access (and sudo where required). Git is needed only to obtain or update files from GitHub; curl is needed only if downloading the release image online. +**Install only an approved, published release.** The version in `VERSION` must match the prebuilt image. Customer installations never compile application code. Do not run failure-injection tests on operational Docker hosts. -Install only a published, approved release. The version in `VERSION` selects the corresponding prebuilt Docker image. Customer installations never compile application code. +## 2. Installation -## Install +Choose **Online** if the host can access GitHub and download the release image. Otherwise follow **Offline**. Each procedure is complete; there are no shared steps to skip or return to. -Get the installation files in either of these ways: +### 2.1 Online installation -**GitHub available** — clone the default `main` branch: +**Step 1 — Download the installation files from `main`.** ```bash git clone https://github.com/Radware/CyberController-Container-Alert.git cd CyberController-Container-Alert ``` -**GitHub unavailable** — obtain the approved offline package from Radware, copy it to the host, and extract it: +**Step 2 — Start the installation wizard.** + +```bash +bash install.sh +``` + +**Step 3 — Answer the wizard questions.** + +- Answer **Yes** when asked whether the host can download the official image from GitHub Releases. The installer downloads `watchdog.tar` and `SHA256SUMS`, checks integrity and the expected image version, and loads the image. +- Choose notification channels: **Syslog**, **SMTP**, **SNMP Trap**, and/or **Slack**. Supply only the destinations and credentials requested for your selection. +- Confirm the hostname displayed in alerts. On upgrades, choose **No** at “Reconfigure from scratch?” to preserve existing settings. +- The installer creates the configuration for new installations, starts the watchdog, and checks its Docker health and image version. + +If the matching GitHub Release is not yet published or is inaccessible, the wizard offers a local image/archive. Obtain the approved offline package before retrying; it will **not** build source code. + +**Step 4 — Confirm the installation.** + +```bash +docker ps --filter name=docker-container-watchdog +docker logs --tail 50 docker-container-watchdog +``` + +Expected: `docker-container-watchdog` is running and its logs show normal monitoring activity. Additional read-only checks are in [Section 3](#3-verify-the-installation). + +### 2.2 Offline installation + +**Step 1 — Obtain the offline package.** Get the approved `container-alert-v1.6.0-offline.tar.gz` from Radware, then transfer it to the target host. The package includes the installer, Compose file, version, prebuilt image, and checksum; no Git or image build is needed. + +**Step 2 — Extract the installation files.** ```bash tar -xzf container-alert-v1.6.0-offline.tar.gz cd CyberController-Container-Alert ``` -Then **run the same command** in either case: +**Step 3 — Start the same installation wizard.** ```bash bash install.sh ``` -The wizard asks whether this host can download the official image from GitHub Releases. - -- **Yes:** Download the Docker image and its checksum, verify the image, then continue. If the download fails, you can choose a local image or archive instead. -- **No:** Check for the matching local image and ask whether to use it. If not selected, load `watchdog.tar` from the installation directory. An offline package includes this archive and its checksum. -- **Neither available:** The installer explains what is missing and exits without changing the running watchdog. Obtain the release package and rerun `bash install.sh`. +**Step 4 — Answer the wizard questions.** -The installer then asks for the desired notification channels (Syslog, SMTP, SNMP, Slack), the necessary connection settings, and the hostname shown in alerts. Existing `.env` and `watchdog-config.yaml` files are kept by default during upgrades. +- Answer **No** to the Internet download question. +- If the matching local `watchdog:1.6.0` image is already available, the installer asks whether to use it. Otherwise it loads the included `watchdog.tar` (and verifies the supplied checksum). +- Select Syslog, SMTP, SNMP Trap and/or Slack, enter their settings, and choose the alert hostname. +- The installer writes the configuration for a new installation, starts the watchdog, and verifies health and image version. -**No Docker image build, manual `docker load`, or Compose editing is required for a normal installation.** +If neither a matching image nor the archive is available, the installer exits **without changing running containers**. Obtain the missing approved package and rerun it. -## Verify the installation +**Step 5 — Confirm the installation.** ```bash docker ps --filter name=docker-container-watchdog -docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' docker logs --tail 50 docker-container-watchdog ``` -Expected: the watchdog container is running, its Docker health check becomes healthy, and recent logs show normal monitoring. Check the configured receiver for previously emitted alerts if available. A working monitor does not need an artificial failure to prove it is alive. +Expected: the watchdog is running. Check the read-only verification in [Section 3](#3-verify-the-installation) if it does not become healthy. + +## 3. Verify the installation + +Run these commands from the installation directory, or use the global `docker` commands from any directory: + +```bash +docker compose ps +docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' +docker inspect docker-container-watchdog --format '{{.Config.Image}}' +docker logs --tail 100 docker-container-watchdog +``` + +The container should be `running` and become `healthy`; the running image should be `watchdog:` matching the package. The installer verifies both status and the actual image ID. Startup may take several health-check intervals. + +**Logs and alert delivery:** + +| Location or command | Purpose | +|---|---| +| `docker logs -f docker-container-watchdog` | Live container output | +| `docker compose logs --tail 100 docker-container-watchdog` | Compose-scoped stdout | +| `./watchdog/watchdog.log` | Persistent host log, rotated at 10 MB with five backups | +| `./watchdog/state.json` | Persistent incident/HA/reboot tracking state; do not hand-edit | +| `grep 'watchdog_alert' watchdog/watchdog.log` | Search existing structured alert messages | + +Also verify that the configured notification receiver can receive traffic from the host, and inspect **existing** Syslog/SMTP/SNMP/Slack records if available. No artificial container failure is necessary to verify ordinary operation. + +**Optional HA installation:** On each node with `ha_monitor.enabled: true`, the local exporter can be inspected read-only using `curl -fsS http://127.0.0.1:9664/metrics` (or the configured exporter URL). The first valid reading sets a quiet baseline; no failover notification should be expected at startup. Controlled HA failover testing belongs to an approved maintenance procedure, not routine installation verification. + +Never stop, kill, stress, or restart **application containers** merely to test the watchdog. -Read-only inspection is sufficient on operational hosts. **Do not stop, kill, restart, or stress application containers to test alerting.** +## 4. Configuration reference -## Upgrade with the same installer +The installer configures notification channels during initial setup. Use this section **after installation** if an authorized operator needs to review or change monitoring settings. No Docker image rebuild is required for a YAML or environment change. -**GitHub installation:** +### 4.1 Configuration files and editing + +| File | What it controls | Permissions and application | +|---|---|---| +| `watchdog-config.yaml` | Monitoring thresholds, health probes, exclusions, logging, HA and notification endpoints | Installer secures it as root-owned, group-readable (mode `640`); edit with `sudoedit` or an authorized privileged editor; restart watchdog | +| `.env` | Channel secrets, alert hostname, Docker group identity and selected image version | Owner-readable only (mode `600`); edit with an authorized editor; recreate watchdog | +| `docker-compose.yaml` | Container runtime mounts, Docker socket, CPU/memory limits, networking and restart policy | Normally unchanged by customers; change only under approved support/engineering instructions | +| `watchdog-config.yaml.example` / `.env.example` | Reference templates | Not loaded by the running watchdog; don't edit these expecting the service to change | + +From the installation directory: ```bash -cd CyberController-Container-Alert -git pull --ff-only -bash install.sh +sudoedit watchdog-config.yaml # only if an authorized configuration change is required +nano .env # only if changing credentials, hostname, etc. +``` + +If running as root, edit directly without `sudo`. Otherwise, if `sudoedit` is unavailable, use your organization's approved privileged-editor method. Make a backup before editing. Never commit `.env`, passwords, or tokens to Git. Changes in one file do not automatically update the other. + +### 4.2 Watchdog monitoring settings + +These are `watchdog-config.yaml` keys. Defaults below are application defaults unless noted; installation-wizard selections can override them. + +| YAML setting | Default | Explanation | +|---|---|---| +| `alert_channels` | `[slack]` in application defaults; installer prompts for selection | List of enabled delivery channel identifiers: `syslog`, `smtp`, `snmp_trap`, `slack`. Select the corresponding channel configuration too | +| `check_interval_seconds` | `60` | Interval between periodic Docker health/restart polls | +| `cooldown_minutes` | `5` | Suppresses repeated alerts for the same incident; changing it may conceal a real recurring failure | +| `restart_threshold` | `5` | Number of Docker restart-counter increases that qualifies as a restart loop | +| `restart_window_minutes` | `10` | Rolling window for restart-loop detection | +| `restart_recovery_stability_seconds` | `120` | Continuous running time needed before reporting recovery from crash/OOM/restart loop | +| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy polls required for an unhealthy alert | +| `alert_on_recovery` | `true` | Emit one informational `recovered` event when an alerted condition clears | +| `lifecycle_detection.enabled` | `true` | Correlate Docker-requested stop events with nonzero exits to reduce false crash alerts | +| `lifecycle_detection.correlation_window_seconds` | `3` | Wait briefly for matching stop evidence; supported range is 0.5–30 seconds | +| `log_level` | `INFO` | Console/file verbosity: `DEBUG`, `INFO`, `WARNING` or `ERROR`. Syslog alert severity is kept independently | +| `log_file` | `/var/log/watchdog/watchdog.log` | Persistent log file within the container, bind-mounted to `./watchdog/`; `null` disables the file log | +| `state_file` | `/var/log/watchdog/state.json` | Persists incidents, restart history and HA/reboot state across watchdog restarts; empty string disables persistence | +| `boot_grace_seconds` | `180` | Grace period after a detected host reboot before summary/normal alerts; `0` removes the grace period | +| `runbook_base_url` | Empty | Optional base URL included with alerts | + +Changes to exclusions, thresholds, probe methods or suppression rules can mask real failures. Review observed evidence and obtain the applicable Radware support/engineering approval before changing those settings. + +### 4.3 Notification channels + +For any channel, its identifier must be present in `alert_channels` **and** its configuration must be enabled. Enabling a channel doesn't automatically prove that its destination received an alert. + +| Channel | YAML field | Purpose | +|---|---|---| +| Syslog | `syslog.enabled` | Enable structured forwarding | +| Syslog | `syslog.host`, `syslog.port` | Receiver hostname/IP and listening port (commonly 514) | +| Syslog | `syslog.protocol` | `udp` or `tcp` | +| Syslog | `syslog.facility` | Syslog facility (for example `local0`) | +| SMTP | `smtp.enabled`, `smtp.auth` | Enable email; set `auth: false` for approved unauthenticated relays | +| SMTP | `smtp.host`, `smtp.port` | Mail server and port | +| SMTP | `smtp.sender`, `smtp.recipients` | Sender address and YAML list of recipient addresses | +| SMTP | `smtp.tls` | Whether to use SMTP TLS | +| SMTP | `smtp.username_env`, `smtp.password_env` | Names of secret environment variables, normally `SMTP_USERNAME` and `SMTP_PASSWORD` | +| Slack | `slack.enabled` | Enable Slack notifications | +| Slack | `slack.webhook_url_env` | Name of the secret environment variable, normally `SLACK_WEBHOOK_URL` | +| SNMP | `snmp_trap.enabled` | Enable trap dispatch | +| SNMP | `snmp_trap.host`, `snmp_trap.port` | NMS/trap receiver and destination UDP port (normally 162) | +| SNMP | `snmp_trap.version`, `snmp_trap.community` | SNMP `v1`/`v2c`/`v3`; community applies to v1/v2c only | +| SNMP | `snmp_trap.trap_oid` | Optional trap OID; defaults to the Radware Container Alert notification OID | +| SNMPv3 | `v3_username`, `v3_auth_protocol`, `v3_priv_protocol` | USM user and authentication/privacy algorithms under `snmp_trap` | +| SNMPv3 | `v3_auth_key_env`, `v3_priv_key_env` | Names of passphrase variables in `.env`; do not embed secrets in YAML | +| SNMPv3 | `v3_local_engine_id` | Optional fixed sender engine ID used by receiving NMS registration | + +Example: changing the Syslog destination **after installation**: + +```yaml +alert_channels: + - syslog + +syslog: + enabled: true + host: syslog.example.com + port: 514 + protocol: udp + facility: local0 ``` -**Offline installation:** obtain the new offline package and update the installation files in the **existing installation directory**, preserving `.env`, `watchdog-config.yaml`, and `watchdog/`. Then run `bash install.sh` there. +When enabling Syslog **alongside** existing channels, add `syslog` to the existing list rather than replacing the list as shown in this single-channel example. Restart the watchdog after a YAML-only change. -On upgrade, the installer retains existing alert settings unless you explicitly request reconfiguration. It verifies the new image before changing deployment state and recreates **only the watchdog service**, preserving unrelated Docker containers. The prior image should be retained for rollback. +For SMTP authentication, store the username/password in `.env`; for Slack, store the webhook in `.env`. SNMPv3 passphrases also belong in `.env` and must meet protocol requirements (at least eight characters). See [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) for SNMPv3 receiver/engine-ID setup and the supported enterprise trap fields. -### Apply configuration changes +### 4.4 Health checks, exclusions, and HA -Changes to `watchdog-config.yaml` require a watchdog restart. Changes to `.env` or Compose settings require recreation to apply the new container environment: +| YAML setting | Default | Explanation | +|---|---|---| +| `auto_health_check.enabled` | `true` in installer | Enable automatic HTTP/TCP/discovery probes for containers without a native Docker health check | +| `auto_health_check.paths` | `/-/healthy`, `/health`, `/healthz`, `/api/health`, `/metrics`, `/` | HTTP paths attempted during endpoint discovery | +| `auto_health_check.timeout_seconds` | `2` in installer | Probe timeout after endpoint discovery | +| `container_health_checks` | `{}` | Per-container `http` or `exec` probe overrides; take precedence over automatic discovery | +| `excluded_containers` | `[]` | Names excluded from alerting entirely; not a routine noise-management setting | +| `ignored_container_events` | R&D-labelled temporary syntax-check event | Narrow, label-based exceptions for intentional temporary operations; preserve normal monitoring | +| `ha_monitor.enabled` | `false` | Optional CyberController HA active-node change notification, installed on both nodes | +| `ha_monitor.exporter_url` | `http://127.0.0.1:9664/metrics` | Local HA metrics exporter endpoint; doesn't install the exporter | + +For an approved custom HTTP probe, the shape is: + +```yaml +container_health_checks: + example-api: + type: http + internal_port: 8080 + path: /health + timeout_seconds: 5 +``` + +The supported per-container probe fields are: + +| Field under `container_health_checks.` | Meaning | +|---|---| +| `type` | `http` (default) or `exec` | +| `internal_port` | Required for HTTP checks; internal container listening port | +| `path` | HTTP request path (defaults to `/`) | +| `command` | Command to run inside the container for `exec` probes; required for this type | +| `timeout_seconds` | Timeout for either method (default 5 seconds) | + +Use the *actual* container name and a supported health endpoint only after validation. Manual `type: exec` checks execute commands inside monitored containers, so they must be approved and carefully reviewed. Do not add exclusions or broad ignore rules merely to silence unexpected alerts. + +HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md) for limitations and approved maintenance testing. + +### 4.5 Environment variables + +Variables in `.env` are read when the container is created. The installer sets the image version, runtime identity and alert hostname automatically; credentials appear only for selected channels. + +| Environment variable | Purpose | When needed | +|---|---|---| +| `WATCHDOG_HOST` | Hostname displayed in alerts | Generated by installer; change only when the alert identity needs correction | +| `WATCHDOG_VERSION` | Pinned local image tag `watchdog:` | Managed by installer; used for controlled upgrades/rollback | +| `WATCHDOG_UID`, `WATCHDOG_GID` | Non-root runtime user and group (normally 1000/1000) | Set by installer; do not change casually | +| `DOCKER_GID` | Group ID of the host Docker socket | Detected automatically by installer | +| `SLACK_WEBHOOK_URL` | Slack incoming webhook secret | If Slack enabled | +| `SMTP_USERNAME`, `SMTP_PASSWORD` | Mail credentials or app token | If SMTP authentication enabled | +| `SNMP_V3_AUTH_KEY` | SNMPv3 authentication passphrase | For SNMPv3 authNoPriv/authPriv | +| `SNMP_V3_PRIV_KEY` | SNMPv3 encryption passphrase | For SNMPv3 authPriv | +| `WATCHDOG_CONFIG` | Container-internal config path set explicitly in Compose | Normally `/etc/watchdog/watchdog-config.yaml`; not an everyday `.env` setting | + +**Logging clarification:** `LOG_LEVEL` in older `.env` files is not used by the application. Change `log_level` in `watchdog-config.yaml` instead. Do not store passwords or webhook values in YAML or version control. + +### 4.6 Apply configuration changes + +First back up the relevant files ([Section 5.2](#52-backup-and-restore)) and have an authorized operator review the change. + +| What changed | Command | New Docker image needed? | +|---|---|---| +| `watchdog-config.yaml` only | `docker compose restart docker-container-watchdog` | No | +| `.env` (secrets, hostname, image version) | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | No rebuild; requested image must already exist | +| `docker-compose.yaml` runtime settings | Same service-only Compose recreation, after engineering approval | No rebuild | +| Application Python code | Deploy a newly tested, versioned release image | **Yes — a different image**, not merely a restart | ```bash +# After an approved YAML-only change: docker compose restart docker-container-watchdog -# For changed environment or Compose settings: + +# After an approved .env or Compose change: docker compose up -d --no-deps --force-recreate docker-container-watchdog ``` -Production monitoring configuration changes should follow the normal authorized change process. +If both YAML and `.env` changed, use the recreation command to apply both. Always confirm health and inspect logs afterward. Updating a Git checkout alone does **not** change Python code inside the running image. -### Roll back an image +## 5. Operations and maintenance -If the previous image is still available, restore its version in `WATCHDOG_VERSION` in `.env` and recreate **only** the watchdog service: +All commands below target **only the watchdog**. Run Compose commands from the directory containing `docker-compose.yaml` and `.env` (use `docker-compose` in place of `docker compose` on a legacy standalone installation). + +### 5.1 Common commands and logs + +| Operation | Command | +|---|---| +| Show container status | `docker compose ps` | +| Show recent output | `docker logs --tail 100 docker-container-watchdog` | +| Follow logs live | `docker logs -f docker-container-watchdog` | +| Inspect running image tag | `docker inspect docker-container-watchdog --format '{{.Config.Image}}'` | +| List installed image versions | `docker image ls watchdog` | +| Start the watchdog | `docker compose start docker-container-watchdog` | +| Stop watchdog for approved maintenance | `docker compose stop docker-container-watchdog` | +| Restart watchdog | `docker compose restart docker-container-watchdog` | +| Recreate only watchdog | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | + +Stopping the watchdog **pauses alert monitoring**. It does not stop application containers. Avoid project-wide `docker compose down` in routine operations. + +### 5.2 Backup and restore + +Before upgrades or authorized configuration changes, preserve the configuration and secrets in a restricted local directory: ```bash -docker image ls watchdog +mkdir -p -m 700 backup +cp -p .env backup/.env.bak +sudo cp -p watchdog-config.yaml backup/watchdog-config.yaml.bak +``` + +The `watchdog/` directory contains persistent logs and incident state: keep it intact across upgrades and rollbacks. The installer also creates a dated backup of both config files when you **explicitly** choose to reconfigure from scratch. + +To restore both files after an approved rollback, preserve their original permissions and recreate **only** the watchdog: + +```bash +cp -p backup/.env.bak .env +sudo cp -p backup/watchdog-config.yaml.bak watchdog-config.yaml docker compose up -d --no-deps --force-recreate docker-container-watchdog ``` -Do not overwrite existing version tags with new code. Obtain the previous approved package if its image was removed. +If running as root, omit `sudo`. Keep backup directories access-restricted because `.env` contains credentials. + +### 5.3 Upgrade the watchdog + +**Online upgrade** + +1. Back up configuration using [Section 5.2](#52-backup-and-restore). +2. In the installation directory, update the checked-out `main` branch: + + ```bash + git pull --ff-only + bash install.sh + ``` + +3. Answer **Yes** for the online image download. When the installer detects the existing config, answer **No** to reconfigure from scratch; it preserves channel settings while updating the image version. +4. Check health, actual running image, and recent logs using [Section 3](#3-verify-the-installation). + +**Offline upgrade** + +1. Back up configuration and retain `watchdog/`. +2. Transfer the **new** approved offline bundle, unpack it into a separate temporary folder, then copy only the new installation/release files into the existing installation directory (never overwrite `.env`, `watchdog-config.yaml`, or `watchdog/`): + + ```bash + mkdir -p /tmp/container-alert-upgrade + tar -xzf container-alert-v1.6.0-offline.tar.gz -C /tmp/container-alert-upgrade + cd /path/to/existing/CyberController-Container-Alert + cp /tmp/container-alert-upgrade/CyberController-Container-Alert/{install.sh,uninstall.sh,docker-compose.yaml,VERSION,watchdog.tar,SHA256SUMS,README.md,DEPLOYMENT.md,.env.example,watchdog-config.yaml.example} . + mkdir -p docs + cp /tmp/container-alert-upgrade/CyberController-Container-Alert/docs/DEPLOYMENT_ADVANCED.md docs/ + bash install.sh + ``` -## Troubleshooting +3. Answer **No** to Internet download. Use the supplied archive (or approved matching local image), and answer **No** to reconfiguration to keep current settings. +4. Verify health and the running image, as with the online upgrade. -| Symptom | Check | +Use the versioned package for the release you are actually upgrading to; `v1.6.0` above is an example. Older image tags should remain available for rollback. **There is no automatic updater**: upgrades happen only when an operator runs the installer. + +### 5.4 Rollback + +If a new watchdog image fails after an upgrade, use the prior approved version, provided it remains available locally: + +1. Record the failure and inspect `docker logs --tail 100 docker-container-watchdog`. +2. Verify the previous image exists: `docker image ls watchdog`. If absent, obtain/load the previous approved image through the authorized offline process. +3. Restore the previous `WATCHDOG_VERSION` in `.env` (for example, `1.5.4` when rolling back from v1.6.0) and, if necessary, restore its saved YAML/credential configuration. +4. Recreate **only** the watchdog and verify it: + + ```bash + docker compose up -d --no-deps --force-recreate docker-container-watchdog + docker ps --filter name=docker-container-watchdog + docker logs --tail 50 docker-container-watchdog + ``` + +An ordinary `git checkout` or restart does **not** roll back running code; the selected image controls it. Running a newer `install.sh` afterward can reset `WATCHDOG_VERSION` to its `VERSION` file, so use the corresponding approved package for a lasting rollback. + +### 5.5 Uninstall + +The installer package includes `uninstall.sh`. Its options govern whether images and historical logs are retained. + +| Desired result | Command | |---|---| -| Online image download failed | GitHub Release for the version in `VERSION` must be published and accessible; otherwise use the offline package | -| Local image rejected | The image must have a matching release version label; supply the official `watchdog.tar` | -| Cannot load image | Check available disk space, image archive, and `SHA256SUMS` | -| Docker/Compose check fails | Confirm the daemon is running and the operator has Docker access | -| Configuration is incomplete | Restore missing `.env` or `watchdog-config.yaml` from backup; installer does not discard the remaining file | -| Watchdog is not healthy | Review `docker logs --tail 100 docker-container-watchdog` and the configured notification endpoints | +| Interactive uninstall with confirmation | `bash uninstall.sh` | +| Remove watchdog, keep image and logs | `bash uninstall.sh --keep-image --keep-logs` | +| Remove watchdog/image, preserve logs | `bash uninstall.sh --keep-logs` | +| Remove container, image, and logs | `bash uninstall.sh --remove-all` | +| Fully non-interactive removal, **including logs** | `bash uninstall.sh --remove-all --force` | -For configuration examples, security considerations, image-building instructions for developers, release packaging, and advanced troubleshooting, see [Advanced Deployment Reference](docs/DEPLOYMENT_ADVANCED.md). Notification-channel examples are also in [README.md](README.md). +Do not use `--remove-all` if historical alerts or state must be retained. Uninstall affects the watchdog, not CyberController application containers. -To remove the watchdog, run `bash uninstall.sh` from its installation directory and choose the desired retention options. +## 6. Troubleshooting + +| Observation | What to check | +|---|---| +| Installer cannot reach GitHub Releases | Host HTTPS access and the published release matching `VERSION`; use the approved offline archive if disconnected | +| Local image/archive rejected | Correct `watchdog:` tag, release version label, archive checksum and disk space | +| Docker or Compose command fails | `docker info`, Docker socket access, `docker compose version` and configured permissions | +| Only one config file exists | Restore the missing `.env` or `watchdog-config.yaml` backup; installer will not overwrite the surviving file | +| Watchdog exits or stays unhealthy | `docker inspect` health state and `docker logs --tail 100 docker-container-watchdog` | +| No alert at receiver | Confirm `alert_channels`, the matching channel's `enabled` flag, endpoints, credentials, and receiver connectivity | +| Syslog recovery not seen in local WARNING-level logs | Console/file level and forwarded Syslog alert priorities are separate; inspect receiver's structured `watchdog_alert` records | +| Unexpected unhealthy reports | Compare Docker native HEALTHCHECK, probe type and detection details; do not suppress alerts before authorized review | +| Excessive restart/crash notifications | Correlate incident and Docker event evidence; do not raise thresholds/exclude containers solely to quiet alerts | +| HA change notification missing | Both watchdogs, local HA exporter, quorum/metrics, notification receivers and saved baseline; see advanced HA guide | + +For authorization or assistance obtaining the offline release, contact the appropriate Radware support channel. Never change production CyberController service settings solely in response to a watchdog alert. + +## 7. Advanced references and support + +- [README](README.md) — product behavior, failure types, architecture and version history. +- [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) — developer image builds, release publication, security, SNMPv3 receiver setup. +- [HA alert design and validation](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md) — HA prerequisites, validation, limitations. +- [Release smoke testing](docs/RELEASE_SMOKE_TESTING.md) — isolated **developer/CI-only** acceptance tests. Never run destructive scenarios on an operational Docker host. +- [Example watchdog YAML](watchdog-config.yaml.example) and [example environment variables](.env.example) — full annotated configuration templates. + +For offline-image access or product-specific guidance, use your designated Radware support contact (the previous deployment guide listed `northamericare@radware.com`). diff --git a/README.md b/README.md index 0a25b2f..73dbb81 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,7 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitori - [Image Size](#image-size) - [Python Dependencies](#python-dependencies) 3. [Configuration](#3-configuration) - - [Configuration Keys](#configuration-keys) - - [Environment Variables](#environment-variables) 4. [Supported Alert Channels](#4-supported-alert-channels) - - [Slack](#slack) - - [Syslog](#syslog) - - [SMTP (Email)](#smtp-email) - - [SNMP Traps](#snmp-traps) 5. [Deployment Instructions](#5-deployment-instructions) 6. [Failure Types & Severities](#6-failure-types--severities) 7. [Verify Watchdog Operation](#7-verify-watchdog-operation) @@ -76,7 +70,7 @@ existing Docker container health checks. Enable the following block on **both** HA nodes, after verifying that each node has a local Pacemaker HA cluster exporter: -~~~yaml +```yaml ha_monitor: enabled: true exporter_url: http://127.0.0.1:9664/metrics @@ -180,121 +174,24 @@ pysnmp>=6.2 ## 3. Configuration -All non-secret settings live in `watchdog-config.yaml`. Secrets (webhook URLs, passwords) are stored in `.env` and never committed to version control. Restart the container after editing either file — no image rebuild is needed. - -### Configuration Keys - -| Key | Default | Description | -|-----|---------|-------------| -| `alert_channels` | `["slack"]` | Active destinations: `slack`, `smtp`, `snmp_trap` | -| `check_interval_seconds` | `60` | How often the poll loop runs | -| `cooldown_minutes` | `5` | Suppress duplicate alerts per container per failure type | -| `restart_threshold` | `5` | Restart count within window that triggers a restart-loop alert | -| `restart_window_minutes` | `10` | Rolling window for restart counting | -| `restart_recovery_stability_seconds` | `120` | Minimum uninterrupted running time before a recovered alert for automatically restarted crash/OOM/restart-loop incidents | -| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy cycles before alerting | -| `alert_on_recovery` | `true` | Send an INFO "recovered" alert once a previously-alarmed container returns to normal | -| `excluded_containers` | `[]` | Container names to never alert on | -| `ignored_container_events` | MariaDB syntax-check label rule | Label-based event suppressions for intentional temporary containers | -| `auto_health_check` | enabled | Probe containers reporting `health=none` by discovering an HTTP/TCP endpoint | -| `container_health_checks` | `{}` | Per-container probe overrides (`type: http` or `type: exec`) for `health=none` containers; takes precedence over `auto_health_check` | -| `log_level` | `INFO` | `DEBUG` / `INFO` / `WARNING` / `ERROR` | -| `log_file` | `/var/log/watchdog/watchdog.log` | Bind-mounted to `./watchdog/watchdog.log` on host. Rotates at 10 MB, 5 backups. Set to `null` to disable | -| `state_file` | `/var/log/watchdog/state.json` | Persists open alerts and the host boot time so recovery and reboot reporting survive a restart. Set to `""` to disable | -| `boot_grace_seconds` | `180` | After a detected host reboot, hold alerts this long while services start, then send one `reboot-summary`. `0` alerts immediately and sends the summary on the first poll | -| `runbook_base_url` | — | URL included in every alert | - -Preferred suppression: Temporary MariaDB HA syntax-check containers should be created with Docker label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The watchdog ignores only configured events for containers carrying that explicit label; normal service containers still alert. - -Temporary fallback: until that label exists, dynamic MariaDB dump helper containers are not alerted as crashes only when all identifying signals match together: image `kvision_infra_mariadb`, container name is not `config_kvision-infra-mariadb_1`, Docker `AutoRemove` is `true`, a mount contains `/mnt/cli/tmp/mariadb_dump.sql.gz`, and the exit code is `137`. The primary MariaDB container remains monitored. - -### Environment Variables - -| Variable | Required | Description | -|----------|----------|-------------| -| `SLACK_WEBHOOK_URL` | Yes (if Slack enabled) | Slack incoming webhook URL | -| `SMTP_USERNAME` | Yes (if SMTP enabled) | SMTP username or login email used for login | -| `SMTP_PASSWORD` | Yes (if SMTP enabled) | SMTP account password or api key value | -| `SNMP_V3_AUTH_KEY` | Yes (if SNMPv3 enabled) | SNMPv3 authentication passphrase — minimum 8 characters (RFC 3414) | -| `SNMP_V3_PRIV_KEY` | Yes (if SNMPv3 authPriv) | SNMPv3 privacy passphrase — minimum 8 characters; requires auth key also set | -| `WATCHDOG_CONFIG` | No | Path to config file (default: `/etc/watchdog/watchdog-config.yaml`) | -| `WATCHDOG_HOST` | No | Hostname shown in alerts (default: system hostname) | - ---- - -## 4. Supported Alert Channels - -Configure at least one channel and add its identifier to `alert_channels` in `watchdog-config.yaml`. Multiple channels can be active simultaneously. - -### Slack - -Add `slack` to `alert_channels` and set `SLACK_WEBHOOK_URL` in `.env`: - -```yaml -slack: - enabled: true - webhook_url_env: SLACK_WEBHOOK_URL -``` - -### Syslog - -```yaml -syslog: - enabled: true - host: - port: 514 - protocol: udp # udp or tcp - facility: local0 -``` - -### SMTP (Email) - -Add `smtp` to `alert_channels` to enable. +The guided installer creates `watchdog-config.yaml` (monitoring policy, probes, logging and notification endpoints) and `.env` (credentials and runtime identity). The configuration is preserved on an upgrade unless the operator explicitly chooses to reconfigure it. -```yaml -smtp: - enabled: true - host: smtp.radware.com - port: 587 - sender: noc-alerts@radware.com - username_env: SMTP_USERNAME # env var: SMTP username or app token - recipients: - - ops-team@radware.com - - oncall@radware.com - tls: true - password_env: SMTP_PASSWORD # env var: SMTP password or app token -``` +**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](DEPLOYMENT.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. -### SNMP Traps +A YAML-only change takes effect after restarting the watchdog. A changed `.env` value requires recreating **only** the watchdog container. Neither action rebuilds Python code into the image. Thresholds, exclusions, custom exec probes, and monitoring suppressions require authorized review; do not use them merely to silence alerts. -Add `snmp_trap` to `alert_channels` to enable. Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. +--- -```yaml -snmp_trap: - enabled: true - host: syslog.example.com # IP or hostname of your SNMP trap receiver - port: 162 # Standard SNMP trap port - version: v2c # SNMP version: v1, v2c, or v3 - community: public # SNMPv1/v2c community string (ignored for v3) - trap_oid: "1.3.6.1.4.1.89.110.0.1" # Radware enterprise container-alert notification OID - # SNMPv3 only — add SNMP_V3_AUTH_KEY / SNMP_V3_PRIV_KEY to .env - # v3_username: watchdog-user - # v3_auth_protocol: SHA # MD5 | SHA | SHA256 | SHA384 | SHA512 - # v3_auth_key_env: SNMP_V3_AUTH_KEY # min 8 chars - # v3_priv_protocol: AES # AES (recommended) | AES192 | AES256 | DES (weak) - # v3_priv_key_env: SNMP_V3_PRIV_KEY # min 8 chars; requires auth key also set - # v3_local_engine_id: "" # hex engine ID; see docs/DEPLOYMENT_ADVANCED.md -``` +## 4. Supported Alert Channels -Each trap carries five var-binds, all under the Radware enterprise OID arc (`1.3.6.1.4.1.89.110`): +| Channel | YAML identifier | Credentials or other setup | +|---|---|---| +| Syslog | `syslog` | Syslog destination, port, protocol and facility | +| SMTP (email) | `smtp` | Mail server, sender, recipients, TLS/auth; credentials in `.env` if applicable | +| SNMP Trap | `snmp_trap` | NMS destination, SNMP version; SNMPv3 keys in `.env` | +| Slack | `slack` | Webhook URL stored in `.env` | -| OID | Object | Value | -|-----|--------|-------| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Hostname of the alerting node | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Human-readable alert summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Probe failure detail | +At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](DEPLOYMENT.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/DEPLOYMENT_ADVANCED.md#snmp-trap-receiver-reference) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. --- @@ -377,20 +274,15 @@ Common causes: 1. Check `alert_channels` in `watchdog-config.yaml` — ensure at least one channel is listed and configured 2. Check watchdog logs for errors: ```bash - grep -E "ERROR|CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log + grep -E "ERROR|CRITICAL" watchdog/watchdog.log ``` -3. For Slack: verify `SLACK_WEBHOOK_URL` is set in `.env` and test manually: - ```bash - curl -X POST -H 'Content-type: application/json' \ - --data '{"text":"Watchdog test"}' "$SLACK_WEBHOOK_URL" - ``` - Expected response: `ok` +3. For Slack: confirm `SLACK_WEBHOOK_URL` is present in `.env` and that the destination is reachable. Use an authorized test receiver if end-to-end validation is required; do not trigger application failures to test alert delivery. **Recovery alert (`recovered`) specifically not received:** - Confirm `alert_on_recovery` is not set to `false` in the live `watchdog-config.yaml`. - Confirm the container was actually alarmed first — `state.alerted_for` must be non-empty; a container that was never flagged won't generate a recovery alert. - For containers with a Docker `HEALTHCHECK`, confirm Docker actually reports `healthy` — `docker inspect --format '{{json .State.Health}}'`. If it's stuck on `starting`, the watchdog is waiting on Docker, not the other way around. -- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, the probe (often `auto_health_check`) is misjudging the service; +- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, review the actual endpoint and probe evidence with the responsible engineering team before changing monitoring. ### Duplicate and Excessive Alert Notifications @@ -401,7 +293,7 @@ Review the affected container's alert history and probe details. Check which probe was cached at startup: ```bash -grep -E "auto-discover|falling back" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +grep -E "auto-discover|falling back" watchdog/watchdog.log ``` ### Viewing Logs @@ -413,20 +305,23 @@ Log destinations (simultaneous): ```bash # Live tail (preferred — host path, no need to exec into the container) -tail -f /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +tail -f watchdog/watchdog.log # Last 100 lines -tail -100 /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +tail -100 watchdog/watchdog.log + +# All structured alerts, including INFO recoveries +grep 'watchdog_alert' watchdog/watchdog.log -# Alerts and errors only -grep -E "ALERT|ERROR|CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +# Errors and critical lines +grep -E "ERROR|CRITICAL" watchdog/watchdog.log # Filter by container name -grep "my-container" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +grep "my-container" watchdog/watchdog.log # Search across rotated logs -grep "CRITICAL" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log \ - /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log.1 +grep "CRITICAL" watchdog/watchdog.log \ + watchdog/watchdog.log.1 # Via Docker (stdout only) docker compose logs -f docker-container-watchdog @@ -484,7 +379,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor > **Verify what was actually cached after startup:** > ```bash -> grep -E "auto-discover|falling back" /opt/radware/storage/scripts/Alert_Container/watchdog/watchdog.log +> grep -E "auto-discover|falling back" watchdog/watchdog.log > ``` ## 10. Version History diff --git a/docs/DEPLOYMENT_ADVANCED.md b/docs/DEPLOYMENT_ADVANCED.md index b7bda29..6214e40 100644 --- a/docs/DEPLOYMENT_ADVANCED.md +++ b/docs/DEPLOYMENT_ADVANCED.md @@ -54,16 +54,44 @@ The wizard generates and retains two files: The installer detects the group ID of `/var/run/docker.sock` and configures its existing non-root runtime permissions. The production Compose file maintains a read-only root filesystem, capability restrictions, CPU/memory limits and persistent log volume. -Review the channel examples and settings in [README](../README.md#4-supported-alert-channels). Slack and SMTP secrets belong in `.env`; Syslog/SNMP destinations and health-check policy belong in `watchdog-config.yaml`. Do not commit local secrets to Git. - -For SNMPv3, the watchdog logs a stable engine ID based on the hostname at first run. Register that engine ID and the USM user with the authorized trap receiver; if setting the optional fixed `v3_local_engine_id`, use the logged hexadecimal value. Authentication/privacy passphrases must each be at least 8 characters. The trap receiver must accept the selected auth and encryption algorithms. +For the full operator-facing channel and settings reference, see [Deployment](../DEPLOYMENT.md#4-configuration-reference). Slack and SMTP secrets belong in `.env`; Syslog/SNMP destinations and health-check policy belong in `watchdog-config.yaml`. Do not commit local secrets to Git. For HA active-node notifications, both nodes need the supported local HA exporter and watchdog. See [HA design and validation](HA_ACTIVE_NODE_ALERT_DESIGN.md) for exporter metrics, alert behavior, and limitations. The installer does not create a Pacemaker exporter or change cluster roles. +## SNMP trap receiver reference + +The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. + +Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): + +```text +createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" +``` + +Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. + +**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: + +| OID | Object | Value | +|---|---|---| +| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | +| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | +| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | +| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | +| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | + +HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. + +## Container-event filtering reference + +The intended MariaDB HA syntax-check helper carries the explicit label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The configured `ignored_container_events` policy ignores only the designated temporary helper `die` event; normal service container failures remain monitored. + +A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Any change to monitoring exceptions belongs to Radware R&D/authorized operations, not routine customer alert handling. + ## Failure handling Image acquisition occurs before changing configuration or running containers. A failed download or missing offline archive does not stop the watchdog. Re-running the installer keeps existing configuration unless the operator explicitly elects to reconfigure. If Compose reports an invalid configuration, installation stops before attempting deployment. If a container recreation fails after it starts, inspect its logs and restore the previous approved version and configuration under the normal operations process; do not assume an automatic rollback occurred. -For operational verification, use read-only Docker status, inspect and log commands in [Deployment](../DEPLOYMENT.md#verify-the-installation). **Never run destructive Docker acceptance tests on customer, shared or operational Docker hosts.** The isolated hosted acceptance instructions are in [Release Smoke Testing](RELEASE_SMOKE_TESTING.md). +For operational verification, use read-only Docker status, inspect and log commands in [Deployment](../DEPLOYMENT.md#3-verify-the-installation). **Never run destructive Docker acceptance tests on customer, shared or operational Docker hosts.** The isolated hosted acceptance instructions are in [Release Smoke Testing](RELEASE_SMOKE_TESTING.md). diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index b140b84..daaa58d 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -15,6 +15,7 @@ This document describes version **1.6.0** and its changes since v1.5.4. Building | Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | | Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Guided release installation | One installer wizard selects verified GitHub Release image download or an approved local image/archive, preserves existing configuration, and recreates only the watchdog. Source builds remain developer-only. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Operator documentation and configuration clarity | Restore independent Online/Offline walkthroughs, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP technical details in advanced reference; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | diff --git a/install.sh b/install.sh index ac6df17..052e7d3 100755 --- a/install.sh +++ b/install.sh @@ -668,7 +668,7 @@ configure_all() { chmod 600 "$ENV_TMP" { printf "# .env — generated by install.sh on %s\n" "$(date '+%Y-%m-%d %H:%M:%S')" - printf "# Secrets only. All other settings are in watchdog-config.yaml.\n\n" + printf "# Secrets and deployment settings; monitoring policy is in watchdog-config.yaml.\n\n" if [ "$SLACK_ENABLED" = "true" ]; then printf "# Slack\nSLACK_WEBHOOK_URL=%s\n\n" "$SLACK_WEBHOOK" fi @@ -684,7 +684,7 @@ configure_all() { "$WATCHDOG_UID" "$WATCHDOG_GID" "$DOCKER_GID" printf "# Deployed release — docker-compose.yaml pins watchdog:\${WATCHDOG_VERSION}\n" printf "WATCHDOG_VERSION=%s\n\n" "$VERSION" - printf "# Tuning\nLOG_LEVEL=INFO\n" + } > "$ENV_TMP" if ! _publish_file "$ENV_TMP" "$ENV_FILE" 600 "$(id -u):$(id -g)"; then rm -f "$ENV_TMP" diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index 33684a0..91da1fe 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -4,8 +4,8 @@ # SECRETS: All passwords and webhook URLs are read from environment # variables, never hardcoded here. Set them in your .env file. # -# Copy this file to /opt/radware/storage/scripts/Alert_Container/watchdog-config.yaml -# and update the values for your environment. +# Reference template only: install.sh creates watchdog-config.yaml in your +# chosen installation directory. Edit that active file after installation. # ── Alert channels ──────────────────────────────────────────────────────────── # Empty for local testing — alerts are still logged to stdout/file. @@ -35,9 +35,16 @@ lifecycle_detection: excluded_containers: [] # - my-temp-container +# Narrow, approved exception for Radware-designated temporary syntax checks. +ignored_container_events: + - label: + com.radware.cybercontroller.role: mariadb-ha-syntax-check + events: + - die + # ── Logging ─────────────────────────────────────────────────────────────────── -log_level: DEBUG # DEBUG | INFO | WARNING | ERROR +log_level: INFO # DEBUG | INFO | WARNING | ERROR (default INFO) log_file: /var/log/watchdog/watchdog.log # bind-mounted to ./watchdog/watchdog.log on host # Alert state is persisted here so recovery alerts still fire after the host or @@ -75,7 +82,7 @@ smtp: # ── SNMP Traps ──────────────────────────────────────────────────────────────── # Sends SNMP traps (v1, v2c, or v3) to your NMS/SIEM on every alert. # Add "snmp_trap" to alert_channels above to enable. -# Requires: pip install pysnmp (already in requirements-watchdog.txt) +# Dependencies are included in the release image; no host pip install. snmp_trap: enabled: false # set to true to enable host: nms.example.com # replace with your NMS receiver From 810b881a2690a440acf4839a75c736ac58c4c280 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 14:28:50 +0000 Subject: [PATCH 10/18] release: add versioned installer bundles and maintainer workflow --- .github/workflows/ci.yml | 1 + .github/workflows/release-package.yml | 41 +++--- .github/workflows/release-smoke.yml | 2 + DEPLOYMENT.md | 88 ++++++------ README.md | 25 ++-- docs/DEPLOYMENT_ADVANCED.md | 17 ++- docs/DEVELOPMENT_AND_RELEASE.md | 199 ++++++++++++++++++++++++++ docs/RELEASE_NOTES_V1.6.0.md | 8 +- scripts/package_release.sh | 53 +++++++ tests/test_release_package.py | 82 +++++++++++ 10 files changed, 433 insertions(+), 83 deletions(-) create mode 100644 docs/DEVELOPMENT_AND_RELEASE.md create mode 100644 scripts/package_release.sh create mode 100644 tests/test_release_package.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2badf6e..3da3888 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: bash -n install.sh bash -n uninstall.sh bash -n tests/release-smoke.sh + bash -n scripts/package_release.sh - name: Validate Python syntax run: | diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index a3a7661..c1ad343 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -55,21 +55,10 @@ jobs: run: | set -euo pipefail docker save "watchdog:$WATCHDOG_VERSION" -o watchdog.tar - if [ "$(stat -c '%s' watchdog.tar)" -ge 2147483648 ]; then - echo "Release image exceeds GitHub's 2 GiB asset limit" >&2 - exit 1 - fi - sha256sum watchdog.tar > SHA256SUMS - - staging="$RUNNER_TEMP/offline/CyberController-Container-Alert" - mkdir -p "$staging/docs" - cp install.sh uninstall.sh docker-compose.yaml VERSION \ - .env.example watchdog-config.yaml.example \ - README.md DEPLOYMENT.md watchdog.tar SHA256SUMS "$staging/" - cp docs/DEPLOYMENT_ADVANCED.md "$staging/docs/" - tar -czf "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ - -C "$RUNNER_TEMP/offline" CyberController-Container-Alert - sha256sum "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" >> SHA256SUMS + bash scripts/package_release.sh watchdog.tar dist + (cd dist && sha256sum --check SHA256SUMS) + tar -tzf "dist/container-alert-v$WATCHDOG_VERSION-online.tar.gz" > /dev/null + tar -tzf "dist/container-alert-v$WATCHDOG_VERSION-offline.tar.gz" > /dev/null - name: Publish GitHub Release env: @@ -77,8 +66,22 @@ jobs: shell: bash run: | set -euo pipefail + notes=(--generate-notes) + if [ -f "docs/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then + notes=(--notes-file "docs/RELEASE_NOTES_V$WATCHDOG_VERSION.md") + fi + + # Keep the release private as a draft until all four assets exist. gh release create "$GITHUB_REF_NAME" \ - watchdog.tar SHA256SUMS \ - "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ - --verify-tag --title "Container Alert $GITHUB_REF_NAME" \ - --generate-notes + dist/watchdog.tar dist/SHA256SUMS \ + "dist/container-alert-v$WATCHDOG_VERSION-online.tar.gz" \ + "dist/container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ + --verify-tag --draft --title "Container Alert $GITHUB_REF_NAME" \ + "${notes[@]}" + + expected=$(printf '%s\n' SHA256SUMS watchdog.tar \ + "container-alert-v$WATCHDOG_VERSION-online.tar.gz" \ + "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" | sort) + actual=$(gh release view "$GITHUB_REF_NAME" --json assets --jq '.assets[].name' | sort) + test "$actual" = "$expected" || { echo "Release assets incomplete" >&2; exit 1; } + gh release edit "$GITHUB_REF_NAME" --draft=false diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index b5a27a8..c3a4bb9 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -16,6 +16,8 @@ on: - "docker-compose.build.yaml" - "tests/release-smoke.sh" - ".github/workflows/release-smoke.yml" + - ".github/workflows/release-package.yml" + - "scripts/package_release.sh" - "Dockerfile" workflow_dispatch: diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index b050407..c6407e8 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -52,75 +52,75 @@ Docker socket access is highly privileged even if a container runs as a non-root ## 2. Installation -Choose **Online** if the host can access GitHub and download the release image. Otherwise follow **Offline**. Each procedure is complete; there are no shared steps to skip or return to. +Choose **Online** if the host can download the official release image. Choose **Offline** if the host cannot reach GitHub. Both use exactly the same `bash install.sh` wizard. **Neither package includes Python source code, and the customer never builds an image.** + +The names below show v1.6.0. For newer releases, select the approved version on the [GitHub Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) and use its matching filenames. **The v1.6.0 packages become available only after that version is officially published.** ### 2.1 Online installation -**Step 1 — Download the installation files from `main`.** +**Step 1 — Obtain the small online installer package.** From the approved release on the [Releases page](https://github.com/Radware/CyberController-Container-Alert/releases), download these two files to the target host: + +- `container-alert-v1.6.0-online.tar.gz` — installer, Compose file, version, templates and documentation (**no Docker image**). +- `SHA256SUMS` — file integrity checks. + +**Step 2 — Verify and extract.** Run these commands where both files were downloaded: ```bash -git clone https://github.com/Radware/CyberController-Container-Alert.git +sha256sum --check --ignore-missing SHA256SUMS +tar -xzf container-alert-v1.6.0-online.tar.gz cd CyberController-Container-Alert ``` -**Step 2 — Start the installation wizard.** +**Step 3 — Run the guided installer.** ```bash bash install.sh ``` -**Step 3 — Answer the wizard questions.** +Answer **Yes** to the GitHub Releases download question. The wizard automatically obtains the matching prebuilt Docker image (`watchdog.tar`), verifies its checksum and version, asks which alert channels to configure (Syslog, SMTP, SNMP and/or Slack), and deploys only the watchdog. No Git clone, source build, or manual `docker load` is needed. -- Answer **Yes** when asked whether the host can download the official image from GitHub Releases. The installer downloads `watchdog.tar` and `SHA256SUMS`, checks integrity and the expected image version, and loads the image. -- Choose notification channels: **Syslog**, **SMTP**, **SNMP Trap**, and/or **Slack**. Supply only the destinations and credentials requested for your selection. -- Confirm the hostname displayed in alerts. On upgrades, choose **No** at “Reconfigure from scratch?” to preserve existing settings. -- The installer creates the configuration for new installations, starts the watchdog, and checks its Docker health and image version. - -If the matching GitHub Release is not yet published or is inaccessible, the wizard offers a local image/archive. Obtain the approved offline package before retrying; it will **not** build source code. - -**Step 4 — Confirm the installation.** +**Step 4 — Confirm that the watchdog is running.** ```bash docker ps --filter name=docker-container-watchdog docker logs --tail 50 docker-container-watchdog ``` -Expected: `docker-container-watchdog` is running and its logs show normal monitoring activity. Additional read-only checks are in [Section 3](#3-verify-the-installation). +The watchdog should be running and report normal monitoring activity. Full read-only verification is in [Section 3](#3-verify-the-installation). ### 2.2 Offline installation -**Step 1 — Obtain the offline package.** Get the approved `container-alert-v1.6.0-offline.tar.gz` from Radware, then transfer it to the target host. The package includes the installer, Compose file, version, prebuilt image, and checksum; no Git or image build is needed. +**Step 1 — Obtain the complete offline installer package.** Download the approved release on a connected machine, then transfer both files to the disconnected target host: + +- `container-alert-v1.6.0-offline.tar.gz` — the **same installer and configuration files** plus the prebuilt `watchdog.tar` and its checksum. +- `SHA256SUMS` — file integrity checks for the transferred package. -**Step 2 — Extract the installation files.** +**Step 2 — Verify and extract.** On the target host, run these commands where both transferred files are located: ```bash +sha256sum --check --ignore-missing SHA256SUMS tar -xzf container-alert-v1.6.0-offline.tar.gz cd CyberController-Container-Alert ``` -**Step 3 — Start the same installation wizard.** +**Step 3 — Run the same guided installer.** ```bash bash install.sh ``` -**Step 4 — Answer the wizard questions.** +Answer **No** to the Internet download question. The wizard selects an approved local versioned image or loads and verifies the included `watchdog.tar`. Configure the alert channels and hostname as prompted. No Git or Internet access is necessary on the target. -- Answer **No** to the Internet download question. -- If the matching local `watchdog:1.6.0` image is already available, the installer asks whether to use it. Otherwise it loads the included `watchdog.tar` (and verifies the supplied checksum). -- Select Syslog, SMTP, SNMP Trap and/or Slack, enter their settings, and choose the alert hostname. -- The installer writes the configuration for a new installation, starts the watchdog, and verifies health and image version. - -If neither a matching image nor the archive is available, the installer exits **without changing running containers**. Obtain the missing approved package and rerun it. - -**Step 5 — Confirm the installation.** +**Step 4 — Confirm that the watchdog is running.** ```bash docker ps --filter name=docker-container-watchdog docker logs --tail 50 docker-container-watchdog ``` -Expected: the watchdog is running. Check the read-only verification in [Section 3](#3-verify-the-installation) if it does not become healthy. +The watchdog should be running. Full read-only verification is in [Section 3](#3-verify-the-installation). + +**If image acquisition fails:** the installer does not modify the running watchdog. Obtain the correct published release package before retrying. Do not build source on production hosts. ## 3. Verify the installation @@ -361,38 +361,40 @@ If running as root, omit `sudo`. Keep backup directories access-restricted becau ### 5.3 Upgrade the watchdog +First use [Section 5.2](#52-backup-and-restore) to back up `.env` and `watchdog-config.yaml`. Retain the `watchdog/` directory containing state and historical alerts. + **Online upgrade** -1. Back up configuration using [Section 5.2](#52-backup-and-restore). -2. In the installation directory, update the checked-out `main` branch: +1. Download the newer approved `container-alert-v-online.tar.gz` and `SHA256SUMS` from the [GitHub Releases page](https://github.com/Radware/CyberController-Container-Alert/releases). +2. Verify the package using `sha256sum --check --ignore-missing SHA256SUMS`. +3. Extract the package **over the existing installation folder** (from its parent directory), then run the installer: ```bash - git pull --ff-only + cd /path/to/parent-of/CyberController-Container-Alert + tar -xzf /path/to/container-alert-v1.6.1-online.tar.gz + cd CyberController-Container-Alert bash install.sh ``` -3. Answer **Yes** for the online image download. When the installer detects the existing config, answer **No** to reconfigure from scratch; it preserves channel settings while updating the image version. -4. Check health, actual running image, and recent logs using [Section 3](#3-verify-the-installation). +4. Answer **Yes** to image download and **No** to reconfigure existing settings. The installer updates the pinned image version and recreates **only** the watchdog. Verify using [Section 3](#3-verify-the-installation). **Offline upgrade** -1. Back up configuration and retain `watchdog/`. -2. Transfer the **new** approved offline bundle, unpack it into a separate temporary folder, then copy only the new installation/release files into the existing installation directory (never overwrite `.env`, `watchdog-config.yaml`, or `watchdog/`): +1. Transfer the newer approved offline package and `SHA256SUMS` to the host. Verify the package with `sha256sum --check --ignore-missing SHA256SUMS`. +2. From the existing installation folder's parent directory, extract the new package over the old installation files: ```bash - mkdir -p /tmp/container-alert-upgrade - tar -xzf container-alert-v1.6.0-offline.tar.gz -C /tmp/container-alert-upgrade - cd /path/to/existing/CyberController-Container-Alert - cp /tmp/container-alert-upgrade/CyberController-Container-Alert/{install.sh,uninstall.sh,docker-compose.yaml,VERSION,watchdog.tar,SHA256SUMS,README.md,DEPLOYMENT.md,.env.example,watchdog-config.yaml.example} . - mkdir -p docs - cp /tmp/container-alert-upgrade/CyberController-Container-Alert/docs/DEPLOYMENT_ADVANCED.md docs/ + cd /path/to/parent-of/CyberController-Container-Alert + tar -xzf /path/to/container-alert-v1.6.1-offline.tar.gz + cd CyberController-Container-Alert bash install.sh ``` -3. Answer **No** to Internet download. Use the supplied archive (or approved matching local image), and answer **No** to reconfiguration to keep current settings. -4. Verify health and the running image, as with the online upgrade. +3. Answer **No** to image download and **No** to reconfiguration. The new archive supplies the approved image. Confirm the running image and watchdog health. + +The commands use v1.6.1 **as a future example**, not as a claim that it exists. Use the actual filename and version supplied by Radware. Both packages intentionally exclude active `.env`, `watchdog-config.yaml`, and `watchdog/` files, so extracting an upgrade over the existing directory preserves customer-specific settings, logs and incident state. Keep earlier image tags for rollback. -Use the versioned package for the release you are actually upgrading to; `v1.6.0` above is an example. Older image tags should remain available for rollback. **There is no automatic updater**: upgrades happen only when an operator runs the installer. +There is **no automatic updater**. Upgrades occur only when an authorized operator obtains a newer release package and runs the installer. Changing Git source files is not an application upgrade. ### 5.4 Rollback diff --git a/README.md b/README.md index 73dbb81..1c68a27 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,10 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ├── requirements-watchdog.txt # Python dependencies (reference; Dockerfile pip-installs inline) ├── install.sh # Guided online/offline installation ├── uninstall.sh # Stop + remove script +├── scripts/package_release.sh # CI release-package assembly (maintainers only) +├── docs/DEVELOPMENT_AND_RELEASE.md # Development and v1.6.0 publication checklist ├── VERSION # Release version — drives the image tag and installer -├── watchdog.tar # Optional: image supplied in offline release package +├── watchdog.tar # Only in offline release package; not tracked by Git └── .env # Secrets — NOT committed to git ``` @@ -197,17 +199,22 @@ At least one channel must be selected in the installer; multiple channels may be ## 5. Deployment Instructions -Use the same installer for Internet-connected and offline hosts. With access to GitHub, clone the default **main** branch: +**Customers install from a numbered [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), not a Git clone.** Both release packages use the same `install.sh` wizard. No Python source is shipped to customers. + +| Host | Download from the approved release | Wizard choice | +|---|---|---| +| **Online** (can reach GitHub Releases) | `container-alert-v1.6.0-online.tar.gz` and `SHA256SUMS` | **Yes** — automatically download the matching prebuilt Docker image | +| **Offline** | `container-alert-v1.6.0-offline.tar.gz` and `SHA256SUMS` | **No** — load the image included in the transferred package | + +Verify the package checksum, extract the archive, enter the `CyberController-Container-Alert` directory and run: ```bash -git clone https://github.com/Radware/CyberController-Container-Alert.git -cd CyberController-Container-Alert bash install.sh ``` -For a disconnected host, extract the approved offline release package and run `bash install.sh` inside the extracted installation directory. The wizard offers download or local-image selection, asks for the selected notification channels, and verifies the watchdog. Customer installations do not build Docker images. +Replace v1.6.0 with the actual approved release. **Release v1.6.0 must be published before those assets become available.** The wizard handles channel configuration and watchdog-only deployment; no Git, image build or manual Docker load is required for a normal installation. -See [Deployment Guide](DEPLOYMENT.md) for the short procedure, and [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) for developer and release-maintainer details. +Read the complete [Online/Offline installation and maintenance guide](DEPLOYMENT.md). Developers/release maintainers: [Development and Release Cycle](docs/DEVELOPMENT_AND_RELEASE.md). --- @@ -386,11 +393,11 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor **Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -**Release procedure:** After validating a release candidate, increment `VERSION`, add a dated entry here, build and verify `watchdog:`, and package the matching `watchdog.tar` for offline installs. Pin deployment with `WATCHDOG_VERSION` in `.env`; verify upgrade and rollback before tagging the exact released commit (`vX.Y.Z`) and publishing release notes. Do not overwrite an old versioned image with new code. GitHub CI checks PRs but does not publish releases automatically; as of this update, no Git tags or GitHub Releases exist. +**Release procedure:** Review/validate the release PR, merge approved code to `main`, then push a matching `vX.Y.Z` Git tag. The hosted [release workflow](.github/workflows/release-package.yml) builds and verifies the image, packages the Online/Offline installers, checks their manifests, and publishes the versioned GitHub Release. `VERSION`, README version history, and versioned [release notes](docs/RELEASE_NOTES_V1.6.0.md) stay aligned. PRs do not publish releases automatically. See the [maintainer release guide](docs/DEVELOPMENT_AND_RELEASE.md) for the full procedure. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Egor Egorov | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, installer fallback, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, guided versioned Online/Offline release installation, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | @@ -399,7 +406,7 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor | 1.4.0 | 2026-08-17 | Rahul Kumar | Added INFO "recovered" alert | | 1.3.3 | 2026-08-05 | Rahul Kumar | Added Auth True/false | | 1.3.2 | 2026-07-28 | Rahul Kumar | Fixed SMTP Auth issue | -| 1.3.1 | 2026-07-21 | Egor Egorov | Updated Readme and Deployment guides | +| 1.3.1 | 2026-07-21 | Radware engineering | Updated Readme and Deployment guides | | 1.3.0 | 2026-07-16 | Rahul Kumar | Added Upgrade Guide | | 1.2.0 | 2026-07-16 | Rahul Kumar | Removed sudo and renamed the container | | 1.1.0 | 2026-07-09 | Rahul Kumar | Added SNMPv3 support | diff --git a/docs/DEPLOYMENT_ADVANCED.md b/docs/DEPLOYMENT_ADVANCED.md index 6214e40..b28e902 100644 --- a/docs/DEPLOYMENT_ADVANCED.md +++ b/docs/DEPLOYMENT_ADVANCED.md @@ -1,6 +1,6 @@ # Container Alert — Advanced Deployment Reference -This page supports engineers and release maintainers. For normal installation or upgrades, use the [Deployment Guide](../DEPLOYMENT.md): one interactive installer for online and offline hosts. +This page supports engineers and release maintainers. For normal installation and upgrades, use the [Deployment Guide](../DEPLOYMENT.md). For branching, CI and release publication, see [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md). ## Image distribution and provenance @@ -15,8 +15,11 @@ https://github.com/Radware/CyberController-Container-Alert/releases/tag/v-offline.tar.gz` — an installation folder containing `install.sh`, `VERSION`, Compose, templates, documentation, checksum, and image. +- `SHA256SUMS` — SHA-256 digests for the image and **both** install packages. +- `container-alert-v-online.tar.gz` — installer, `VERSION`, Compose, templates and documentation **without the Docker image or Python source**. +- `container-alert-v-offline.tar.gz` — exactly the same installation files **plus** `watchdog.tar` and its internal image checksum. + +The [release packaging script](../scripts/package_release.sh) builds both packages from the same list of files. Only the offline bundle includes the Docker image. Operator configuration (`.env`, `watchdog-config.yaml` and `watchdog/`) is deliberately absent so upgrades don't overwrite settings, secrets or history. The installer verifies downloaded archive checksum and the image's `org.opencontainers.image.version` label. When an operator explicitly accepts an existing local versioned image, the installer verifies its version label. The operator remains responsible for ensuring the local image came from an approved source. An offline archive with `SHA256SUMS` is independently checksum-checked before load. @@ -24,13 +27,9 @@ Image tags must not be reused for different source code. Build new code under a ## Release publication (maintainers) -1. Finish review and merge the release changes into `main`. Do not publish directly from an unmerged release branch. -2. Confirm the existing PR CI and isolated release smoke workflow passed for the reviewed candidate. -3. Tag the approved commit as `v` and push the tag. The `Publish Container Alert release` GitHub Actions workflow checks that the tag matches `VERSION` and that the commit is on `main`. -4. The hosted workflow runs non-destructive tests, builds one version-labelled image, exports `watchdog.tar`, creates the offline bundle/checksums, and publishes the GitHub Release. -5. Verify the release page assets and checksum before announcing customer availability. +Use the full [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md) checklist. In brief: approve and merge the release PR to `main`, confirm CI and isolated smoke on the reviewed SHA, obtain approval for **public** distribution, and only then push the `v` tag. GitHub Actions builds and validates the image, assembles both packages, checks their SHA-256 digests, creates a draft GitHub Release, verifies all four assets, then publishes. -Publication requires repository permission for GitHub Actions to create releases (contents: write), and approval to distribute the image publicly. Until that is approved and the release workflow succeeds, use a controlled offline package instead. No release publication happens merely by opening or merging a PR. +Tagging does not automatically deploy anything to an operational host. Until the release workflow succeeds, its online/offline assets do not exist for customers. Do not build source on customer systems. ## Developer-only source build diff --git a/docs/DEVELOPMENT_AND_RELEASE.md b/docs/DEVELOPMENT_AND_RELEASE.md new file mode 100644 index 0000000..0a16e5f --- /dev/null +++ b/docs/DEVELOPMENT_AND_RELEASE.md @@ -0,0 +1,199 @@ +# Container Alert — Development and Release Guide + +**For repository maintainers.** This is the repeatable development → testing → approval → GitHub Release process, including the exact finalization checklist for **v1.6.0**. Customer installation instructions belong in [Deployment](../DEPLOYMENT.md). + +## Table of contents + +1. [How source, images and packages relate](#1-how-source-images-and-packages-relate) +2. [Normal development cycle](#2-normal-development-cycle) +3. [Preparing the next numbered release](#3-preparing-the-next-numbered-release) +4. [v1.6.0 finalization checklist](#4-v160-finalization-checklist) +5. [Publishing and verifying v1.6.0](#5-publishing-and-verifying-v160) +6. [After release and rollback](#6-after-release-and-rollback) +7. [Release safety and best practices](#7-release-safety-and-best-practices) + +## 1. How source, images and packages relate + +| Item | v1.6.0 example | Purpose | +|---|---|---| +| Development/stabilization branch | `release/v1.6.0` | Code and documents under review | +| Source version file | `VERSION` = `1.6.0` | Single source of truth for the image and asset filenames | +| Final Git tag | `v1.6.0` | Immutable reference to the approved `main` commit | +| Docker image | `watchdog:1.6.0` | Prebuilt application and Python dependencies; customers do not build this | +| Online package | `container-alert-v1.6.0-online.tar.gz` | Installer, Compose, templates and docs; no code or image | +| Offline package | `container-alert-v1.6.0-offline.tar.gz` | Identical installer plus the saved Docker image and checksum | +| Release integrity manifest | `SHA256SUMS` | SHA-256 for `watchdog.tar` and both release packages | + +**Source code changes do not upgrade customers.** Customer upgrades occur only when a new numbered, tested image is published and the installer deploys that image. + +The hosted tag workflow reads `VERSION`, builds and tests the image, packages both installers, generates checksums, creates a **draft** GitHub Release, verifies all four assets exist, then publishes it. The published assets are: + +```text +watchdog.tar +SHA256SUMS +container-alert-v1.6.0-online.tar.gz +container-alert-v1.6.0-offline.tar.gz +``` + +The published release also uses the version-specific release notes when `docs/RELEASE_NOTES_V1.6.0.md` exists. No changes to `install.sh` or download URLs are required for v1.6.1 and later. + +## 2. Normal development cycle + +**For a new fix or feature after v1.6.0 is released:** + +1. Start with the latest `main`, and create a short-lived feature branch: + + ```bash + git switch main + git pull --ff-only origin main + git switch -c fix/describe-change + ``` + +2. Change Python code, unit tests, configuration examples or documentation. Do not reuse an existing release tag or overwrite a versioned image with different code. + +3. Validate with **non-destructive** checks only: + + ```bash + bash -n install.sh uninstall.sh scripts/package_release.sh + python3 -m unittest discover -s tests -q + ``` + + Run Python tests in a development environment with the required packages installed. Unit tests and fake-Docker installer/package tests must not use an operational Docker daemon. Do not run `tests/release-smoke.sh` directly on an appliance or shared host. + +4. Push and open a PR targeting `main`; obtain code review: + + ```bash + git push -u origin fix/describe-change + gh pr create --base main --head fix/describe-change --fill + ``` + +5. Check normal GitHub CI. For major changes or the final release candidate, require the **[Release smoke (isolated)](../.github/workflows/release-smoke.yml)** hosted workflow on the release PR. It is the only supported location for destructive Docker acceptance tests. + +**While v1.6.0 is being stabilized:** feature/fix PRs may target `release/v1.6.0`; the release branch itself is reviewed through [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) targeting `main`. Don't merge unreviewed work directly into a published release. + +## 3. Preparing the next numbered release + +After v1.6.0 is published, the next compatible maintenance release would typically be **v1.6.1**. A feature release might instead be v1.7.0; breaking changes require special compatibility review. + +1. Choose the version and update `VERSION` (e.g. `1.6.1`) in the release candidate. +2. Add the matching version entry in the README version history (required by CI). +3. Create/update `docs/RELEASE_NOTES_V1.6.1.md` describing all changes since the previous published release, customer-visible behavior, tests and limitations. +4. Use a release PR to `main` if batching fixes. Validate the actual final PR head in CI and hosted isolated smoke. +5. Merge only with approval. Then create the matching tag from `main`. **Do not** tag a release branch or publish from an unmerged PR. +6. The **same** `scripts/package_release.sh`, GitHub Actions workflow and installer produce the correct versioned assets automatically. + +**Development iterations are not releases:** changes can be committed and tested many times without publishing an image for each commit. Release numbering happens when preparing an approved delivery. + +## 4. v1.6.0 finalization checklist + +**Status when this guide was prepared (October 16, 2026):** `main` is still v1.5.4, `release/v1.6.0` contains `VERSION=1.6.0`, and PR #9 is **open/unmerged**. The GitHub Release for v1.6.0 has not been published by this work. + +Do **not** push the tag until every applicable item below is satisfied: + +- [ ] PR #9 receives final engineering review and approval, including lifecycle, alerting, HA, security and release packaging. +- [ ] Normal CI and [isolated hosted Docker smoke](RELEASE_SMOKE_TESTING.md) both pass **for the current PR head SHA**. Rerun if release-impacting code changes. +- [ ] The release notes and customer alert text describe all changes since v1.5.4; there are no private lab hostnames/addresses or unapproved secrets in release files. +- [ ] Release approval explicitly permits publication of **Docker image and packages to the public GitHub repository**. A published release can be downloaded by anyone. +- [ ] Deployment and rollback instructions are reviewed; HA failover and customer notification destinations receive any required authorized, separate acceptance. A passing hosted smoke does not prove live customer SMTP/Slack/SNMP delivery. +- [ ] The `main` branch contains the release workflow and the full approved PR content. Tagging before merge will not publish this release as intended. + +Useful inspection commands (read-only): + +```bash +gh pr view 9 -R Radware/CyberController-Container-Alert +gh pr checks 9 -R Radware/CyberController-Container-Alert +gh run list -R Radware/CyberController-Container-Alert \ + --workflow release-smoke.yml --limit 5 +``` + +If checks fail or public distribution is not authorized, **do not merge/tag/publish**. Fix the PR and rerun checks. + +## 5. Publishing and verifying v1.6.0 + +The commands below are **release-maintainer actions**. Run them only **after** approval. They are documented here; they are **not** executed as part of this documentation update. + +### Step A — Merge and verify main + +Merge [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) using the approved GitHub review process. Then update a clean repository checkout: + +```bash +git fetch origin main --tags +git switch main +git pull --ff-only origin main +cat VERSION # must be 1.6.0 +git status --short # must be empty +git log -1 --oneline # record final release commit SHA +git ls-remote --tags origin refs/tags/v1.6.0 +``` + +The final command must return **no existing v1.6.0 tag**. If it exists, investigate; do not move or overwrite release tags. + +### Step B — Create the release tag + +From the clean, approved `main` commit: + +```bash +test "$(cat VERSION)" = "1.6.0" || exit 1 +git tag -a v1.6.0 -m "Container Alert v1.6.0" +git push origin v1.6.0 +``` + +This tag push triggers [Publish Container Alert release](../.github/workflows/release-package.yml) on a GitHub-hosted runner. The workflow verifies that the tag matches `VERSION` and points into `main` history, runs unit tests, builds the version-labelled Docker image, checks imports, creates both release packages and checksum manifest, and publishes **after** checking all assets. + +### Step C — Check the GitHub workflow and release + +```bash +gh run list -R Radware/CyberController-Container-Alert \ + --workflow release-package.yml --limit 5 +gh release view v1.6.0 -R Radware/CyberController-Container-Alert +``` + +Check that the workflow completed successfully and that the release contains **all four assets**, the intended release notes, and no customer-specific secrets. + +Download and verify the release files **without installing or touching Docker**: + +```bash +mkdir -p /tmp/container-alert-v1.6.0-verification +gh release download v1.6.0 -R Radware/CyberController-Container-Alert \ + --dir /tmp/container-alert-v1.6.0-verification +cd /tmp/container-alert-v1.6.0-verification +sha256sum --check SHA256SUMS +tar -tzf container-alert-v1.6.0-online.tar.gz | head +tar -tzf container-alert-v1.6.0-offline.tar.gz | grep watchdog.tar +``` + +Verify the online archive contains no Python application source and no `watchdog.tar`; the offline archive must contain the same installer plus `watchdog.tar` and its internal checksum. Confirm both carry `VERSION=1.6.0`. + +**If publication fails:** investigate the GitHub Actions logs and any draft release. Do not tell customers the release is ready. Do not force-move an existing published tag; follow the organization's correction/versioning process. + +### Step D — Announce and distribute + +- **Online operators** download `container-alert-v1.6.0-online.tar.gz` plus `SHA256SUMS` from the published release page. They run the contained `bash install.sh` wizard and answer **Yes** to download the image. +- **Offline operators** receive `container-alert-v1.6.0-offline.tar.gz` plus `SHA256SUMS` through an approved transfer method. They run the **same wizard** and answer **No**. +- Keep the previous approved image available on target systems for rollback. Do not stop or alter CyberController application containers during installation. + +See [Deployment](../DEPLOYMENT.md#2-installation) for the complete customer procedures. + +## 6. After release and rollback + +An online/offline customer upgrade obtains the **new release package** and runs its installer in the existing installation directory; the package intentionally omits active `.env`, `watchdog-config.yaml` and the persistent `watchdog/` directory so customer settings and incident history are retained. The installer recreates only the watchdog container. + +For YAML-only monitoring changes, restart the watchdog. For `.env` or Compose changes, recreate just the watchdog. For Python changes, build and publish **a different numbered image**. A source-code pull or restart does not update Python inside the running image. + +If rollback is authorized, select the last approved `WATCHDOG_VERSION` and its corresponding stored image/configuration. See [Deployment — Rollback](../DEPLOYMENT.md#54-rollback). Never rewrite an old published tag to mean new code. + +## 7. Release safety and best practices + +This project uses a standard GitHub Flow / tagged-release approach: review changes by PR, test them in CI, merge approved code to `main`, assign a semantic version/tag, and distribute **prebuilt artifacts** rather than compiling on customer systems. + +Essential rules: + +- **Approval before publication:** this repository is public. Confirm Radware authorizes distributing the code, Docker image, release notes and package contents. +- **Traceability:** align `VERSION`, the Git tag, the Docker image label/tag, both installer package names, CI results and published checksums. Do not reuse image tags. +- **Safety:** all Docker failure injection must stay inside [isolated hosted CI](RELEASE_SMOKE_TESTING.md), never an operational or shared Docker daemon. Test installers/packages with fake commands and tiny dummy image files. +- **Integrity:** SHA-256 manifests check whether package bytes match the release assets; they are not a replacement for trusted distribution, source review, or stronger artifact signatures. +- **Production restraint:** no source builds, no automatic updates, no service-wide Docker shutdowns, and no customer-directed changes to CyberController Compose/resource policy. + +For mature supply-chain requirements, Radware may later add protected `v*` tags, immutable GitHub Releases, pinned image dependencies and signed provenance. The release image is built anew from the approved tag, so it is **not guaranteed byte-for-byte identical** to the image tested in the PR while its base/dependency versions can change. Record its final digest and obtain the appropriate release sign-off. Those harder supply-chain enhancements are separate from this deliberately small v1.6.0 delivery flow. + +Official references: [Managing GitHub Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), and [immutable releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index daaa58d..91bf2a9 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -14,7 +14,7 @@ This document describes version **1.6.0** and its changes since v1.5.4. Building | Release documentation | Clarify merged vs unreleased features and explicit image version/tag process | [PR #8](https://github.com/Radware/CyberController-Container-Alert/pull/8) | | Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | | Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Guided release installation | One installer wizard selects verified GitHub Release image download or an approved local image/archive, preserves existing configuration, and recreates only the watchdog. Source builds remain developer-only. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Guided release installation | Versioned online/offline release packages share one installer wizard. Online downloads the verified release image, offline includes it; existing settings are preserved and only the watchdog is recreated. No customer-side code builds or Git checkout. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Operator documentation and configuration clarity | Restore independent Online/Offline walkthroughs, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP technical details in advanced reference; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | @@ -48,7 +48,9 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C ## Image distribution and deployment -The v1.6.0 release publication workflow prepares a version-labelled `watchdog.tar`, `SHA256SUMS`, and a complete offline installation bundle through GitHub Releases. Publishing is triggered only after the approved code is merged to `main` and a matching `v1.6.0` tag is pushed; opening or merging this PR does not publish artifacts. Operators obtain files from `main` or the offline bundle and run the same `bash install.sh` wizard. Production installations no longer build Docker images from source. See [Deployment](../DEPLOYMENT.md) and [Advanced Deployment](DEPLOYMENT_ADVANCED.md). +The approved v1.6.0 GitHub Release is prepared by pushing tag `v1.6.0` **only after PR #9 is approved and merged into `main`**. The hosted release workflow builds a version-labelled image and publishes four assets: `watchdog.tar`, `SHA256SUMS`, `container-alert-v1.6.0-online.tar.gz`, and `container-alert-v1.6.0-offline.tar.gz`. The online package contains only installer/Compose/templates/docs and downloads the image during installation. The offline package contains the same files and the prebuilt image. + +Neither package contains application Python source, and neither installer builds code on production systems. The release workflow prepares a draft and publishes only after verifying that all four assets are uploaded. Release packaging is **not** triggered merely by opening or merging a PR. See [Deployment](../DEPLOYMENT.md) and [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md). ## Exclusions, limitations and release gating @@ -56,4 +58,4 @@ The v1.6.0 release publication workflow prepares a version-labelled `watchdog.ta The HA notification dispatch path has no persistent retry queue (intentional v1 limitation). Docker planned-stop correlation is not an availability policy; a deliberately stopped long-running service does not automatically produce a downtime alert. Unexpected external Docker restarts were seen on a lab; the initiator was not determined. These are documented operational limitations, not release regressions. -**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, package signed/verified artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. None of those final publish steps is automatic. +**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, verify release checksums and packaged artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. Merge/approval and tagging are manual release-maintainer actions; the package build and publication run automatically from the approved tag. diff --git a/scripts/package_release.sh b/scripts/package_release.sh new file mode 100644 index 0000000..a28edb2 --- /dev/null +++ b/scripts/package_release.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Assemble tested release installation bundles without running Docker. +# Usage: bash scripts/package_release.sh +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$(dirname "$0")/.." && pwd)" +version="$(tr -d '[:space:]' < "$repo/VERSION")" +if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid VERSION: $version" >&2 + exit 1 +fi + +image="$(realpath "$1")" +test -f "$image" && test -s "$image" || { echo "Image archive missing or empty" >&2; exit 1; } +mkdir -p "$2" +out="$(cd "$2" && pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +dir="$tmp/CyberController-Container-Alert" +mkdir -p "$dir/docs" +for file in install.sh uninstall.sh docker-compose.yaml VERSION \ + .env.example watchdog-config.yaml.example README.md DEPLOYMENT.md; do + cp "$repo/$file" "$dir/$file" +done +cp "$repo"/docs/*.md "$dir/docs/" # text-only references; never Python source + +online="container-alert-v$version-online.tar.gz" +offline="container-alert-v$version-offline.tar.gz" + +# Both packages have exactly the same installer and configuration templates. +tar -C "$tmp" -czf "$out/$online" CyberController-Container-Alert + +# The offline package adds the prebuilt image and its own checksum manifest. +cp "$image" "$out/watchdog.tar" +cp "$out/watchdog.tar" "$dir/watchdog.tar" +(cd "$out" && sha256sum watchdog.tar) > "$dir/SHA256SUMS" +tar -C "$tmp" -czf "$out/$offline" CyberController-Container-Alert + +# Public release checksum file covers every downloadable binary asset. +(cd "$out" && sha256sum watchdog.tar "$online" "$offline") > "$out/SHA256SUMS" +for asset in watchdog.tar "$online" "$offline" SHA256SUMS; do + if [ "$(stat -c '%s' "$out/$asset")" -ge 2147483648 ]; then + echo "GitHub release asset exceeds 2 GiB: $asset" >&2 + exit 1 + fi +done +echo "Built release bundles for v$version in $out" diff --git a/tests/test_release_package.py b/tests/test_release_package.py new file mode 100644 index 0000000..9821d38 --- /dev/null +++ b/tests/test_release_package.py @@ -0,0 +1,82 @@ +"""Test release package composition without Docker, image builds, or live hosts.""" + +import hashlib +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "package_release.sh" +VERSION = (ROOT / "VERSION").read_text().strip() +PREFIX = "CyberController-Container-Alert/" + + +class ReleasePackageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="container-alert-release-fake-") + self.addCleanup(self.temp.cleanup) + self.workspace = Path(self.temp.name) + self.fake_image = self.workspace / "test-watchdog.tar" + self.fake_image.write_bytes(b"FAKE ARCHIVE - NOT A DOCKER IMAGE\n") + self.output = self.workspace / "dist" + result = subprocess.run( + ["bash", str(SCRIPT), str(self.fake_image), str(self.output)], + capture_output=True, text=True, timeout=15, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.online = self.output / f"container-alert-v{VERSION}-online.tar.gz" + self.offline = self.output / f"container-alert-v{VERSION}-offline.tar.gz" + + @staticmethod + def members(path): + with tarfile.open(path, "r:gz") as archive: + return {member.name: archive.extractfile(member).read() + for member in archive if member.isfile()} + + def test_online_bundle_contains_only_installer_files(self): + contents = self.members(self.online) + for filename in ("install.sh", "uninstall.sh", "VERSION", + "docker-compose.yaml", "README.md", "DEPLOYMENT.md", + ".env.example", "watchdog-config.yaml.example", + "docs/DEPLOYMENT_ADVANCED.md", + "docs/DEVELOPMENT_AND_RELEASE.md", + "docs/RELEASE_NOTES_V1.6.0.md"): + self.assertIn(PREFIX + filename, contents) + self.assertNotIn(PREFIX + "watchdog.tar", contents) + self.assertNotIn(PREFIX + "watchdog.py", contents) + self.assertNotIn(PREFIX + "Dockerfile", contents) + + def test_offline_bundle_adds_image_and_internal_checksum(self): + contents = self.members(self.offline) + self.assertEqual(contents[PREFIX + "watchdog.tar"], + self.fake_image.read_bytes()) + expected = hashlib.sha256(self.fake_image.read_bytes()).hexdigest() + self.assertIn(expected, contents[PREFIX + "SHA256SUMS"].decode()) + self.assertIn("watchdog.tar", contents[PREFIX + "SHA256SUMS"].decode()) + self.assertNotIn(PREFIX + "watchdog.py", contents) + + def test_both_bundles_share_identical_installer_and_version(self): + online = self.members(self.online) + offline = self.members(self.offline) + for path, content in online.items(): + self.assertEqual(offline[path], content, path) + self.assertEqual(online[PREFIX + "VERSION"].decode().strip(), VERSION) + + def test_release_checksum_manifest_covers_every_asset(self): + manifest = (self.output / "SHA256SUMS").read_text().splitlines() + self.assertEqual(len(manifest), 3) + names = set() + for line in manifest: + digest, name = line.split(maxsplit=1) + names.add(name) + path = self.output / name + self.assertTrue(path.is_file(), name) + self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), digest) + self.assertEqual(names, {"watchdog.tar", self.online.name, self.offline.name}) + + +if __name__ == "__main__": + unittest.main() From 3ae5a0e2586a04df0e8a7d2f0bfb4c240e60ec54 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 15:00:31 +0000 Subject: [PATCH 11/18] release: use single installer bundle and evergreen developer guide --- .github/workflows/release-package.yml | 19 +-- DEPLOYMENT.md | 107 ++++-------- README.md | 29 ++-- docs/DEPLOYMENT_ADVANCED.md | 18 +- docs/DEVELOPMENT.md | 170 +++++++++++++++++++ docs/DEVELOPMENT_AND_RELEASE.md | 199 ---------------------- docs/RELEASE_NOTES_V1.6.0.md | 8 +- docs/RELEASE_V1.6.0_CHECKLIST.md | 64 +++++++ install.sh | 102 +++-------- scripts/package_release.sh | 50 +++--- tests/test_installer_image.py | 235 ++++++++++++-------------- tests/test_release_package.py | 98 +++++------ 12 files changed, 498 insertions(+), 601 deletions(-) create mode 100644 docs/DEVELOPMENT.md delete mode 100644 docs/DEVELOPMENT_AND_RELEASE.md create mode 100644 docs/RELEASE_V1.6.0_CHECKLIST.md diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index c1ad343..8fe5a1c 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -50,15 +50,17 @@ jobs: "watchdog:$WATCHDOG_VERSION" -c \ "import docker, requests, yaml, pysnmp, cryptography, watchdog, lifecycle, ha_monitor" - - name: Package online and offline downloads + - name: Assemble and validate single customer package shell: bash run: | set -euo pipefail docker save "watchdog:$WATCHDOG_VERSION" -o watchdog.tar bash scripts/package_release.sh watchdog.tar dist (cd dist && sha256sum --check SHA256SUMS) - tar -tzf "dist/container-alert-v$WATCHDOG_VERSION-online.tar.gz" > /dev/null - tar -tzf "dist/container-alert-v$WATCHDOG_VERSION-offline.tar.gz" > /dev/null + package="dist/container-alert-v$WATCHDOG_VERSION.tar.gz" + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/watchdog.tar' + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/SHA256SUMS' + tar -tzf "$package" | grep -qx 'CyberController-Container-Alert/install.sh' - name: Publish GitHub Release env: @@ -71,17 +73,14 @@ jobs: notes=(--notes-file "docs/RELEASE_NOTES_V$WATCHDOG_VERSION.md") fi - # Keep the release private as a draft until all four assets exist. + # Check both assets before making this public release visible. gh release create "$GITHUB_REF_NAME" \ - dist/watchdog.tar dist/SHA256SUMS \ - "dist/container-alert-v$WATCHDOG_VERSION-online.tar.gz" \ - "dist/container-alert-v$WATCHDOG_VERSION-offline.tar.gz" \ + dist/SHA256SUMS "dist/container-alert-v$WATCHDOG_VERSION.tar.gz" \ --verify-tag --draft --title "Container Alert $GITHUB_REF_NAME" \ "${notes[@]}" - expected=$(printf '%s\n' SHA256SUMS watchdog.tar \ - "container-alert-v$WATCHDOG_VERSION-online.tar.gz" \ - "container-alert-v$WATCHDOG_VERSION-offline.tar.gz" | sort) + expected=$(printf '%s\n' SHA256SUMS \ + "container-alert-v$WATCHDOG_VERSION.tar.gz" | sort) actual=$(gh release view "$GITHUB_REF_NAME" --json assets --jq '.assets[].name' | sort) test "$actual" = "$expected" || { echo "Release assets incomplete" >&2; exit 1; } gh release edit "$GITHUB_REF_NAME" --draft=false diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index c6407e8..e0a989d 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,13 +1,13 @@ # CyberController Container Watchdog — Deployment Guide -This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. The same `install.sh` wizard is used for online and offline installations. It deploys only the `docker-container-watchdog` service; it never restarts CyberController application containers. +This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. One complete, versioned release package and one `install.sh` wizard work on hosts with or without Internet access. The installer deploys only `docker-container-watchdog`; it never restarts CyberController application containers. ## Table of Contents 1. [Prerequisites](#1-prerequisites) 2. [Installation](#2-installation) - - [Online installation](#21-online-installation) - - [Offline installation](#22-offline-installation) + - [Get and verify the release package](#21-get-and-verify-the-release-package) + - [Run the installer](#22-run-the-installer) 3. [Verify the installation](#3-verify-the-installation) 4. [Configuration reference](#4-configuration-reference) - [Configuration files and editing](#41-configuration-files-and-editing) @@ -33,8 +33,8 @@ This guide covers **installation, configuration, day-to-day operations, upgrades | Docker Engine | 20.10+ and a running Docker daemon | | Docker Compose | Version 2 plugin (`docker compose`) recommended; `install.sh` also supports standalone `docker-compose` | | Host permissions | Root or an operator with Docker access; `sudo` may be required to secure configuration files | -| Online installation | Git access to this repository **and** HTTPS access to its published GitHub Release image; `curl` is used for the image download | -| Offline installation | Approved release package copied to the host; Git and Internet are not required | +| Release package | Approved complete `container-alert-v.tar.gz` with matching `SHA256SUMS`, copied to the host | +| Network access | **Neither Git nor Internet access is required on the target host**; download from GitHub Releases elsewhere if needed | | Alert delivery | Network connectivity to the chosen Syslog, SMTP, SNMP, or Slack destination | Read-only prerequisite checks: @@ -52,75 +52,45 @@ Docker socket access is highly privileged even if a container runs as a non-root ## 2. Installation -Choose **Online** if the host can download the official release image. Choose **Offline** if the host cannot reach GitHub. Both use exactly the same `bash install.sh` wizard. **Neither package includes Python source code, and the customer never builds an image.** +**One process on every host.** Download the approved versioned package directly, or have an authorized operator transfer the same package to a disconnected CyberController host. The package already contains the prebuilt Docker image, installer, Compose files, configuration templates and documentation. No Git clone, customer-side image download or source build is needed. -The names below show v1.6.0. For newer releases, select the approved version on the [GitHub Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) and use its matching filenames. **The v1.6.0 packages become available only after that version is officially published.** +The filename below uses v1.6.0 as an example. Release assets exist only after that version has been officially [published on GitHub](https://github.com/Radware/CyberController-Container-Alert/releases). -### 2.1 Online installation +### 2.1 Get and verify the release package -**Step 1 — Obtain the small online installer package.** From the approved release on the [Releases page](https://github.com/Radware/CyberController-Container-Alert/releases), download these two files to the target host: +**Step 1 — Obtain these two files** from the approved [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases) (or through Radware's authorized offline transfer method): -- `container-alert-v1.6.0-online.tar.gz` — installer, Compose file, version, templates and documentation (**no Docker image**). -- `SHA256SUMS` — file integrity checks. +- `container-alert-v1.6.0.tar.gz` — complete install package, including `watchdog.tar`. +- `SHA256SUMS` — checksum to verify the downloaded/transferred package. -**Step 2 — Verify and extract.** Run these commands where both files were downloaded: +**Step 2 — Copy both files to the target host**, into the same directory. Verify the package and extract it: ```bash -sha256sum --check --ignore-missing SHA256SUMS -tar -xzf container-alert-v1.6.0-online.tar.gz +sha256sum --check SHA256SUMS +tar -xzf container-alert-v1.6.0.tar.gz cd CyberController-Container-Alert ``` -**Step 3 — Run the guided installer.** +The extracted directory contains `VERSION`, `install.sh`, `docker-compose.yaml`, `watchdog.tar` and its own image checksum. Use the matching filenames for later versions. -```bash -bash install.sh -``` - -Answer **Yes** to the GitHub Releases download question. The wizard automatically obtains the matching prebuilt Docker image (`watchdog.tar`), verifies its checksum and version, asks which alert channels to configure (Syslog, SMTP, SNMP and/or Slack), and deploys only the watchdog. No Git clone, source build, or manual `docker load` is needed. - -**Step 4 — Confirm that the watchdog is running.** - -```bash -docker ps --filter name=docker-container-watchdog -docker logs --tail 50 docker-container-watchdog -``` - -The watchdog should be running and report normal monitoring activity. Full read-only verification is in [Section 3](#3-verify-the-installation). - -### 2.2 Offline installation - -**Step 1 — Obtain the complete offline installer package.** Download the approved release on a connected machine, then transfer both files to the disconnected target host: - -- `container-alert-v1.6.0-offline.tar.gz` — the **same installer and configuration files** plus the prebuilt `watchdog.tar` and its checksum. -- `SHA256SUMS` — file integrity checks for the transferred package. - -**Step 2 — Verify and extract.** On the target host, run these commands where both transferred files are located: +### 2.2 Run the installer -```bash -sha256sum --check --ignore-missing SHA256SUMS -tar -xzf container-alert-v1.6.0-offline.tar.gz -cd CyberController-Container-Alert -``` - -**Step 3 — Run the same guided installer.** +**Step 3 — Start the configuration wizard:** ```bash bash install.sh ``` -Answer **No** to the Internet download question. The wizard selects an approved local versioned image or loads and verifies the included `watchdog.tar`. Configure the alert channels and hostname as prompted. No Git or Internet access is necessary on the target. +The installer automatically verifies the included Docker image checksum and version, loads the image, then asks you to select and configure the notification channels (Syslog, SMTP, SNMP and/or Slack) and alert hostname. On an upgrade, answer **No** to “Reconfigure from scratch?” to preserve existing channel settings. It recreates **only the watchdog**, not other application containers. -**Step 4 — Confirm that the watchdog is running.** +**Step 4 — Confirm normal operation:** ```bash docker ps --filter name=docker-container-watchdog docker logs --tail 50 docker-container-watchdog ``` -The watchdog should be running. Full read-only verification is in [Section 3](#3-verify-the-installation). - -**If image acquisition fails:** the installer does not modify the running watchdog. Obtain the correct published release package before retrying. Do not build source on production hosts. +Expected: the watchdog is running and logging normal monitoring. For further read-only checks, see [Section 3](#3-verify-the-installation). A failed or incomplete image verification stops installation before touching the running watchdog. ## 3. Verify the installation @@ -361,40 +331,25 @@ If running as root, omit `sudo`. Keep backup directories access-restricted becau ### 5.3 Upgrade the watchdog -First use [Section 5.2](#52-backup-and-restore) to back up `.env` and `watchdog-config.yaml`. Retain the `watchdog/` directory containing state and historical alerts. - -**Online upgrade** - -1. Download the newer approved `container-alert-v-online.tar.gz` and `SHA256SUMS` from the [GitHub Releases page](https://github.com/Radware/CyberController-Container-Alert/releases). -2. Verify the package using `sha256sum --check --ignore-missing SHA256SUMS`. -3. Extract the package **over the existing installation folder** (from its parent directory), then run the installer: - - ```bash - cd /path/to/parent-of/CyberController-Container-Alert - tar -xzf /path/to/container-alert-v1.6.1-online.tar.gz - cd CyberController-Container-Alert - bash install.sh - ``` - -4. Answer **Yes** to image download and **No** to reconfigure existing settings. The installer updates the pinned image version and recreates **only** the watchdog. Verify using [Section 3](#3-verify-the-installation). - -**Offline upgrade** +Upgrades use **the same complete release package and installer** as first-time installation. -1. Transfer the newer approved offline package and `SHA256SUMS` to the host. Verify the package with `sha256sum --check --ignore-missing SHA256SUMS`. -2. From the existing installation folder's parent directory, extract the new package over the old installation files: +1. Back up `.env` and `watchdog-config.yaml` as in [Section 5.2](#52-backup-and-restore). Keep the persistent `watchdog/` directory and earlier approved Docker image available for rollback. +2. Obtain the **new approved** `container-alert-v.tar.gz` and matching `SHA256SUMS` via GitHub Releases or authorized transfer. Copy both files into the **parent directory of the existing** `CyberController-Container-Alert` installation. +3. From that parent directory, verify and extract the new package **over the existing installation files**, then run the installer: ```bash cd /path/to/parent-of/CyberController-Container-Alert - tar -xzf /path/to/container-alert-v1.6.1-offline.tar.gz + sha256sum --check SHA256SUMS + tar -xzf container-alert-v1.6.1.tar.gz cd CyberController-Container-Alert bash install.sh ``` -3. Answer **No** to image download and **No** to reconfiguration. The new archive supplies the approved image. Confirm the running image and watchdog health. +4. Answer **No** to “Reconfigure from scratch?” to preserve configuration. The installer verifies/loads the new image, updates the pinned version, and recreates **only the watchdog**. Check its health, actual image and recent logs. -The commands use v1.6.1 **as a future example**, not as a claim that it exists. Use the actual filename and version supplied by Radware. Both packages intentionally exclude active `.env`, `watchdog-config.yaml`, and `watchdog/` files, so extracting an upgrade over the existing directory preserves customer-specific settings, logs and incident state. Keep earlier image tags for rollback. +The filename v1.6.1 above is an **illustrative future version**, not a claim it exists. Replace it with the actual approved release. The archive deliberately excludes active `.env`, `watchdog-config.yaml` and `watchdog/` so upgrades preserve credentials, monitoring policy, historical logs and incident state. -There is **no automatic updater**. Upgrades occur only when an authorized operator obtains a newer release package and runs the installer. Changing Git source files is not an application upgrade. +There is **no automatic updater**; upgrading requires an authorized operator to run the installer. Updating Git source code is not an application upgrade. ### 5.4 Rollback @@ -431,8 +386,8 @@ Do not use `--remove-all` if historical alerts or state must be retained. Uninst | Observation | What to check | |---|---| -| Installer cannot reach GitHub Releases | Host HTTPS access and the published release matching `VERSION`; use the approved offline archive if disconnected | -| Local image/archive rejected | Correct `watchdog:` tag, release version label, archive checksum and disk space | +| Release package missing | Obtain the complete archive and checksum from the approved GitHub Release or Radware transfer method | +| Included image/archive rejected | Check the complete package checksum, included `watchdog.tar` and internal `SHA256SUMS`, image version label and disk space | | Docker or Compose command fails | `docker info`, Docker socket access, `docker compose version` and configured permissions | | Only one config file exists | Restore the missing `.env` or `watchdog-config.yaml` backup; installer will not overwrite the surviving file | | Watchdog exits or stays unhealthy | `docker inspect` health state and `docker logs --tail 100 docker-container-watchdog` | diff --git a/README.md b/README.md index 1c68a27..d69735c 100644 --- a/README.md +++ b/README.md @@ -132,12 +132,13 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ├── docker-compose.build.yaml # Developer-only source build ├── Dockerfile # Image build definition (python:3.11-slim) ├── requirements-watchdog.txt # Python dependencies (reference; Dockerfile pip-installs inline) -├── install.sh # Guided online/offline installation +├── install.sh # Guided installation from complete release package ├── uninstall.sh # Stop + remove script ├── scripts/package_release.sh # CI release-package assembly (maintainers only) -├── docs/DEVELOPMENT_AND_RELEASE.md # Development and v1.6.0 publication checklist +├── docs/DEVELOPMENT.md # Version-independent source-to-release workflow +├── docs/RELEASE_V1.6.0_CHECKLIST.md # v1.6.0-specific release gates ├── VERSION # Release version — drives the image tag and installer -├── watchdog.tar # Only in offline release package; not tracked by Git +├── watchdog.tar # Included in published release package; not tracked by Git └── .env # Secrets — NOT committed to git ``` @@ -199,22 +200,20 @@ At least one channel must be selected in the installer; multiple channels may be ## 5. Deployment Instructions -**Customers install from a numbered [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), not a Git clone.** Both release packages use the same `install.sh` wizard. No Python source is shipped to customers. +Install **one complete versioned package** from an approved [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), or obtain the same package through authorized offline transfer. No Git clone, separate Docker image download, or source build is required on the customer host. -| Host | Download from the approved release | Wizard choice | -|---|---|---| -| **Online** (can reach GitHub Releases) | `container-alert-v1.6.0-online.tar.gz` and `SHA256SUMS` | **Yes** — automatically download the matching prebuilt Docker image | -| **Offline** | `container-alert-v1.6.0-offline.tar.gz` and `SHA256SUMS` | **No** — load the image included in the transferred package | - -Verify the package checksum, extract the archive, enter the `CyberController-Container-Alert` directory and run: +Download/transfer both `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (substitute your approved version). From the directory containing those two files: ```bash +sha256sum --check SHA256SUMS +tar -xzf container-alert-v1.6.0.tar.gz +cd CyberController-Container-Alert bash install.sh ``` -Replace v1.6.0 with the actual approved release. **Release v1.6.0 must be published before those assets become available.** The wizard handles channel configuration and watchdog-only deployment; no Git, image build or manual Docker load is required for a normal installation. +The wizard verifies and loads the included Docker image, asks for the selected notification channels, and deploys **only** the watchdog. **The v1.6.0 release archive becomes downloadable after the release is published.** -Read the complete [Online/Offline installation and maintenance guide](DEPLOYMENT.md). Developers/release maintainers: [Development and Release Cycle](docs/DEVELOPMENT_AND_RELEASE.md). +Read the full [Deployment Guide](DEPLOYMENT.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](docs/DEVELOPMENT.md) and the [v1.6.0 finalization checklist](docs/RELEASE_V1.6.0_CHECKLIST.md). --- @@ -274,7 +273,7 @@ docker compose logs docker-container-watchdog Common causes: - **Missing `.env` file** — complete the initial installer wizard, or restore it from a previous-installation backup - **Docker socket not accessible** — ensure `/var/run/docker.sock` exists and the container has read access -- **Release image unavailable** — rerun `bash install.sh` and select online download, or supply the approved offline `watchdog.tar` and choose the local-image option (see [Deployment Guide](DEPLOYMENT.md)) +- **Release image/package unavailable** — obtain the complete approved versioned package, verify `SHA256SUMS`, extract it and run `bash install.sh` (see [Deployment Guide](DEPLOYMENT.md)) ### Alert Notifications Not Received @@ -393,11 +392,11 @@ Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitor **Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -**Release procedure:** Review/validate the release PR, merge approved code to `main`, then push a matching `vX.Y.Z` Git tag. The hosted [release workflow](.github/workflows/release-package.yml) builds and verifies the image, packages the Online/Offline installers, checks their manifests, and publishes the versioned GitHub Release. `VERSION`, README version history, and versioned [release notes](docs/RELEASE_NOTES_V1.6.0.md) stay aligned. PRs do not publish releases automatically. See the [maintainer release guide](docs/DEVELOPMENT_AND_RELEASE.md) for the full procedure. +**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](docs/DEVELOPMENT.md); version-specific approval for v1.6.0 is in its [finalization checklist](docs/RELEASE_V1.6.0_CHECKLIST.md). | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, guided versioned Online/Offline release installation, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/docs/DEPLOYMENT_ADVANCED.md b/docs/DEPLOYMENT_ADVANCED.md index b28e902..ac3f240 100644 --- a/docs/DEPLOYMENT_ADVANCED.md +++ b/docs/DEPLOYMENT_ADVANCED.md @@ -1,6 +1,6 @@ # Container Alert — Advanced Deployment Reference -This page supports engineers and release maintainers. For normal installation and upgrades, use the [Deployment Guide](../DEPLOYMENT.md). For branching, CI and release publication, see [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md). +This page supports engineers and release maintainers. For normal installation and upgrades, use the [Deployment Guide](../DEPLOYMENT.md). For branching, CI and release publication, see [Development Guide](DEVELOPMENT.md). ## Image distribution and provenance @@ -12,24 +12,22 @@ Official image delivery uses **GitHub Releases** at: https://github.com/Radware/CyberController-Container-Alert/releases/tag/v ``` -Each approved release contains: +Each approved release contains **two downloadable assets**: -- `watchdog.tar` — the prebuilt, version-tagged Docker image. -- `SHA256SUMS` — SHA-256 digests for the image and **both** install packages. -- `container-alert-v-online.tar.gz` — installer, `VERSION`, Compose, templates and documentation **without the Docker image or Python source**. -- `container-alert-v-offline.tar.gz` — exactly the same installation files **plus** `watchdog.tar` and its internal image checksum. +- `container-alert-v.tar.gz` — complete installer, `VERSION`, Compose, templates, documentation, the prebuilt `watchdog.tar` image, and its internal `SHA256SUMS`. +- `SHA256SUMS` — checksum of the entire packaged archive, checked before extraction. -The [release packaging script](../scripts/package_release.sh) builds both packages from the same list of files. Only the offline bundle includes the Docker image. Operator configuration (`.env`, `watchdog-config.yaml` and `watchdog/`) is deliberately absent so upgrades don't overwrite settings, secrets or history. +The [release packaging script](../scripts/package_release.sh) produces the same complete package for connected and disconnected customer hosts. No app Python source or active operator configuration (`.env`, `watchdog-config.yaml` or `watchdog/`) is included. -The installer verifies downloaded archive checksum and the image's `org.opencontainers.image.version` label. When an operator explicitly accepts an existing local versioned image, the installer verifies its version label. The operator remains responsible for ensuring the local image came from an approved source. An offline archive with `SHA256SUMS` is independently checksum-checked before load. +The operator first checks the package checksum, then extracts the archive and runs `bash install.sh`. The installer **requires** the included image archive and internal checksum, verifies them and the image's `org.opencontainers.image.version` label, and loads `watchdog:`. No installer-side Git, Internet connection or source build is needed. Image tags must not be reused for different source code. Build new code under a new version so rollbacks are unambiguous. ## Release publication (maintainers) -Use the full [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md) checklist. In brief: approve and merge the release PR to `main`, confirm CI and isolated smoke on the reviewed SHA, obtain approval for **public** distribution, and only then push the `v` tag. GitHub Actions builds and validates the image, assembles both packages, checks their SHA-256 digests, creates a draft GitHub Release, verifies all four assets, then publishes. +Use the version-independent [Development Guide](DEVELOPMENT.md) and, for v1.6.0, the [finalization checklist](RELEASE_V1.6.0_CHECKLIST.md). Approve and merge the release PR into `main`, check latest CI and isolated smoke results, obtain authorization for public distribution, then push the approved matching `v` tag. The hosted release workflow validates/tests/builds the image, assembles the complete archive and external checksum, uploads both to a draft GitHub Release, verifies the assets, and publishes. -Tagging does not automatically deploy anything to an operational host. Until the release workflow succeeds, its online/offline assets do not exist for customers. Do not build source on customer systems. +Tagging does not deploy to an operational host. Customer packages do not exist until the tagged release workflow succeeds. ## Developer-only source build diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..05fba17 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,170 @@ +# Container Alert — Development and Release Guide + +**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../DEPLOYMENT.md). + +## Table of contents + +1. [What we publish](#1-what-we-publish) +2. [Make and test a change](#2-make-and-test-a-change) +3. [Prepare the next version](#3-prepare-the-next-version) +4. [Review, merge and tag](#4-review-merge-and-tag) +5. [Verify the published package](#5-verify-the-published-package) +6. [Customer delivery and rollback](#6-customer-delivery-and-rollback) +7. [Safety and best practices](#7-safety-and-best-practices) + +## 1. What we publish + +The source repository contains the Python application, tests, installer and release automation. **Customer hosts get none of the application Python source.** A tagged release builds one versioned image and creates two public release assets: + +| Asset | Contents | +|---|---| +| `container-alert-v.tar.gz` | Installer, VERSION, Compose, templates, operator docs, prebuilt `watchdog.tar` and the image's internal `SHA256SUMS` | +| `SHA256SUMS` | SHA-256 checksum of the complete release archive | + +All versions use the **same workflow, package format and installer**. A release such as `1.6.1` uses `VERSION=1.6.1`, Git tag `v1.6.1`, image `watchdog:1.6.1` and package `container-alert-v1.6.1.tar.gz`. These are examples, not claims that a version has been published. + +Changes in Git do **not** upgrade running customer containers. Publication and installation are separate actions. + +## 2. Make and test a change + +**Step 1 — Create a development branch in a clean checkout.** + +```bash +git switch main +git pull --ff-only origin main +git switch -c fix/describe-change +``` + +Use `feature/...`, `fix/...` or `docs/...`. If there is an active stabilization branch for an upcoming release, coordinate whether your PR should target that release branch instead of main. Multiple worktrees are supported. + +**Step 2 — Change code, tests and documentation.** Typical files include `watchdog.py`, `lifecycle.py`, `ha_monitor.py`, `install.sh`, `scripts/package_release.sh`, and `tests/`. Don't reuse a previously published Docker image version for changed Python code. + +**Step 3 — Run non-destructive tests on a development workstation.** + +```bash +bash -n install.sh uninstall.sh scripts/package_release.sh +python3 -m unittest discover -s tests -q +git diff --check +``` + +The Python test environment needs the repo's test dependencies (including Docker SDK, Requests, PyYAML). Installer/package tests use fake Docker. **Never** run real failure-injection tests or `tests/release-smoke.sh` on operational/shared Docker hosts. + +**Step 4 — Commit, push, request review.** + +```bash +git add +git commit -m "fix: describe the change" +git push -u origin fix/describe-change +gh pr create --base main --head fix/describe-change --fill +``` + +Substitute your branch/files. PRs should cover behavior, tests, customer impact and safety. Review results: + +```bash +gh pr view -R Radware/CyberController-Container-Alert +gh pr checks -R Radware/CyberController-Container-Alert +``` + +Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. Code review and approval are still required when checks pass. + +## 3. Prepare the next version + +Several changes may ship together. When ready to publish, choose a **new** version (e.g. `1.6.1` for a compatible fix; `1.7.0` for a compatible feature). + +```bash +NEXT_VER=1.6.1 # set the desired NEW version here each time +git switch main +git pull --ff-only origin main +git switch -c "release/v$NEXT_VER" +printf '%s\n' "$NEXT_VER" > VERSION +``` + +Update the following to match the chosen version: + +1. `VERSION` — set the new number (the command above does this). +2. `README.md` — add the matching version-history row; CI checks this. +3. `docs/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. +4. Deployment and examples if operator procedures/settings changed; keep private lab/customer identifiers and secrets out. +5. Tests and hosted smoke approval for the release PR. + +```bash +git add VERSION README.md "docs/RELEASE_NOTES_V$NEXT_VER.md" +git commit -m "release: prepare v$NEXT_VER" +git push -u origin "release/v$NEXT_VER" +gh pr create --base main --head "release/v$NEXT_VER" --title "Release v$NEXT_VER" --fill +``` + +Change only `NEXT_VER` at the start of the release commands and use the same shell session, or set it again later. Write actual release notes before staging; the quoted filename in `git add` must exist. Stage all other modified files explicitly. If the release branch already exists, update it and its PR instead of creating a second one. + +## 4. Review, merge and tag + +Before release approval: verify independent review, complete notes, current PR CI and isolated smoke, rollback guidance, image/package contents, and **Radware approval to publish assets publicly** (this GitHub repository is public). + +Merge the approved release PR using GitHub's normal authorized process. Then, on a **clean checkout**: + +```bash +git switch main +git pull --ff-only origin main +git status --short # must be empty +cat VERSION # record approved version +git log -1 --oneline # record main's release commit +``` + +Confirm the tag does not already exist; the next command should print **nothing**: + +```bash +VER="$(cat VERSION)" +git ls-remote --tags origin "refs/tags/v$VER" +``` + +**Only after explicit approval**, tag the exact approved main commit: + +```bash +git tag -a "v$VER" -m "Container Alert v$VER" +git push origin "v$VER" +``` + +The tag triggers [Publish Container Alert release](../.github/workflows/release-package.yml). Its GitHub-hosted runner checks version/ancestry, runs tests, builds the prebuilt image, creates the package and checksum, uploads both assets as a **draft release**, checks the assets, then publishes. PR merges do not automatically create customer packages. + +## 5. Verify the published package + +Confirm the tagged workflow passed and exactly two assets exist: + +```bash +gh run list -R Radware/CyberController-Container-Alert --workflow release-package.yml --limit 5 +gh release view "v$VER" -R Radware/CyberController-Container-Alert +``` + +Download and inspect **without touching Docker**: + +```bash +mkdir -p "/tmp/container-alert-check-$VER" +cd "/tmp/container-alert-check-$VER" +gh release download "v$VER" -R Radware/CyberController-Container-Alert +sha256sum --check SHA256SUMS +tar -tzf "container-alert-v$VER.tar.gz" | grep -E 'VERSION|install.sh|watchdog.tar|SHA256SUMS' +mkdir -p extracted +tar -xzf "container-alert-v$VER.tar.gz" -C extracted +(cd extracted/CyberController-Container-Alert && sha256sum --check SHA256SUMS) +cat extracted/CyberController-Container-Alert/VERSION +``` + +Record the Git tag/commit, CI/smoke URLs, release link, package digest and final image digest. The tagged build may not be byte-identical to the PR build while image bases/dependencies can float; pinning/signing is a separate supply-chain improvement. + +If the workflow fails or the release is incomplete, do not distribute it. Inspect the failed workflow/draft and follow the approved correction process; **never rewrite an existing published tag**. + +## 6. Customer delivery and rollback + +Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or approved internal transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. + +YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../DEPLOYMENT.md#54-rollback). + +## 7. Safety and best practices + +- Never run destructive Docker tests on customer, production or shared operational hosts. Use mock Docker unit tests locally and isolated GitHub-hosted Docker acceptance. +- Do not stop or alter CyberController application containers during installation or tests. +- The SHA-256 manifest verifies contents, not the publisher's identity. Only distribute approved release packages. +- Never publish personal lab hostnames, IP addresses, passwords or customer-specific data. +- Require code review and explicit public-distribution approval before tagging. Image builds use prebuilt, versioned artifacts rather than compiling on customer systems. + +References: [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), [Managing Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [About releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). diff --git a/docs/DEVELOPMENT_AND_RELEASE.md b/docs/DEVELOPMENT_AND_RELEASE.md deleted file mode 100644 index 0a16e5f..0000000 --- a/docs/DEVELOPMENT_AND_RELEASE.md +++ /dev/null @@ -1,199 +0,0 @@ -# Container Alert — Development and Release Guide - -**For repository maintainers.** This is the repeatable development → testing → approval → GitHub Release process, including the exact finalization checklist for **v1.6.0**. Customer installation instructions belong in [Deployment](../DEPLOYMENT.md). - -## Table of contents - -1. [How source, images and packages relate](#1-how-source-images-and-packages-relate) -2. [Normal development cycle](#2-normal-development-cycle) -3. [Preparing the next numbered release](#3-preparing-the-next-numbered-release) -4. [v1.6.0 finalization checklist](#4-v160-finalization-checklist) -5. [Publishing and verifying v1.6.0](#5-publishing-and-verifying-v160) -6. [After release and rollback](#6-after-release-and-rollback) -7. [Release safety and best practices](#7-release-safety-and-best-practices) - -## 1. How source, images and packages relate - -| Item | v1.6.0 example | Purpose | -|---|---|---| -| Development/stabilization branch | `release/v1.6.0` | Code and documents under review | -| Source version file | `VERSION` = `1.6.0` | Single source of truth for the image and asset filenames | -| Final Git tag | `v1.6.0` | Immutable reference to the approved `main` commit | -| Docker image | `watchdog:1.6.0` | Prebuilt application and Python dependencies; customers do not build this | -| Online package | `container-alert-v1.6.0-online.tar.gz` | Installer, Compose, templates and docs; no code or image | -| Offline package | `container-alert-v1.6.0-offline.tar.gz` | Identical installer plus the saved Docker image and checksum | -| Release integrity manifest | `SHA256SUMS` | SHA-256 for `watchdog.tar` and both release packages | - -**Source code changes do not upgrade customers.** Customer upgrades occur only when a new numbered, tested image is published and the installer deploys that image. - -The hosted tag workflow reads `VERSION`, builds and tests the image, packages both installers, generates checksums, creates a **draft** GitHub Release, verifies all four assets exist, then publishes it. The published assets are: - -```text -watchdog.tar -SHA256SUMS -container-alert-v1.6.0-online.tar.gz -container-alert-v1.6.0-offline.tar.gz -``` - -The published release also uses the version-specific release notes when `docs/RELEASE_NOTES_V1.6.0.md` exists. No changes to `install.sh` or download URLs are required for v1.6.1 and later. - -## 2. Normal development cycle - -**For a new fix or feature after v1.6.0 is released:** - -1. Start with the latest `main`, and create a short-lived feature branch: - - ```bash - git switch main - git pull --ff-only origin main - git switch -c fix/describe-change - ``` - -2. Change Python code, unit tests, configuration examples or documentation. Do not reuse an existing release tag or overwrite a versioned image with different code. - -3. Validate with **non-destructive** checks only: - - ```bash - bash -n install.sh uninstall.sh scripts/package_release.sh - python3 -m unittest discover -s tests -q - ``` - - Run Python tests in a development environment with the required packages installed. Unit tests and fake-Docker installer/package tests must not use an operational Docker daemon. Do not run `tests/release-smoke.sh` directly on an appliance or shared host. - -4. Push and open a PR targeting `main`; obtain code review: - - ```bash - git push -u origin fix/describe-change - gh pr create --base main --head fix/describe-change --fill - ``` - -5. Check normal GitHub CI. For major changes or the final release candidate, require the **[Release smoke (isolated)](../.github/workflows/release-smoke.yml)** hosted workflow on the release PR. It is the only supported location for destructive Docker acceptance tests. - -**While v1.6.0 is being stabilized:** feature/fix PRs may target `release/v1.6.0`; the release branch itself is reviewed through [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) targeting `main`. Don't merge unreviewed work directly into a published release. - -## 3. Preparing the next numbered release - -After v1.6.0 is published, the next compatible maintenance release would typically be **v1.6.1**. A feature release might instead be v1.7.0; breaking changes require special compatibility review. - -1. Choose the version and update `VERSION` (e.g. `1.6.1`) in the release candidate. -2. Add the matching version entry in the README version history (required by CI). -3. Create/update `docs/RELEASE_NOTES_V1.6.1.md` describing all changes since the previous published release, customer-visible behavior, tests and limitations. -4. Use a release PR to `main` if batching fixes. Validate the actual final PR head in CI and hosted isolated smoke. -5. Merge only with approval. Then create the matching tag from `main`. **Do not** tag a release branch or publish from an unmerged PR. -6. The **same** `scripts/package_release.sh`, GitHub Actions workflow and installer produce the correct versioned assets automatically. - -**Development iterations are not releases:** changes can be committed and tested many times without publishing an image for each commit. Release numbering happens when preparing an approved delivery. - -## 4. v1.6.0 finalization checklist - -**Status when this guide was prepared (October 16, 2026):** `main` is still v1.5.4, `release/v1.6.0` contains `VERSION=1.6.0`, and PR #9 is **open/unmerged**. The GitHub Release for v1.6.0 has not been published by this work. - -Do **not** push the tag until every applicable item below is satisfied: - -- [ ] PR #9 receives final engineering review and approval, including lifecycle, alerting, HA, security and release packaging. -- [ ] Normal CI and [isolated hosted Docker smoke](RELEASE_SMOKE_TESTING.md) both pass **for the current PR head SHA**. Rerun if release-impacting code changes. -- [ ] The release notes and customer alert text describe all changes since v1.5.4; there are no private lab hostnames/addresses or unapproved secrets in release files. -- [ ] Release approval explicitly permits publication of **Docker image and packages to the public GitHub repository**. A published release can be downloaded by anyone. -- [ ] Deployment and rollback instructions are reviewed; HA failover and customer notification destinations receive any required authorized, separate acceptance. A passing hosted smoke does not prove live customer SMTP/Slack/SNMP delivery. -- [ ] The `main` branch contains the release workflow and the full approved PR content. Tagging before merge will not publish this release as intended. - -Useful inspection commands (read-only): - -```bash -gh pr view 9 -R Radware/CyberController-Container-Alert -gh pr checks 9 -R Radware/CyberController-Container-Alert -gh run list -R Radware/CyberController-Container-Alert \ - --workflow release-smoke.yml --limit 5 -``` - -If checks fail or public distribution is not authorized, **do not merge/tag/publish**. Fix the PR and rerun checks. - -## 5. Publishing and verifying v1.6.0 - -The commands below are **release-maintainer actions**. Run them only **after** approval. They are documented here; they are **not** executed as part of this documentation update. - -### Step A — Merge and verify main - -Merge [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) using the approved GitHub review process. Then update a clean repository checkout: - -```bash -git fetch origin main --tags -git switch main -git pull --ff-only origin main -cat VERSION # must be 1.6.0 -git status --short # must be empty -git log -1 --oneline # record final release commit SHA -git ls-remote --tags origin refs/tags/v1.6.0 -``` - -The final command must return **no existing v1.6.0 tag**. If it exists, investigate; do not move or overwrite release tags. - -### Step B — Create the release tag - -From the clean, approved `main` commit: - -```bash -test "$(cat VERSION)" = "1.6.0" || exit 1 -git tag -a v1.6.0 -m "Container Alert v1.6.0" -git push origin v1.6.0 -``` - -This tag push triggers [Publish Container Alert release](../.github/workflows/release-package.yml) on a GitHub-hosted runner. The workflow verifies that the tag matches `VERSION` and points into `main` history, runs unit tests, builds the version-labelled Docker image, checks imports, creates both release packages and checksum manifest, and publishes **after** checking all assets. - -### Step C — Check the GitHub workflow and release - -```bash -gh run list -R Radware/CyberController-Container-Alert \ - --workflow release-package.yml --limit 5 -gh release view v1.6.0 -R Radware/CyberController-Container-Alert -``` - -Check that the workflow completed successfully and that the release contains **all four assets**, the intended release notes, and no customer-specific secrets. - -Download and verify the release files **without installing or touching Docker**: - -```bash -mkdir -p /tmp/container-alert-v1.6.0-verification -gh release download v1.6.0 -R Radware/CyberController-Container-Alert \ - --dir /tmp/container-alert-v1.6.0-verification -cd /tmp/container-alert-v1.6.0-verification -sha256sum --check SHA256SUMS -tar -tzf container-alert-v1.6.0-online.tar.gz | head -tar -tzf container-alert-v1.6.0-offline.tar.gz | grep watchdog.tar -``` - -Verify the online archive contains no Python application source and no `watchdog.tar`; the offline archive must contain the same installer plus `watchdog.tar` and its internal checksum. Confirm both carry `VERSION=1.6.0`. - -**If publication fails:** investigate the GitHub Actions logs and any draft release. Do not tell customers the release is ready. Do not force-move an existing published tag; follow the organization's correction/versioning process. - -### Step D — Announce and distribute - -- **Online operators** download `container-alert-v1.6.0-online.tar.gz` plus `SHA256SUMS` from the published release page. They run the contained `bash install.sh` wizard and answer **Yes** to download the image. -- **Offline operators** receive `container-alert-v1.6.0-offline.tar.gz` plus `SHA256SUMS` through an approved transfer method. They run the **same wizard** and answer **No**. -- Keep the previous approved image available on target systems for rollback. Do not stop or alter CyberController application containers during installation. - -See [Deployment](../DEPLOYMENT.md#2-installation) for the complete customer procedures. - -## 6. After release and rollback - -An online/offline customer upgrade obtains the **new release package** and runs its installer in the existing installation directory; the package intentionally omits active `.env`, `watchdog-config.yaml` and the persistent `watchdog/` directory so customer settings and incident history are retained. The installer recreates only the watchdog container. - -For YAML-only monitoring changes, restart the watchdog. For `.env` or Compose changes, recreate just the watchdog. For Python changes, build and publish **a different numbered image**. A source-code pull or restart does not update Python inside the running image. - -If rollback is authorized, select the last approved `WATCHDOG_VERSION` and its corresponding stored image/configuration. See [Deployment — Rollback](../DEPLOYMENT.md#54-rollback). Never rewrite an old published tag to mean new code. - -## 7. Release safety and best practices - -This project uses a standard GitHub Flow / tagged-release approach: review changes by PR, test them in CI, merge approved code to `main`, assign a semantic version/tag, and distribute **prebuilt artifacts** rather than compiling on customer systems. - -Essential rules: - -- **Approval before publication:** this repository is public. Confirm Radware authorizes distributing the code, Docker image, release notes and package contents. -- **Traceability:** align `VERSION`, the Git tag, the Docker image label/tag, both installer package names, CI results and published checksums. Do not reuse image tags. -- **Safety:** all Docker failure injection must stay inside [isolated hosted CI](RELEASE_SMOKE_TESTING.md), never an operational or shared Docker daemon. Test installers/packages with fake commands and tiny dummy image files. -- **Integrity:** SHA-256 manifests check whether package bytes match the release assets; they are not a replacement for trusted distribution, source review, or stronger artifact signatures. -- **Production restraint:** no source builds, no automatic updates, no service-wide Docker shutdowns, and no customer-directed changes to CyberController Compose/resource policy. - -For mature supply-chain requirements, Radware may later add protected `v*` tags, immutable GitHub Releases, pinned image dependencies and signed provenance. The release image is built anew from the approved tag, so it is **not guaranteed byte-for-byte identical** to the image tested in the PR while its base/dependency versions can change. Record its final digest and obtain the appropriate release sign-off. Those harder supply-chain enhancements are separate from this deliberately small v1.6.0 delivery flow. - -Official references: [Managing GitHub Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), and [immutable releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/RELEASE_NOTES_V1.6.0.md index 91bf2a9..2413d57 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/RELEASE_NOTES_V1.6.0.md @@ -14,8 +14,8 @@ This document describes version **1.6.0** and its changes since v1.5.4. Building | Release documentation | Clarify merged vs unreleased features and explicit image version/tag process | [PR #8](https://github.com/Radware/CyberController-Container-Alert/pull/8) | | Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | | Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Guided release installation | Versioned online/offline release packages share one installer wizard. Online downloads the verified release image, offline includes it; existing settings are preserved and only the watchdog is recreated. No customer-side code builds or Git checkout. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Operator documentation and configuration clarity | Restore independent Online/Offline walkthroughs, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP technical details in advanced reference; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Guided release installation | One complete versioned package includes the installer and prebuilt image; operators verify/extract and run one wizard regardless of network availability. Existing settings are preserved and only the watchdog is recreated. No customer-side source build, Git checkout or image download. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Operator documentation and configuration clarity | Restore one self-contained installation/upgrade path, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP technical details in advanced reference; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | @@ -48,9 +48,9 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C ## Image distribution and deployment -The approved v1.6.0 GitHub Release is prepared by pushing tag `v1.6.0` **only after PR #9 is approved and merged into `main`**. The hosted release workflow builds a version-labelled image and publishes four assets: `watchdog.tar`, `SHA256SUMS`, `container-alert-v1.6.0-online.tar.gz`, and `container-alert-v1.6.0-offline.tar.gz`. The online package contains only installer/Compose/templates/docs and downloads the image during installation. The offline package contains the same files and the prebuilt image. +The approved v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is approved and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. -Neither package contains application Python source, and neither installer builds code on production systems. The release workflow prepares a draft and publishes only after verifying that all four assets are uploaded. Release packaging is **not** triggered merely by opening or merging a PR. See [Deployment](../DEPLOYMENT.md) and [Development and Release Cycle](DEVELOPMENT_AND_RELEASE.md). +The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](DEVELOPMENT.md) and [v1.6.0 finalization checklist](RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](../DEPLOYMENT.md). ## Exclusions, limitations and release gating diff --git a/docs/RELEASE_V1.6.0_CHECKLIST.md b/docs/RELEASE_V1.6.0_CHECKLIST.md new file mode 100644 index 0000000..f344339 --- /dev/null +++ b/docs/RELEASE_V1.6.0_CHECKLIST.md @@ -0,0 +1,64 @@ +# Container Alert v1.6.0 — Final Release Checklist + +**This checklist applies only to v1.6.0.** For every subsequent release, use the permanent [Development and Release Guide](DEVELOPMENT.md) and create a corresponding version-specific approval checklist only if needed. + +**Status on October 16, 2026:** `main` still contains v1.5.4; `release/v1.6.0` contains `VERSION=1.6.0` and is reviewed through [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9), which is open/unmerged. The v1.6.0 GitHub Release has **not** been published by this work. + +## Approval and verification — before merge + +- [ ] Final engineering review of PR #9: alert/incident lifecycle, notifications, HA behavior, security and image packaging. +- [ ] Normal GitHub CI **and** [Release smoke (isolated)](RELEASE_SMOKE_TESTING.md) both pass against the **latest PR head SHA**. The smoke is a disposable GitHub-hosted test; never run it against an operational Docker host. +- [ ] Document all changes since v1.5.4 in [release notes](RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. +- [ ] Confirm exact supported customer message text, severity mappings, and downstream Syslog/SNMP compatibility. +- [ ] Confirm there are no private lab hostnames, IP addresses, personal information or credentials in the **published installer archive**, documentation and release notes. +- [ ] Approve distributing the Docker image and complete release package through this **public** GitHub repository. +- [ ] Confirm upgrade/rollback procedure and identify any additional authorized HA/customer-delivery verification needed. CI success alone does not establish real customer SMTP/Slack/SNMP delivery. + +## Finalize and publish + +After **all** approvals above: + +1. Merge PR #9 to `main` through the reviewed GitHub process. +2. In a clean checkout, run: + + ```bash + git switch main + git pull --ff-only origin main + cat VERSION # expect 1.6.0 + git status --short # expect empty + git ls-remote --tags origin refs/tags/v1.6.0 + ``` + + The final command must show **no existing tag**. If it exists, investigate; don't overwrite tags. + +3. With approval to publish, tag the approved `main` commit: + + ```bash + test "$(cat VERSION)" = "1.6.0" || exit 1 + git tag -a v1.6.0 -m "Container Alert v1.6.0" + git push origin v1.6.0 + ``` + +4. Confirm the tagged [release workflow](../.github/workflows/release-package.yml) passed: + + ```bash + gh run list -R Radware/CyberController-Container-Alert --workflow release-package.yml --limit 5 + gh release view v1.6.0 -R Radware/CyberController-Container-Alert + ``` + +5. Verify the **two published assets**, `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`, without installing or starting containers: + + ```bash + mkdir -p /tmp/container-alert-v1.6.0-verify + cd /tmp/container-alert-v1.6.0-verify + gh release download v1.6.0 -R Radware/CyberController-Container-Alert + sha256sum --check SHA256SUMS + mkdir -p unpacked + tar -xzf container-alert-v1.6.0.tar.gz -C unpacked + (cd unpacked/CyberController-Container-Alert && sha256sum --check SHA256SUMS) + cat unpacked/CyberController-Container-Alert/VERSION + ``` + +6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../DEPLOYMENT.md). + +**Stop if a release gate fails.** Don't distribute a draft, incomplete or unapproved release. Do not run installation, Docker failure injection or HA failover tests on production systems as part of this checklist. diff --git a/install.sh b/install.sh index 052e7d3..c1f984e 100755 --- a/install.sh +++ b/install.sh @@ -335,26 +335,24 @@ verify_release_image() { fi } -# Verify archive checksum when supplied (mandatory for online downloads). +# Verify the image archive checksum shipped in every release package. load_release_archive() { local archive="$1" sums="$2" expected actual loaded - if [ -n "$sums" ]; then - if [ ! -r "$sums" ] || ! command -v sha256sum >/dev/null 2>&1; then - print_error "Cannot verify the release archive checksum" - return 1 - fi - expected=$(awk '$2 == "watchdog.tar" || $2 == "*watchdog.tar" {print $1}' "$sums" | tail -n1) - if ! [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]]; then - print_error "SHA256SUMS has no valid digest for watchdog.tar" - return 1 - fi - actual=$(sha256sum "$archive" | awk '{print $1}') || return 1 - if [ "$actual" != "$expected" ]; then - print_error "Archive checksum mismatch; image not loaded" - return 1 - fi - print_success "Archive checksum verified" + if [ ! -r "$archive" ] || [ ! -r "$sums" ] || ! command -v sha256sum >/dev/null 2>&1; then + print_error "Release image or checksum is missing; image not loaded" + return 1 + fi + expected=$(awk '$2 == "watchdog.tar" || $2 == "*watchdog.tar" {print $1}' "$sums" | tail -n1) + if ! [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]]; then + print_error "SHA256SUMS has no valid digest for watchdog.tar" + return 1 fi + actual=$(sha256sum "$archive" | awk '{print $1}') || return 1 + if [ "$actual" != "$expected" ]; then + print_error "Archive checksum mismatch; image not loaded" + return 1 + fi + print_success "Archive checksum verified" if ! loaded=$(docker load -i "$archive" 2>&1); then print_error "Docker could not load the image" printf '%s\n' "$loaded" @@ -371,69 +369,17 @@ load_release_archive() { print_success "Loaded and verified $IMAGE_NAME" } -download_release_image() { - local tempdir base - if ! command -v curl >/dev/null 2>&1; then - print_warning "curl is not installed" - return 1 - fi - tempdir=$(mktemp -d) || return 1 - base="https://github.com/Radware/CyberController-Container-Alert/releases/download/v$VERSION" - print_info "Downloading official release image $IMAGE_NAME" - if ! curl --fail --location --silent --show-error --retry 2 --connect-timeout 15 \ - --output "$tempdir/SHA256SUMS" "$base/SHA256SUMS" || \ - ! curl --fail --location --silent --show-error --retry 2 --connect-timeout 15 \ - --output "$tempdir/watchdog.tar" "$base/watchdog.tar"; then - rm -rf "$tempdir" - print_warning "Release download failed" - return 1 - fi - if load_release_archive "$tempdir/watchdog.tar" "$tempdir/SHA256SUMS"; then - rm -rf "$tempdir" - return 0 - fi - rm -rf "$tempdir" - return 1 -} - -use_local_image() { - if docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then - if verify_release_image; then - if [ "$(_prompt_yn "Use existing local $IMAGE_NAME?" yes)" = true ]; then - print_success "Using local verified image $IMAGE_NAME" - return 0 - fi - else - print_warning "Local image is not a verified release build" - fi - fi - if [ -f "$IMAGE_ARCHIVE" ]; then - local sums="" - [ ! -f "$INSTALL_DIR/SHA256SUMS" ] || sums="$INSTALL_DIR/SHA256SUMS" - print_info "Loading supplied watchdog.tar" - load_release_archive "$IMAGE_ARCHIVE" "$sums" && return 0 - return 1 - fi - print_error "No matching local $IMAGE_NAME or watchdog.tar available" - echo " Obtain the official v$VERSION offline package from Radware." - echo " Copy watchdog.tar to $INSTALL_DIR and rerun bash install.sh." - echo " No running containers were changed." - return 1 -} - +# One release bundle contains its own approved, version-labelled Docker image. +# A missing or invalid archive aborts before changing application configuration. load_docker_image() { - print_section "Select Docker Image" - if [ "$(_prompt_yn "Can this host download an image from GitHub Releases?" yes)" = true ]; then - if download_release_image; then - return 0 - fi - print_warning "Image download unavailable; source builds are not used in production" - if [ "$(_prompt_yn "Try a local image or archive instead?" yes)" != true ]; then - print_error "Installation cancelled; running containers unchanged" - return 1 - fi + print_section "Load Release Image" + if [ ! -f "$IMAGE_ARCHIVE" ] || [ ! -f "$INSTALL_DIR/SHA256SUMS" ]; then + print_error "Incomplete release package: watchdog.tar and SHA256SUMS are required" + echo " Download and extract the complete container-alert-v$VERSION.tar.gz package." + echo " No running containers have been changed." + return 1 fi - use_local_image + load_release_archive "$IMAGE_ARCHIVE" "$INSTALL_DIR/SHA256SUMS" } ################################################################################ diff --git a/scripts/package_release.sh b/scripts/package_release.sh index a28edb2..494fb22 100644 --- a/scripts/package_release.sh +++ b/scripts/package_release.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -# Assemble tested release installation bundles without running Docker. -# Usage: bash scripts/package_release.sh +# Build one customer package from an already-built release image. +# No Docker commands; safe to test with a disposable fake archive. +# Usage: bash scripts/package_release.sh set -euo pipefail if [ "$#" -ne 2 ]; then - echo "Usage: $0 " >&2 + echo "Usage: $0 " >&2 exit 2 fi @@ -16,38 +17,35 @@ if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then fi image="$(realpath "$1")" -test -f "$image" && test -s "$image" || { echo "Image archive missing or empty" >&2; exit 1; } +if [ ! -s "$image" ]; then + echo "Prebuilt image archive is missing or empty" >&2 + exit 1 +fi + mkdir -p "$2" out="$(cd "$2" && pwd)" -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT -dir="$tmp/CyberController-Container-Alert" +dir="$work/CyberController-Container-Alert" mkdir -p "$dir/docs" -for file in install.sh uninstall.sh docker-compose.yaml VERSION \ - .env.example watchdog-config.yaml.example README.md DEPLOYMENT.md; do - cp "$repo/$file" "$dir/$file" +for f in install.sh uninstall.sh docker-compose.yaml VERSION \ + .env.example watchdog-config.yaml.example README.md DEPLOYMENT.md; do + cp "$repo/$f" "$dir/$f" done -cp "$repo"/docs/*.md "$dir/docs/" # text-only references; never Python source - -online="container-alert-v$version-online.tar.gz" -offline="container-alert-v$version-offline.tar.gz" +cp "$repo"/docs/*.md "$dir/docs/" -# Both packages have exactly the same installer and configuration templates. -tar -C "$tmp" -czf "$out/$online" CyberController-Container-Alert +cp "$image" "$dir/watchdog.tar" +(cd "$dir" && sha256sum watchdog.tar > SHA256SUMS) -# The offline package adds the prebuilt image and its own checksum manifest. -cp "$image" "$out/watchdog.tar" -cp "$out/watchdog.tar" "$dir/watchdog.tar" -(cd "$out" && sha256sum watchdog.tar) > "$dir/SHA256SUMS" -tar -C "$tmp" -czf "$out/$offline" CyberController-Container-Alert +package="container-alert-v$version.tar.gz" +tar -C "$work" -czf "$out/$package" CyberController-Container-Alert +(cd "$out" && sha256sum "$package" > SHA256SUMS) -# Public release checksum file covers every downloadable binary asset. -(cd "$out" && sha256sum watchdog.tar "$online" "$offline") > "$out/SHA256SUMS" -for asset in watchdog.tar "$online" "$offline" SHA256SUMS; do +for asset in "$package" SHA256SUMS; do if [ "$(stat -c '%s' "$out/$asset")" -ge 2147483648 ]; then - echo "GitHub release asset exceeds 2 GiB: $asset" >&2 + echo "Release asset exceeds the 2 GiB GitHub limit: $asset" >&2 exit 1 fi done -echo "Built release bundles for v$version in $out" +echo "Packaged Container Alert v$version: $out/$package" diff --git a/tests/test_installer_image.py b/tests/test_installer_image.py index 1c2f434..2ed2821 100644 --- a/tests/test_installer_image.py +++ b/tests/test_installer_image.py @@ -1,4 +1,4 @@ -"""Installer decisions use fake Docker and fake downloads; never a real daemon.""" +"""Validate the single bundled installer using fake Docker only.""" import hashlib import os @@ -9,28 +9,32 @@ import unittest -SOURCE = Path(__file__).resolve().parents[1] / "install.sh" -VERSION = "1.6.0" +ROOT = Path(__file__).resolve().parents[1] +VERSION = (ROOT / "VERSION").read_text().strip() class InstallerImageTests(unittest.TestCase): def setUp(self): - self.t = tempfile.TemporaryDirectory(prefix="watchdog-installer-fake-") - self.addCleanup(self.t.cleanup) - self.root = Path(self.t.name) + self.workspace = tempfile.TemporaryDirectory(prefix="watchdog-installer-fake-") + self.addCleanup(self.workspace.cleanup) + self.root = Path(self.workspace.name) self.bin = self.root / "fake-bin" self.bin.mkdir() - self.release = self.root / "fake-release" - self.release.mkdir() self.state = self.root / "fake-state" self.state.mkdir() - shutil.copy2(SOURCE, self.root / "install.sh") + shutil.copy2(ROOT / "install.sh", self.root / "install.sh") (self.root / "VERSION").write_text(VERSION + "\n") self.log = self.root / "fake-docker-calls.log" self.log.write_text("") - self.archive = b"small fake release archive, never a real Docker image" - self.write_release_archive(self.release) - + self.archive = b"fake image bytes, not a real Docker image" + self.env = os.environ.copy() + self.env.update( + PATH=str(self.bin) + os.pathsep + os.environ["PATH"], + FAKE_DOCKER_LOG=str(self.log), + FAKE_STATE_DIR=str(self.state), + FAKE_VERSION=VERSION, + FAKE_IMAGE_PRESENT="0", + ) docker = self.bin / "docker" docker.write_text("""#!/usr/bin/env bash printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" @@ -39,7 +43,7 @@ def setUp(self): if [ "$2" = inspect ]; then if [ "$FAKE_IMAGE_PRESENT" = 1 ] || [ -f "$FAKE_STATE_DIR/loaded" ]; then case "$*" in - *org.opencontainers.image.version*) printf '%s\\n' "$FAKE_VERSION" ;; + *org.opencontainers.image.version*) echo "$FAKE_VERSION" ;; *'{{.Id}}'*) echo 'sha256:fake-image-id' ;; esac exit 0 @@ -48,136 +52,105 @@ def setUp(self): fi ;; load) touch "$FAKE_STATE_DIR/loaded" - echo 'Loaded image: watchdog:1.6.0' - exit 0 ;; - config|up) + echo "Loaded image: watchdog:$FAKE_TAG_VERSION" exit 0 ;; + config|up) exit 0 ;; esac echo "Unexpected fake Docker arguments: $*" >&2 exit 97 """) docker.chmod(0o755) - curl = self.bin / "curl" - curl.write_text("""#!/usr/bin/env bash -[ "$FAKE_CURL_FAIL" != 1 ] || exit 22 -out="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - --output) out="$2"; shift 2 ;; - *) url="$1"; shift ;; - esac -done -case "$url" in - */watchdog.tar) cp "$FAKE_RELEASE_DIR/watchdog.tar" "$out" ;; - */SHA256SUMS) cp "$FAKE_RELEASE_DIR/SHA256SUMS" "$out" ;; - *) echo "Unexpected download URL" >&2; exit 22 ;; -esac -""") - curl.chmod(0o755) - - self.env = os.environ.copy() - self.env.update( - PATH=str(self.bin) + os.pathsep + os.environ["PATH"], - FAKE_DOCKER_LOG=str(self.log), - FAKE_STATE_DIR=str(self.state), - FAKE_RELEASE_DIR=str(self.release), - FAKE_VERSION=VERSION, - FAKE_IMAGE_PRESENT="0", - FAKE_CURL_FAIL="0", + self.env["FAKE_TAG_VERSION"] = VERSION + + def write_archive(self, with_checksum=True, valid_checksum=True): + (self.root / "watchdog.tar").write_bytes(self.archive) + if with_checksum: + digest = hashlib.sha256(self.archive).hexdigest() + if not valid_checksum: + digest = "0" * 64 + (self.root / "SHA256SUMS").write_text( + f"{digest} watchdog.tar\n" + ) + + def run_function(self, body="load_docker_image"): + result = subprocess.run( + ["bash", "-c", "source ./install.sh; " + body], + cwd=self.root, env=self.env, + text=True, capture_output=True, timeout=10, ) + result.output = result.stdout + result.stderr + return result - def write_release_archive(self, folder, valid=True): - (folder / "watchdog.tar").write_bytes(self.archive) - digest = hashlib.sha256(self.archive).hexdigest() - if not valid: - digest = "0" * 64 - (folder / "SHA256SUMS").write_text(f"{digest} watchdog.tar\n") - - def run_installer(self, task="load_docker_image", online=False): - # Source install.sh. Overriding prompts never invokes main or Docker. - prompt = ( - '_prompt_yn() { case "$1" in ' - '*"Can this host download"*) printf "' - + ("true" if online else "false") - + '";; *) printf "true";; esac; }; ' - ) - command = "source ./install.sh; " + prompt + task - return subprocess.run( - ["bash", "-c", command], - env=self.env, - cwd=self.root, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - timeout=10, - ) - - def docker_calls(self): + def calls(self): return self.log.read_text().splitlines() - def test_offline_uses_matching_local_image_without_loading(self): - self.env["FAKE_IMAGE_PRESENT"] = "1" - res = self.run_installer() - self.assertEqual(res.returncode, 0, res.stdout) - self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) - - def test_offline_loads_checksum_verified_archive(self): - shutil.copy2(self.release / "watchdog.tar", self.root / "watchdog.tar") - shutil.copy2(self.release / "SHA256SUMS", self.root / "SHA256SUMS") - res = self.run_installer() - self.assertEqual(res.returncode, 0, res.stdout) - self.assertTrue(any(s.startswith("load ") for s in self.docker_calls())) - - def test_bad_offline_checksum_is_rejected_before_docker_load(self): - self.write_release_archive(self.root, valid=False) - res = self.run_installer() - self.assertNotEqual(res.returncode, 0) - self.assertIn("checksum mismatch", res.stdout) - self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) - - def test_unverified_local_image_is_rejected(self): - self.env["FAKE_IMAGE_PRESENT"] = "1" - self.env["FAKE_VERSION"] = "old-version" - res = self.run_installer() - self.assertNotEqual(res.returncode, 0) - self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) - - def test_online_download_and_load_never_builds(self): - res = self.run_installer(online=True) - self.assertEqual(res.returncode, 0, res.stdout) - self.assertTrue(any(s.startswith("load ") for s in self.docker_calls())) - self.assertFalse(any("build" in s for s in self.docker_calls())) - - def test_online_failure_can_use_matching_local_image(self): - self.env["FAKE_CURL_FAIL"] = "1" + def test_valid_package_loads_image(self): + self.write_archive() + out = self.run_function() + self.assertEqual(out.returncode, 0, out.output) + self.assertTrue(any(c.startswith("load ") for c in self.calls())) + + def test_existing_local_image_is_not_silently_trusted(self): + self.write_archive() self.env["FAKE_IMAGE_PRESENT"] = "1" - res = self.run_installer(online=True) - self.assertEqual(res.returncode, 0, res.stdout) - self.assertFalse(any(s.startswith("load ") for s in self.docker_calls())) - - def test_missing_image_fails_without_docker_mutations(self): - res = self.run_installer() - self.assertNotEqual(res.returncode, 0) - self.assertIn("No running containers were changed", res.stdout) - self.assertFalse(any(s.startswith(("load ", "up ", "run ", "stop ", "build ")) for s in self.docker_calls())) - - def test_incomplete_existing_configuration_does_not_reset(self): - (self.root / ".env").write_text("SENSITIVE=keep\n") - res = self.run_installer(task="configure_all") - self.assertNotEqual(res.returncode, 0) - self.assertEqual((self.root / ".env").read_text(), "SENSITIVE=keep\n") + out = self.run_function() + self.assertEqual(out.returncode, 0, out.output) + self.assertTrue(any(c.startswith("load ") for c in self.calls())) + + def test_missing_package_fails_without_docker_mutations(self): + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("Incomplete release package", out.output) + self.assertEqual(self.calls(), []) + + def test_missing_checksum_rejected_before_loading(self): + self.write_archive(with_checksum=False) + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertEqual(self.calls(), []) + + def test_invalid_checksum_rejected_before_loading(self): + self.write_archive(valid_checksum=False) + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("checksum mismatch", out.output) + self.assertEqual(self.calls(), []) + + def test_wrong_image_version_label_rejected(self): + self.write_archive() + self.env["FAKE_VERSION"] = "unexpected-version" + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("does not match", out.output) + + def test_wrong_image_tag_rejected(self): + self.write_archive() + self.env["FAKE_TAG_VERSION"] = "old" + out = self.run_function() + self.assertNotEqual(out.returncode, 0) + self.assertIn("expected tag", out.output) + + def test_existing_configuration_is_not_regenerated(self): + (self.root / ".env").write_text("SECRET=keep\n") + out = self.run_function("configure_all") + self.assertNotEqual(out.returncode, 0) + self.assertEqual((self.root / ".env").read_text(), "SECRET=keep\n") self.assertFalse((self.root / "watchdog-config.yaml").exists()) - self.assertEqual(self.docker_calls(), []) - - def test_start_recreates_watchdog_only(self): - # The service call below is a fake Docker executable, not a host daemon. - res = self.run_installer(task='DC=docker; start_watchdog') - self.assertEqual(res.returncode, 0, res.stdout) - calls = self.docker_calls() - self.assertIn("config -q", calls) - self.assertIn("up -d --no-deps --force-recreate docker-container-watchdog", calls) - self.assertFalse(any("down" in s for s in calls)) + self.assertEqual(self.calls(), []) + + def test_recreate_targets_watchdog_only(self): + out = self.run_function("DC=docker; start_watchdog") + self.assertEqual(out.returncode, 0, out.output) + self.assertIn("config -q", self.calls()) + self.assertIn("up -d --no-deps --force-recreate docker-container-watchdog", + self.calls()) + self.assertFalse(any("down" in c for c in self.calls())) + + def test_no_network_image_download_path(self): + source = (ROOT / "install.sh").read_text() + self.assertNotIn("download_release_image()", source) + self.assertNotIn("curl --fail", source) + self.assertNotIn('docker build', source) if __name__ == "__main__": diff --git a/tests/test_release_package.py b/tests/test_release_package.py index 9821d38..838782a 100644 --- a/tests/test_release_package.py +++ b/tests/test_release_package.py @@ -1,4 +1,4 @@ -"""Test release package composition without Docker, image builds, or live hosts.""" +"""Validate single customer release package with a fake image; never use Docker.""" import hashlib from pathlib import Path @@ -16,66 +16,60 @@ class ReleasePackageTests(unittest.TestCase): def setUp(self): - self.temp = tempfile.TemporaryDirectory(prefix="container-alert-release-fake-") - self.addCleanup(self.temp.cleanup) - self.workspace = Path(self.temp.name) - self.fake_image = self.workspace / "test-watchdog.tar" - self.fake_image.write_bytes(b"FAKE ARCHIVE - NOT A DOCKER IMAGE\n") - self.output = self.workspace / "dist" + self.workspace = tempfile.TemporaryDirectory(prefix="container-alert-package-fake-") + self.addCleanup(self.workspace.cleanup) + self.path = Path(self.workspace.name) + self.fake_image = self.path / "fake-watchdog.tar" + self.fake_image.write_bytes(b"FAKE BINARY, NOT A DOCKER IMAGE") + self.dist = self.path / "dist" result = subprocess.run( - ["bash", str(SCRIPT), str(self.fake_image), str(self.output)], + ["bash", str(SCRIPT), str(self.fake_image), str(self.dist)], capture_output=True, text=True, timeout=15, ) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.online = self.output / f"container-alert-v{VERSION}-online.tar.gz" - self.offline = self.output / f"container-alert-v{VERSION}-offline.tar.gz" + self.archive = self.dist / f"container-alert-v{VERSION}.tar.gz" - @staticmethod - def members(path): - with tarfile.open(path, "r:gz") as archive: - return {member.name: archive.extractfile(member).read() - for member in archive if member.isfile()} + def package_files(self): + with tarfile.open(self.archive, "r:gz") as tf: + return { + item.name: tf.extractfile(item).read() + for item in tf if item.isfile() + } - def test_online_bundle_contains_only_installer_files(self): - contents = self.members(self.online) - for filename in ("install.sh", "uninstall.sh", "VERSION", - "docker-compose.yaml", "README.md", "DEPLOYMENT.md", - ".env.example", "watchdog-config.yaml.example", - "docs/DEPLOYMENT_ADVANCED.md", - "docs/DEVELOPMENT_AND_RELEASE.md", - "docs/RELEASE_NOTES_V1.6.0.md"): - self.assertIn(PREFIX + filename, contents) - self.assertNotIn(PREFIX + "watchdog.tar", contents) - self.assertNotIn(PREFIX + "watchdog.py", contents) - self.assertNotIn(PREFIX + "Dockerfile", contents) + def test_one_complete_release_package(self): + files = self.package_files() + required = ("install.sh", "uninstall.sh", "docker-compose.yaml", "VERSION", + "README.md", "DEPLOYMENT.md", ".env.example", + "watchdog-config.yaml.example", "watchdog.tar", "SHA256SUMS", + "docs/DEPLOYMENT_ADVANCED.md", "docs/DEVELOPMENT.md", + f"docs/RELEASE_NOTES_V{VERSION}.md") + for name in required: + self.assertIn(PREFIX + name, files) + self.assertEqual(files[PREFIX + "VERSION"].decode().strip(), VERSION) - def test_offline_bundle_adds_image_and_internal_checksum(self): - contents = self.members(self.offline) - self.assertEqual(contents[PREFIX + "watchdog.tar"], - self.fake_image.read_bytes()) - expected = hashlib.sha256(self.fake_image.read_bytes()).hexdigest() - self.assertIn(expected, contents[PREFIX + "SHA256SUMS"].decode()) - self.assertIn("watchdog.tar", contents[PREFIX + "SHA256SUMS"].decode()) - self.assertNotIn(PREFIX + "watchdog.py", contents) + def test_bundle_contains_image_not_python_source(self): + files = self.package_files() + self.assertEqual(files[PREFIX + "watchdog.tar"], self.fake_image.read_bytes()) + self.assertNotIn(PREFIX + "watchdog.py", files) + self.assertNotIn(PREFIX + "lifecycle.py", files) + self.assertNotIn(PREFIX + "Dockerfile", files) + self.assertNotIn(PREFIX + ".env", files) + self.assertNotIn(PREFIX + "watchdog-config.yaml", files) - def test_both_bundles_share_identical_installer_and_version(self): - online = self.members(self.online) - offline = self.members(self.offline) - for path, content in online.items(): - self.assertEqual(offline[path], content, path) - self.assertEqual(online[PREFIX + "VERSION"].decode().strip(), VERSION) + def test_image_checksum_inside_package(self): + files = self.package_files() + expected = hashlib.sha256(self.fake_image.read_bytes()).hexdigest() + self.assertEqual(files[PREFIX + "SHA256SUMS"].decode().strip(), + f"{expected} watchdog.tar") - def test_release_checksum_manifest_covers_every_asset(self): - manifest = (self.output / "SHA256SUMS").read_text().splitlines() - self.assertEqual(len(manifest), 3) - names = set() - for line in manifest: - digest, name = line.split(maxsplit=1) - names.add(name) - path = self.output / name - self.assertTrue(path.is_file(), name) - self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), digest) - self.assertEqual(names, {"watchdog.tar", self.online.name, self.offline.name}) + def test_release_checksum_covers_only_package(self): + self.assertTrue(self.archive.is_file()) + lines = (self.dist / "SHA256SUMS").read_text().splitlines() + self.assertEqual(lines, [ + f"{hashlib.sha256(self.archive.read_bytes()).hexdigest()} {self.archive.name}" + ]) + self.assertEqual(sorted(p.name for p in self.dist.iterdir()), + ["SHA256SUMS", self.archive.name]) if __name__ == "__main__": From 43b0d444298b8f06674e823fcab0108b3f8a8a3a Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 15:43:55 +0000 Subject: [PATCH 12/18] docs: organize customer and engineering guides and audit references --- .env.example | 2 +- .github/workflows/release-package.yml | 4 +- README.md | 171 +++--------------- docker-compose.yaml | 4 +- docs/DEPLOYMENT_ADVANCED.md | 94 ---------- .../customer/INSTALLATION_AND_OPERATIONS.md | 80 ++++++-- docs/{ => customer}/RELEASE_NOTES_V1.6.0.md | 6 +- .../DEVELOPMENT_AND_RELEASE.md} | 48 ++++- .../HA_MONITORING_DESIGN.md} | 11 +- .../HA_VALIDATION_2026-10-13.md} | 9 +- docs/engineering/PROBE_MAP_SNAPSHOT.md | 59 ++++++ .../RELEASE_ACCEPTANCE_V1.6.0.md} | 10 +- .../RELEASE_SMOKE_TESTING.md | 2 +- .../RELEASE_V1.6.0_CHECKLIST.md | 8 +- .../RELEASE_VALIDATION_V1.6.0.md | 12 +- .../RELEASE_SMOKE_V1.6.0_20261014.json | 0 ...ELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json | 0 ...ASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json | 0 ...E_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json | 0 scripts/package_release.sh | 6 +- tests/test_documentation.py | 110 +++++++++++ tests/test_release_package.py | 26 ++- watchdog-config.yaml.example | 8 +- 23 files changed, 378 insertions(+), 292 deletions(-) delete mode 100644 docs/DEPLOYMENT_ADVANCED.md rename DEPLOYMENT.md => docs/customer/INSTALLATION_AND_OPERATIONS.md (84%) rename docs/{ => customer}/RELEASE_NOTES_V1.6.0.md (85%) rename docs/{DEVELOPMENT.md => engineering/DEVELOPMENT_AND_RELEASE.md} (74%) rename docs/{HA_ACTIVE_NODE_ALERT_DESIGN.md => engineering/HA_MONITORING_DESIGN.md} (96%) rename docs/{HA_VALIDATION.md => engineering/HA_VALIDATION_2026-10-13.md} (89%) create mode 100644 docs/engineering/PROBE_MAP_SNAPSHOT.md rename docs/{RELEASE_ACCEPTANCE_V1.6.0_FINAL.md => engineering/RELEASE_ACCEPTANCE_V1.6.0.md} (96%) rename docs/{ => engineering}/RELEASE_SMOKE_TESTING.md (90%) rename docs/{ => engineering}/RELEASE_V1.6.0_CHECKLIST.md (89%) rename docs/{ => engineering}/RELEASE_VALIDATION_V1.6.0.md (93%) rename docs/{ => engineering/evidence}/RELEASE_SMOKE_V1.6.0_20261014.json (100%) rename docs/{ => engineering/evidence}/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json (100%) rename docs/{ => engineering/evidence}/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json (100%) rename docs/{ => engineering/evidence}/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json (100%) create mode 100644 tests/test_documentation.py diff --git a/.env.example b/.env.example index e08fb26..e209345 100644 --- a/.env.example +++ b/.env.example @@ -48,5 +48,5 @@ SMTP_PASSWORD=your-smtp-password/api key value # ── Image version ───────────────────────────────────────────────────────────── # Release tag the container runs (see the repository VERSION file). install.sh -# sets this to the verified release image version (downloaded or loaded locally). +# sets this to the verified release image version loaded from packaged watchdog.tar. # WATCHDOG_VERSION= diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index 8fe5a1c..390aae7 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -69,8 +69,8 @@ jobs: run: | set -euo pipefail notes=(--generate-notes) - if [ -f "docs/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then - notes=(--notes-file "docs/RELEASE_NOTES_V$WATCHDOG_VERSION.md") + if [ -f "docs/customer/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then + notes=(--notes-file "docs/customer/RELEASE_NOTES_V$WATCHDOG_VERSION.md") fi # Check both assets before making this public release visible. diff --git a/README.md b/README.md index d69735c..3facae1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitoring service that continuously tracks the health of all Docker containers on the host. It detects failures — including crashes, out-of-memory kills, prolonged unhealthy states, and restart loops — and dispatches real-time alerts through one or more configurable channels (**Slack**, **SMTP**, **SNMP Traps**, or **Syslog**). It also sends a one-time INFO **recovered** alert through the same channels once a previously-alarmed container returns to normal operation. +**Documentation:** [Customer installation and operations](docs/customer/INSTALLATION_AND_OPERATIONS.md) · [v1.6.0 release notes](docs/customer/RELEASE_NOTES_V1.6.0.md). + +**For maintainers (source repository):** [Development and release workflow](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md). Historical design, test reports, and CI procedures live under `docs/engineering/` and are **not included** in customer installation packages. + --- ## Table of Contents @@ -17,13 +21,7 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitori 6. [Failure Types & Severities](#6-failure-types--severities) 7. [Verify Watchdog Operation](#7-verify-watchdog-operation) 8. [Troubleshooting](#8-troubleshooting) - - [Service Fails to Start](#service-fails-to-start) - - [Alert Notifications Not Received](#alert-notifications-not-received) - - [Duplicate and Excessive Alert Notifications](#duplicate-and-excessive-alert-notifications) - - [Probe Type Not as Expected](#probe-type-not-as-expected) - - [Viewing Logs](#viewing-logs) -9. [Container Probe Map (This Deployment)](#9-container-probe-map-this-deployment) -10. [Version History](#10-version-history) +9. [Version History](#9-version-history) --- @@ -115,7 +113,7 @@ CyberController services have recovered. The exporter may use port **9664** even when the built-in CyberController Prometheus configuration references **9002**. This optional check reads the exporter directly and makes **no changes** to native Prometheus configuration. -See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). +See [customer HA configuration and expected behavior](docs/customer/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). --- @@ -123,11 +121,13 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ### Project Structure +The following is the **source repository** layout, not the customer archive. The published package includes the installer, runtime Compose file, version, templates, customer documentation, and prebuilt `watchdog.tar` — **not** Python source, Dockerfile, or engineering reports. + ``` . ├── watchdog.py # Main monitor script ├── lifecycle.py # Bounded Docker stop/die evidence correlation -├── watchdog-config.yaml # All configuration (no secrets) +├── watchdog-config.yaml.example # Configuration template (active file is generated) ├── docker-compose.yaml # Production / offline runtime (image only) ├── docker-compose.build.yaml # Developer-only source build ├── Dockerfile # Image build definition (python:3.11-slim) @@ -135,10 +135,10 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). ├── install.sh # Guided installation from complete release package ├── uninstall.sh # Stop + remove script ├── scripts/package_release.sh # CI release-package assembly (maintainers only) -├── docs/DEVELOPMENT.md # Version-independent source-to-release workflow -├── docs/RELEASE_V1.6.0_CHECKLIST.md # v1.6.0-specific release gates +├── docs/customer/ # Installation/operations and release notes +├── docs/engineering/ # Development, release, HA and test evidence ├── VERSION # Release version — drives the image tag and installer -├── watchdog.tar # Included in published release package; not tracked by Git +├── watchdog.tar # Prebuilt image only in published package └── .env # Secrets — NOT committed to git ``` @@ -150,13 +150,13 @@ See [design, test plan and release gates](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md). |------|------| | Docker image (`watchdog:`) | ~180 MB (python:3.11-slim base + dependencies) | | Running container (memory) | ~50–80 MB baseline; capped at `mem_limit: 256m` in `docker-compose.yaml` | -| `watchdog.tar` export | ~170 MB | +| `watchdog.tar` export | Approximate; inspect the actual published release | -> **Note:** Image size and memory usage vary with the installed release. If resource usage is abnormal, capture Docker health, logs, and usage observations for engineering review before making production resource-limit changes. +> **Note:** Size and memory figures above are prior estimates, not guaranteed release specifications. Actual image size varies by build. If resource usage is abnormal, capture Docker health, logs, and usage observations for engineering review before making production resource-limit changes. ### Container Hardening -By default the container runs as **root** so `docker compose up` works with zero configuration — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md#configuration-files)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. +When running Compose manually without installer-generated `.env`, the default container user is **root** — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/customer/INSTALLATION_AND_OPERATIONS.md#41-configuration-files-and-editing)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. ### Python Dependencies @@ -166,6 +166,7 @@ docker==7.1.0 requests==2.32.3 PyYAML==6.0.2 pysnmp>=6.2 +cryptography>=42.0 ``` @@ -179,7 +180,7 @@ pysnmp>=6.2 The guided installer creates `watchdog-config.yaml` (monitoring policy, probes, logging and notification endpoints) and `.env` (credentials and runtime identity). The configuration is preserved on an upgrade unless the operator explicitly chooses to reconfigure it. -**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](DEPLOYMENT.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. +**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](docs/customer/INSTALLATION_AND_OPERATIONS.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. A YAML-only change takes effect after restarting the watchdog. A changed `.env` value requires recreating **only** the watchdog container. Neither action rebuilds Python code into the image. Thresholds, exclusions, custom exec probes, and monitoring suppressions require authorized review; do not use them merely to silence alerts. @@ -194,7 +195,7 @@ A YAML-only change takes effect after restarting the watchdog. A changed `.env` | SNMP Trap | `snmp_trap` | NMS destination, SNMP version; SNMPv3 keys in `.env` | | Slack | `slack` | Webhook URL stored in `.env` | -At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](DEPLOYMENT.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/DEPLOYMENT_ADVANCED.md#snmp-trap-receiver-reference) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. +At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/customer/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/customer/INSTALLATION_AND_OPERATIONS.md#7-snmp-trap-integration) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. --- @@ -213,7 +214,7 @@ bash install.sh The wizard verifies and loads the included Docker image, asks for the selected notification channels, and deploys **only** the watchdog. **The v1.6.0 release archive becomes downloadable after the release is published.** -Read the full [Deployment Guide](DEPLOYMENT.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](docs/DEVELOPMENT.md) and the [v1.6.0 finalization checklist](docs/RELEASE_V1.6.0_CHECKLIST.md). +Read the full [Deployment Guide](docs/customer/INSTALLATION_AND_OPERATIONS.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md) and the [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md). --- @@ -260,143 +261,27 @@ docker logs --tail 50 docker-container-watchdog Expected: the watchdog is running/healthy and monitoring logs are current. Check the configured notification receiver for any existing alerts. A healthy system does not need to generate a new alert just for verification. -Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](docs/RELEASE_SMOKE_TESTING.md). +Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_SMOKE_TESTING.md). ## 8. Troubleshooting -### Service Fails to Start - -```bash -docker compose logs docker-container-watchdog -``` - -Common causes: -- **Missing `.env` file** — complete the initial installer wizard, or restore it from a previous-installation backup -- **Docker socket not accessible** — ensure `/var/run/docker.sock` exists and the container has read access -- **Release image/package unavailable** — obtain the complete approved versioned package, verify `SHA256SUMS`, extract it and run `bash install.sh` (see [Deployment Guide](DEPLOYMENT.md)) - -### Alert Notifications Not Received - -1. Check `alert_channels` in `watchdog-config.yaml` — ensure at least one channel is listed and configured -2. Check watchdog logs for errors: - ```bash - grep -E "ERROR|CRITICAL" watchdog/watchdog.log - ``` -3. For Slack: confirm `SLACK_WEBHOOK_URL` is present in `.env` and that the destination is reachable. Use an authorized test receiver if end-to-end validation is required; do not trigger application failures to test alert delivery. - -**Recovery alert (`recovered`) specifically not received:** -- Confirm `alert_on_recovery` is not set to `false` in the live `watchdog-config.yaml`. -- Confirm the container was actually alarmed first — `state.alerted_for` must be non-empty; a container that was never flagged won't generate a recovery alert. -- For containers with a Docker `HEALTHCHECK`, confirm Docker actually reports `healthy` — `docker inspect --format '{{json .State.Health}}'`. If it's stuck on `starting`, the watchdog is waiting on Docker, not the other way around. -- For containers on active probes (HTTP/TCP/`/proc`), confirm the poll loop reports `health=healthy` for the container in the log — if it stays `unhealthy`, review the actual endpoint and probe evidence with the responsible engineering team before changing monitoring. - -### Duplicate and Excessive Alert Notifications - -Review the affected container's alert history and probe details. - -### Probe Type Not as Expected - -Check which probe was cached at startup: - -```bash -grep -E "auto-discover|falling back" watchdog/watchdog.log -``` - -### Viewing Logs - -Log destinations (simultaneous): -- **stdout** — always on; captured by `docker compose logs` -- **file** — `./watchdog/watchdog.log` on the host (bind-mounted), rotates at 10 MB, keeps 5 backups -- **syslog** — remote server if `syslog.enabled: true` in config - -```bash -# Live tail (preferred — host path, no need to exec into the container) -tail -f watchdog/watchdog.log - -# Last 100 lines -tail -100 watchdog/watchdog.log - -# All structured alerts, including INFO recoveries -grep 'watchdog_alert' watchdog/watchdog.log - -# Errors and critical lines -grep -E "ERROR|CRITICAL" watchdog/watchdog.log - -# Filter by container name -grep "my-container" watchdog/watchdog.log - -# Search across rotated logs -grep "CRITICAL" watchdog/watchdog.log \ - watchdog/watchdog.log.1 - -# Via Docker (stdout only) -docker compose logs -f docker-container-watchdog -``` +For installation failures, alert delivery, missing recovery, probe selection, +persistent logs, maintenance, upgrades or rollback, use the +[Customer Installation and Operations Guide](docs/customer/INSTALLATION_AND_OPERATIONS.md#6-troubleshooting). +Use read-only status/log checks on operational hosts, and do not simulate +failures on application containers. --- -## 9. Container Probe Map (This Deployment) - -Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitored passively via events; all others get an active probe auto-discovered on the first poll and cached for the container's lifetime. - -| Container | Probe Type | Endpoint | Why | -|-----------|-----------|----------|-----| -| `config_kvision-infra-fluentd_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-dp-inline-config_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-configuration-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-anomaly-detection-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-persist-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-rt-alert_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_policy-editor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-alerts_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-reporter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-vrm_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-redis_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-health_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-webui_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-tor-feed_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-efk_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-polling-mgr_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-formatter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-data-polling-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-infra-cadvisor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-vdirect_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-help_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-ted_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_postgres_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-snmp-trap-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-auto-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-lls_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-assist-service_1` | HTTP | `:3005` (path auto-discovered) | No Docker HEALTHCHECK; Node.js service; path cached on first successful response | -| `config_kvision-infra-rabbitmq_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `config_kvision-ha-operator_1` | HTTP | `:8080` (path auto-discovered) | No Docker HEALTHCHECK; Java HTTP service; path cached on first successful response | -| `config_kvision-dc-nginx_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | -| `process-exporter` | HTTP | `:9256/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `prometheus` | HTTP | `:9090/-/ready` | No Docker HEALTHCHECK; standard Prometheus readiness endpoint | -| `alertmanager` | HTTP | `:9093/-/healthy` | No Docker HEALTHCHECK; standard Alertmanager health endpoint | -| `postgres-exporter` | HTTP | `:9187/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `mysql-exporter` | HTTP | `:9104/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `es-exporter` | HTTP | `:9114/metrics` | No Docker HEALTHCHECK; port 7979 HTTP fails; port 9114 serves `/metrics` | -| `node-exporter` | HTTP | `:9100/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | -| `grafana` | HTTP | `:3000/api/health` | No Docker HEALTHCHECK; Grafana health endpoint returns 200 | -| `config_kvision-infra-mariadb_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | - -> **Verify what was actually cached after startup:** -> ```bash -> grep -E "auto-discover|falling back" watchdog/watchdog.log -> ``` - -## 10. Version History +## 9. Version History **Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](docs/DEVELOPMENT.md); version-specific approval for v1.6.0 is in its [finalization checklist](docs/RELEASE_V1.6.0_CHECKLIST.md). +**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md); version-specific approval for v1.6.0 is in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md). | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/customer/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/docker-compose.yaml b/docker-compose.yaml index 66eab9f..cc33cd5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,6 @@ # PRODUCTION RUNTIME — use one verified, prebuilt versioned Docker image. -# bash install.sh obtains it from GitHub Releases, a local image, or watchdog.tar. -# Normal customer installation does not build source or require docker load. +# The complete customer release archive includes watchdog.tar and its checksum. +# install.sh verifies/loads that image; customers do not build source or manually load images. # For authorized manual operations: docker compose up -d --no-deps docker-container-watchdog services: diff --git a/docs/DEPLOYMENT_ADVANCED.md b/docs/DEPLOYMENT_ADVANCED.md deleted file mode 100644 index ac3f240..0000000 --- a/docs/DEPLOYMENT_ADVANCED.md +++ /dev/null @@ -1,94 +0,0 @@ -# Container Alert — Advanced Deployment Reference - -This page supports engineers and release maintainers. For normal installation and upgrades, use the [Deployment Guide](../DEPLOYMENT.md). For branching, CI and release publication, see [Development Guide](DEVELOPMENT.md). - -## Image distribution and provenance - -The production installer reads the root `VERSION` file (for example `1.6.0`) and deploys only `watchdog:`. It never builds Docker images from source. - -Official image delivery uses **GitHub Releases** at: - -```text -https://github.com/Radware/CyberController-Container-Alert/releases/tag/v -``` - -Each approved release contains **two downloadable assets**: - -- `container-alert-v.tar.gz` — complete installer, `VERSION`, Compose, templates, documentation, the prebuilt `watchdog.tar` image, and its internal `SHA256SUMS`. -- `SHA256SUMS` — checksum of the entire packaged archive, checked before extraction. - -The [release packaging script](../scripts/package_release.sh) produces the same complete package for connected and disconnected customer hosts. No app Python source or active operator configuration (`.env`, `watchdog-config.yaml` or `watchdog/`) is included. - -The operator first checks the package checksum, then extracts the archive and runs `bash install.sh`. The installer **requires** the included image archive and internal checksum, verifies them and the image's `org.opencontainers.image.version` label, and loads `watchdog:`. No installer-side Git, Internet connection or source build is needed. - -Image tags must not be reused for different source code. Build new code under a new version so rollbacks are unambiguous. - -## Release publication (maintainers) - -Use the version-independent [Development Guide](DEVELOPMENT.md) and, for v1.6.0, the [finalization checklist](RELEASE_V1.6.0_CHECKLIST.md). Approve and merge the release PR into `main`, check latest CI and isolated smoke results, obtain authorization for public distribution, then push the approved matching `v` tag. The hosted release workflow validates/tests/builds the image, assembles the complete archive and external checksum, uploads both to a draft GitHub Release, verifies the assets, and publishes. - -Tagging does not deploy to an operational host. Customer packages do not exist until the tagged release workflow succeeds. - -## Developer-only source build - -Production operators should never build source as part of installation. In a development environment only: - -```bash -export WATCHDOG_VERSION="$(cat VERSION)" -docker compose -f docker-compose.build.yaml build -``` - -The build file passes `WATCHDOG_VERSION` into the image metadata. This produces a local image; it does **not** publish a GitHub Release or update an existing running container. The Dockerfile packages `watchdog.py`, `lifecycle.py`, and `ha_monitor.py` inside the image; a container restart alone does not apply host-side Python edits. - -## Configuration files - -The wizard generates and retains two files: - -| File | Purpose | Apply changes | -|---|---|---| -| `watchdog-config.yaml` | Alert channels, endpoints, checks and thresholds | Restart only the watchdog | -| `.env` | Authentication secrets, image version, hostname and Docker group settings | Recreate only the watchdog | - -The installer detects the group ID of `/var/run/docker.sock` and configures its existing non-root runtime permissions. The production Compose file maintains a read-only root filesystem, capability restrictions, CPU/memory limits and persistent log volume. - -For the full operator-facing channel and settings reference, see [Deployment](../DEPLOYMENT.md#4-configuration-reference). Slack and SMTP secrets belong in `.env`; Syslog/SNMP destinations and health-check policy belong in `watchdog-config.yaml`. Do not commit local secrets to Git. - -For HA active-node notifications, both nodes need the supported local HA exporter and watchdog. See [HA design and validation](HA_ACTIVE_NODE_ALERT_DESIGN.md) for exporter metrics, alert behavior, and limitations. The installer does not create a Pacemaker exporter or change cluster roles. - -## SNMP trap receiver reference - -The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. - -Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): - -```text -createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" -``` - -Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. - -**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: - -| OID | Object | Value | -|---|---|---| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | - -HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. - -## Container-event filtering reference - -The intended MariaDB HA syntax-check helper carries the explicit label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The configured `ignored_container_events` policy ignores only the designated temporary helper `die` event; normal service container failures remain monitored. - -A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Any change to monitoring exceptions belongs to Radware R&D/authorized operations, not routine customer alert handling. - -## Failure handling - -Image acquisition occurs before changing configuration or running containers. A failed download or missing offline archive does not stop the watchdog. Re-running the installer keeps existing configuration unless the operator explicitly elects to reconfigure. - -If Compose reports an invalid configuration, installation stops before attempting deployment. If a container recreation fails after it starts, inspect its logs and restore the previous approved version and configuration under the normal operations process; do not assume an automatic rollback occurred. - -For operational verification, use read-only Docker status, inspect and log commands in [Deployment](../DEPLOYMENT.md#3-verify-the-installation). **Never run destructive Docker acceptance tests on customer, shared or operational Docker hosts.** The isolated hosted acceptance instructions are in [Release Smoke Testing](RELEASE_SMOKE_TESTING.md). diff --git a/DEPLOYMENT.md b/docs/customer/INSTALLATION_AND_OPERATIONS.md similarity index 84% rename from DEPLOYMENT.md rename to docs/customer/INSTALLATION_AND_OPERATIONS.md index e0a989d..d6e78ba 100644 --- a/DEPLOYMENT.md +++ b/docs/customer/INSTALLATION_AND_OPERATIONS.md @@ -1,4 +1,4 @@ -# CyberController Container Watchdog — Deployment Guide +# Container Alert — Customer Installation and Operations This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. One complete, versioned release package and one `install.sh` wizard work on hosts with or without Internet access. The installer deploys only `docker-container-watchdog`; it never restarts CyberController application containers. @@ -23,7 +23,10 @@ This guide covers **installation, configuration, day-to-day operations, upgrades - [Rollback](#54-rollback) - [Uninstall](#55-uninstall) 6. [Troubleshooting](#6-troubleshooting) -7. [Advanced references and support](#7-advanced-references-and-support) + - [Missing recovery notifications](#61-missing-recovery-notifications) + - [Read-only logs and probe diagnostics](#62-read-only-logs-and-probe-diagnostics) +7. [SNMP trap integration](#7-snmp-trap-integration) +8. [References and support](#8-references-and-support) ## 1. Prerequisites @@ -208,7 +211,7 @@ syslog: When enabling Syslog **alongside** existing channels, add `syslog` to the existing list rather than replacing the list as shown in this single-channel example. Restart the watchdog after a YAML-only change. -For SMTP authentication, store the username/password in `.env`; for Slack, store the webhook in `.env`. SNMPv3 passphrases also belong in `.env` and must meet protocol requirements (at least eight characters). See [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) for SNMPv3 receiver/engine-ID setup and the supported enterprise trap fields. +For SMTP authentication, store the username/password in `.env`; for Slack, store the webhook in `.env`. SNMPv3 passphrases also belong in `.env` and must meet protocol requirements (at least eight characters). See [SNMP trap integration](#7-snmp-trap-integration) for SNMPv3 receiver/engine-ID setup and the supported trap OIDs. ### 4.4 Health checks, exclusions, and HA @@ -246,7 +249,7 @@ The supported per-container probe fields are: Use the *actual* container name and a supported health endpoint only after validation. Manual `type: exec` checks execute commands inside monitored containers, so they must be approved and carefully reviewed. Do not add exclusions or broad ignore rules merely to silence unexpected alerts. -HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md) for limitations and approved maintenance testing. +HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/HA_MONITORING_DESIGN.md) for limitations and approved maintenance testing. ### 4.5 Environment variables @@ -395,16 +398,67 @@ Do not use `--remove-all` if historical alerts or state must be retained. Uninst | Syslog recovery not seen in local WARNING-level logs | Console/file level and forwarded Syslog alert priorities are separate; inspect receiver's structured `watchdog_alert` records | | Unexpected unhealthy reports | Compare Docker native HEALTHCHECK, probe type and detection details; do not suppress alerts before authorized review | | Excessive restart/crash notifications | Correlate incident and Docker event evidence; do not raise thresholds/exclude containers solely to quiet alerts | -| HA change notification missing | Both watchdogs, local HA exporter, quorum/metrics, notification receivers and saved baseline; see advanced HA guide | +| HA change notification missing | Check both watchdogs, local HA exporter, quorum/metrics, notification receivers and saved baseline; see [HA settings](#44-health-checks-exclusions-and-ha) | -For authorization or assistance obtaining the offline release, contact the appropriate Radware support channel. Never change production CyberController service settings solely in response to a watchdog alert. +### 6.1 Missing recovery notifications -## 7. Advanced references and support +- Confirm `alert_on_recovery: true` and that an incident previously generated an alert. A never-alarmed container has no recovery notification to send. +- For Docker-native health, inspect status using `docker inspect --format '{{json .State.Health}}'`; `starting` is not `healthy`. +- For HTTP/TCP/`/proc` probes, review cached probe evidence. Crash/OOM/restart-loop recovery requires the configured stable-running interval. Do not change thresholds to force recovery. -- [README](README.md) — product behavior, failure types, architecture and version history. -- [Advanced Deployment](docs/DEPLOYMENT_ADVANCED.md) — developer image builds, release publication, security, SNMPv3 receiver setup. -- [HA alert design and validation](docs/HA_ACTIVE_NODE_ALERT_DESIGN.md) — HA prerequisites, validation, limitations. -- [Release smoke testing](docs/RELEASE_SMOKE_TESTING.md) — isolated **developer/CI-only** acceptance tests. Never run destructive scenarios on an operational Docker host. -- [Example watchdog YAML](watchdog-config.yaml.example) and [example environment variables](.env.example) — full annotated configuration templates. +### 6.2 Read-only logs and probe diagnostics -For offline-image access or product-specific guidance, use your designated Radware support contact (the previous deployment guide listed `northamericare@radware.com`). +From the installation directory: + +```bash +tail -f watchdog/watchdog.log +grep 'watchdog_alert' watchdog/watchdog.log +grep -E 'ERROR|CRITICAL' watchdog/watchdog.log +grep -E 'auto-discover|falling back' watchdog/watchdog.log +grep 'CRITICAL' watchdog/watchdog.log watchdog/watchdog.log.1 +docker compose logs --tail 100 docker-container-watchdog +``` + +Logs rotate at approximately 10 MB with five backups. Retain `watchdog/` and +incident state; do not delete alert history simply to suppress notifications. + +For authorization or assistance obtaining the release, contact the appropriate Radware support channel. Never change production CyberController service settings solely in response to a watchdog alert. + +## 7. SNMP trap integration + +The watchdog supports SNMP v1/v2c/v3 and uses the existing Radware +Container Alert enterprise notification OID. An NMS operator can configure +a receiver using this reference without modifying CyberController services. + +The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. + +Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): + +```text +createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" +``` + +Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. + +**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: + +| OID | Object | Value | +|---|---|---| +| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | +| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | +| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | +| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | +| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | + +HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. + +## 8. References and support + +- [Project overview](../../README.md) — product behavior, alert categories and version history. +- [Release notes](RELEASE_NOTES_V1.6.0.md) — changes, compatibility and limitations for v1.6.0. +- [HA monitoring](#44-health-checks-exclusions-and-ha) — operator prerequisites and expected behavior. + +- [Example watchdog YAML](../../watchdog-config.yaml.example) and [example environment variables](../../.env.example) — full annotated configuration templates. + +For release access, installation assistance or product-specific guidance, +use your organization's designated Radware support contact. diff --git a/docs/RELEASE_NOTES_V1.6.0.md b/docs/customer/RELEASE_NOTES_V1.6.0.md similarity index 85% rename from docs/RELEASE_NOTES_V1.6.0.md rename to docs/customer/RELEASE_NOTES_V1.6.0.md index 2413d57..6bec6fb 100644 --- a/docs/RELEASE_NOTES_V1.6.0.md +++ b/docs/customer/RELEASE_NOTES_V1.6.0.md @@ -41,8 +41,8 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C ## Verification and safety boundaries - Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. -- Destructive real Docker failure testing is **only** through the manually dispatched [Release smoke (isolated)](../.github/workflows/release-smoke.yml) workflow on an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](RELEASE_SMOKE_TESTING.md). -- Historical test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md), [earlier live HA-lab findings](RELEASE_VALIDATION_V1.6.0.md), and JSON files in this directory. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. +- Destructive real Docker failure testing is **only** through the [Release smoke (isolated)](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/.github/workflows/release-smoke.yml) workflow, automatically triggered for qualifying release PR changes and manually rerunnable when available on the default branch. It uses an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_SMOKE_TESTING.md). +- Historical **engineering-only** test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/engineering/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. - Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. - Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. @@ -50,7 +50,7 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C The approved v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is approved and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. -The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](DEVELOPMENT.md) and [v1.6.0 finalization checklist](RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](../DEPLOYMENT.md). +The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/DEVELOPMENT_AND_RELEASE.md) and [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/customer/INSTALLATION_AND_OPERATIONS.md). ## Exclusions, limitations and release gating diff --git a/docs/DEVELOPMENT.md b/docs/engineering/DEVELOPMENT_AND_RELEASE.md similarity index 74% rename from docs/DEVELOPMENT.md rename to docs/engineering/DEVELOPMENT_AND_RELEASE.md index 05fba17..719ceff 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/engineering/DEVELOPMENT_AND_RELEASE.md @@ -1,6 +1,6 @@ # Container Alert — Development and Release Guide -**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../DEPLOYMENT.md). +**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../customer/INSTALLATION_AND_OPERATIONS.md). ## Table of contents @@ -11,6 +11,7 @@ 5. [Verify the published package](#5-verify-the-published-package) 6. [Customer delivery and rollback](#6-customer-delivery-and-rollback) 7. [Safety and best practices](#7-safety-and-best-practices) +8. [Developer-only builds and monitoring notes](#8-developer-only-builds-and-monitoring-notes) ## 1. What we publish @@ -65,7 +66,7 @@ gh pr view -R Radware/CyberController-Container-Alert gh pr checks -R Radware/CyberController-Container-Alert ``` -Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. Code review and approval are still required when checks pass. +Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. Code review and approval are still required when checks pass. ## 3. Prepare the next version @@ -83,12 +84,12 @@ Update the following to match the chosen version: 1. `VERSION` — set the new number (the command above does this). 2. `README.md` — add the matching version-history row; CI checks this. -3. `docs/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. +3. `docs/customer/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. 4. Deployment and examples if operator procedures/settings changed; keep private lab/customer identifiers and secrets out. 5. Tests and hosted smoke approval for the release PR. ```bash -git add VERSION README.md "docs/RELEASE_NOTES_V$NEXT_VER.md" +git add VERSION README.md "docs/customer/RELEASE_NOTES_V$NEXT_VER.md" git commit -m "release: prepare v$NEXT_VER" git push -u origin "release/v$NEXT_VER" gh pr create --base main --head "release/v$NEXT_VER" --title "Release v$NEXT_VER" --fill @@ -124,7 +125,7 @@ git tag -a "v$VER" -m "Container Alert v$VER" git push origin "v$VER" ``` -The tag triggers [Publish Container Alert release](../.github/workflows/release-package.yml). Its GitHub-hosted runner checks version/ancestry, runs tests, builds the prebuilt image, creates the package and checksum, uploads both assets as a **draft release**, checks the assets, then publishes. PR merges do not automatically create customer packages. +The tag triggers [Publish Container Alert release](../../.github/workflows/release-package.yml). Its GitHub-hosted runner checks version/ancestry, runs tests, builds the prebuilt image, creates the package and checksum, uploads both assets as a **draft release**, checks the assets, then publishes. PR merges do not automatically create customer packages. ## 5. Verify the published package @@ -157,7 +158,7 @@ If the workflow fails or the release is incomplete, do not distribute it. Inspec Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or approved internal transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. -YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../DEPLOYMENT.md#54-rollback). +YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../customer/INSTALLATION_AND_OPERATIONS.md#54-rollback). ## 7. Safety and best practices @@ -168,3 +169,38 @@ YAML-only changes require a watchdog restart; `.env`/Compose changes require rec - Require code review and explicit public-distribution approval before tagging. Image builds use prebuilt, versioned artifacts rather than compiling on customer systems. References: [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), [Managing Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [About releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). + + +## 8. Developer-only builds and monitoring notes + +### Local image build (never a customer installation) + +For authorized development: + +```bash +export WATCHDOG_VERSION="$(cat VERSION)" +docker compose -f docker-compose.build.yaml build +``` + +The Dockerfile packages `watchdog.py`, `lifecycle.py`, and `ha_monitor.py` +inside the image. A restart alone does **not** apply Python source edits. +Production deployment requires a published, prebuilt release package. + +### Narrow CyberController event exceptions + + +The intended MariaDB HA syntax-check helper carries the explicit label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The configured `ignored_container_events` policy ignores only the designated temporary helper `die` event; normal service container failures remain monitored. + +A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Any change to monitoring exceptions belongs to Radware R&D/authorized operations, not routine customer alert handling. + + +These exceptions are not blanket MariaDB or OOM alert suppression. Changes +to monitoring exceptions require R&D review and regression validation. + +### Historical design and validation + +The [HA design](HA_MONITORING_DESIGN.md) and [dated HA snapshot](HA_VALIDATION_2026-10-13.md) +preserve earlier evidence. The [release acceptance](RELEASE_ACCEPTANCE_V1.6.0.md) +and [earlier failures](RELEASE_VALIDATION_V1.6.0.md) are historical reports, +not current installation instructions or release approval. Use the +[release checklist](RELEASE_V1.6.0_CHECKLIST.md) and actual GitHub checks. diff --git a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md b/docs/engineering/HA_MONITORING_DESIGN.md similarity index 96% rename from docs/HA_ACTIVE_NODE_ALERT_DESIGN.md rename to docs/engineering/HA_MONITORING_DESIGN.md index 513ca1f..cce12fc 100644 --- a/docs/HA_ACTIVE_NODE_ALERT_DESIGN.md +++ b/docs/engineering/HA_MONITORING_DESIGN.md @@ -1,6 +1,13 @@ +> **Historical design proposal — 2026-10-09.** This is the planning snapshot for +> the optional HA feature, **not its current release status or installation +> procedure**. PR #4 was subsequently merged into `main` on 2026-10-14. +> Later failover/failback acceptance appears in the +> [release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). +> Unchecked design checkboxes below are kept for traceability, not current release gates. + # Optional CyberController HA Active-Node Alert — Design Proposal -**Status:** Feature under review; manual switchover verified, failback and alert delivery pending +**Status on 2026-10-09:** Feature under review; failback and delivery validation pending at the time **Date:** 2026-10-09 **Target:** Radware/CyberController-Container-Alert, baseline v1.5.4 (main: a4ddfed) **Delivery:** One small implementation PR, with release blocked on two-node QA @@ -172,4 +179,4 @@ The implementation is ready for review/release when all automated tests and CI p **Out of scope:** HA orchestration/control, restarting or promoting resources, automatic remediation, logging every Pacemaker transition, live log parsing, new monitoring dashboards/exporters, historical HA event reconstruction, generalized multi-service health monitoring, and remediation of the native Prometheus 9002 vs 9664 mismatch. The latter should be handled as a separate CyberController native monitoring correction. -**Implementation status:** Approved by the user on 2026-10-09 and implemented on the feature branch. Do not merge or release until the two-node acceptance gates in [HA_VALIDATION.md](HA_VALIDATION.md) are satisfied. +**Implementation status:** Approved by the user on 2026-10-09 and implemented on the feature branch. Do not merge or release until the two-node acceptance gates in [HA_VALIDATION.md](HA_VALIDATION_2026-10-13.md) are satisfied. diff --git a/docs/HA_VALIDATION.md b/docs/engineering/HA_VALIDATION_2026-10-13.md similarity index 89% rename from docs/HA_VALIDATION.md rename to docs/engineering/HA_VALIDATION_2026-10-13.md index 846ba11..c519581 100644 --- a/docs/HA_VALIDATION.md +++ b/docs/engineering/HA_VALIDATION_2026-10-13.md @@ -1,7 +1,14 @@ +> **Historical validation snapshot — 2026-10-13.** PR #4 was subsequently +> merged into `main` on 2026-10-14, and later HA failover/failback +> acceptance is recorded in the +> [v1.6.0 release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). +> The pending gates below are preserved as historical findings, **not** the +> current status of PR #4. + # Optional HA active-node alert — validation status **Branch:** feature/optional-ha-failover-alert -**Status:** Draft PR #4; tested in the two-node lab, not approved for merge +**Status at the time:** Draft PR #4; two-node lab tested, pending merge **Updated:** 2026-10-13 ## Automated tests and build diff --git a/docs/engineering/PROBE_MAP_SNAPSHOT.md b/docs/engineering/PROBE_MAP_SNAPSHOT.md new file mode 100644 index 0000000..a6bafdf --- /dev/null +++ b/docs/engineering/PROBE_MAP_SNAPSHOT.md @@ -0,0 +1,59 @@ +# Historical Container Probe Map — example only + +**Engineering reference, not an installation configuration or a current product guarantee.** +This is a snapshot from one earlier CyberController environment. Image HEALTHCHECK +definitions, service ports and probe discovery can differ by product version and +deployment. Never copy these entries into monitoring overrides without confirming +the current Docker state and obtaining the normal authorization. + + +Probe selection is automatic: containers with a Docker `HEALTHCHECK` are monitored passively via events; all others get an active probe auto-discovered on the first poll and cached for the container's lifetime. + +| Container | Probe Type | Endpoint | Why | +|-----------|-----------|----------|-----| +| `config_kvision-infra-fluentd_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-dp-inline-config_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-configuration-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-anomaly-detection-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-persist-service_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-rt-alert_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_policy-editor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-alerts_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-reporter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-vrm_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-redis_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-health_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-webui_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-tor-feed_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-efk_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-polling-mgr_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-formatter_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-data-polling-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-infra-cadvisor_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-scheduler_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-vdirect_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-help_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-ted_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_postgres_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-snmp-trap-collector_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-auto-engine_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-lls_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-assist-service_1` | HTTP | `:3005` (path auto-discovered) | No Docker HEALTHCHECK; Node.js service; path cached on first successful response | +| `config_kvision-infra-rabbitmq_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `config_kvision-ha-operator_1` | HTTP | `:8080` (path auto-discovered) | No Docker HEALTHCHECK; Java HTTP service; path cached on first successful response | +| `config_kvision-dc-nginx_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | +| `process-exporter` | HTTP | `:9256/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `prometheus` | HTTP | `:9090/-/ready` | No Docker HEALTHCHECK; standard Prometheus readiness endpoint | +| `alertmanager` | HTTP | `:9093/-/healthy` | No Docker HEALTHCHECK; standard Alertmanager health endpoint | +| `postgres-exporter` | HTTP | `:9187/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `mysql-exporter` | HTTP | `:9104/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `es-exporter` | HTTP | `:9114/metrics` | No Docker HEALTHCHECK; port 7979 HTTP fails; port 9114 serves `/metrics` | +| `node-exporter` | HTTP | `:9100/metrics` | No Docker HEALTHCHECK; Prometheus exporter — `/metrics` returns 200 | +| `grafana` | HTTP | `:3000/api/health` | No Docker HEALTHCHECK; Grafana health endpoint returns 200 | +| `config_kvision-infra-mariadb_1` | Docker HEALTHCHECK | Docker managed | Image has HEALTHCHECK directive | + +> **Verify what was actually cached after startup:** +> ```bash +> grep -E "auto-discover|falling back" watchdog/watchdog.log +> ``` diff --git a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md b/docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md similarity index 96% rename from docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md rename to docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md index 9ec3add..7505283 100644 --- a/docs/RELEASE_ACCEPTANCE_V1.6.0_FINAL.md +++ b/docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md @@ -6,7 +6,7 @@ > customer-facing message updates. Do **not** rerun their Docker commands on > any shared or operational Docker Engine. Earlier nested testing triggered > real notifications on its parent host. -> See [current release notes](RELEASE_NOTES_V1.6.0.md) and +> See [current release notes](../customer/RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). **Validation date:** October 14, 2026 (UTC) @@ -100,9 +100,9 @@ Installer source-build and offline archive `docker save`/`docker load` procedure ## Evidence and replay -- [Final full suite JSON](RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json) - 15/15 and cleanup evidence -- [Before-fix real failure reproduction](RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json) -- [Real OOM/manual-restart 120.6s proof](RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json) +- [Final full suite JSON](evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json) - 15/15 and cleanup evidence +- [Before-fix real failure reproduction](evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json) +- [Real OOM/manual-restart 120.6s proof](evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json) - [Earlier HA-lab conditional report](RELEASE_VALIDATION_V1.6.0.md) -- [Reusable real Docker smoke runner](../tests/release-smoke.sh) +- [Reusable real Docker smoke runner](../../tests/release-smoke.sh) - Historical runtime evidence: temporary smoke-run directories containing test events, Syslog records and observer state were retained during validation. diff --git a/docs/RELEASE_SMOKE_TESTING.md b/docs/engineering/RELEASE_SMOKE_TESTING.md similarity index 90% rename from docs/RELEASE_SMOKE_TESTING.md rename to docs/engineering/RELEASE_SMOKE_TESTING.md index 145516a..f90543c 100644 --- a/docs/RELEASE_SMOKE_TESTING.md +++ b/docs/engineering/RELEASE_SMOKE_TESTING.md @@ -6,7 +6,7 @@ **NEVER run `tests/release-smoke.sh` on CyberController, customer systems, or any shared Docker host.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. -Instead run the workflow [Release smoke (isolated)](../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to any operational host. All test-created resources are disposable and contained within that hosted worker. +Instead run the workflow [Release smoke (isolated)](../../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to any operational host. All test-created resources are disposable and contained within that hosted worker. The workflow has no corporate customer credentials, no HA exporter dependency, no CyberController Docker socket, no external notification destinations, and no network access from the disposable failure fixtures. diff --git a/docs/RELEASE_V1.6.0_CHECKLIST.md b/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md similarity index 89% rename from docs/RELEASE_V1.6.0_CHECKLIST.md rename to docs/engineering/RELEASE_V1.6.0_CHECKLIST.md index f344339..ad47038 100644 --- a/docs/RELEASE_V1.6.0_CHECKLIST.md +++ b/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md @@ -1,6 +1,6 @@ # Container Alert v1.6.0 — Final Release Checklist -**This checklist applies only to v1.6.0.** For every subsequent release, use the permanent [Development and Release Guide](DEVELOPMENT.md) and create a corresponding version-specific approval checklist only if needed. +**This checklist applies only to v1.6.0.** For every subsequent release, use the permanent [Development and Release Guide](DEVELOPMENT_AND_RELEASE.md) and create a corresponding version-specific approval checklist only if needed. **Status on October 16, 2026:** `main` still contains v1.5.4; `release/v1.6.0` contains `VERSION=1.6.0` and is reviewed through [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9), which is open/unmerged. The v1.6.0 GitHub Release has **not** been published by this work. @@ -8,7 +8,7 @@ - [ ] Final engineering review of PR #9: alert/incident lifecycle, notifications, HA behavior, security and image packaging. - [ ] Normal GitHub CI **and** [Release smoke (isolated)](RELEASE_SMOKE_TESTING.md) both pass against the **latest PR head SHA**. The smoke is a disposable GitHub-hosted test; never run it against an operational Docker host. -- [ ] Document all changes since v1.5.4 in [release notes](RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. +- [ ] Document all changes since v1.5.4 in [release notes](../customer/RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. - [ ] Confirm exact supported customer message text, severity mappings, and downstream Syslog/SNMP compatibility. - [ ] Confirm there are no private lab hostnames, IP addresses, personal information or credentials in the **published installer archive**, documentation and release notes. - [ ] Approve distributing the Docker image and complete release package through this **public** GitHub repository. @@ -39,7 +39,7 @@ After **all** approvals above: git push origin v1.6.0 ``` -4. Confirm the tagged [release workflow](../.github/workflows/release-package.yml) passed: +4. Confirm the tagged [release workflow](../../.github/workflows/release-package.yml) passed: ```bash gh run list -R Radware/CyberController-Container-Alert --workflow release-package.yml --limit 5 @@ -59,6 +59,6 @@ After **all** approvals above: cat unpacked/CyberController-Container-Alert/VERSION ``` -6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../DEPLOYMENT.md). +6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../customer/INSTALLATION_AND_OPERATIONS.md). **Stop if a release gate fails.** Don't distribute a draft, incomplete or unapproved release. Do not run installation, Docker failure injection or HA failover tests on production systems as part of this checklist. diff --git a/docs/RELEASE_VALIDATION_V1.6.0.md b/docs/engineering/RELEASE_VALIDATION_V1.6.0.md similarity index 93% rename from docs/RELEASE_VALIDATION_V1.6.0.md rename to docs/engineering/RELEASE_VALIDATION_V1.6.0.md index 5ba66e3..51be1b0 100644 --- a/docs/RELEASE_VALIDATION_V1.6.0.md +++ b/docs/engineering/RELEASE_VALIDATION_V1.6.0.md @@ -6,10 +6,10 @@ > customer-facing message updates. Do **not** rerun their Docker commands on > any shared or operational Docker Engine. Earlier nested testing triggered > real notifications on its parent host. -> See [current release notes](RELEASE_NOTES_V1.6.0.md) and +> See [current release notes](../customer/RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). -> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on a dedicated development host, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0_FINAL.md). Retain this document as evidence of the original findings, not the current release verdict. +> **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on a dedicated development host, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). Retain this document as evidence of the original findings, not the current release verdict. **Date:** October 14, 2026 (UTC) **Repository:** `Radware/CyberController-Container-Alert` @@ -25,7 +25,7 @@ - A separate resource-limited v1.6.0 observer used its own Docker volume/state, 2-second Docker polling, real 5-restart threshold, and **real 120-second** recovery requirement. SMTP disabled only on the observer; Syslog sent to the **existing** DFC UDP/514 receiver. Production-style main watchdog and cluster were not reconfigured. - Used uniquely prefixed, labeled disposable containers, restricted to 0.2 CPU and 64–128 MiB each, with `--network none`. Verified actual Docker state, events, watchdog structured alert records, and DFC `/var/log/syslog` deliveries. - No host packages, new Docker networks, Pacemaker operations, or host service modifications. Cleanup removed all smoke-owned containers and volumes. Original watchdog stayed healthy. -- Runner: [`tests/release-smoke.sh`](../tests/release-smoke.sh). It saves `report.json`, `report.md`, `watchdog.log`, `docker-events-live.jsonl`, and `dfc-syslog.txt` to `/tmp/container-alert-smoke-/` inside the authorized RDC client. +- Runner: [`tests/release-smoke.sh`](../../tests/release-smoke.sh). It saves `report.json`, `report.md`, `watchdog.log`, `docker-events-live.jsonl`, and `dfc-syslog.txt` to `/tmp/container-alert-smoke-/` inside the authorized RDC client. ## 1. Automated, packaging, and release checks @@ -110,6 +110,8 @@ The DFC Syslog pipeline itself had **100% message-count correspondence in the fi - Lab RDC container: `/tmp/container-alert-smoke-261014214436754/` - Raw files: `report.json`, `report.md`, `docker-events-live.jsonl`, `watchdog.log`, `dfc-syslog.txt`, and `observer-state.json`. -- Source-controlled runner: [`tests/release-smoke.sh`](../tests/release-smoke.sh). Use `bash release-smoke.sh` from the authorized lab admin environment; its `/tmp` filesystem is mounted `noexec`. +- Source-controlled runner: [`tests/release-smoke.sh`](../../tests/release-smoke.sh). The historical lab smoke command must **not be repeated**; current destructive +acceptance is permitted only through the +[disposable GitHub-hosted workflow](RELEASE_SMOKE_TESTING.md). -Machine-readable final run: [`RELEASE_SMOKE_V1.6.0_20261014.json`](RELEASE_SMOKE_V1.6.0_20261014.json). +Machine-readable final run: [`RELEASE_SMOKE_V1.6.0_20261014.json`](evidence/RELEASE_SMOKE_V1.6.0_20261014.json). diff --git a/docs/RELEASE_SMOKE_V1.6.0_20261014.json b/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_20261014.json similarity index 100% rename from docs/RELEASE_SMOKE_V1.6.0_20261014.json rename to docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_20261014.json diff --git a/docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json b/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json similarity index 100% rename from docs/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json rename to docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json diff --git a/docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json b/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json similarity index 100% rename from docs/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json rename to docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json diff --git a/docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json b/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json similarity index 100% rename from docs/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json rename to docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json diff --git a/scripts/package_release.sh b/scripts/package_release.sh index 494fb22..f9e47dd 100644 --- a/scripts/package_release.sh +++ b/scripts/package_release.sh @@ -28,12 +28,12 @@ work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT dir="$work/CyberController-Container-Alert" -mkdir -p "$dir/docs" +mkdir -p "$dir/docs/customer" for f in install.sh uninstall.sh docker-compose.yaml VERSION \ - .env.example watchdog-config.yaml.example README.md DEPLOYMENT.md; do + .env.example watchdog-config.yaml.example README.md; do cp "$repo/$f" "$dir/$f" done -cp "$repo"/docs/*.md "$dir/docs/" +cp "$repo"/docs/customer/*.md "$dir/docs/customer/" # only customer guides ship cp "$image" "$dir/watchdog.tar" (cd "$dir" && sha256sum watchdog.tar > SHA256SUMS) diff --git a/tests/test_documentation.py b/tests/test_documentation.py new file mode 100644 index 0000000..732bf14 --- /dev/null +++ b/tests/test_documentation.py @@ -0,0 +1,110 @@ +"""Read-only documentation consistency checks; no Docker or network access.""" + +import re +from pathlib import Path +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +DOCS = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] +LINK_RE = re.compile(r"\]\(([^)#]+)?(?:#([a-zA-Z0-9\-]+))?\)") + + +def heading_anchors(markdown): + counts = {} + anchors = set() + fence = None + for line in markdown.splitlines(): + marker = line.strip()[:3] + if marker in ("~~~", chr(96) * 3): + fence = None if fence == marker else marker + continue + if fence: + continue + matched = re.match(r"^#{1,6}\s+(.+?)\s*#*\s*$", line) + if not matched: + continue + slug = re.sub(r"[^\w\- ]", "", matched.group(1).lower()).replace(" ", "-") + count = counts.get(slug, 0) + counts[slug] = count + 1 + anchors.add(slug + (f"-{count}" if count else "")) + if fence: + raise AssertionError("Unclosed Markdown code fence") + return anchors + + +class DocumentationTests(unittest.TestCase): + def test_all_relative_links_and_headings_resolve(self): + for doc in DOCS: + source = doc.read_text() + heading_anchors(source) + for match in LINK_RE.finditer(source): + relative, anchor = match.groups() + if relative and relative.startswith(("http:", "https:", "mailto:")): + continue + target = (doc.parent / (relative or "")).resolve() if relative else doc + with self.subTest(document=str(doc.relative_to(ROOT)), link=match.group(0)): + self.assertTrue(target.exists(), f"Broken link to {relative}") + if anchor and target.suffix == ".md": + self.assertIn(anchor, heading_anchors(target.read_text())) + + def test_customer_docs_are_self_contained(self): + docs = [ROOT / "README.md", *(ROOT / "docs/customer").glob("*.md")] + for doc in docs: + for match in LINK_RE.finditer(doc.read_text()): + relative, _ = match.groups() + if not relative or relative.startswith(("http:", "https:", "mailto:")): + continue + target = (doc.parent / relative).resolve() + self.assertNotIn( + ROOT / "docs/engineering", + target.parents, + f"Customer bundle refers to internal engineering file: {doc}", + ) + self.assertFalse((ROOT / "DEPLOYMENT.md").exists()) + self.assertFalse((ROOT / "docs/DEPLOYMENT_ADVANCED.md").exists()) + + def test_release_notes_links_work_when_published_as_github_release(self): + # GitHub Release notes are rendered at the release URL, not at + # docs/customer/, so links to other files must be absolute. + for notes in (ROOT / "docs/customer").glob("RELEASE_NOTES_V*.md"): + for match in LINK_RE.finditer(notes.read_text()): + destination, _ = match.groups() + if not destination or destination.startswith("#"): + continue + with self.subTest(release_notes=notes.name, link=destination): + self.assertTrue( + destination.startswith(("https:", "http:", "mailto:")), + "Relative links are broken when copied into GitHub Releases", + ) + + def test_operator_guide_covers_configuration_template_fields(self): + config = yaml.safe_load((ROOT / "watchdog-config.yaml.example").read_text()) + guide = (ROOT / "docs/customer/INSTALLATION_AND_OPERATIONS.md").read_text() + + def keys(item, prefix=""): + if isinstance(item, dict): + for key, value in item.items(): + field = prefix + str(key) + yield field + yield from keys(value, field + ".") + + for key in keys(config): + # Detailed event matching and individual custom-probe entries are + # supported by the operator guide without documenting test fixtures. + if key.startswith(("ignored_container_events", "container_health_checks")): + continue + with self.subTest(yaml_setting=key): + self.assertIn(key, guide) + + for line in (ROOT / ".env.example").read_text().splitlines(): + matched = re.match(r"^\s*#?\s*([A-Z][A-Z0-9_]+)=", line) + if matched: + with self.subTest(env_setting=matched.group(1)): + self.assertIn(matched.group(1), guide) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_package.py b/tests/test_release_package.py index 838782a..d211e1e 100644 --- a/tests/test_release_package.py +++ b/tests/test_release_package.py @@ -1,6 +1,8 @@ """Validate single customer release package with a fake image; never use Docker.""" import hashlib +import posixpath +import re from pathlib import Path import subprocess import tarfile @@ -39,10 +41,10 @@ def package_files(self): def test_one_complete_release_package(self): files = self.package_files() required = ("install.sh", "uninstall.sh", "docker-compose.yaml", "VERSION", - "README.md", "DEPLOYMENT.md", ".env.example", + "README.md", ".env.example", "watchdog-config.yaml.example", "watchdog.tar", "SHA256SUMS", - "docs/DEPLOYMENT_ADVANCED.md", "docs/DEVELOPMENT.md", - f"docs/RELEASE_NOTES_V{VERSION}.md") + "docs/customer/INSTALLATION_AND_OPERATIONS.md", + f"docs/customer/RELEASE_NOTES_V{VERSION}.md") for name in required: self.assertIn(PREFIX + name, files) self.assertEqual(files[PREFIX + "VERSION"].decode().strip(), VERSION) @@ -55,6 +57,24 @@ def test_bundle_contains_image_not_python_source(self): self.assertNotIn(PREFIX + "Dockerfile", files) self.assertNotIn(PREFIX + ".env", files) self.assertNotIn(PREFIX + "watchdog-config.yaml", files) + self.assertFalse(any(path.startswith(PREFIX + "docs/engineering/") for path in files)) + self.assertNotIn(PREFIX + "docs/DEVELOPMENT.md", files) + + def test_every_local_customer_link_is_packaged(self): + files = self.package_files() + links = re.compile(r"\\]\\(([^)#]+)?(?:#[a-zA-Z0-9\\-]+)?\\)") + for name, content in files.items(): + if not name.endswith(".md"): + continue + for match in links.finditer(content.decode("utf-8")): + relative = match.group(1) + if not relative or relative.startswith(("http:", "https:", "mailto:")): + continue + target = posixpath.normpath( + posixpath.join(posixpath.dirname(name), relative) + ) + with self.subTest(markdown=name, target=target): + self.assertIn(target, files) def test_image_checksum_inside_package(self): files = self.package_files() diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index 91da1fe..96c2e8a 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -69,13 +69,13 @@ syslog: smtp: enabled: true # set to true to enable auth: false - host: smtp.radware.com + host: smtp.example.com port: 587 - sender: noc-alerts@radware.com + sender: alerts@example.com username_env: SMTP_USERNAME # env var: SMTP username or app token (Mailtrap, Gmail app password, etc.) recipients: - - ops-team@radware.com - - oncall@radware.com + - ops@example.com + - oncall@example.com tls: true password_env: SMTP_PASSWORD # env var: SMTP password or app token From ff86a05eefa4ca9c2974de27151674e4e68e2a5c Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 16:02:00 +0000 Subject: [PATCH 13/18] folders reorg --- docs/{engineering => dev}/DEVELOPMENT_AND_RELEASE.md | 0 docs/{engineering => dev}/HA_MONITORING_DESIGN.md | 0 docs/{engineering => dev}/HA_VALIDATION_2026-10-13.md | 0 docs/{engineering => dev}/PROBE_MAP_SNAPSHOT.md | 0 docs/{engineering => dev}/RELEASE_ACCEPTANCE_V1.6.0.md | 0 docs/{engineering => dev}/RELEASE_SMOKE_TESTING.md | 0 docs/{engineering => dev}/RELEASE_V1.6.0_CHECKLIST.md | 0 docs/{engineering => dev}/RELEASE_VALIDATION_V1.6.0.md | 0 .../evidence/RELEASE_SMOKE_V1.6.0_20261014.json | 0 .../evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json | 0 .../evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json | 0 .../evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json | 0 docs/{customer => operations}/INSTALLATION_AND_OPERATIONS.md | 0 docs/{customer => operations}/RELEASE_NOTES_V1.6.0.md | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename docs/{engineering => dev}/DEVELOPMENT_AND_RELEASE.md (100%) rename docs/{engineering => dev}/HA_MONITORING_DESIGN.md (100%) rename docs/{engineering => dev}/HA_VALIDATION_2026-10-13.md (100%) rename docs/{engineering => dev}/PROBE_MAP_SNAPSHOT.md (100%) rename docs/{engineering => dev}/RELEASE_ACCEPTANCE_V1.6.0.md (100%) rename docs/{engineering => dev}/RELEASE_SMOKE_TESTING.md (100%) rename docs/{engineering => dev}/RELEASE_V1.6.0_CHECKLIST.md (100%) rename docs/{engineering => dev}/RELEASE_VALIDATION_V1.6.0.md (100%) rename docs/{engineering => dev}/evidence/RELEASE_SMOKE_V1.6.0_20261014.json (100%) rename docs/{engineering => dev}/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json (100%) rename docs/{engineering => dev}/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json (100%) rename docs/{engineering => dev}/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json (100%) rename docs/{customer => operations}/INSTALLATION_AND_OPERATIONS.md (100%) rename docs/{customer => operations}/RELEASE_NOTES_V1.6.0.md (100%) diff --git a/docs/engineering/DEVELOPMENT_AND_RELEASE.md b/docs/dev/DEVELOPMENT_AND_RELEASE.md similarity index 100% rename from docs/engineering/DEVELOPMENT_AND_RELEASE.md rename to docs/dev/DEVELOPMENT_AND_RELEASE.md diff --git a/docs/engineering/HA_MONITORING_DESIGN.md b/docs/dev/HA_MONITORING_DESIGN.md similarity index 100% rename from docs/engineering/HA_MONITORING_DESIGN.md rename to docs/dev/HA_MONITORING_DESIGN.md diff --git a/docs/engineering/HA_VALIDATION_2026-10-13.md b/docs/dev/HA_VALIDATION_2026-10-13.md similarity index 100% rename from docs/engineering/HA_VALIDATION_2026-10-13.md rename to docs/dev/HA_VALIDATION_2026-10-13.md diff --git a/docs/engineering/PROBE_MAP_SNAPSHOT.md b/docs/dev/PROBE_MAP_SNAPSHOT.md similarity index 100% rename from docs/engineering/PROBE_MAP_SNAPSHOT.md rename to docs/dev/PROBE_MAP_SNAPSHOT.md diff --git a/docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md b/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md similarity index 100% rename from docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md rename to docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md diff --git a/docs/engineering/RELEASE_SMOKE_TESTING.md b/docs/dev/RELEASE_SMOKE_TESTING.md similarity index 100% rename from docs/engineering/RELEASE_SMOKE_TESTING.md rename to docs/dev/RELEASE_SMOKE_TESTING.md diff --git a/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md b/docs/dev/RELEASE_V1.6.0_CHECKLIST.md similarity index 100% rename from docs/engineering/RELEASE_V1.6.0_CHECKLIST.md rename to docs/dev/RELEASE_V1.6.0_CHECKLIST.md diff --git a/docs/engineering/RELEASE_VALIDATION_V1.6.0.md b/docs/dev/RELEASE_VALIDATION_V1.6.0.md similarity index 100% rename from docs/engineering/RELEASE_VALIDATION_V1.6.0.md rename to docs/dev/RELEASE_VALIDATION_V1.6.0.md diff --git a/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_20261014.json similarity index 100% rename from docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_20261014.json rename to docs/dev/evidence/RELEASE_SMOKE_V1.6.0_20261014.json diff --git a/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json similarity index 100% rename from docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json rename to docs/dev/evidence/RELEASE_SMOKE_V1.6.0_ISOLATED_20261014.json diff --git a/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json similarity index 100% rename from docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json rename to docs/dev/evidence/RELEASE_SMOKE_V1.6.0_OOM_RESTART_20261014.json diff --git a/docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json b/docs/dev/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json similarity index 100% rename from docs/engineering/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json rename to docs/dev/evidence/RELEASE_SMOKE_V1.6.0_PRE_FIX_REPRO_20261014.json diff --git a/docs/customer/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md similarity index 100% rename from docs/customer/INSTALLATION_AND_OPERATIONS.md rename to docs/operations/INSTALLATION_AND_OPERATIONS.md diff --git a/docs/customer/RELEASE_NOTES_V1.6.0.md b/docs/operations/RELEASE_NOTES_V1.6.0.md similarity index 100% rename from docs/customer/RELEASE_NOTES_V1.6.0.md rename to docs/operations/RELEASE_NOTES_V1.6.0.md From 6b6a3e6228b52693d0c9b414752977fe47dbe8b6 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 16:08:24 +0000 Subject: [PATCH 14/18] fix: align docs and release packaging with renamed folders --- .github/workflows/release-package.yml | 4 +-- README.md | 26 +++++++++---------- docs/dev/DEVELOPMENT_AND_RELEASE.md | 8 +++--- docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md | 2 +- docs/dev/RELEASE_V1.6.0_CHECKLIST.md | 4 +-- docs/dev/RELEASE_VALIDATION_V1.6.0.md | 2 +- .../operations/INSTALLATION_AND_OPERATIONS.md | 2 +- docs/operations/RELEASE_NOTES_V1.6.0.md | 6 ++--- scripts/package_release.sh | 4 +-- tests/test_documentation.py | 10 +++---- tests/test_release_package.py | 6 ++--- 11 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml index 390aae7..ba06a62 100644 --- a/.github/workflows/release-package.yml +++ b/.github/workflows/release-package.yml @@ -69,8 +69,8 @@ jobs: run: | set -euo pipefail notes=(--generate-notes) - if [ -f "docs/customer/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then - notes=(--notes-file "docs/customer/RELEASE_NOTES_V$WATCHDOG_VERSION.md") + if [ -f "docs/operations/RELEASE_NOTES_V$WATCHDOG_VERSION.md" ]; then + notes=(--notes-file "docs/operations/RELEASE_NOTES_V$WATCHDOG_VERSION.md") fi # Check both assets before making this public release visible. diff --git a/README.md b/README.md index 3facae1..e85e7ad 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ The **CyberController Container Watchdog** is a lightweight, autonomous monitoring service that continuously tracks the health of all Docker containers on the host. It detects failures — including crashes, out-of-memory kills, prolonged unhealthy states, and restart loops — and dispatches real-time alerts through one or more configurable channels (**Slack**, **SMTP**, **SNMP Traps**, or **Syslog**). It also sends a one-time INFO **recovered** alert through the same channels once a previously-alarmed container returns to normal operation. -**Documentation:** [Customer installation and operations](docs/customer/INSTALLATION_AND_OPERATIONS.md) · [v1.6.0 release notes](docs/customer/RELEASE_NOTES_V1.6.0.md). +**Documentation:** [Customer installation and operations](docs/operations/INSTALLATION_AND_OPERATIONS.md) · [v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). -**For maintainers (source repository):** [Development and release workflow](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md). Historical design, test reports, and CI procedures live under `docs/engineering/` and are **not included** in customer installation packages. +**For maintainers (source repository):** [Development and release workflow](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md). Historical design, test reports, and CI procedures live under `docs/dev/` and are **not included** in customer installation packages. --- @@ -113,7 +113,7 @@ CyberController services have recovered. The exporter may use port **9664** even when the built-in CyberController Prometheus configuration references **9002**. This optional check reads the exporter directly and makes **no changes** to native Prometheus configuration. -See [customer HA configuration and expected behavior](docs/customer/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). +See [customer HA configuration and expected behavior](docs/operations/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). --- @@ -135,8 +135,8 @@ The following is the **source repository** layout, not the customer archive. The ├── install.sh # Guided installation from complete release package ├── uninstall.sh # Stop + remove script ├── scripts/package_release.sh # CI release-package assembly (maintainers only) -├── docs/customer/ # Installation/operations and release notes -├── docs/engineering/ # Development, release, HA and test evidence +├── docs/operations/ # Installation/operations and release notes +├── docs/dev/ # Development, release, HA and test evidence ├── VERSION # Release version — drives the image tag and installer ├── watchdog.tar # Prebuilt image only in published package └── .env # Secrets — NOT committed to git @@ -156,7 +156,7 @@ The following is the **source repository** layout, not the customer archive. The ### Container Hardening -When running Compose manually without installer-generated `.env`, the default container user is **root** — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/customer/INSTALLATION_AND_OPERATIONS.md#41-configuration-files-and-editing)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. +When running Compose manually without installer-generated `.env`, the default container user is **root** — no host-specific group setup needed to read `/var/run/docker.sock`. [install.sh](install.sh) instead auto-detects the socket's GID and opts into running the container as **non-root UID/GID 1000** (joined to that group via `group_add`), with no manual input required; manual installs can do the same by setting `WATCHDOG_UID`/`WATCHDOG_GID`/`DOCKER_GID` in `.env` (see [Advanced Deployment](docs/operations/INSTALLATION_AND_OPERATIONS.md#41-configuration-files-and-editing)). Either way, the container is constrained by a read-only root filesystem, no extra Linux capabilities (`cap_drop: ALL`), `no-new-privileges`, and CPU/memory/PID caps (`cpus: "0.50"`, `pids_limit: 200`, `mem_limit: 256m`), which contain a leak or runaway condition to this container instead of the host. Note that access to `/var/run/docker.sock` is effectively host-root-equivalent for **any** user (root or not): anything that can reach the socket can create privileged containers or bind-mount the host filesystem, so a compromise of the watchdog process is **not** contained by these controls — treat `docker.sock` access as the primary trust boundary when deciding who/what can reach this host. ### Python Dependencies @@ -180,7 +180,7 @@ cryptography>=42.0 The guided installer creates `watchdog-config.yaml` (monitoring policy, probes, logging and notification endpoints) and `.env` (credentials and runtime identity). The configuration is preserved on an upgrade unless the operator explicitly chooses to reconfigure it. -**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](docs/customer/INSTALLATION_AND_OPERATIONS.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. +**For the complete list of settings, defaults, YAML examples, and instructions for applying changes after installation, see [Deployment — Configuration Reference](docs/operations/INSTALLATION_AND_OPERATIONS.md#4-configuration-reference).** The annotated [YAML template](watchdog-config.yaml.example) and [environment template](.env.example) are also available. A YAML-only change takes effect after restarting the watchdog. A changed `.env` value requires recreating **only** the watchdog container. Neither action rebuilds Python code into the image. Thresholds, exclusions, custom exec probes, and monitoring suppressions require authorized review; do not use them merely to silence alerts. @@ -195,7 +195,7 @@ A YAML-only change takes effect after restarting the watchdog. A changed `.env` | SNMP Trap | `snmp_trap` | NMS destination, SNMP version; SNMPv3 keys in `.env` | | Slack | `slack` | Webhook URL stored in `.env` | -At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/customer/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/customer/INSTALLATION_AND_OPERATIONS.md#7-snmp-trap-integration) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. +At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/operations/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/operations/INSTALLATION_AND_OPERATIONS.md#7-snmp-trap-integration) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. --- @@ -214,7 +214,7 @@ bash install.sh The wizard verifies and loads the included Docker image, asks for the selected notification channels, and deploys **only** the watchdog. **The v1.6.0 release archive becomes downloadable after the release is published.** -Read the full [Deployment Guide](docs/customer/INSTALLATION_AND_OPERATIONS.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md) and the [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md). +Read the full [Deployment Guide](docs/operations/INSTALLATION_AND_OPERATIONS.md) for installation, configuration and maintenance. Developers and release maintainers: use the version-independent [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md) and the [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). --- @@ -261,13 +261,13 @@ docker logs --tail 50 docker-container-watchdog Expected: the watchdog is running/healthy and monitoring logs are current. Check the configured notification receiver for any existing alerts. A healthy system does not need to generate a new alert just for verification. -Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_SMOKE_TESTING.md). +Do not simulate failures on operational hosts. Developers can use the isolated release test procedure in [Release Smoke Testing](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_SMOKE_TESTING.md). ## 8. Troubleshooting For installation failures, alert delivery, missing recovery, probe selection, persistent logs, maintenance, upgrades or rollback, use the -[Customer Installation and Operations Guide](docs/customer/INSTALLATION_AND_OPERATIONS.md#6-troubleshooting). +[Customer Installation and Operations Guide](docs/operations/INSTALLATION_AND_OPERATIONS.md#6-troubleshooting). Use read-only status/log checks on operational hosts, and do not simulate failures on application containers. @@ -277,11 +277,11 @@ failures on application containers. **Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/DEVELOPMENT_AND_RELEASE.md); version-specific approval for v1.6.0 is in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md). +**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md); version-specific approval for v1.6.0 is in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/customer/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | diff --git a/docs/dev/DEVELOPMENT_AND_RELEASE.md b/docs/dev/DEVELOPMENT_AND_RELEASE.md index 719ceff..56529c5 100644 --- a/docs/dev/DEVELOPMENT_AND_RELEASE.md +++ b/docs/dev/DEVELOPMENT_AND_RELEASE.md @@ -1,6 +1,6 @@ # Container Alert — Development and Release Guide -**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../customer/INSTALLATION_AND_OPERATIONS.md). +**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../operations/INSTALLATION_AND_OPERATIONS.md). ## Table of contents @@ -84,12 +84,12 @@ Update the following to match the chosen version: 1. `VERSION` — set the new number (the command above does this). 2. `README.md` — add the matching version-history row; CI checks this. -3. `docs/customer/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. +3. `docs/operations/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. 4. Deployment and examples if operator procedures/settings changed; keep private lab/customer identifiers and secrets out. 5. Tests and hosted smoke approval for the release PR. ```bash -git add VERSION README.md "docs/customer/RELEASE_NOTES_V$NEXT_VER.md" +git add VERSION README.md "docs/operations/RELEASE_NOTES_V$NEXT_VER.md" git commit -m "release: prepare v$NEXT_VER" git push -u origin "release/v$NEXT_VER" gh pr create --base main --head "release/v$NEXT_VER" --title "Release v$NEXT_VER" --fill @@ -158,7 +158,7 @@ If the workflow fails or the release is incomplete, do not distribute it. Inspec Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or approved internal transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. -YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../customer/INSTALLATION_AND_OPERATIONS.md#54-rollback). +YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../operations/INSTALLATION_AND_OPERATIONS.md#54-rollback). ## 7. Safety and best practices diff --git a/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md b/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md index 7505283..73e39e7 100644 --- a/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md +++ b/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md @@ -6,7 +6,7 @@ > customer-facing message updates. Do **not** rerun their Docker commands on > any shared or operational Docker Engine. Earlier nested testing triggered > real notifications on its parent host. -> See [current release notes](../customer/RELEASE_NOTES_V1.6.0.md) and +> See [current release notes](../operations/RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). **Validation date:** October 14, 2026 (UTC) diff --git a/docs/dev/RELEASE_V1.6.0_CHECKLIST.md b/docs/dev/RELEASE_V1.6.0_CHECKLIST.md index ad47038..a3de0c7 100644 --- a/docs/dev/RELEASE_V1.6.0_CHECKLIST.md +++ b/docs/dev/RELEASE_V1.6.0_CHECKLIST.md @@ -8,7 +8,7 @@ - [ ] Final engineering review of PR #9: alert/incident lifecycle, notifications, HA behavior, security and image packaging. - [ ] Normal GitHub CI **and** [Release smoke (isolated)](RELEASE_SMOKE_TESTING.md) both pass against the **latest PR head SHA**. The smoke is a disposable GitHub-hosted test; never run it against an operational Docker host. -- [ ] Document all changes since v1.5.4 in [release notes](../customer/RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. +- [ ] Document all changes since v1.5.4 in [release notes](../operations/RELEASE_NOTES_V1.6.0.md). Verify the README history includes `1.6.0`. - [ ] Confirm exact supported customer message text, severity mappings, and downstream Syslog/SNMP compatibility. - [ ] Confirm there are no private lab hostnames, IP addresses, personal information or credentials in the **published installer archive**, documentation and release notes. - [ ] Approve distributing the Docker image and complete release package through this **public** GitHub repository. @@ -59,6 +59,6 @@ After **all** approvals above: cat unpacked/CyberController-Container-Alert/VERSION ``` -6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../customer/INSTALLATION_AND_OPERATIONS.md). +6. Record tag, commit SHA, workflow link, package hash and final image digest. Inform operators that the *same* complete package works on connected and disconnected hosts, as described in [Deployment](../operations/INSTALLATION_AND_OPERATIONS.md). **Stop if a release gate fails.** Don't distribute a draft, incomplete or unapproved release. Do not run installation, Docker failure injection or HA failover tests on production systems as part of this checklist. diff --git a/docs/dev/RELEASE_VALIDATION_V1.6.0.md b/docs/dev/RELEASE_VALIDATION_V1.6.0.md index 51be1b0..65bddd9 100644 --- a/docs/dev/RELEASE_VALIDATION_V1.6.0.md +++ b/docs/dev/RELEASE_VALIDATION_V1.6.0.md @@ -6,7 +6,7 @@ > customer-facing message updates. Do **not** rerun their Docker commands on > any shared or operational Docker Engine. Earlier nested testing triggered > real notifications on its parent host. -> See [current release notes](../customer/RELEASE_NOTES_V1.6.0.md) and +> See [current release notes](../operations/RELEASE_NOTES_V1.6.0.md) and > [safe smoke workflow](RELEASE_SMOKE_TESTING.md). > **Historical baseline (pre-fix).** This page records the original HA-lab smoke-test failures. They were subsequently investigated using a separate real Docker Engine on a dedicated development host, and a minimal recovery fix passed the complete [final release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). Retain this document as evidence of the original findings, not the current release verdict. diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index d6e78ba..9d2e31d 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -249,7 +249,7 @@ The supported per-container probe fields are: Use the *actual* container name and a supported health endpoint only after validation. Manual `type: exec` checks execute commands inside monitored containers, so they must be approved and carefully reviewed. Do not add exclusions or broad ignore rules merely to silence unexpected alerts. -HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/engineering/HA_MONITORING_DESIGN.md) for limitations and approved maintenance testing. +HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/HA_MONITORING_DESIGN.md) for limitations and approved maintenance testing. ### 4.5 Environment variables diff --git a/docs/operations/RELEASE_NOTES_V1.6.0.md b/docs/operations/RELEASE_NOTES_V1.6.0.md index 6bec6fb..27c2b51 100644 --- a/docs/operations/RELEASE_NOTES_V1.6.0.md +++ b/docs/operations/RELEASE_NOTES_V1.6.0.md @@ -41,8 +41,8 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C ## Verification and safety boundaries - Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. -- Destructive real Docker failure testing is **only** through the [Release smoke (isolated)](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/.github/workflows/release-smoke.yml) workflow, automatically triggered for qualifying release PR changes and manually rerunnable when available on the default branch. It uses an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_SMOKE_TESTING.md). -- Historical **engineering-only** test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/engineering/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. +- Destructive real Docker failure testing is **only** through the [Release smoke (isolated)](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/.github/workflows/release-smoke.yml) workflow, automatically triggered for qualifying release PR changes and manually rerunnable when available on the default branch. It uses an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_SMOKE_TESTING.md). +- Historical **engineering-only** test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/dev/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. - Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. - Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. @@ -50,7 +50,7 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C The approved v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is approved and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. -The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/DEVELOPMENT_AND_RELEASE.md) and [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/engineering/RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/customer/INSTALLATION_AND_OPERATIONS.md). +The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/DEVELOPMENT_AND_RELEASE.md) and [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/operations/INSTALLATION_AND_OPERATIONS.md). ## Exclusions, limitations and release gating diff --git a/scripts/package_release.sh b/scripts/package_release.sh index f9e47dd..58e2ef0 100644 --- a/scripts/package_release.sh +++ b/scripts/package_release.sh @@ -28,12 +28,12 @@ work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT dir="$work/CyberController-Container-Alert" -mkdir -p "$dir/docs/customer" +mkdir -p "$dir/docs/operations" for f in install.sh uninstall.sh docker-compose.yaml VERSION \ .env.example watchdog-config.yaml.example README.md; do cp "$repo/$f" "$dir/$f" done -cp "$repo"/docs/customer/*.md "$dir/docs/customer/" # only customer guides ship +cp "$repo"/docs/operations/*.md "$dir/docs/operations/" # only customer guides ship cp "$image" "$dir/watchdog.tar" (cd "$dir" && sha256sum watchdog.tar > SHA256SUMS) diff --git a/tests/test_documentation.py b/tests/test_documentation.py index 732bf14..c6dc86c 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -51,7 +51,7 @@ def test_all_relative_links_and_headings_resolve(self): self.assertIn(anchor, heading_anchors(target.read_text())) def test_customer_docs_are_self_contained(self): - docs = [ROOT / "README.md", *(ROOT / "docs/customer").glob("*.md")] + docs = [ROOT / "README.md", *(ROOT / "docs/operations").glob("*.md")] for doc in docs: for match in LINK_RE.finditer(doc.read_text()): relative, _ = match.groups() @@ -59,7 +59,7 @@ def test_customer_docs_are_self_contained(self): continue target = (doc.parent / relative).resolve() self.assertNotIn( - ROOT / "docs/engineering", + ROOT / "docs/dev", target.parents, f"Customer bundle refers to internal engineering file: {doc}", ) @@ -68,8 +68,8 @@ def test_customer_docs_are_self_contained(self): def test_release_notes_links_work_when_published_as_github_release(self): # GitHub Release notes are rendered at the release URL, not at - # docs/customer/, so links to other files must be absolute. - for notes in (ROOT / "docs/customer").glob("RELEASE_NOTES_V*.md"): + # docs/operations/, so links to other files must be absolute. + for notes in (ROOT / "docs/operations").glob("RELEASE_NOTES_V*.md"): for match in LINK_RE.finditer(notes.read_text()): destination, _ = match.groups() if not destination or destination.startswith("#"): @@ -82,7 +82,7 @@ def test_release_notes_links_work_when_published_as_github_release(self): def test_operator_guide_covers_configuration_template_fields(self): config = yaml.safe_load((ROOT / "watchdog-config.yaml.example").read_text()) - guide = (ROOT / "docs/customer/INSTALLATION_AND_OPERATIONS.md").read_text() + guide = (ROOT / "docs/operations/INSTALLATION_AND_OPERATIONS.md").read_text() def keys(item, prefix=""): if isinstance(item, dict): diff --git a/tests/test_release_package.py b/tests/test_release_package.py index d211e1e..4395156 100644 --- a/tests/test_release_package.py +++ b/tests/test_release_package.py @@ -43,8 +43,8 @@ def test_one_complete_release_package(self): required = ("install.sh", "uninstall.sh", "docker-compose.yaml", "VERSION", "README.md", ".env.example", "watchdog-config.yaml.example", "watchdog.tar", "SHA256SUMS", - "docs/customer/INSTALLATION_AND_OPERATIONS.md", - f"docs/customer/RELEASE_NOTES_V{VERSION}.md") + "docs/operations/INSTALLATION_AND_OPERATIONS.md", + f"docs/operations/RELEASE_NOTES_V{VERSION}.md") for name in required: self.assertIn(PREFIX + name, files) self.assertEqual(files[PREFIX + "VERSION"].decode().strip(), VERSION) @@ -57,7 +57,7 @@ def test_bundle_contains_image_not_python_source(self): self.assertNotIn(PREFIX + "Dockerfile", files) self.assertNotIn(PREFIX + ".env", files) self.assertNotIn(PREFIX + "watchdog-config.yaml", files) - self.assertFalse(any(path.startswith(PREFIX + "docs/engineering/") for path in files)) + self.assertFalse(any(path.startswith(PREFIX + "docs/dev/") for path in files)) self.assertNotIn(PREFIX + "docs/DEVELOPMENT.md", files) def test_every_local_customer_link_is_packaged(self): From 3ac48f7ae6890591eb2c54699645508edc00062c Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 16:23:00 +0000 Subject: [PATCH 15/18] docs: group all notification channels under one reference --- README.md | 2 +- .../operations/INSTALLATION_AND_OPERATIONS.md | 86 +++++++++++-------- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index e85e7ad..6d5b585 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ A YAML-only change takes effect after restarting the watchdog. A changed `.env` | SNMP Trap | `snmp_trap` | NMS destination, SNMP version; SNMPv3 keys in `.env` | | Slack | `slack` | Webhook URL stored in `.env` | -At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/operations/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [advanced reference](docs/operations/INSTALLATION_AND_OPERATIONS.md#7-snmp-trap-integration) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. +At least one channel must be selected in the installer; multiple channels may be enabled together. **See [Deployment — Notification Channels](docs/operations/INSTALLATION_AND_OPERATIONS.md#43-notification-channels) for all configuration fields and examples.** The [SNMP trap receiver details](docs/operations/INSTALLATION_AND_OPERATIONS.md#snmp-traps) retains SNMPv3 receiver setup and the Radware trap OID/var-bind definitions. --- diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index 9d2e31d..27f83c1 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -13,6 +13,10 @@ This guide covers **installation, configuration, day-to-day operations, upgrades - [Configuration files and editing](#41-configuration-files-and-editing) - [Watchdog monitoring settings](#42-watchdog-monitoring-settings) - [Notification channels](#43-notification-channels) + - [Syslog](#syslog) + - [SMTP (email)](#smtp-email) + - [Slack](#slack) + - [SNMP traps](#snmp-traps) - [Health checks, exclusions, and HA](#44-health-checks-exclusions-and-ha) - [Environment variables](#45-environment-variables) - [Apply configuration changes](#46-apply-configuration-changes) @@ -25,8 +29,7 @@ This guide covers **installation, configuration, day-to-day operations, upgrades 6. [Troubleshooting](#6-troubleshooting) - [Missing recovery notifications](#61-missing-recovery-notifications) - [Read-only logs and probe diagnostics](#62-read-only-logs-and-probe-diagnostics) -7. [SNMP trap integration](#7-snmp-trap-integration) -8. [References and support](#8-references-and-support) +7. [References and support](#7-references-and-support) ## 1. Prerequisites @@ -172,7 +175,7 @@ Changes to exclusions, thresholds, probe methods or suppression rules can mask r ### 4.3 Notification channels -For any channel, its identifier must be present in `alert_channels` **and** its configuration must be enabled. Enabling a channel doesn't automatically prove that its destination received an alert. +The installer records the selected delivery channels and their settings. **Slack, SMTP, and SNMP traps** use both `alert_channels` and their channel's `enabled` setting (Slack defaults to enabled if omitted). **Syslog** instead forwards through the logging subsystem when `syslog.enabled: true`; listing it in `alert_channels` alone does not activate forwarding. All channels require a reachable receiver, and secret-based channels require credentials in `.env`. | Channel | YAML field | Purpose | |---|---|---| @@ -195,12 +198,13 @@ For any channel, its identifier must be present in `alert_channels` **and** its | SNMPv3 | `v3_auth_key_env`, `v3_priv_key_env` | Names of passphrase variables in `.env`; do not embed secrets in YAML | | SNMPv3 | `v3_local_engine_id` | Optional fixed sender engine ID used by receiving NMS registration | -Example: changing the Syslog destination **after installation**: +#### Syslog -```yaml -alert_channels: - - syslog +Set `syslog.enabled: true` and provide the receiver host, port, protocol and facility. Syslog forwards structured alert records (including INFO recoveries) independently of the local console/file log threshold. The installer may list `syslog` in `alert_channels` for consistency, but the actual forwarding switch is `syslog.enabled`. + +Example receiver settings after installation: +```yaml syslog: enabled: true host: syslog.example.com @@ -209,9 +213,43 @@ syslog: facility: local0 ``` -When enabling Syslog **alongside** existing channels, add `syslog` to the existing list rather than replacing the list as shown in this single-channel example. Restart the watchdog after a YAML-only change. +Restart the watchdog after an approved YAML-only change. Confirm receipt at the configured Syslog receiver. + +#### SMTP (email) + +Include `smtp` in `alert_channels` and set `smtp.enabled: true`. Configure the mail host, port, sender, recipients and `smtp.tls` in YAML. With `tls: true`, the watchdog uses SMTP STARTTLS. Use `smtp.auth: false` only for an approved unauthenticated relay. If authentication is enabled, store `SMTP_USERNAME` and `SMTP_PASSWORD` in `.env` (the YAML fields `username_env` and `password_env` reference those variable names). + +#### Slack + +Include `slack` in `alert_channels` and set `slack.enabled: true` (or omit `enabled`, which defaults to true for compatibility). Store the incoming webhook URL in `.env` as `SLACK_WEBHOOK_URL`; the YAML `slack.webhook_url_env` names that variable. Do not embed the webhook URL in YAML or commit it to the repository. + +#### SNMP traps + +Include `snmp_trap` in `alert_channels`, set `snmp_trap.enabled: true`, and configure the NMS receiver host, UDP port (normally 162) and SNMP version. SNMPv1/v2c use the configured community string; SNMPv3 credentials are referenced from `.env`. The additional details below are **receiver-side SNMP integration**, not a separate watchdog feature. + +The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. + +Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): + +```text +createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" +``` + +Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. + +**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: -For SMTP authentication, store the username/password in `.env`; for Slack, store the webhook in `.env`. SNMPv3 passphrases also belong in `.env` and must meet protocol requirements (at least eight characters). See [SNMP trap integration](#7-snmp-trap-integration) for SNMPv3 receiver/engine-ID setup and the supported trap OIDs. +| OID | Object | Value | +|---|---|---| +| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | +| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | +| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | +| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | +| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | + +HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. + +For a YAML-only channel change, restart the watchdog. For changed `.env` secrets, recreate **only** the watchdog as described in [Apply configuration changes](#46-apply-configuration-changes). Do not restart any monitored application container to verify channel delivery. ### 4.4 Health checks, exclusions, and HA @@ -424,35 +462,7 @@ incident state; do not delete alert history simply to suppress notifications. For authorization or assistance obtaining the release, contact the appropriate Radware support channel. Never change production CyberController service settings solely in response to a watchdog alert. -## 7. SNMP trap integration - -The watchdog supports SNMP v1/v2c/v3 and uses the existing Radware -Container Alert enterprise notification OID. An NMS operator can configure -a receiver using this reference without modifying CyberController services. - -The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. - -Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): - -```text -createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" -``` - -Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. - -**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: - -| OID | Object | Value | -|---|---|---| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | - -HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. - -## 8. References and support +## 7. References and support - [Project overview](../../README.md) — product behavior, alert categories and version history. - [Release notes](RELEASE_NOTES_V1.6.0.md) — changes, compatibility and limitations for v1.6.0. From 8732fda7f260cf64ac9dfcd58368f8a5795d2da5 Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 16:34:52 +0000 Subject: [PATCH 16/18] docs: add direct CLI release download for connected hosts --- docs/operations/INSTALLATION_AND_OPERATIONS.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index 27f83c1..1d57eee 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -64,12 +64,21 @@ The filename below uses v1.6.0 as an example. Release assets exist only after th ### 2.1 Get and verify the release package -**Step 1 — Obtain these two files** from the approved [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases) (or through Radware's authorized offline transfer method): +**Step 1 — Obtain the release package and checksum.** Use either method to get the **same two files** on the target host: -- `container-alert-v1.6.0.tar.gz` — complete install package, including `watchdog.tar`. -- `SHA256SUMS` — checksum to verify the downloaded/transferred package. +**Host has Internet access (simplest):** Run these commands directly on the target host, in your chosen installation directory: -**Step 2 — Copy both files to the target host**, into the same directory. Verify the package and extract it: +```bash +VER=1.6.0 +URL="https://github.com/Radware/CyberController-Container-Alert/releases/download/v${VER}" +curl -fL -O "$URL/container-alert-v${VER}.tar.gz" -O "$URL/SHA256SUMS" +``` + +No Git clone, GitHub CLI, browser download, or SCP is needed. `curl` requires HTTPS access to the GitHub Release and its redirected download host. + +**Disconnected host:** Obtain `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` from the [approved GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases) on a connected machine, then transfer both to the target host using your authorized method. + +**Step 2 — Verify and extract** both files in the same directory: ```bash sha256sum --check SHA256SUMS From cb5610bfda005e98a5ed1cd6a54e2a782a837d3b Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Fri, 16 Oct 2026 16:43:01 +0000 Subject: [PATCH 17/18] docs: choose published release version before downloading --- .../operations/INSTALLATION_AND_OPERATIONS.md | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index 1d57eee..0d37498 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -64,33 +64,36 @@ The filename below uses v1.6.0 as an example. Release assets exist only after th ### 2.1 Get and verify the release package -**Step 1 — Obtain the release package and checksum.** Use either method to get the **same two files** on the target host: +**Step 1 — Choose an approved release version.** Open the [Container Alert Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) to see all published versions. The release marked **Latest** is usually the newest stable version, but use the version approved for this environment; do not install a pre-release unless specifically authorized. For example, the tag `v1.6.0` means you set `VER=1.6.0` (without the `v`). Versions appear on the page only after they are published. -**Host has Internet access (simplest):** Run these commands directly on the target host, in your chosen installation directory: +**Step 2 — Obtain the release package and checksum.** Use either method to get the **same two files** on the target host: + +**Host has Internet access (simplest):** Run on the target host in the intended installation directory: ```bash -VER=1.6.0 +VER=1.6.0 # Replace with the approved published version from Step 1 URL="https://github.com/Radware/CyberController-Container-Alert/releases/download/v${VER}" curl -fL -O "$URL/container-alert-v${VER}.tar.gz" -O "$URL/SHA256SUMS" ``` -No Git clone, GitHub CLI, browser download, or SCP is needed. `curl` requires HTTPS access to the GitHub Release and its redirected download host. +No Git clone, GitHub CLI, browser download, or SCP is required. `curl` needs HTTPS access to GitHub Releases and its redirected download host. -**Disconnected host:** Obtain `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` from the [approved GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases) on a connected machine, then transfer both to the target host using your authorized method. +**Disconnected host:** Download `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (adjust for the selected version) from that release on a connected workstation, then transfer both to the target host using the approved method. -**Step 2 — Verify and extract** both files in the same directory: +**Step 3 — Verify and extract** the two files from the same directory: ```bash +VER=1.6.0 # Use the same version chosen in Step 1 sha256sum --check SHA256SUMS -tar -xzf container-alert-v1.6.0.tar.gz +tar -xzf "container-alert-v${VER}.tar.gz" cd CyberController-Container-Alert ``` -The extracted directory contains `VERSION`, `install.sh`, `docker-compose.yaml`, `watchdog.tar` and its own image checksum. Use the matching filenames for later versions. +The extracted directory contains `VERSION`, `install.sh`, `docker-compose.yaml`, `watchdog.tar` and its own image checksum. ### 2.2 Run the installer -**Step 3 — Start the configuration wizard:** +**Step 4 — Start the configuration wizard:** ```bash bash install.sh @@ -98,7 +101,7 @@ bash install.sh The installer automatically verifies the included Docker image checksum and version, loads the image, then asks you to select and configure the notification channels (Syslog, SMTP, SNMP and/or Slack) and alert hostname. On an upgrade, answer **No** to “Reconfigure from scratch?” to preserve existing channel settings. It recreates **only the watchdog**, not other application containers. -**Step 4 — Confirm normal operation:** +**Step 5 — Confirm normal operation:** ```bash docker ps --filter name=docker-container-watchdog From 673040c1512d50ba2cd424fe50e8d79fef62776a Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Mon, 19 Oct 2026 14:55:02 +0000 Subject: [PATCH 18/18] v1.6.0 --- README.md | 12 +- docs/dev/DEVELOPMENT_AND_RELEASE.md | 63 ++- docs/dev/HA_MONITORING.md | 57 +++ docs/dev/HA_MONITORING_DESIGN.md | 182 ------- docs/dev/HA_VALIDATION_2026-10-13.md | 46 -- docs/dev/RELEASE_SMOKE_TESTING.md | 4 +- .../operations/INSTALLATION_AND_OPERATIONS.md | 468 ++++++++---------- docs/operations/RELEASE_NOTES_V1.6.0.md | 14 +- install.sh | 16 +- watchdog-config.yaml.example | 2 +- 10 files changed, 327 insertions(+), 537 deletions(-) create mode 100644 docs/dev/HA_MONITORING.md delete mode 100644 docs/dev/HA_MONITORING_DESIGN.md delete mode 100644 docs/dev/HA_VALIDATION_2026-10-13.md diff --git a/README.md b/README.md index 6d5b585..0228ed3 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ The following is the **source repository** layout, not the customer archive. The | Running container (memory) | ~50–80 MB baseline; capped at `mem_limit: 256m` in `docker-compose.yaml` | | `watchdog.tar` export | Approximate; inspect the actual published release | -> **Note:** Size and memory figures above are prior estimates, not guaranteed release specifications. Actual image size varies by build. If resource usage is abnormal, capture Docker health, logs, and usage observations for engineering review before making production resource-limit changes. +> **Note:** Size and memory figures above are prior estimates, not guaranteed release specifications. Actual image size varies by build. If resource usage is abnormal, collect Docker health, logs and resource metrics. Do not change resource limits based only on a monitoring alert. ### Container Hardening @@ -201,9 +201,9 @@ At least one channel must be selected in the installer; multiple channels may be ## 5. Deployment Instructions -Install **one complete versioned package** from an approved [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), or obtain the same package through authorized offline transfer. No Git clone, separate Docker image download, or source build is required on the customer host. +Install **one complete versioned package** from a published [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases), or transfer the same package from another machine. No Git clone, separate Docker image download, or source build is required on the customer host. -Download/transfer both `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (substitute your approved version). From the directory containing those two files: +Download/transfer both `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (substitute the version you selected). From the directory containing those two files: ```bash sha256sum --check SHA256SUMS @@ -277,11 +277,11 @@ failures on application containers. **Version 1.6.0 (2026-10-15):** This section documents v1.6.0 and changes since v1.5.4. Release artifacts are built, tested and published separately; updating the source checkout does not upgrade running containers. -**Release procedure:** Merge reviewed release changes into `main`, then push the approved matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md); version-specific approval for v1.6.0 is in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). +**Release procedure:** Merge reviewed release changes into `main`, then push the matching `vX.Y.Z` tag. GitHub Actions builds the image, assembles one complete customer package and its checksum, and publishes the [GitHub Release](https://github.com/Radware/CyberController-Container-Alert/releases). PR merges do **not** publish automatically. Follow the reusable [Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/DEVELOPMENT_AND_RELEASE.md); v1.6.0 publication checks are in its [finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/RELEASE_V1.6.0_CHECKLIST.md). | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.0 | 2026-10-15 | Radware engineering | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, approved informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). | +| 1.6.0 | 2026-10-15 | — | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, revised informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | | 1.5.2 | 2026-08-31 | Rahul Kumar | Updated error message"Suppressing expected Cyber Controller SQL dump syntax-check container termination (exit 137) alert" | @@ -290,7 +290,7 @@ failures on application containers. | 1.4.0 | 2026-08-17 | Rahul Kumar | Added INFO "recovered" alert | | 1.3.3 | 2026-08-05 | Rahul Kumar | Added Auth True/false | | 1.3.2 | 2026-07-28 | Rahul Kumar | Fixed SMTP Auth issue | -| 1.3.1 | 2026-07-21 | Radware engineering | Updated Readme and Deployment guides | +| 1.3.1 | 2026-07-21 | — | Updated Readme and Deployment guides | | 1.3.0 | 2026-07-16 | Rahul Kumar | Added Upgrade Guide | | 1.2.0 | 2026-07-16 | Rahul Kumar | Removed sudo and renamed the container | | 1.1.0 | 2026-07-09 | Rahul Kumar | Added SNMPv3 support | diff --git a/docs/dev/DEVELOPMENT_AND_RELEASE.md b/docs/dev/DEVELOPMENT_AND_RELEASE.md index 56529c5..7556941 100644 --- a/docs/dev/DEVELOPMENT_AND_RELEASE.md +++ b/docs/dev/DEVELOPMENT_AND_RELEASE.md @@ -1,6 +1,6 @@ # Container Alert — Development and Release Guide -**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. For the specific approval status of v1.6.0, see [its release checklist](RELEASE_V1.6.0_CHECKLIST.md). Customers follow [Deployment](../operations/INSTALLATION_AND_OPERATIONS.md). +**Reusable for every release.** This guide goes from changing source code to publishing and verifying a customer installation package. To deploy, follow [Installation and Operations Guide](../operations/INSTALLATION_AND_OPERATIONS.md). ## Table of contents @@ -26,6 +26,10 @@ All versions use the **same workflow, package format and installer**. A release Changes in Git do **not** upgrade running customer containers. Publication and installation are separate actions. +**End-to-end sequence:** change code → open a development PR → review/test → merge the development PR → prepare and review a release PR → **merge the release PR into `main`** → tag the resulting `main` commit → GitHub publishes the package → verify the release. + +If a development PR targets an existing `release/vX.Y.Z` branch, it merges into that branch first. The release PR later brings those changes into `main`. Do not merge the same changes twice. + ## 2. Make and test a change **Step 1 — Create a development branch in a clean checkout.** @@ -36,7 +40,7 @@ git pull --ff-only origin main git switch -c fix/describe-change ``` -Use `feature/...`, `fix/...` or `docs/...`. If there is an active stabilization branch for an upcoming release, coordinate whether your PR should target that release branch instead of main. Multiple worktrees are supported. +Use `feature/...`, `fix/...` or `docs/...`. If there is an active stabilization branch for an upcoming release, check whether your PR should target that release branch instead of main. Multiple worktrees are supported. **Step 2 — Change code, tests and documentation.** Typical files include `watchdog.py`, `lifecycle.py`, `ha_monitor.py`, `install.sh`, `scripts/package_release.sh`, and `tests/`. Don't reuse a previously published Docker image version for changed Python code. @@ -66,7 +70,11 @@ gh pr view -R Radware/CyberController-Container-Alert gh pr checks -R Radware/CyberController-Container-Alert ``` -Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. Code review and approval are still required when checks pass. +Normal GitHub CI runs unit/static/build checks. Only the [hosted isolated smoke workflow](../../.github/workflows/release-smoke.yml) may perform destructive real-Docker acceptance. For release-critical changes it must pass at the latest reviewed PR head. + +**Step 5 — Merge the development PR.** Once the latest changes have been reviewed and checks pass, open the PR on GitHub, confirm its **base branch** (`main` or the active release branch), and choose **Merge pull request**. GitHub applies any branch protection rules. + +A PR targeting `main` puts its changes directly into `main`; a PR targeting `release/vX.Y.Z` updates that release branch. **Do not merge the release PR at this step**—its final merge happens in [Section 4](#4-review-merge-and-tag), after release preparation and testing. ## 3. Prepare the next version @@ -86,7 +94,7 @@ Update the following to match the chosen version: 2. `README.md` — add the matching version-history row; CI checks this. 3. `docs/operations/RELEASE_NOTES_V.md` — create the matching version-specific release notes, including all changes since the previous published release, operational impact, limitations and test evidence. 4. Deployment and examples if operator procedures/settings changed; keep private lab/customer identifiers and secrets out. -5. Tests and hosted smoke approval for the release PR. +5. Tests and hosted smoke results for the release PR. ```bash git add VERSION README.md "docs/operations/RELEASE_NOTES_V$NEXT_VER.md" @@ -97,28 +105,36 @@ gh pr create --base main --head "release/v$NEXT_VER" --title "Release v$NEXT_VER Change only `NEXT_VER` at the start of the release commands and use the same shell session, or set it again later. Write actual release notes before staging; the quoted filename in `git add` must exist. Stage all other modified files explicitly. If the release branch already exists, update it and its PR instead of creating a second one. +**Leave this release PR open** while you review and run CI and isolated smoke. The next step is the final merge into `main`. + ## 4. Review, merge and tag -Before release approval: verify independent review, complete notes, current PR CI and isolated smoke, rollback guidance, image/package contents, and **Radware approval to publish assets publicly** (this GitHub repository is public). +**This is where you merge the release PR into `main`—after final validation, before tagging.** + +**Step 1 — Review the release PR.** Confirm its base branch is `main`, the release notes and rollback instructions are ready, and regular CI plus isolated smoke passed on the **latest PR head commit**. GitHub release assets are public in this repository, so check the contents before publishing. + +**Step 2 — Merge the release PR.** On GitHub, open the release PR created in Section 3, review **Files changed** and **Checks**, then choose **Merge pull request** and confirm. You do not need to merge earlier feature PRs again. + +This merges `release/vX.Y.Z` into `main`. **It does not publish the release yet.** -Merge the approved release PR using GitHub's normal authorized process. Then, on a **clean checkout**: +**Step 3 — Sync your local `main`.** Use a clean checkout; commit, set aside, or finish any uncommitted work before switching branches: ```bash git switch main git pull --ff-only origin main git status --short # must be empty -cat VERSION # record approved version -git log -1 --oneline # record main's release commit +cat VERSION # check intended release version +git log -1 --oneline # record merged main commit ``` -Confirm the tag does not already exist; the next command should print **nothing**: +**Step 4 — Check for an existing tag.** The following tag query should print **nothing**: ```bash VER="$(cat VERSION)" git ls-remote --tags origin "refs/tags/v$VER" ``` -**Only after explicit approval**, tag the exact approved main commit: +**Step 5 — Create and push the release tag.** Tag the **merged `main` commit**, not the release branch: ```bash git tag -a "v$VER" -m "Container Alert v$VER" @@ -152,21 +168,21 @@ cat extracted/CyberController-Container-Alert/VERSION Record the Git tag/commit, CI/smoke URLs, release link, package digest and final image digest. The tagged build may not be byte-identical to the PR build while image bases/dependencies can float; pinning/signing is a separate supply-chain improvement. -If the workflow fails or the release is incomplete, do not distribute it. Inspect the failed workflow/draft and follow the approved correction process; **never rewrite an existing published tag**. +If the workflow fails or the release is incomplete, do not distribute it. Inspect the failed workflow and draft release before retrying; **never rewrite an existing published tag**. ## 6. Customer delivery and rollback -Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or approved internal transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. +Operators obtain `container-alert-v.tar.gz` and `SHA256SUMS` from the release or through file transfer. They verify, extract, and run `bash install.sh`. The archive works equally well on connected and disconnected hosts. It doesn't include active `.env`, `watchdog-config.yaml` or persistent `watchdog/`, so upgrades preserve customer settings and history and recreate **only** the watchdog service. -YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the last approved image and configuration; follow [Deployment — Rollback](../operations/INSTALLATION_AND_OPERATIONS.md#54-rollback). +YAML-only changes require a watchdog restart; `.env`/Compose changes require recreating that service. Python changes require a new tested and published image. Rollback means selecting the previous release image and a compatible configuration; follow [Deployment — Rollback](../operations/INSTALLATION_AND_OPERATIONS.md#54-rollback). ## 7. Safety and best practices - Never run destructive Docker tests on customer, production or shared operational hosts. Use mock Docker unit tests locally and isolated GitHub-hosted Docker acceptance. - Do not stop or alter CyberController application containers during installation or tests. -- The SHA-256 manifest verifies contents, not the publisher's identity. Only distribute approved release packages. +- The SHA-256 manifest verifies contents, not the publisher's identity. Use the expected release source and verify the package checksum. - Never publish personal lab hostnames, IP addresses, passwords or customer-specific data. -- Require code review and explicit public-distribution approval before tagging. Image builds use prebuilt, versioned artifacts rather than compiling on customer systems. +- Review the code, test results and public visibility of release assets before tagging. Image builds use prebuilt, versioned artifacts rather than compiling on customer systems. References: [GitHub Flow](https://docs.github.com/en/get-started/using-github/github-flow), [Managing Releases](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository), [About releases](https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases). @@ -175,7 +191,7 @@ References: [GitHub Flow](https://docs.github.com/en/get-started/using-github/gi ### Local image build (never a customer installation) -For authorized development: +For development on a disposable build environment: ```bash export WATCHDOG_VERSION="$(cat VERSION)" @@ -191,16 +207,15 @@ Production deployment requires a published, prebuilt release package. The intended MariaDB HA syntax-check helper carries the explicit label `com.radware.cybercontroller.role=mariadb-ha-syntax-check`. The configured `ignored_container_events` policy ignores only the designated temporary helper `die` event; normal service container failures remain monitored. -A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Any change to monitoring exceptions belongs to Radware R&D/authorized operations, not routine customer alert handling. +A narrow compatibility fallback exists for dynamic MariaDB dump helpers: it applies only when the helper image, non-primary container name, Docker `AutoRemove` status, expected SQL dump file mount and exit status 137 **all** match. This is not a general rule to suppress MariaDB failures. Do not alter these narrow exceptions simply to suppress alerts; validate changes with regression tests. -These exceptions are not blanket MariaDB or OOM alert suppression. Changes -to monitoring exceptions require R&D review and regression validation. +These exceptions are not blanket MariaDB or OOM alert suppression. Changes to monitoring exceptions require regression validation. ### Historical design and validation -The [HA design](HA_MONITORING_DESIGN.md) and [dated HA snapshot](HA_VALIDATION_2026-10-13.md) -preserve earlier evidence. The [release acceptance](RELEASE_ACCEPTANCE_V1.6.0.md) -and [earlier failures](RELEASE_VALIDATION_V1.6.0.md) are historical reports, -not current installation instructions or release approval. Use the -[release checklist](RELEASE_V1.6.0_CHECKLIST.md) and actual GitHub checks. +The [HA monitoring design and verification notes](HA_MONITORING.md) retain the reasons behind +the HA decisions and the validation that supports them. The +[release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md) and +[earlier test findings](RELEASE_VALIDATION_V1.6.0.md) are release-specific test +records, not alternative installation instructions. Use the diff --git a/docs/dev/HA_MONITORING.md b/docs/dev/HA_MONITORING.md new file mode 100644 index 0000000..1b5ce4e --- /dev/null +++ b/docs/dev/HA_MONITORING.md @@ -0,0 +1,57 @@ +# HA Monitoring — Design Decisions and Verification + +**Purpose:** Explain why the optional active-node change alert works as it does, what it detects, and what its limits are. This is an engineering reference, not an installation procedure or a release checklist. Operator settings are in [Installation and Operations](../operations/INSTALLATION_AND_OPERATIONS.md#44-health-checks-exclusions-and-ha). + +## Decision summary + +| Decision | Why | +|---|---| +| Observe the **promoted/active HA Operator** (`HAOperatorRep-Res`), not the Pacemaker Designated Controller (DC) | The DC is the Pacemaker coordinator; it can be on the standby node and is **not** evidence of the active CyberController role | +| Use the existing **local Pacemaker HA exporter** at `http://127.0.0.1:9664/metrics` | No additional daemon, privileged host command, container, firewall opening, cross-node API call, or exporter is needed | +| Run HA polling in a **separate thread and HTTP session** from Docker polling | Slow or blocked Docker API calls must not delay HA observations; HA exporter failures must not interrupt container monitoring | +| Require **quorum, a successful Pacemaker scrape, one local node and exactly one promoted owner** | Missing or conflicting telemetry is not proof of a role change; fail closed instead of inventing a failover | +| Require **two consecutive valid observations** of a new active node | Avoid reporting transient/promoted-resource ambiguity while the cluster is transitioning | +| **Only the newly active node** emits the alert | Watchdogs run on both nodes; duplicate notifications from old and new active nodes would be misleading | +| Persist the confirmed active node using the existing state file | Survive watchdog restarts without replaying an old role change or creating a second state store | +| Reuse the existing notification channels and SNMP MIB | Treat HA as another monitored event without adding a new alert transport or breaking container notifications | + +## How the monitor decides + +1. Disabled by default. When enabled, each node reads its **own** exporter using the normal `check_interval_seconds` (default 60 seconds). Requests have a 3-second timeout, a 256 KiB response cap, and permit only loopback HTTP `/metrics` URLs with no redirects. +2. It reads `ha_cluster_pacemaker_resources{resource="HAOperatorRep-Res",role="promoted",status="active"} == 1` to identify the active owner and `ha_cluster_corosync_member_votes{local="true"}` to identify the local node. The Pacemaker scrape-success and quorum metrics must both be 1. +3. The **first valid reading** establishes a silent baseline. An active-node change must appear in **two valid consecutive checks** before it is confirmed. +4. On a confirmed transition A → B, both nodes can update their saved baseline, but **only the watchdog local to B** issues one `WARNING` `ha-failover` event with previous/new node details. +5. Invalid/unavailable metrics leave the last confirmed baseline unchanged and reset a pending candidate. Exporter errors are logged at most once every five minutes; they never trigger a speculative failover alert. +6. HA state is kept separately from container incidents in `watchdog/state.json`. Restart with monitoring enabled restores that state. Running the watchdog with HA monitoring disabled clears its baseline, so a later re-enable starts silently. +7. During the host reboot grace interval, HA role checks are deferred. HA events are not included in container recovery or reboot-summary logic. + +The alert means **a promoted active role was observed to change**. It does **not** identify the cause (planned switchover versus unexpected failover) or guarantee all applications are healthy. The detection timestamp is the observation time, not necessarily the Pacemaker event time. + +## Notification and state trade-offs + +- Existing Syslog, SMTP, Slack and SNMP delivery paths are reused. The alert is `ha-failover`, not a container crash, and must not produce a fake container recovery. +- SNMP retains the existing notification OID and five var-binds: `cwContainerName=(HA cluster)` and `cwFailureType=ha-failover` distinguish HA events; previous/new owners appear in the summary and details. +- Delivery is **best effort**: the confirmed baseline is persisted before dispatch, and failed delivery is logged but not automatically retried. This avoids duplicates from repeated polls; it does **not** guarantee exactly-once receiver delivery. Failed persistence can allow a duplicate after restart. Adding a durable delivery queue would change the notification architecture and is intentionally out of scope. + +## What verification established + +| Check | Result / implication | +|---|---| +| Parser, state and failure-mode unit tests | Covered correct promotion/local identity, bad metrics, no quorum, startup silence, two-poll confirmation, failback, disabled/re-enabled baseline, persistence, and notification formatting | +| Docker polling deliberately blocked in a mocked test | The independent HA thread continued; Docker and HA checks do not depend on each other's progress | +| Real two-node **switchover and failback** | Observed one new-owner WARNING per direction and no old-owner duplicate | +| Actual Syslog delivery on both nodes | HA failover and failback records reached the configured Syslog receivers | +| Transient ambiguous exporter reading during a transition | An invalid reading did not cause a false HA event; later valid readings produced the correct notification | + +The [release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md#ha-acceptance-carried-forward-actual-pacemaker-operations) contains the supporting lab results. Future regressions are in [`tests/test_ha_monitor.py`](../../tests/test_ha_monitor.py). These are prior validation results, not evidence that every delivery channel or later code/image version has been validated. + +## Limits and lessons retained + +- Both HA nodes need a running watchdog, accessible **local** exporter, and configured notification path. No access from one node to the other's exporter is necessary. +- If a surviving node has never learned the earlier active owner, it starts with a silent baseline. The monitor cannot reconstruct older transitions. Fast changes that happen entirely between polls can also be missed. +- Unexpected active-node loss, network partitions, fencing/STONITH and SMTP/Slack/physical-NMS SNMP delivery were **not** demonstrated end to end by the recorded two-node acceptance. Do not infer coverage from successful switchover/failback tests. +- During a planned HA demotion, removal of dependent application containers previously produced separate CRITICAL container alerts. HA role-change detection is intentionally separate from Docker failure detection; suppressing those application events requires a distinct, narrowly scoped rule and its own tests. +- The older native Prometheus HA rule used location-score information that did not reliably prove promotion; its exporter scrape also targeted a different port. Rather than modifying that native monitoring, this feature reads the validated local HA Operator metrics directly. +- No HA control, resource promotion, remediation, Pacemaker writes, cluster configuration changes, or new ports are part of this feature. + +**When revisiting the design:** first confirm exporter metric names and HA Operator resource identity on the target CyberController version. Preserve fail-closed checks, independent polling and quiet baseline. Run mocked regression tests and use only a controlled, separate HA test environment for real switchovers; never induce failover in an operational cluster to test documentation. diff --git a/docs/dev/HA_MONITORING_DESIGN.md b/docs/dev/HA_MONITORING_DESIGN.md deleted file mode 100644 index cce12fc..0000000 --- a/docs/dev/HA_MONITORING_DESIGN.md +++ /dev/null @@ -1,182 +0,0 @@ -> **Historical design proposal — 2026-10-09.** This is the planning snapshot for -> the optional HA feature, **not its current release status or installation -> procedure**. PR #4 was subsequently merged into `main` on 2026-10-14. -> Later failover/failback acceptance appears in the -> [release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). -> Unchecked design checkboxes below are kept for traceability, not current release gates. - -# Optional CyberController HA Active-Node Alert — Design Proposal - -**Status on 2026-10-09:** Feature under review; failback and delivery validation pending at the time -**Date:** 2026-10-09 -**Target:** Radware/CyberController-Container-Alert, baseline v1.5.4 (main: a4ddfed) -**Delivery:** One small implementation PR, with release blocked on two-node QA - -## 1. Goal and decision - -Customer request: alert when CyberController HA changes its active node, including a manual switchover, automatic failover, or failback. The alert should identify the old and new active nodes, not speculate about the cause. - -**Decision:** Add one opt-in HA role-change check to the existing Container Watchdog. Read the existing HA exporter directly over host-local HTTP. Reuse the watchdog's polling interval, persistent state, and existing syslog/SMTP/SNMP/Slack delivery. Do not introduce another daemon, Docker container, metric exporter, firewall port, or Pacemaker control capability. - -This is an **observed active-node change** alert, not a claim to identify why a failover happened or that all CyberController applications are healthy. - -## 2. HA validation evidence (read-only inspection) - -Node addresses below are anonymized using documentation-only IPs. - -In the two-node HA test environment, Pacemaker reports nodes 192.0.2.41 and 192.0.2.42 online. The HA Operator, OpenSearch, PostgreSQL, MariaDB, and Failback promotable resources are currently **Promoted on .41** and **Unpromoted on .42**. The Pacemaker DC is .42; **DC is not the CyberController active node**. - -- HA exporter: **ha_cluster_exporter.service**, active on .41, serving port **9664** and reachable at **http://127.0.0.1:9664/metrics** (HTTP 200). -- Promoted role metric: **ha_cluster_pacemaker_resources** includes resource=HAOperatorRep-Res, role=promoted, status=active, node=192.0.2.41, value 1. -- Host identity metric: **ha_cluster_corosync_member_votes** includes local=true, node=192.0.2.41. -- Data-quality signals: **ha_cluster_scrape_success{collector="pacemaker"} = 1** and **ha_cluster_corosync_quorate = 1**. -- Existing Watchdog v1.5.4 runs on .41 in **host network** mode, polls every 60 seconds, has persistent state, and currently delivers syslog. -- Prometheus's native HA target is misconfigured: it scrapes **localhost:9002**, while the exporter serves **9664**. The existing rule **ha_failover_occured** compares per-node location-constraint sum vs average and is not reliable for an actual role change. Correcting the native configuration is a **separate Radware/R&D action**, not part of this feature. -- Standby .42 did **not** accept cross-node connections to 9664/9099/9093. It has not been inspected locally, so its exporter/watchdog installation is **unverified**. Release testing must cover both nodes. - -## 3. Small architecture - -~~~text -Per CyberController node: - Existing ha_cluster_exporter :9664/metrics (localhost only) - | - | bounded HTTP GET from independent HA polling thread (60s) - v - Optional HA role check: - identify local node + unique promoted HA Operator - validate quorum / scrape health - compare with persisted last confirmed active node - confirm role change on 2 consecutive successful polls - | - | only the newly promoted node sends - v - Existing watchdog alert dispatcher - syslog / SMTP / SNMP / Slack - -Both HA nodes must run the watchdog and have their own local exporter. -No inter-node polling or new firewall opening is required. -~~~ - -The HA check runs in its own lightweight thread with a dedicated HTTP session. Slow/blocking Docker probes cannot delay HA polling, and failed HA reads cannot break Docker monitoring. This is a single additional thread within the existing process, not a new service, container, queue, or worker framework. - -### Signal selection - -**Active owner:** accept exactly one node for which the HA exporter reports: - -~~~text -ha_cluster_pacemaker_resources{ - resource="HAOperatorRep-Res", - role="promoted", - status="active", - node="" -} 1 -~~~ - -The check must not use the Pacemaker DC, host virtual IP alone, or location-constraint scores as evidence of an actual promotion. This resource was verified in the current CC version; release testing must confirm the resource name is consistent on the targeted CC versions. - -**Sender identity:** derive the local node from the exporter's **ha_cluster_corosync_member_votes{local="true",node=...}** series. Do not guess local identity from WATCHDOG_HOST, container hostname, or the Prometheus instance label. - -**Validity:** require a successful HTTP response, valid/parseable metrics, successful Pacemaker collection, quorum present, exactly one promoted HA Operator resource, and a unique local-node identity. Otherwise retain previous state, log a bounded diagnostic warning, and do not announce a completed failover. - -### Proposed configuration (disabled by default) - -~~~yaml -ha_monitor: - enabled: false - exporter_url: http://127.0.0.1:9664/metrics -~~~ - -Inherit **check_interval_seconds** (60 seconds in current deployments). Keep the implementation constants small: HTTP timeout 3 seconds, a bounded response size, and two consecutive good polls to confirm a new owner. Avoid extra user-configurable knobs unless validation demonstrates a need. - -Use only the local exporter in v1. No firewall rule, container exec, host bind mount, database query, REST credential, or Pacemaker write command is required. - -## 4. State and alert behavior - -Retain a **small optional ha_monitor section in the existing state.json** (backward-compatible with current container alert state). Track the last confirmed active node and the last notified transition. This state must survive watchdog restart, use existing atomic/serialized persistence, and must not be treated as a Docker container entry. - -| Observation | Required behavior | -|---|---| -| Disabled, or no HA configured | Existing watchdog behavior unchanged; no HA requests | -| First valid poll without prior HA state | Record baseline silently; no historical or startup alert | -| Same active node | Refresh/retain baseline; no notification | -| Candidate active node changes A -> B | Confirm on next good poll (two consecutive observations) | -| Confirmed A -> B | Update persistent baseline; **only watchdog on B** sends one WARNING | -| Confirmed B -> A (failback) | Same behavior; notify once from A | -| Watchdog restarted while HA stays enabled | Restore confirmed owner; do not repeat previous transition | -| HA intentionally disabled and watchdog restarted | Clear saved HA owner; re-enable establishes a silent new baseline | -| Exporter timeout, 404/500, invalid/incomplete series, lost quorum, zero/multiple promoted owners | Do not change baseline or claim a completed failover; log a rate-limited local warning | -| Watchdog newly installed after the transition | Baseline silently; historical detection not guaranteed | -| Host reboot/grace window | Respect existing boot-grace behavior; do not mix HA alerts into the container reboot-summary. Reconfirm role after grace | -| Both nodes running watchdog | Both can observe role state, but only the newly promoted node emits the role-change notification | - -**Operator notification** (example, not an observed event): - -~~~text -[WARNING] CyberController HA active node changed -Previous active: 192.0.2.41 -New active: 192.0.2.42 -Observed at: -Evidence: HAOperatorRep-Res promoted on 192.0.2.42 (Pacemaker exporter) -Next step: Confirm expected switchover and application health. -~~~ - -Treat this as an event, **not a persistent container failure**: do not send a spurious "container recovered" alert. Keep the normal container alert subject/fields unchanged. Make only the minimum HA-specific display adjustment needed so email/Slack/syslog/SNMP do not misleadingly label it as a crashed container. Preserve existing notification formats for container alerts. - -**Delivery semantics (accepted v1 limitation):** one best-effort notification per confirmed observed transition, with state-based deduplication across restarts. The confirmed owner is persisted before dispatch; a delivery failure is logged but is not automatically retried. External syslog/SNMP delivery acknowledgement and exactly-once semantics are not available; adding a persisted retry queue would require changing the existing dispatcher contract and is deferred. A failed state-file save can lead to a duplicate after restart. Verify alert routing on both nodes. Detection time is an observation timestamp, not necessarily the exact Pacemaker event time. - -**SNMP compatibility (accepted v1 mapping):** the existing Container Watchdog MIB is reused. For HA notifications, cwContainerName is `(HA cluster)`, cwFailureType is `ha-failover`, and cwSummary/cwProbeDetail contain the old/new active nodes. An NMS should distinguish the HA event by cwFailureType; no new trap identity or OIDs are introduced in this release. - -## 5. Implementation scope — one development PR, after approval - -| Existing component | Small change | -|---|---| -| **watchdog.py / DEFAULT_CONFIG** | Optional ha_monitor block, disabled by default | -| **watchdog.py / Watchdog.start, _ha_poll_loop** | Start independent HA polling thread with its own HTTP session and stop event; Docker polling remains unchanged | -| **HA reader/transition helper** | Bounded local HTTP read; parse only required metrics; validate; confirm active change; deduplicate | -| **WatchdogState persistence** | Optional HA baseline and notified transition, without breaking existing v1 state files or container recovery | -| **AlertPayload/dispatch_alert** | Reuse channels; ensure HA event is clearly labelled and not folded into container reboot/recovery logic | -| **watchdog-config.yaml.example, README.md** | Add brief opt-in example, behavior, limitations, and both-node setup requirements | -| **tests/test_ha_monitor.py + existing regression suite** | Unit tests, mocks, notification checks | - -No changes to **install.sh**, Compose files, Python dependencies, port mappings, Docker privileges, Pacemaker services, or Prometheus are expected. If validation reveals a real need for one, bring it back for design review rather than expanding scope silently. - -## 6. Tests and Definition of Done (single implementation increment) - -### Automated tests (CI, no live HA action) - -- [ ] Feature disabled by default: zero exporter requests and no changes to existing container alerts. -- [ ] Parse metrics with different label orders; select only HAOperatorRep-Res that is active and promoted (value 1), and identify local=true node. -- [ ] Initial valid sample establishes baseline without alert. -- [ ] Stable A -> B across two polls sends **one** HA alert from B; the old node A sends none. -- [ ] B -> A produces one new notification; stable subsequent polls do not repeat. -- [ ] Duplicate/wrong-role resource series, missing local identity, missing Pacemaker-success metric, lost quorum, and zero/multiple promoted owners never create false failover alerts. -- [ ] HTTP timeout, connection refused, malformed metrics and empty response are bounded, leave baseline intact, and do not prevent Docker polling. -- [ ] Persist and restore role and notification deduplication; existing state v1 loads unchanged, and concurrent state writes remain valid. -- [ ] Restart/reboot/grace-window behavior does not generate stale duplicate alerts or contaminate the container reboot-summary. -- [ ] Notification shows previous/new node and UTC observation, is formatted correctly for the configured channels, and does not report fake container failure/recovery. -- [ ] Existing regression tests, Python syntax checks, and Compose/CI checks pass with no new dependencies. - -### Controlled two-node validation (manual QA, authorized window) - -- [ ] Watchdog and local HA exporter are **running and reachable locally on both nodes**; active role is visible from both local metric endpoints. -- [ ] Confirm normal steady state emits no failover alerts; initial installation/restart remains quiet. -- [ ] With an **explicitly authorized controlled switchover**, verify one notification from newly active node with correct A/B details and no duplicate from old active. -- [ ] Controlled failback produces one opposite-direction notification. -- [ ] Verify a loss-of-active-node scenario, or an approved equivalent fault simulation, demonstrates the surviving node can alert without the original node being reachable. -- [ ] Temporarily unavailable local exporter (test fixture or approved fault scenario) does not produce false failover alerts and Docker monitoring continues. -- [ ] Verify syslog end-to-end; test other configured alert channels as applicable. -- [ ] Observe bounded CPU/memory/HTTP impact and document expected limitations, rollback, and compatibility. - -### Definition of Done - -The implementation is ready for review/release when all automated tests and CI pass, controlled two-node role-change and failback tests produce the expected one-per-transition notifications, an active-node loss is covered by the surviving watchdog, resource use remains within current watchdog limits, documentation shows how to enable/disable the option on both nodes, and existing non-HA functionality is unchanged. No HA commands that modify cluster state or customer production services are introduced. - -## 7. Release gates, risks, and intentional exclusions - -**Before release:** confirm both HA nodes can send notifications end-to-end. Both watchdogs and local exporters were confirmed running during the manual .42 -> .41 test. The standby's earlier *remote* exporter connection refusal reflected an inaccessible cross-node port, not a missing local service. No inter-node exporter access is required. - -**Known limitations:** If the surviving node has no running watchdog/exporter or has never recorded the original active node, this check cannot reconstruct the failover after the fact. Polling can miss a transition that changes twice between samples. Metrics cannot distinguish planned switchover vs unexpected failover, and promotion does not guarantee the entire application is healthy. - -**Out of scope:** HA orchestration/control, restarting or promoting resources, automatic remediation, logging every Pacemaker transition, live log parsing, new monitoring dashboards/exporters, historical HA event reconstruction, generalized multi-service health monitoring, and remediation of the native Prometheus 9002 vs 9664 mismatch. The latter should be handled as a separate CyberController native monitoring correction. - -**Implementation status:** Approved by the user on 2026-10-09 and implemented on the feature branch. Do not merge or release until the two-node acceptance gates in [HA_VALIDATION.md](HA_VALIDATION_2026-10-13.md) are satisfied. diff --git a/docs/dev/HA_VALIDATION_2026-10-13.md b/docs/dev/HA_VALIDATION_2026-10-13.md deleted file mode 100644 index c519581..0000000 --- a/docs/dev/HA_VALIDATION_2026-10-13.md +++ /dev/null @@ -1,46 +0,0 @@ -> **Historical validation snapshot — 2026-10-13.** PR #4 was subsequently -> merged into `main` on 2026-10-14, and later HA failover/failback -> acceptance is recorded in the -> [v1.6.0 release acceptance report](RELEASE_ACCEPTANCE_V1.6.0.md). -> The pending gates below are preserved as historical findings, **not** the -> current status of PR #4. - -# Optional HA active-node alert — validation status - -**Branch:** feature/optional-ha-failover-alert -**Status at the time:** Draft PR #4; two-node lab tested, pending merge -**Updated:** 2026-10-13 - -## Automated tests and build - -- **26 tests pass** after the PR review fixes. Added regression coverage for an HA thread running while the Docker poll is blocked and raises, and disable -> role change -> re-enable with a quiet baseline. -- The new HA thread owns its HTTP session, uses the existing stop event and global polling interval, and does not change Docker health checks or require new dependencies. -- Earlier PR #4 CI passed, including the clean Docker build. **CI must pass again on the updated commit** before merging. -- The existing state.json v1 format and Docker container alert state are retained. - -## Real two-node environment — user-initiated test (2026-10-10 UTC) - -- **Both nodes have HA exporters and watchdog builds running** during the user test, not just the original active node. These installations were done by the user, not by this code review. -- At about 02:53:46 UTC the user initiated a manual switchover on .42, which moved the promoted HA Operator resource to .41. -- .42 logged at **02:55:17**: "HA role changed 192.0.2.42 -> 192.0.2.41; new owner will notify." -- .41 logged at **02:55:37**: one WARNING, "CyberController HA active node changed: 192.0.2.42 -> 192.0.2.41". -- Both nodes stayed online and .41's promoted HA resources became active. The actual switch was detected correctly, without duplicate HA events observed. -- **Actual notification delivery still needs verification.** .41 had syslog configured to 127.0.0.1:514; receiving NMS confirmation was not established. .42 used SMTP, but smtp.radware.com did not resolve, so its container failure messages were not emailed. -- During the planned demotion, the HA Operator explicitly removed vDirect, TED and Assist Service on .42. The watchdog generated separate CRITICAL crash alerts for vDirect and Assist Service. Narrowly scoped suppression is **out of scope** for PR #4 and should have a separate code review. -- A **reverse failback (.41 -> .42)**, end-to-end receiver confirmation, and a safe surviving-node test after failure of the original active node have not been confirmed. - -## Release gates still open - -1. CI passes for the current head, with new concurrency/re-enable tests. -2. Test a controlled **failback** from .41 to .42, verifying a single reverse alert. -3. Verify the configured receiving systems, not just local watchdog logs; resolve .42 SMTP DNS or configure a working syslog route. -4. Validate a supported active-node-loss scenario without deliberately partitioning a cluster where STONITH is disabled. -5. Confirm targeted CC-version compatibility and ordinary watchdog restart/disabled option behavior. - -## Accepted v1 limitations - -- One-shot best-effort dispatch: transition state is persisted **before** notification. Failed SMTP/Slack/SNMP/syslog delivery does not automatically retry. A failed state write may cause a duplicate after restart. Do not add a durable delivery queue to this PR without changing the dispatch API. -- HA SNMP events use the existing Container Watchdog MIB: cwContainerName=(HA cluster), cwFailureType=ha-failover. NMS integrations should classify by failure type. A separate MIB identity is not included in v1. -- Planned HA service removals currently produce container failure alerts. Suppression is a separate, narrowly scoped enhancement. - -**No HA host, Docker service, Pacemaker, firewall, or monitoring configuration changes were made during this code-review update.** diff --git a/docs/dev/RELEASE_SMOKE_TESTING.md b/docs/dev/RELEASE_SMOKE_TESTING.md index f90543c..de5e01c 100644 --- a/docs/dev/RELEASE_SMOKE_TESTING.md +++ b/docs/dev/RELEASE_SMOKE_TESTING.md @@ -4,7 +4,7 @@ ## Safety rule -**NEVER run `tests/release-smoke.sh` on CyberController, customer systems, or any shared Docker host.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. +**NEVER run `tests/release-smoke.sh` on production CyberController, customer production systems, or any shared Docker host.** Do not try to make a live host "safe" by changing its watchdog exclusions, switching off its notifications, labelling dummy containers, or setting a live-host opt-in flag. Instead run the workflow [Release smoke (isolated)](../../.github/workflows/release-smoke.yml). It executes on a **GitHub-hosted disposable runner** and creates a second, empty, short-lived, uniquely labelled Docker-in-Docker Engine. Only that Engine's Docker socket is mounted inside the test observer. The temporary outer Engine is visible to the GitHub runner's Docker daemon, never to any operational host. All test-created resources are disposable and contained within that hosted worker. @@ -28,7 +28,7 @@ workflow file exists on the default branch. 3. Confirm the workflow's commit SHA matches the release PR head. It builds the watchdog image on the **hosted runner only**, loads it into a freshly created nested Docker Engine, and checks that the image files match the checked-out source. 4. Download the attached structured evidence artifact (Docker events, alerts, raw UDP receiver packets, state snapshot, run reports). The workflow runs both the full acceptance suite and the additional kernel OOM/manual restart scenario. -5. All cases must PASS; the final cleanup must report zero test-labelled containers and volumes. Keep the actual workflow URL/commit in the release approval record. +5. All cases must PASS; the final cleanup must report zero test-labelled containers and volumes. Keep the actual workflow URL/commit in the release verification record. ## Automatic fail-closed checks diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index 0d37498..98983a6 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -1,6 +1,6 @@ # Container Alert — Customer Installation and Operations -This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. One complete, versioned release package and one `install.sh` wizard work on hosts with or without Internet access. The installer deploys only `docker-container-watchdog`; it never restarts CyberController application containers. +This guide covers **installation, configuration, day-to-day operations, upgrades, rollback, and troubleshooting**. ## Table of Contents @@ -27,66 +27,55 @@ This guide covers **installation, configuration, day-to-day operations, upgrades - [Rollback](#54-rollback) - [Uninstall](#55-uninstall) 6. [Troubleshooting](#6-troubleshooting) - - [Missing recovery notifications](#61-missing-recovery-notifications) - - [Read-only logs and probe diagnostics](#62-read-only-logs-and-probe-diagnostics) -7. [References and support](#7-references-and-support) +7. [Related documentation](#7-related-documentation) ## 1. Prerequisites | Requirement | What is needed | |---|---| | Operating system | Supported Linux host with Docker (Ubuntu 20.04 / RHEL 8 or later are the documented baseline) | -| Docker Engine | 20.10+ and a running Docker daemon | -| Docker Compose | Version 2 plugin (`docker compose`) recommended; `install.sh` also supports standalone `docker-compose` | +| Docker and Compose | Docker Engine 20.10+ and Docker Compose (plugin or standalone) | | Host permissions | Root or an operator with Docker access; `sudo` may be required to secure configuration files | -| Release package | Approved complete `container-alert-v.tar.gz` with matching `SHA256SUMS`, copied to the host | -| Network access | **Neither Git nor Internet access is required on the target host**; download from GitHub Releases elsewhere if needed | +| Release package | Published `container-alert-v.tar.gz` and matching `SHA256SUMS` | +| Network access | Optional; download from [GitHub Releases](https://github.com/Radware/CyberController-Container-Alert/releases) or transfer both files from another machine | | Alert delivery | Network connectivity to the chosen Syslog, SMTP, SNMP, or Slack destination | -Read-only prerequisite checks: - -```bash -docker info -docker ps -docker compose version # or: docker-compose --version -df -h . -``` - -Docker socket access is highly privileged even if a container runs as a non-root user. Restrict access to authorized operators. The installer automatically detects the socket group and configures the watchdog's existing non-root execution mode. - -**Install only an approved, published release.** The version in `VERSION` must match the prebuilt image. Customer installations never compile application code. Do not run failure-injection tests on operational Docker hosts. ## 2. Installation -**One process on every host.** Download the approved versioned package directly, or have an authorized operator transfer the same package to a disconnected CyberController host. The package already contains the prebuilt Docker image, installer, Compose files, configuration templates and documentation. No Git clone, customer-side image download or source build is needed. +Download the installation package directly, or transfer the same package to a CyberController host `opt/radware/storage/scripts/` directory. The release package includes the installer, prebuilt Docker image and configuration templates. -The filename below uses v1.6.0 as an example. Release assets exist only after that version has been officially [published on GitHub](https://github.com/Radware/CyberController-Container-Alert/releases). +The examples use v1.6.0; download a version from [GitHub Releases](https://github.com/Radware/CyberController-Container-Alert/releases). ### 2.1 Get and verify the release package -**Step 1 — Choose an approved release version.** Open the [Container Alert Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) to see all published versions. The release marked **Latest** is usually the newest stable version, but use the version approved for this environment; do not install a pre-release unless specifically authorized. For example, the tag `v1.6.0` means you set `VER=1.6.0` (without the `v`). Versions appear on the page only after they are published. +**Step 1 — Choose a release version.** Open the [Container Alert Releases page](https://github.com/Radware/CyberController-Container-Alert/releases) to see all published versions. The release marked **Latest** is usually the newest stable version. **Step 2 — Obtain the release package and checksum.** Use either method to get the **same two files** on the target host: -**Host has Internet access (simplest):** Run on the target host in the intended installation directory: +**Host has Internet access:** From a root shell (or account with write access), download into the scripts directory: ```bash -VER=1.6.0 # Replace with the approved published version from Step 1 +mkdir -p /opt/radware/storage/scripts +cd /opt/radware/storage/scripts +VER=1.6.0 # Example: GitHub tag v1.6.0 URL="https://github.com/Radware/CyberController-Container-Alert/releases/download/v${VER}" curl -fL -O "$URL/container-alert-v${VER}.tar.gz" -O "$URL/SHA256SUMS" ``` -No Git clone, GitHub CLI, browser download, or SCP is required. `curl` needs HTTPS access to GitHub Releases and its redirected download host. +`curl` needs HTTPS access to GitHub Releases and its redirected download host. -**Disconnected host:** Download `container-alert-v1.6.0.tar.gz` and `SHA256SUMS` (adjust for the selected version) from that release on a connected workstation, then transfer both to the target host using the approved method. +**Host without Internet access:** Download the chosen release package and `SHA256SUMS` on another computer. Transfer both files to `/opt/radware/storage/scripts/` on the CyberController host. **Step 3 — Verify and extract** the two files from the same directory: ```bash -VER=1.6.0 # Use the same version chosen in Step 1 +mkdir -p /opt/radware/storage/scripts +cd /opt/radware/storage/scripts +VER=1.6.0 # Use the version chosen in Step 1 sha256sum --check SHA256SUMS tar -xzf "container-alert-v${VER}.tar.gz" -cd CyberController-Container-Alert +cd /opt/radware/storage/scripts/CyberController-Container-Alert ``` The extracted directory contains `VERSION`, `install.sh`, `docker-compose.yaml`, `watchdog.tar` and its own image checksum. @@ -101,182 +90,152 @@ bash install.sh The installer automatically verifies the included Docker image checksum and version, loads the image, then asks you to select and configure the notification channels (Syslog, SMTP, SNMP and/or Slack) and alert hostname. On an upgrade, answer **No** to “Reconfigure from scratch?” to preserve existing channel settings. It recreates **only the watchdog**, not other application containers. -**Step 5 — Confirm normal operation:** - -```bash -docker ps --filter name=docker-container-watchdog -docker logs --tail 50 docker-container-watchdog -``` - -Expected: the watchdog is running and logging normal monitoring. For further read-only checks, see [Section 3](#3-verify-the-installation). A failed or incomplete image verification stops installation before touching the running watchdog. - ## 3. Verify the installation -Run these commands from the installation directory, or use the global `docker` commands from any directory: +Run these commands after installation: ```bash docker compose ps docker inspect docker-container-watchdog --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' docker inspect docker-container-watchdog --format '{{.Config.Image}}' -docker logs --tail 100 docker-container-watchdog ``` -The container should be `running` and become `healthy`; the running image should be `watchdog:` matching the package. The installer verifies both status and the actual image ID. Startup may take several health-check intervals. +The watchdog should be `running` and become `healthy`; the image should be `watchdog:` matching the package. -**Logs and alert delivery:** +**Logs and alert delivery** | Location or command | Purpose | |---|---| | `docker logs -f docker-container-watchdog` | Live container output | -| `docker compose logs --tail 100 docker-container-watchdog` | Compose-scoped stdout | -| `./watchdog/watchdog.log` | Persistent host log, rotated at 10 MB with five backups | -| `./watchdog/state.json` | Persistent incident/HA/reboot tracking state; do not hand-edit | -| `grep 'watchdog_alert' watchdog/watchdog.log` | Search existing structured alert messages | +| `docker compose logs --tail 100 docker-container-watchdog` | Recent Compose logs | +| `./watchdog/watchdog.log` | Persistent host log (rotates at 10 MB, five backups) | +| `./watchdog/state.json` | Incident, HA and reboot state; do not edit manually | +| `grep 'watchdog_alert' watchdog/watchdog.log` | Find existing alert records | + +Check that the selected Syslog, SMTP, SNMP or Slack receiver can be reached and inspect any alerts already received. No artificial container failure is needed. -Also verify that the configured notification receiver can receive traffic from the host, and inspect **existing** Syslog/SMTP/SNMP/Slack records if available. No artificial container failure is necessary to verify ordinary operation. +**Optional HA check:** If `ha_monitor.enabled: true`, check the local exporter on **each HA node**: + +```bash +curl -fsS http://127.0.0.1:9664/metrics +``` -**Optional HA installation:** On each node with `ha_monitor.enabled: true`, the local exporter can be inspected read-only using `curl -fsS http://127.0.0.1:9664/metrics` (or the configured exporter URL). The first valid reading sets a quiet baseline; no failover notification should be expected at startup. Controlled HA failover testing belongs to an approved maintenance procedure, not routine installation verification. +Use the configured exporter URL if different. The first successful reading establishes a baseline; it does not send a failover alert. Test a real switchover only during planned HA maintenance. -Never stop, kill, stress, or restart **application containers** merely to test the watchdog. ## 4. Configuration reference -The installer configures notification channels during initial setup. Use this section **after installation** if an authorized operator needs to review or change monitoring settings. No Docker image rebuild is required for a YAML or environment change. +The installer creates the active configuration files. Edit them only when needed; most installations require no manual changes. ### 4.1 Configuration files and editing -| File | What it controls | Permissions and application | +| File | What it contains | Apply changes | |---|---|---| -| `watchdog-config.yaml` | Monitoring thresholds, health probes, exclusions, logging, HA and notification endpoints | Installer secures it as root-owned, group-readable (mode `640`); edit with `sudoedit` or an authorized privileged editor; restart watchdog | -| `.env` | Channel secrets, alert hostname, Docker group identity and selected image version | Owner-readable only (mode `600`); edit with an authorized editor; recreate watchdog | -| `docker-compose.yaml` | Container runtime mounts, Docker socket, CPU/memory limits, networking and restart policy | Normally unchanged by customers; change only under approved support/engineering instructions | -| `watchdog-config.yaml.example` / `.env.example` | Reference templates | Not loaded by the running watchdog; don't edit these expecting the service to change | +| `watchdog-config.yaml` | Monitoring, alerts, health checks and HA | Restart watchdog | +| `.env` | Credentials, hostname and Docker image version | Recreate watchdog | +| `docker-compose.yaml` | Container runtime settings; generally managed by Radware | Recreate watchdog | +| `*.example` | Reference templates, not active settings | No effect on the running service | -From the installation directory: - -```bash -sudoedit watchdog-config.yaml # only if an authorized configuration change is required -nano .env # only if changing credentials, hostname, etc. -``` - -If running as root, edit directly without `sudo`. Otherwise, if `sudoedit` is unavailable, use your organization's approved privileged-editor method. Make a backup before editing. Never commit `.env`, passwords, or tokens to Git. Changes in one file do not automatically update the other. +The installer creates `watchdog-config.yaml` and `.env`. From the installation directory, you can edit `watchdog-config.yaml` or `.env`. Back up first ([Section 5.2](#52-backup-and-restore)) ### 4.2 Watchdog monitoring settings -These are `watchdog-config.yaml` keys. Defaults below are application defaults unless noted; installation-wizard selections can override them. +These settings are in `watchdog-config.yaml`. Defaults shown are the application defaults; the installer may set channel-specific values. -| YAML setting | Default | Explanation | +| Setting | Default | Purpose | |---|---|---| -| `alert_channels` | `[slack]` in application defaults; installer prompts for selection | List of enabled delivery channel identifiers: `syslog`, `smtp`, `snmp_trap`, `slack`. Select the corresponding channel configuration too | -| `check_interval_seconds` | `60` | Interval between periodic Docker health/restart polls | -| `cooldown_minutes` | `5` | Suppresses repeated alerts for the same incident; changing it may conceal a real recurring failure | -| `restart_threshold` | `5` | Number of Docker restart-counter increases that qualifies as a restart loop | -| `restart_window_minutes` | `10` | Rolling window for restart-loop detection | -| `restart_recovery_stability_seconds` | `120` | Continuous running time needed before reporting recovery from crash/OOM/restart loop | -| `unhealthy_cycles_threshold` | `3` | Consecutive unhealthy polls required for an unhealthy alert | -| `alert_on_recovery` | `true` | Emit one informational `recovered` event when an alerted condition clears | -| `lifecycle_detection.enabled` | `true` | Correlate Docker-requested stop events with nonzero exits to reduce false crash alerts | -| `lifecycle_detection.correlation_window_seconds` | `3` | Wait briefly for matching stop evidence; supported range is 0.5–30 seconds | -| `log_level` | `INFO` | Console/file verbosity: `DEBUG`, `INFO`, `WARNING` or `ERROR`. Syslog alert severity is kept independently | -| `log_file` | `/var/log/watchdog/watchdog.log` | Persistent log file within the container, bind-mounted to `./watchdog/`; `null` disables the file log | -| `state_file` | `/var/log/watchdog/state.json` | Persists incidents, restart history and HA/reboot state across watchdog restarts; empty string disables persistence | -| `boot_grace_seconds` | `180` | Grace period after a detected host reboot before summary/normal alerts; `0` removes the grace period | -| `runbook_base_url` | Empty | Optional base URL included with alerts | - -Changes to exclusions, thresholds, probe methods or suppression rules can mask real failures. Review observed evidence and obtain the applicable Radware support/engineering approval before changing those settings. +| `alert_channels` | Installer selection (code default: `[slack]`) | Lists Slack, SMTP and SNMP notification channels | +| `check_interval_seconds` | `60` | Poll interval in seconds | +| `cooldown_minutes` | `5` | Reduce repeat alerts for the same incident | +| `restart_threshold` | `5` | Restarts needed to report a restart loop | +| `restart_window_minutes` | `10` | Time window for counting restarts | +| `restart_recovery_stability_seconds` | `120` | Stable running time before crash/OOM/restart-loop recovery | +| `unhealthy_cycles_threshold` | `3` | Consecutive failing health checks before alert | +| `alert_on_recovery` | `true` | Notify when an alerted condition clears | +| `lifecycle_detection.enabled` | `true` | Recognize intentional Docker stops | +| `lifecycle_detection.correlation_window_seconds` | `3` | Match stop and exit events (0.5–30 seconds) | +| `boot_grace_seconds` | `180` | Delay normal alerts after reboot; `0` disables the grace period | +| `log_level` | `INFO` | Local verbosity: DEBUG, INFO, WARNING or ERROR; forwarded Syslog priorities are separate | +| `log_file` | `/var/log/watchdog/watchdog.log` | Persistent log, mounted under `./watchdog/`; `null` disables file logging | +| `state_file` | `/var/log/watchdog/state.json` | Preserves incidents across restarts; empty value disables persistence | +| `runbook_base_url` | Empty | Optional runbook link in alerts | + +Changing thresholds, exclusions or probe types can hide failures. Check their impact carefully; do not change them just to suppress alerts. ### 4.3 Notification channels -The installer records the selected delivery channels and their settings. **Slack, SMTP, and SNMP traps** use both `alert_channels` and their channel's `enabled` setting (Slack defaults to enabled if omitted). **Syslog** instead forwards through the logging subsystem when `syslog.enabled: true`; listing it in `alert_channels` alone does not activate forwarding. All channels require a reachable receiver, and secret-based channels require credentials in `.env`. +Choose channels in the installer. For later changes, edit the following YAML settings. To enable SMTP, Slack or SNMP manually, add `smtp`, `slack` or `snmp_trap` to the `alert_channels` list and enable the corresponding channel. Syslog is enabled separately with `syslog.enabled`. -| Channel | YAML field | Purpose | +| Channel | Settings in `watchdog-config.yaml` | Credentials in `.env` | |---|---|---| -| Syslog | `syslog.enabled` | Enable structured forwarding | -| Syslog | `syslog.host`, `syslog.port` | Receiver hostname/IP and listening port (commonly 514) | -| Syslog | `syslog.protocol` | `udp` or `tcp` | -| Syslog | `syslog.facility` | Syslog facility (for example `local0`) | -| SMTP | `smtp.enabled`, `smtp.auth` | Enable email; set `auth: false` for approved unauthenticated relays | -| SMTP | `smtp.host`, `smtp.port` | Mail server and port | -| SMTP | `smtp.sender`, `smtp.recipients` | Sender address and YAML list of recipient addresses | -| SMTP | `smtp.tls` | Whether to use SMTP TLS | -| SMTP | `smtp.username_env`, `smtp.password_env` | Names of secret environment variables, normally `SMTP_USERNAME` and `SMTP_PASSWORD` | -| Slack | `slack.enabled` | Enable Slack notifications | -| Slack | `slack.webhook_url_env` | Name of the secret environment variable, normally `SLACK_WEBHOOK_URL` | -| SNMP | `snmp_trap.enabled` | Enable trap dispatch | -| SNMP | `snmp_trap.host`, `snmp_trap.port` | NMS/trap receiver and destination UDP port (normally 162) | -| SNMP | `snmp_trap.version`, `snmp_trap.community` | SNMP `v1`/`v2c`/`v3`; community applies to v1/v2c only | -| SNMP | `snmp_trap.trap_oid` | Optional trap OID; defaults to the Radware Container Alert notification OID | -| SNMPv3 | `v3_username`, `v3_auth_protocol`, `v3_priv_protocol` | USM user and authentication/privacy algorithms under `snmp_trap` | -| SNMPv3 | `v3_auth_key_env`, `v3_priv_key_env` | Names of passphrase variables in `.env`; do not embed secrets in YAML | -| SNMPv3 | `v3_local_engine_id` | Optional fixed sender engine ID used by receiving NMS registration | +| **Syslog** | `syslog.enabled`, `syslog.host`, `syslog.port`, `syslog.protocol` (UDP/TCP), `syslog.facility` | None | +| **SMTP** | `smtp.enabled`, `smtp.auth`, `smtp.host`, `smtp.port`, `smtp.sender`, `smtp.recipients`, `smtp.tls`, `smtp.username_env` / `smtp.password_env` (**variable names only**) | `SMTP_USERNAME`, `SMTP_PASSWORD` if authentication is enabled | +| **Slack** | `slack.enabled`, `slack.webhook_url_env` | `SLACK_WEBHOOK_URL` | +| **SNMP traps** | `snmp_trap.enabled`, `snmp_trap.host`, `snmp_trap.port`, `snmp_trap.version`, `snmp_trap.community` (v1/v2c), optional `snmp_trap.trap_oid` | `SNMP_V3_AUTH_KEY`, `SNMP_V3_PRIV_KEY` for SNMPv3 | #### Syslog -Set `syslog.enabled: true` and provide the receiver host, port, protocol and facility. Syslog forwards structured alert records (including INFO recoveries) independently of the local console/file log threshold. The installer may list `syslog` in `alert_channels` for consistency, but the actual forwarding switch is `syslog.enabled`. +Set `syslog.enabled: true` and specify the receiver. Syslog forwards structured alerts, including INFO recoveries, regardless of local `log_level`. + +#### SMTP (email) + +Enable SMTP in `alert_channels` and set `smtp.enabled: true`. Set host, port, sender and recipients in YAML. `smtp.tls: true` uses STARTTLS; `smtp.auth: false` is for mail relays that do not require login. -Example receiver settings after installation: +**Important:** `username_env` and `password_env` contain **environment variable names**, not your username or password. Example: ```yaml -syslog: - enabled: true - host: syslog.example.com - port: 514 - protocol: udp - facility: local0 +# watchdog-config.yaml — references only +smtp: + username_env: SMTP_USERNAME + password_env: SMTP_PASSWORD ``` -Restart the watchdog after an approved YAML-only change. Confirm receipt at the configured Syslog receiver. - -#### SMTP (email) +```dotenv +# .env — actual values, never committed to Git +SMTP_USERNAME=your-mail-login +SMTP_PASSWORD=your-mail-password +``` -Include `smtp` in `alert_channels` and set `smtp.enabled: true`. Configure the mail host, port, sender, recipients and `smtp.tls` in YAML. With `tls: true`, the watchdog uses SMTP STARTTLS. Use `smtp.auth: false` only for an approved unauthenticated relay. If authentication is enabled, store `SMTP_USERNAME` and `SMTP_PASSWORD` in `.env` (the YAML fields `username_env` and `password_env` reference those variable names). +The installer normally writes these settings for you. #### Slack -Include `slack` in `alert_channels` and set `slack.enabled: true` (or omit `enabled`, which defaults to true for compatibility). Store the incoming webhook URL in `.env` as `SLACK_WEBHOOK_URL`; the YAML `slack.webhook_url_env` names that variable. Do not embed the webhook URL in YAML or commit it to the repository. +Include `slack` in `alert_channels` and set `slack.enabled: true`. In YAML, `slack.webhook_url_env: SLACK_WEBHOOK_URL` names the variable; the **actual** webhook URL belongs in `.env`. #### SNMP traps -Include `snmp_trap` in `alert_channels`, set `snmp_trap.enabled: true`, and configure the NMS receiver host, UDP port (normally 162) and SNMP version. SNMPv1/v2c use the configured community string; SNMPv3 credentials are referenced from `.env`. The additional details below are **receiver-side SNMP integration**, not a separate watchdog feature. +Include `snmp_trap` in `alert_channels` and set `snmp_trap.enabled: true`. Enter the NMS host and UDP trap port (usually 162); versions v1/v2c use `community`. -The configured `snmp_trap.version` accepts v1, v2c, or v3. For SNMPv3, the watchdog logs a stable engine ID derived from the hostname (look for `SNMP engine ID: 0x...`). Register the engine ID and the USM user at the receiving NMS; `v3_local_engine_id` can pin this identity when necessary. +For **SNMPv3**, configure `v3_username`, `v3_auth_protocol` (MD5, SHA, SHA256, SHA384 or SHA512) and, for encrypted traps, `v3_priv_protocol` (AES, AES192 or AES256). The `v3_auth_key_env` and `v3_priv_key_env` fields reference passphrases stored in `.env`, not plaintext in YAML. Passphrases must be at least eight characters; encryption requires authentication. DES is weak and not recommended. -Example receiver-side USM registration (adapt for the actual engine ID and secret values through the authorized NMS process): +Register the sender engine ID from the watchdog startup logs (`SNMP engine ID: 0x...`) with your NMS. Use `v3_local_engine_id` when a fixed sender identity is needed. -```text -createUser -e 0x watchdog-user SHA "auth-passphrase" AES "priv-passphrase" -``` +The default notification OID is `1.3.6.1.4.1.89.110.0.1`. NMS integrations receive five fields: -Both auth/privacy keys must contain at least eight characters. A privacy key requires an authentication key; SNMPv3 has no privacy-only mode. Supported authentication names include `MD5`, `SHA`, `SHA256`, `SHA384` and `SHA512`. Privacy supports `AES`, `AES192` and `AES256`; `DES` is weak and not recommended. - -**Radware Container Alert trap OID:** `1.3.6.1.4.1.89.110.0.1`. The five existing var-binds are: - -| OID | Object | Value | +| Var-bind | OID | Value | |---|---|---| -| `1.3.6.1.4.1.89.110.1.1.0` | `cwHost` | Alerting host | -| `1.3.6.1.4.1.89.110.1.2.0` | `cwSummary` | Summary | -| `1.3.6.1.4.1.89.110.1.3.0` | `cwContainerName` | Container name | -| `1.3.6.1.4.1.89.110.1.4.0` | `cwFailureType` | `crashed` / `oom` / `unhealthy` / `restart-loop` / `recovered` / `ha-failover` | -| `1.3.6.1.4.1.89.110.1.5.0` | `cwProbeDetail` | Detection detail | - -HA events reuse this MIB. They use `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`; old/new active-node names appear in summary/detail. NMS rules should match the event type, not assume every alert is a failing Docker container. +| `cwHost` | `1.3.6.1.4.1.89.110.1.1.0` | Hostname | +| `cwSummary` | `1.3.6.1.4.1.89.110.1.2.0` | Alert summary | +| `cwContainerName` | `1.3.6.1.4.1.89.110.1.3.0` | Container or HA cluster | +| `cwFailureType` | `1.3.6.1.4.1.89.110.1.4.0` | Failure type, recovery or `ha-failover` | +| `cwProbeDetail` | `1.3.6.1.4.1.89.110.1.5.0` | Detection details | -For a YAML-only channel change, restart the watchdog. For changed `.env` secrets, recreate **only** the watchdog as described in [Apply configuration changes](#46-apply-configuration-changes). Do not restart any monitored application container to verify channel delivery. +HA traps use the same OID and var-binds with `cwFailureType=ha-failover` and `cwContainerName=(HA cluster)`. Check receiver-side SNMP user/engine-ID configuration if traps are rejected. Do not trigger application failures just to test notifications. ### 4.4 Health checks, exclusions, and HA -| YAML setting | Default | Explanation | +| Setting | Default | Purpose | |---|---|---| -| `auto_health_check.enabled` | `true` in installer | Enable automatic HTTP/TCP/discovery probes for containers without a native Docker health check | -| `auto_health_check.paths` | `/-/healthy`, `/health`, `/healthz`, `/api/health`, `/metrics`, `/` | HTTP paths attempted during endpoint discovery | -| `auto_health_check.timeout_seconds` | `2` in installer | Probe timeout after endpoint discovery | -| `container_health_checks` | `{}` | Per-container `http` or `exec` probe overrides; take precedence over automatic discovery | -| `excluded_containers` | `[]` | Names excluded from alerting entirely; not a routine noise-management setting | -| `ignored_container_events` | R&D-labelled temporary syntax-check event | Narrow, label-based exceptions for intentional temporary operations; preserve normal monitoring | -| `ha_monitor.enabled` | `false` | Optional CyberController HA active-node change notification, installed on both nodes | -| `ha_monitor.exporter_url` | `http://127.0.0.1:9664/metrics` | Local HA metrics exporter endpoint; doesn't install the exporter | +| `auto_health_check.enabled` | `true` | Probe containers without a Docker health check | +| `auto_health_check.paths` | `/-/healthy`, `/health`, `/healthz`, `/api/health`, `/metrics`, `/` | Discovery paths | +| `auto_health_check.timeout_seconds` | `2` | Probe timeout | +| `container_health_checks` | `{}` | Per-container `http` / `exec` probe overrides | +| `excluded_containers` | `[]` | Containers excluded from alerts | +| `ignored_container_events` | Radware-labelled temporary check | Narrow exceptions for known transient operations | +| `ha_monitor.enabled` | `false` | Optional active-node change alerts | +| `ha_monitor.exporter_url` | `http://127.0.0.1:9664/metrics` | Local HA exporter address | -For an approved custom HTTP probe, the shape is: +A custom probe can override discovery for a specific container. Example: ```yaml container_health_checks: @@ -287,200 +246,187 @@ container_health_checks: timeout_seconds: 5 ``` -The supported per-container probe fields are: +Fields are `type` (`http` or `exec`), `internal_port` and `path` for HTTP, `command` for exec, and optional `timeout_seconds` (default 5). Exec probes run commands **inside monitored containers**. Check the impact of probe, exclusion and ignore-rule changes before applying them. -| Field under `container_health_checks.` | Meaning | -|---|---| -| `type` | `http` (default) or `exec` | -| `internal_port` | Required for HTTP checks; internal container listening port | -| `path` | HTTP request path (defaults to `/`) | -| `command` | Command to run inside the container for `exec` probes; required for this type | -| `timeout_seconds` | Timeout for either method (default 5 seconds) | - -Use the *actual* container name and a supported health endpoint only after validation. Manual `type: exec` checks execute commands inside monitored containers, so they must be approved and carefully reviewed. Do not add exclusions or broad ignore rules merely to silence unexpected alerts. - -HA monitoring is **disabled by default**. Enabling it requires an existing supported local exporter on both nodes. The first successful reading establishes a baseline silently. Failover/failback alert delivery is best effort; consult the [HA design and validation guide](https://github.com/Radware/CyberController-Container-Alert/blob/main/docs/dev/HA_MONITORING_DESIGN.md) for limitations and approved maintenance testing. +For HA alerts, enable `ha_monitor` on both nodes with a working local exporter. Check the configured exporter URL read-only (normally `http://127.0.0.1:9664/metrics`). The first valid reading sets a baseline without sending an alert. ### 4.5 Environment variables -Variables in `.env` are read when the container is created. The installer sets the image version, runtime identity and alert hostname automatically; credentials appear only for selected channels. +The installer manages most `.env` values. SMTP passwords, Slack webhooks and SNMPv3 passphrases belong here, not in YAML. The SNMPv1/v2c community is configured in YAML, so protect that file too. -| Environment variable | Purpose | When needed | -|---|---|---| -| `WATCHDOG_HOST` | Hostname displayed in alerts | Generated by installer; change only when the alert identity needs correction | -| `WATCHDOG_VERSION` | Pinned local image tag `watchdog:` | Managed by installer; used for controlled upgrades/rollback | -| `WATCHDOG_UID`, `WATCHDOG_GID` | Non-root runtime user and group (normally 1000/1000) | Set by installer; do not change casually | -| `DOCKER_GID` | Group ID of the host Docker socket | Detected automatically by installer | -| `SLACK_WEBHOOK_URL` | Slack incoming webhook secret | If Slack enabled | -| `SMTP_USERNAME`, `SMTP_PASSWORD` | Mail credentials or app token | If SMTP authentication enabled | -| `SNMP_V3_AUTH_KEY` | SNMPv3 authentication passphrase | For SNMPv3 authNoPriv/authPriv | -| `SNMP_V3_PRIV_KEY` | SNMPv3 encryption passphrase | For SNMPv3 authPriv | -| `WATCHDOG_CONFIG` | Container-internal config path set explicitly in Compose | Normally `/etc/watchdog/watchdog-config.yaml`; not an everyday `.env` setting | - -**Logging clarification:** `LOG_LEVEL` in older `.env` files is not used by the application. Change `log_level` in `watchdog-config.yaml` instead. Do not store passwords or webhook values in YAML or version control. +| Variable | Purpose | +|---|---| +| `WATCHDOG_HOST` | Hostname shown in alerts | +| `WATCHDOG_VERSION` | Image version number only (for example `1.6.0`, without `watchdog:`) | +| `WATCHDOG_UID`, `WATCHDOG_GID`, `DOCKER_GID` | Runtime user and Docker socket permissions; installer-managed | +| `SLACK_WEBHOOK_URL` | Slack webhook | +| `SMTP_USERNAME`, `SMTP_PASSWORD` | SMTP credentials, when required | +| `SNMP_V3_AUTH_KEY`, `SNMP_V3_PRIV_KEY` | SNMPv3 passphrases | + +`WATCHDOG_CONFIG` is set in Compose, not normally in `.env`. Set logging with YAML `log_level`, not the obsolete `LOG_LEVEL` environment variable. ### 4.6 Apply configuration changes -First back up the relevant files ([Section 5.2](#52-backup-and-restore)) and have an authorized operator review the change. +Back up the configuration before editing. Run these from `/opt/radware/storage/scripts/CyberController-Container-Alert`: -| What changed | Command | New Docker image needed? | -|---|---|---| -| `watchdog-config.yaml` only | `docker compose restart docker-container-watchdog` | No | -| `.env` (secrets, hostname, image version) | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | No rebuild; requested image must already exist | -| `docker-compose.yaml` runtime settings | Same service-only Compose recreation, after engineering approval | No rebuild | -| Application Python code | Deploy a newly tested, versioned release image | **Yes — a different image**, not merely a restart | +**YAML-only change — restart the watchdog:** ```bash -# After an approved YAML-only change: docker compose restart docker-container-watchdog +``` -# After an approved .env or Compose change: +**`.env` or Compose change — recreate the watchdog:** + +```bash docker compose up -d --no-deps --force-recreate docker-container-watchdog ``` -If both YAML and `.env` changed, use the recreation command to apply both. Always confirm health and inspect logs afterward. Updating a Git checkout alone does **not** change Python code inside the running image. +A Python code change requires a new versioned Docker image; restarting will not update code. Neither command restarts other application containers. Check the watchdog afterward using [Section 3](#3-verify-the-installation). ## 5. Operations and maintenance -All commands below target **only the watchdog**. Run Compose commands from the directory containing `docker-compose.yaml` and `.env` (use `docker-compose` in place of `docker compose` on a legacy standalone installation). +Run these commands from `/opt/radware/storage/scripts/CyberController-Container-Alert`. They manage **only the watchdog**. ### 5.1 Common commands and logs -| Operation | Command | +| Task | Command | |---|---| -| Show container status | `docker compose ps` | -| Show recent output | `docker logs --tail 100 docker-container-watchdog` | +| Check status | `docker compose ps` | +| View recent logs | `docker logs --tail 100 docker-container-watchdog` | | Follow logs live | `docker logs -f docker-container-watchdog` | -| Inspect running image tag | `docker inspect docker-container-watchdog --format '{{.Config.Image}}'` | +| Show running image | `docker inspect docker-container-watchdog --format '{{.Config.Image}}'` | | List installed image versions | `docker image ls watchdog` | -| Start the watchdog | `docker compose start docker-container-watchdog` | -| Stop watchdog for approved maintenance | `docker compose stop docker-container-watchdog` | -| Restart watchdog | `docker compose restart docker-container-watchdog` | -| Recreate only watchdog | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | +| Start / stop | `docker compose start docker-container-watchdog` / `docker compose stop docker-container-watchdog` | +| Restart | `docker compose restart docker-container-watchdog` | +| Recreate | `docker compose up -d --no-deps --force-recreate docker-container-watchdog` | -Stopping the watchdog **pauses alert monitoring**. It does not stop application containers. Avoid project-wide `docker compose down` in routine operations. +Stopping the watchdog **pauses monitoring** but does not stop application containers. Avoid `docker compose down` during routine maintenance. ### 5.2 Backup and restore -Before upgrades or authorized configuration changes, preserve the configuration and secrets in a restricted local directory: +Before an upgrade or configuration change, back up `.env` and `watchdog-config.yaml`: ```bash -mkdir -p -m 700 backup +sudo install -d -m 700 backup cp -p .env backup/.env.bak -sudo cp -p watchdog-config.yaml backup/watchdog-config.yaml.bak +cp -p watchdog-config.yaml backup/watchdog-config.yaml.bak + +chmod 600 backup/.env.bak +chmod 600 backup/watchdog-config.yaml.bak ``` -The `watchdog/` directory contains persistent logs and incident state: keep it intact across upgrades and rollbacks. The installer also creates a dated backup of both config files when you **explicitly** choose to reconfigure from scratch. +Backups contain credentials; keep the backup directory restricted. Preserve the `watchdog/` directory as it stores persistent logs and incident/HA state. Omit `sudo` if already running as root. -To restore both files after an approved rollback, preserve their original permissions and recreate **only** the watchdog: +To restore both files, then recreate **only** the watchdog: ```bash +# Run from the watchdog installation directory cp -p backup/.env.bak .env -sudo cp -p backup/watchdog-config.yaml.bak watchdog-config.yaml +cp -p backup/watchdog-config.yaml.bak watchdog-config.yaml + +# Protect credentials +chmod 600 .env + +# Validate Compose configuration +docker compose config --quiet + +# Recreate only the watchdog docker compose up -d --no-deps --force-recreate docker-container-watchdog -``` -If running as root, omit `sudo`. Keep backup directories access-restricted because `.env` contains credentials. +# Verify +docker compose ps docker-container-watchdog +docker compose logs --tail 50 docker-container-watchdog +``` ### 5.3 Upgrade the watchdog -Upgrades use **the same complete release package and installer** as first-time installation. +The upgrade uses the same installer as a new installation. -1. Back up `.env` and `watchdog-config.yaml` as in [Section 5.2](#52-backup-and-restore). Keep the persistent `watchdog/` directory and earlier approved Docker image available for rollback. -2. Obtain the **new approved** `container-alert-v.tar.gz` and matching `SHA256SUMS` via GitHub Releases or authorized transfer. Copy both files into the **parent directory of the existing** `CyberController-Container-Alert` installation. -3. From that parent directory, verify and extract the new package **over the existing installation files**, then run the installer: +1. [Back up your configuration](#52-backup-and-restore). +2. Download or transfer the **new published version** and its `SHA256SUMS` into `/opt/radware/storage/scripts/`. +3. From that directory, verify and extract the new release **over the existing installation directory**, then run its installer: ```bash - cd /path/to/parent-of/CyberController-Container-Alert + cd /opt/radware/storage/scripts + VER=1.7.0 # Example: use the version you downloaded sha256sum --check SHA256SUMS - tar -xzf container-alert-v1.6.1.tar.gz - cd CyberController-Container-Alert + tar -xzf "container-alert-v${VER}.tar.gz" + cd /opt/radware/storage/scripts/CyberController-Container-Alert bash install.sh ``` -4. Answer **No** to “Reconfigure from scratch?” to preserve configuration. The installer verifies/loads the new image, updates the pinned version, and recreates **only the watchdog**. Check its health, actual image and recent logs. - -The filename v1.6.1 above is an **illustrative future version**, not a claim it exists. Replace it with the actual approved release. The archive deliberately excludes active `.env`, `watchdog-config.yaml` and `watchdog/` so upgrades preserve credentials, monitoring policy, historical logs and incident state. +4. Choose **No** when asked `Reconfigure from scratch?` to keep existing settings. Check the watchdog health and image tag using [Section 3](#3-verify-the-installation). -There is **no automatic updater**; upgrading requires an authorized operator to run the installer. Updating Git source code is not an application upgrade. +The package does not replace active `.env`, `watchdog-config.yaml` or `watchdog/`. Only the watchdog container is recreated. There are no automatic upgrades. ### 5.4 Rollback -If a new watchdog image fails after an upgrade, use the prior approved version, provided it remains available locally: +If you need to return to an earlier version: -1. Record the failure and inspect `docker logs --tail 100 docker-container-watchdog`. -2. Verify the previous image exists: `docker image ls watchdog`. If absent, obtain/load the previous approved image through the authorized offline process. -3. Restore the previous `WATCHDOG_VERSION` in `.env` (for example, `1.5.4` when rolling back from v1.6.0) and, if necessary, restore its saved YAML/credential configuration. -4. Recreate **only** the watchdog and verify it: +1. Check that its image is still installed: `docker image ls watchdog`. +2. If the image exists, update `WATCHDOG_VERSION` in `.env`. For example, to return to v1.5.4: ```bash + cd /opt/radware/storage/scripts/CyberController-Container-Alert + sudo sed -i 's/^WATCHDOG_VERSION=.*/WATCHDOG_VERSION=1.5.4/' .env + grep '^WATCHDOG_VERSION=' .env docker compose up -d --no-deps --force-recreate docker-container-watchdog - docker ps --filter name=docker-container-watchdog - docker logs --tail 50 docker-container-watchdog + docker inspect docker-container-watchdog --format '{{.Config.Image}}' ``` -An ordinary `git checkout` or restart does **not** roll back running code; the selected image controls it. Running a newer `install.sh` afterward can reset `WATCHDOG_VERSION` to its `VERSION` file, so use the corresponding approved package for a lasting rollback. + The `.env` value is `1.5.4`, **not** `watchdog:1.5.4`. Restore the matching configuration backup if necessary. -### 5.5 Uninstall +3. If the earlier image is **not** installed, obtain its release package and follow the [upgrade procedure](#53-upgrade-the-watchdog) using that older version; its installer loads the image. Choose **No** to keep existing configuration. -The installer package includes `uninstall.sh`. Its options govern whether images and historical logs are retained. +Verify status and logs after rollback. Restarting without selecting the older image does **not** change the code. Running a newer installer again will select the version packaged with that installer. -| Desired result | Command | +### 5.5 Uninstall + +| Task | Command | |---|---| -| Interactive uninstall with confirmation | `bash uninstall.sh` | +| Interactive uninstall | `bash uninstall.sh` | | Remove watchdog, keep image and logs | `bash uninstall.sh --keep-image --keep-logs` | -| Remove watchdog/image, preserve logs | `bash uninstall.sh --keep-logs` | -| Remove container, image, and logs | `bash uninstall.sh --remove-all` | -| Fully non-interactive removal, **including logs** | `bash uninstall.sh --remove-all --force` | +| Remove watchdog and image, keep logs | `bash uninstall.sh --keep-logs` | +| Remove watchdog, image and logs | `bash uninstall.sh --remove-all` | -Do not use `--remove-all` if historical alerts or state must be retained. Uninstall affects the watchdog, not CyberController application containers. +Logs are preserved by default. `--force` skips confirmation; use it only when unattended removal is intended. Uninstalling the watchdog does not remove CyberController application containers. ## 6. Troubleshooting -| Observation | What to check | -|---|---| -| Release package missing | Obtain the complete archive and checksum from the approved GitHub Release or Radware transfer method | -| Included image/archive rejected | Check the complete package checksum, included `watchdog.tar` and internal `SHA256SUMS`, image version label and disk space | -| Docker or Compose command fails | `docker info`, Docker socket access, `docker compose version` and configured permissions | -| Only one config file exists | Restore the missing `.env` or `watchdog-config.yaml` backup; installer will not overwrite the surviving file | -| Watchdog exits or stays unhealthy | `docker inspect` health state and `docker logs --tail 100 docker-container-watchdog` | -| No alert at receiver | Confirm `alert_channels`, the matching channel's `enabled` flag, endpoints, credentials, and receiver connectivity | -| Syslog recovery not seen in local WARNING-level logs | Console/file level and forwarded Syslog alert priorities are separate; inspect receiver's structured `watchdog_alert` records | -| Unexpected unhealthy reports | Compare Docker native HEALTHCHECK, probe type and detection details; do not suppress alerts before authorized review | -| Excessive restart/crash notifications | Correlate incident and Docker event evidence; do not raise thresholds/exclude containers solely to quiet alerts | -| HA change notification missing | Check both watchdogs, local HA exporter, quorum/metrics, notification receivers and saved baseline; see [HA settings](#44-health-checks-exclusions-and-ha) | - -### 6.1 Missing recovery notifications +Start with the watchdog status and logs; you do not need to disrupt an application container to investigate. -- Confirm `alert_on_recovery: true` and that an incident previously generated an alert. A never-alarmed container has no recovery notification to send. -- For Docker-native health, inspect status using `docker inspect --format '{{json .State.Health}}'`; `starting` is not `healthy`. -- For HTTP/TCP/`/proc` probes, review cached probe evidence. Crash/OOM/restart-loop recovery requires the configured stable-running interval. Do not change thresholds to force recovery. - -### 6.2 Read-only logs and probe diagnostics +```bash +docker inspect docker-container-watchdog --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' +docker logs --tail 100 docker-container-watchdog +``` -From the installation directory: +| Problem | What to check | +|---|---| +| Release download fails | The version exists on the [Releases page](https://github.com/Radware/CyberController-Container-Alert/releases); host HTTPS access, or use an offline transfer | +| Image or checksum rejected | The archive and `SHA256SUMS` come from the same release; check disk space and re-download if needed | +| Docker/Compose errors | `docker info`, `docker compose version` (or `docker-compose --version`), Docker socket permissions | +| Watchdog exits or remains unhealthy | `docker logs --tail 100 docker-container-watchdog` and `docker inspect` health status; `starting` is not `healthy` | +| No notifications | Verify receiver address, port and connectivity. For Syslog check `syslog.enabled`; for SMTP/Slack/SNMP check `alert_channels` and the channel's `enabled` field | +| SMTP/Slack authentication fails | Check the expected variable names in YAML and the actual secret values in `.env`; recreate the watchdog after `.env` changes | +| SNMPv3 traps rejected | Match NMS USM user, auth/privacy settings and sender engine ID (logged at watchdog startup) | +| INFO recovery missing | Check `alert_on_recovery`, whether an initial alert occurred, and the required stable-running interval | +| Repeated crash/restart alerts | Examine container status, Docker events and watchdog logs; do not suppress or exclude containers just to silence alerts | +| Unexpected unhealthy alerts | Compare Docker's native HEALTHCHECK with auto/manual probe details before changing any monitoring settings | +| Missing HA change alert | Check both watchdogs and the local HA exporter. The first reading creates a quiet baseline; a change should alert only after a confirmed active-node transition | + +For additional information, inspect the persistent log in the installation directory: ```bash -tail -f watchdog/watchdog.log grep 'watchdog_alert' watchdog/watchdog.log grep -E 'ERROR|CRITICAL' watchdog/watchdog.log -grep -E 'auto-discover|falling back' watchdog/watchdog.log -grep 'CRITICAL' watchdog/watchdog.log watchdog/watchdog.log.1 -docker compose logs --tail 100 docker-container-watchdog ``` -Logs rotate at approximately 10 MB with five backups. Retain `watchdog/` and -incident state; do not delete alert history simply to suppress notifications. - -For authorization or assistance obtaining the release, contact the appropriate Radware support channel. Never change production CyberController service settings solely in response to a watchdog alert. - -## 7. References and support +A `WARNING` local log level can hide INFO recovery messages from the local file/console, while forwarded Syslog alerts keep their own severity. Check the receiver when investigating missing INFO events. -- [Project overview](../../README.md) — product behavior, alert categories and version history. -- [Release notes](RELEASE_NOTES_V1.6.0.md) — changes, compatibility and limitations for v1.6.0. -- [HA monitoring](#44-health-checks-exclusions-and-ha) — operator prerequisites and expected behavior. +Do not stop, stress or restart monitored application containers merely to test notifications or silence alerts. Preserve the evidence and investigate the underlying problem before changing application settings. -- [Example watchdog YAML](../../watchdog-config.yaml.example) and [example environment variables](../../.env.example) — full annotated configuration templates. +## 7. Related documentation -For release access, installation assistance or product-specific guidance, -use your organization's designated Radware support contact. +- [Project overview](../../README.md) — product features and alert types. +- [v1.6.0 release notes](RELEASE_NOTES_V1.6.0.md) — changes and known limitations. +- [Configuration template](../../watchdog-config.yaml.example) and [environment template](../../.env.example) — settings examples. diff --git a/docs/operations/RELEASE_NOTES_V1.6.0.md b/docs/operations/RELEASE_NOTES_V1.6.0.md index 27c2b51..c55603a 100644 --- a/docs/operations/RELEASE_NOTES_V1.6.0.md +++ b/docs/operations/RELEASE_NOTES_V1.6.0.md @@ -15,8 +15,8 @@ This document describes version **1.6.0** and its changes since v1.5.4. Building | Notification consistency | Preserve INFO recovery and WARNING HA Syslog independently of local logging threshold; consistent Syslog priorities and alert fields, explicit HA/reboot event identifiers; retain SNMP format and five var-binds | [PR #10](https://github.com/Radware/CyberController-Container-Alert/pull/10) | | Crash/OOM recovery correction | Apply the existing configurable `restart_recovery_stability_seconds` gate to CRASH, OOM and RESTART-LOOP recoveries, even when a policy-`no` container was started manually or by an external agent | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Guided release installation | One complete versioned package includes the installer and prebuilt image; operators verify/extract and run one wizard regardless of network availability. Existing settings are preserved and only the watchdog is recreated. No customer-side source build, Git checkout or image download. | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Operator documentation and configuration clarity | Restore one self-contained installation/upgrade path, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP technical details in advanced reference; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | -| Customer-facing alert descriptions | Approved factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Operator documentation and configuration clarity | Restore one self-contained installation/upgrade path, table of contents, full YAML/environment setting reference, operations, backups, upgrades and rollback. Keep SNMP receiver configuration alongside other notification channels; remove unused `.env` `LOG_LEVEL` (logging is set by YAML `log_level`). | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | +| Customer-facing alert descriptions | Factual wording for CRASH, OOM, UNHEALTHY, RESTART-LOOP, RECOVERED, REBOOT-SUMMARY and unknown fallback. Keep HA-FAILOVER advice unchanged. Email/Slack use `Context` rather than `Action`; structured Syslog `action=` remains unchanged for compatibility | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | | Test isolation and reliability | Eliminate live-Docker destructive test execution. Smoke suite requires a verified, labelled, empty Engine on a GitHub-hosted disposable runner, not an arbitrary DOCKER_HOST. Validate fixture exit events, cleanup ownership and absence of leftovers, and lazily resolve Docker socket GID | [PR #9](https://github.com/Radware/CyberController-Container-Alert/pull/9) | ## Customer-facing message catalogue (v1.6.0) @@ -34,7 +34,7 @@ These strings are informational, not instructions to change production configura | `ha-failover` | WARNING | Verify whether the HA switchover was expected and check service health. **Unchanged.** | | Unknown type | WARNING | A container monitoring event was reported. The failure type and available detection details provide additional context. | -The CRASH read-only logs command is intended for authorized diagnostic access. Customers must not modify Docker Compose, memory limits, restart policies, or production resources based solely on an alert. Configuration remediation is determined by Radware R&D or its authorized operational process. +The CRASH log command is read-only. An alert alone is not a reason to modify Docker Compose, memory limits, restart policies or application resources. Investigate the cause before considering changes. **Compatibility:** No change to machine event identifiers, severities, HA identities, container IDs, OIDs/var-binds, probe details, or Syslog key `action`. Only human-facing text and email/Slack label `Context` change. No new alert transport, background service or message queue. @@ -42,13 +42,13 @@ The CRASH read-only logs command is intended for authorized diagnostic access. C - Automated regression tests run in CI without destructive Docker operations. The additional smoke guard tests use a fake Docker executable and must confirm unsafe endpoints are rejected without contacting Docker. - Destructive real Docker failure testing is **only** through the [Release smoke (isolated)](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/.github/workflows/release-smoke.yml) workflow, automatically triggered for qualifying release PR changes and manually rerunnable when available on the default branch. It uses an ephemeral GitHub-hosted runner with an empty labelled nested Engine. See [procedure](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_SMOKE_TESTING.md). -- Historical **engineering-only** test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/dev/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. +- Historical test evidence is retained: [v1.6.0 final 2026-10-14 acceptance](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_ACCEPTANCE_V1.6.0.md), [earlier live HA-lab findings](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_VALIDATION_V1.6.0.md), and JSON files under `docs/dev/evidence/`. **That evidence predates the latest notification wording and fail-closed test-runner changes**. A fresh hosted workflow result must be associated with the current PR commit. - Earlier nested-engine tests on a shared host generated real watchdog alerts. Destructive acceptance is therefore restricted to disposable hosted CI. -- Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the approved merge commit, identified by image digest, and deployed through the existing versioned release process. +- Existing lab watchdog images are **not automatically upgraded** by a PR push. The final image must be built from the release commit, identified by image digest, and deployed through the existing versioned release process. ## Image distribution and deployment -The approved v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is approved and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. +The v1.6.0 release is triggered only by pushing tag `v1.6.0` **after PR #9 is reviewed and merged into `main`**. GitHub Actions builds the version-labelled image, then publishes two assets: `container-alert-v1.6.0.tar.gz` and `SHA256SUMS`. The archive includes `watchdog.tar` plus the installer, Compose files, templates, documentation and an internal checksum. Neither customer installation nor upgrade needs Git, Internet access or a Python source build. The workflow creates a draft release, verifies both assets, then publishes. PR creation or merge alone does not publish a release. For maintainers, see the [reusable Development Guide](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/DEVELOPMENT_AND_RELEASE.md) and [v1.6.0 finalization checklist](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/dev/RELEASE_V1.6.0_CHECKLIST.md); for customers see [Deployment](https://github.com/Radware/CyberController-Container-Alert/blob/v1.6.0/docs/operations/INSTALLATION_AND_OPERATIONS.md). @@ -58,4 +58,4 @@ The workflow creates a draft release, verifies both assets, then publishes. PR c The HA notification dispatch path has no persistent retry queue (intentional v1 limitation). Docker planned-stop correlation is not an availability policy; a deliberately stopped long-running service does not automatically produce a downtime alert. Unexpected external Docker restarts were seen on a lab; the initiator was not determined. These are documented operational limitations, not release regressions. -**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review approved, merge to `main`, build/verify exact `watchdog:1.6.0`, verify release checksums and packaged artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. Merge/approval and tagging are manual release-maintainer actions; the package build and publication run automatically from the approved tag. +**Before shipping:** PR #9 CI green, separate hosted real-Docker smoke green at the **same candidate commit**, code review completed, merge to `main`, build/verify exact `watchdog:1.6.0`, verify release checksums and packaged artifacts per release process, validate upgrade/rollback, tag `v1.6.0` and publish notes. Merging and tagging are manual steps; the package build and publication run automatically after the release tag is pushed. diff --git a/install.sh b/install.sh index c1f984e..2e1cac1 100755 --- a/install.sh +++ b/install.sh @@ -369,7 +369,7 @@ load_release_archive() { print_success "Loaded and verified $IMAGE_NAME" } -# One release bundle contains its own approved, version-labelled Docker image. +# One release bundle contains its version-labelled Docker image. # A missing or invalid archive aborts before changing application configuration. load_docker_image() { print_section "Load Release Image" @@ -537,16 +537,16 @@ configure_all() { fi # ── SMTP ────────────────────────────────────────────────────────────────── - local SMTP_HOST="smtp.radware.com" SMTP_PORT="587" - local SMTP_SENDER="noc-alerts@radware.com" SMTP_RECIPIENTS="ops-team@radware.com" + local SMTP_HOST="smtp.example.com" SMTP_PORT="587" + local SMTP_SENDER="sender@example.com" SMTP_RECIPIENTS="alerts@example.com" local SMTP_TLS="true" SMTP_AUTH="true" SMTP_USERNAME="" SMTP_PASSWORD="" SMTP_ENABLED="false" if _in_array "smtp" "${SELECTED_CHANNELS[@]}"; then SMTP_ENABLED="true" print_info "SMTP (email)" - SMTP_HOST=$(_prompt "Server host" "smtp.radware.com") + SMTP_HOST=$(_prompt "Server host" "smtp.example.com") SMTP_PORT=$(_prompt "Port" "587") - SMTP_SENDER=$(_prompt "Sender address" "noc-alerts@radware.com") - SMTP_RECIPIENTS=$(_prompt "Recipients (comma-separated)" "ops-team@radware.com") + SMTP_SENDER=$(_prompt "Sender address" "sender@example.com") + SMTP_RECIPIENTS=$(_prompt "Recipients (comma-separated)" "alerts@example.com") SMTP_TLS=$(_prompt_yn "Use TLS?" "yes") SMTP_AUTH=$(_prompt_yn "Server requires authentication (login)?" "yes") if [ "$SMTP_AUTH" = "true" ]; then @@ -652,7 +652,7 @@ configure_all() { r=$(echo "$r" | tr -d '[:space:]') [ -n "$r" ] && RECIPIENTS_YAML+=" - ${r}"$'\n' done - [ -z "$RECIPIENTS_YAML" ] && RECIPIENTS_YAML=" - ops-team@radware.com"$'\n' + [ -z "$RECIPIENTS_YAML" ] && RECIPIENTS_YAML=" - alerts@example.com"$'\n' # ── Write watchdog-config.yaml ──────────────────────────────────────────── local CONFIG_TMP @@ -673,7 +673,7 @@ configure_all() { printf " com.radware.cybercontroller.role: mariadb-ha-syntax-check\n" printf " events:\n" printf " - die\n" - printf "# Preferred: R&D labels temporary MariaDB HA syntax-check containers as above.\n" + printf "# Temporary MariaDB HA syntax-check containers should carry the label shown above.\n" printf "# Fallback: suppress kvision_infra_mariadb AutoRemove dump helpers exiting 137\n" printf "# only when /mnt/cli/tmp/mariadb_dump.sql.gz is mounted; primary MariaDB stays monitored.\n" printf "\nrunbook_base_url: \"\"\n" diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index 96c2e8a..1398a02 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -35,7 +35,7 @@ lifecycle_detection: excluded_containers: [] # - my-temp-container -# Narrow, approved exception for Radware-designated temporary syntax checks. +# Narrow exception for labelled temporary syntax-check containers. ignored_container_events: - label: com.radware.cybercontroller.role: mariadb-ha-syntax-check