From 0993565fa01470fb536e0518889d3b9d2d3b44f3 Mon Sep 17 00:00:00 2001 From: Ronak Pansuriya Date: Mon, 2 Nov 2026 13:10:51 +0100 Subject: [PATCH] Pin image versions, move deploy files out, restore failure streak after restart - no more floating latest tag in CI; images are tagged with the release version (0.1.2) - docker compose and Caddy files moved to the openedx-status-deploy repo - an ongoing outage still opens its incident on time after an API restart --- .github/workflows/docker.yml | 1 - README.md | 22 ++--------- deploy/.env.example | 17 -------- deploy/.gitignore | 2 - deploy/Caddyfile | 6 --- deploy/checks.example.yml | 11 ------ deploy/docker-compose.yml | 70 --------------------------------- openedx_status_api/incidents.py | 5 +++ openedx_status_api/runner.py | 2 + openedx_status_api/store.py | 12 ++++++ pyproject.toml | 2 +- tests/test_core.py | 12 ++++++ 12 files changed, 36 insertions(+), 126 deletions(-) delete mode 100644 deploy/.env.example delete mode 100644 deploy/.gitignore delete mode 100644 deploy/Caddyfile delete mode 100644 deploy/checks.example.yml delete mode 100644 deploy/docker-compose.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cccb022..cebcfe0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,6 @@ jobs: images: ghcr.io/${{ github.repository_owner }}/openedx-status-api tags: | type=semver,pattern={{version}} - type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }} - uses: docker/build-push-action@v6 with: context: . diff --git a/README.md b/README.md index d736b8f..8022ec7 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,9 @@ Django, so it keeps working when Open edX is down. ## Quick start -This runs the API, the web page ([openedx-status-web](https://github.com/Abstract-Tech/openedx-status-web)) -and a Caddy proxy with Docker Compose: - - cd deploy - cp .env.example .env # set STATUS_DOMAIN, and point its DNS at this machine - cp checks.example.yml checks.yml # put your own addresses in - docker compose up -d - -Then open `https://`. To try it locally without HTTPS, set `STATUS_DOMAIN=:80`, -`STATUS_BIND=127.0.0.1` and `STATUS_HTTP_PORT=8088` in `.env`. - -Day to day: - -- Change the checks: edit `checks.yml`, then `docker compose up -d --force-recreate status-api` (a plain restart can keep a stale copy of the file). -- Logs: `docker compose logs -f status-api`. -- Backup: the history is the SQLite file in the `status-data` volume. -- Already have a reverse proxy? Remove the `caddy` service and point your proxy at `status-web:8080`. +To run the API, the web page ([openedx-status-web](https://github.com/Abstract-Tech/openedx-status-web)) +and a Caddy proxy together, use [openedx-status-deploy](https://github.com/Abstract-Tech/openedx-status-deploy). +This repo only contains the API and its Docker image (`ghcr.io/abstract-tech/openedx-status-api`). To run the API alone without Docker: @@ -44,7 +30,7 @@ If you mount a host folder as `/data`, run `chown 10001` on it, because the cont - `params`: extra options for some kinds To keep a secret out of the file, write `_env: SOME_VARIABLE` instead of the value, for example -`password_env: DB_PASSWORD`. With the compose file in `deploy/`, put `DB_PASSWORD=...` in `deploy/.env` +`password_env: DB_PASSWORD`. With [openedx-status-deploy](https://github.com/Abstract-Tech/openedx-status-deploy), put `DB_PASSWORD=...` in its `.env` (it is passed to the API container) and recreate the container. This works for the http `expected_status` and `verify_tls`, the mysql `user`, `password` and `database`, the mongo `username` and `password`, the redis `password`, and the ssl `warn_days`. diff --git a/deploy/.env.example b/deploy/.env.example deleted file mode 100644 index 1d1d4d3..0000000 --- a/deploy/.env.example +++ /dev/null @@ -1,17 +0,0 @@ -# The public address of the status page (DNS A/AAAA record must point at this machine). -STATUS_DOMAIN=status.example.com -STATUS_TITLE=Example Open edX status -STATUS_FAIL_THRESHOLD=3 -# Optional, defaults shown: -# STATUS_RETENTION_DAYS=90 -# STATUS_HTTP_PORT=80 -# STATUS_HTTPS_PORT=443 -# Address the proxy ports bind to; use 127.0.0.1 for local tests. -# STATUS_BIND=0.0.0.0 -# Image versions to pull (default: latest): -# STATUS_API_VERSION=latest -# STATUS_WEB_VERSION=latest -# For a quick local try-out without TLS: STATUS_DOMAIN=:80 STATUS_BIND=127.0.0.1 STATUS_HTTP_PORT=8088 -# Secrets for `_env` options in checks.yml (placeholders only, set real values in .env): -# STATUS_MYSQL_PASSWORD=change-me -# STATUS_REDIS_PASSWORD=change-me diff --git a/deploy/.gitignore b/deploy/.gitignore deleted file mode 100644 index e269681..0000000 --- a/deploy/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.env -checks.yml diff --git a/deploy/Caddyfile b/deploy/Caddyfile deleted file mode 100644 index 01fde5b..0000000 --- a/deploy/Caddyfile +++ /dev/null @@ -1,6 +0,0 @@ -{$STATUS_DOMAIN} { - encode zstd gzip - # Ignored by browsers over plain HTTP (STATUS_DOMAIN=:80), effective once real TLS is on. - header Strict-Transport-Security "max-age=31536000" - reverse_proxy status-web:8080 -} diff --git a/deploy/checks.example.yml b/deploy/checks.example.yml deleted file mode 100644 index 92f7aa7..0000000 --- a/deploy/checks.example.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Copy to checks.yml. Only PUBLIC addresses: this machine is not inside your Open edX server. -# Schema: id, name, group, kind (http|tcp|mysql|mongo|redis|ssl), target, interval (s), timeout (s), critical. -checks: - - {id: lms, name: LMS, group: Open edX, kind: http, target: "https://lms.example.com/heartbeat", interval: 60, timeout: 10} - - {id: studio, name: Studio (CMS), group: Open edX, kind: http, target: "https://studio.example.com/heartbeat", interval: 60, timeout: 10} - - {id: web, name: Public website, group: Open edX, kind: http, target: "https://lms.example.com/", interval: 60, timeout: 10} - - {id: mfe, name: Learning micro-frontends, group: Open edX, kind: http, target: "https://apps.lms.example.com/learning/", interval: 60, timeout: 10} - - {id: tls, name: TLS certificate, group: Open edX, kind: ssl, target: "lms.example.com:443", interval: 3600, timeout: 10, critical: false} - # Databases cannot be checked from outside without exposing them. Either leave the "Databases" group out - # (an outage still shows up through the LMS/Studio checks) or reach them over a VPN/private network: - # - {id: mysql, name: MySQL, group: Databases, kind: tcp, target: "db.internal.example.com:3306", interval: 60} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml deleted file mode 100644 index 313b56a..0000000 --- a/deploy/docker-compose.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Standalone deployment of the Open edX status page. -# Runs OUTSIDE the Tutor stack (own compose project, own TLS), so `tutor local stop/restart/upgrade` -# or an Open edX outage never takes it down. For a full-server outage, run it on a DIFFERENT machine. -name: openedx-status - -services: - status-api: - image: ghcr.io/abstract-tech/openedx-status-api:${STATUS_API_VERSION:-latest} - restart: unless-stopped - read_only: true - tmpfs: [/tmp] - cap_drop: [ALL] - security_opt: ["no-new-privileges:true"] - env_file: - - path: .env # lets `password_env: SOME_VAR` in checks.yml read SOME_VAR from .env - required: false - environment: - STATUS_TITLE: "${STATUS_TITLE:-Open edX status}" - STATUS_HOST: "0.0.0.0" # listen address; set here so .env can never change it - STATUS_PORT: "8000" - STATUS_CHECKS_FILE: /config/checks.yml - STATUS_DATA_DIR: /data - STATUS_FAIL_THRESHOLD: "${STATUS_FAIL_THRESHOLD:-3}" - STATUS_RETENTION_DAYS: "${STATUS_RETENTION_DAYS:-90}" - volumes: - - ./checks.yml:/config/checks.yml:ro - - status-data:/data - healthcheck: - test: ["CMD", "python", "-c", "import socket; socket.create_connection(('127.0.0.1', 8000), 3).close()"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 20s - - status-web: - image: ghcr.io/abstract-tech/openedx-status-web:${STATUS_WEB_VERSION:-latest} - restart: unless-stopped - read_only: true - tmpfs: [/tmp, /var/cache/nginx, /var/run] - volumes: - - status-cache:/var/cache/status-cache # last good API answer survives a restart of this container - cap_drop: [ALL] - security_opt: ["no-new-privileges:true"] - healthcheck: - test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/healthz || exit 1"] - interval: 30s - timeout: 5s - retries: 3 - - # TLS front door (automatic Let's Encrypt). Skip this service and publish status-web yourself if you - # already have a reverse proxy. - caddy: - image: caddy:2-alpine - restart: unless-stopped - ports: - - "${STATUS_BIND:-0.0.0.0}:${STATUS_HTTP_PORT:-80}:80" - - "${STATUS_BIND:-0.0.0.0}:${STATUS_HTTPS_PORT:-443}:443" - - "${STATUS_BIND:-0.0.0.0}:${STATUS_HTTPS_PORT:-443}:443/udp" # HTTP/3 - environment: - STATUS_DOMAIN: "${STATUS_DOMAIN:?set STATUS_DOMAIN, e.g. status.example.com (use :80 for plain HTTP tests)}" - volumes: - - ./Caddyfile:/etc/caddy/Caddyfile:ro - - caddy-data:/data - - caddy-config:/config - -volumes: - status-data: - status-cache: - caddy-data: - caddy-config: diff --git a/openedx_status_api/incidents.py b/openedx_status_api/incidents.py index a22a06c..f3d2853 100644 --- a/openedx_status_api/incidents.py +++ b/openedx_status_api/incidents.py @@ -10,6 +10,11 @@ def __init__(self, store: Store, threshold: int): self._fails: dict[str, int] = {} self._first_fail: dict[str, int] = {} # ts of the first failure in the current streak + def restore(self, check: Check): + n, first = self.store.failure_streak(check.id, self.threshold) + if n: + self._fails[check.id], self._first_fail[check.id] = n, first + def observe(self, check: Check, r: Result): open_id = self.store.open_incident_for(check.id) if r.ok: diff --git a/openedx_status_api/runner.py b/openedx_status_api/runner.py index cc7908f..7dff1ab 100644 --- a/openedx_status_api/runner.py +++ b/openedx_status_api/runner.py @@ -22,6 +22,8 @@ def start(self): n = self.store.resolve_orphaned_incidents([c.id for c in self.checks], int(time.time())) if n: log.info("resolved %d incident(s) for checks no longer configured", n) + for c in self.checks: + self.tracker.restore(c) self._tasks = [asyncio.create_task(self._loop(c), name=f"check-{c.id}") for c in self.checks] self._tasks.append(asyncio.create_task(self._prune_loop(), name="prune")) diff --git a/openedx_status_api/store.py b/openedx_status_api/store.py index 248711f..3bd3165 100644 --- a/openedx_status_api/store.py +++ b/openedx_status_api/store.py @@ -55,6 +55,18 @@ def is_failing(self, check_id: str, n: int) -> bool: ).fetchall() return len(rows) == n and not any(r["ok"] for r in rows) + def failure_streak(self, check_id: str, limit: int) -> tuple[int, int | None]: + with self._lock: + rows = self._db.execute( + "SELECT ts, ok FROM results WHERE check_id=? ORDER BY ts DESC, rowid DESC LIMIT ?", (check_id, limit) + ).fetchall() + streak = [] + for r in rows: + if r["ok"]: + break + streak.append(r["ts"]) + return len(streak), (streak[-1] if streak else None) + def days(self, check_id: str, retention: int = 90, now: int | None = None) -> list[dict]: """Exactly `retention` entries oldest->newest, zero-filled.""" today = datetime.fromtimestamp(now or time.time(), timezone.utc).date() diff --git a/pyproject.toml b/pyproject.toml index 1022dab..c314587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "openedx-status-api" -version = "0.1.0" +version = "0.1.2" description = "Standalone status-page backend for Open edX" readme = "README.md" license = {text = "MIT"} diff --git a/tests/test_core.py b/tests/test_core.py index fe74771..b724fc5 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -154,3 +154,15 @@ def test_never_reported_check_waits_but_stale_one_degrades(store, settings): assert build_status(settings, cs, store, NOW)["groups"][0]["status"] == "operational" # b has not reported yet store.record("b", Result(NOW - 1000, True, "ok")) assert build_status(settings, cs, store, NOW)["groups"][0]["status"] == "degraded" # b went stale + + +def test_failure_streak_survives_restart(store): + c = Check("a", "A", "G", "tcp", "x:1") + for i in range(2): + store.record("a", Result(NOW + i, False, "")) + t = IncidentTracker(store, 3) + t.restore(c) + store.record("a", Result(NOW + 2, False, "")) + t.observe(c, Result(NOW + 2, False, "")) + assert store.open_incident_for("a") is not None + assert store.incidents(0)[0]["started"] == NOW