diff --git a/.github/scripts/check-quickstart-compose.py b/.github/scripts/check-quickstart-compose.py new file mode 100644 index 0000000..2067c42 --- /dev/null +++ b/.github/scripts/check-quickstart-compose.py @@ -0,0 +1,133 @@ +"""Exercise the root Compose, optional UI and memory lab on a disposable runner.""" + +import base64 +import json +import os +from pathlib import Path +import secrets +import ssl +import subprocess +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[2] +BASE_FILES = ['docker-compose.yml'] +LAB_FILES = BASE_FILES + ['docker-compose.ui.yml', 'examples/tutorial-memory/compose.yaml'] +WORKER = 'monit-docker-tutorial-memory' +AUDIT_FILE = '/var/lib/monit-docker/audit/events.jsonl' +ENV = dict(os.environ, COMPOSE_PROJECT_NAME='monit-quickstart-ci', + DEMO_SCENARIO='demo-observe', MONIT_DOCKER_PORT='19808', UI_PORT='18443') + + +def compose(files, *args): + command = ['docker', 'compose'] + for filename in files: + command.extend(['-f', filename]) + return subprocess.check_output(command + list(args), cwd=ROOT, env=ENV, text=True) + + +def wait_for(check, description, timeout=150): + deadline = time.monotonic() + timeout + last_error = None + while time.monotonic() < deadline: + try: + result = check() + if result: + print('Verified: ' + description, flush=True) + return result + except (URLError, OSError, KeyError) as error: + last_error = error + time.sleep(2) + raise AssertionError('Timed out: %s; last error: %r' % (description, last_error)) + + +def main(): + secret_dir = ROOT / 'examples/ui/secrets.local' + if secret_dir.exists(): + raise RuntimeError('Use a clean disposable checkout; secrets.local already exists') + password = secrets.token_hex(24) + subprocess.run(['python3', 'examples/ui/prepare.py', '--self-signed'], + input='compose-ci\n' + password + '\n' + password + '\n', + text=True, cwd=ROOT, check=True) + tls = ssl.create_default_context(cafile=str(secret_dir / 'tls.crt')) + auth = 'Basic ' + base64.b64encode(('compose-ci:' + password).encode()).decode() + + def request(path='/v1/status', browser=False, authenticated=True): + base = 'https://localhost:18443' if browser else 'http://127.0.0.1:19808' + headers = {'Authorization': auth} if browser and authenticated else {} + with urlopen(Request(base + path, headers=headers), + context=tls if browser else None, timeout=5) as response: + return json.load(response) + + def started(): + return subprocess.check_output(['docker', 'inspect', '--format', + '{{.State.StartedAt}}', WORKER], text=True).strip() + + try: + compose(BASE_FILES, 'up', '-d', '--wait', '--wait-timeout', '180') + wait_for(lambda: request()['ready'], 'minimal agent collects actual containers') + assert not request()['manual_actions']['enabled'] + compose(BASE_FILES, 'down') + + compose(LAB_FILES, 'up', '-d', '--wait', '--wait-timeout', '180') + try: + request(browser=True, authenticated=False) + except HTTPError as error: + assert error.code == 401 + else: + raise AssertionError('UI allowed anonymous access') + status = wait_for(lambda: request(browser=True), 'authenticated UI reaches the agent') + assert [c['name'] for c in status['containers']] == [WORKER] + assert not status['manual_actions']['enabled'] + original_start = started() + compose(LAB_FILES, 'kill', '--signal', 'SIGUSR1', 'demo-worker') + wait_for(lambda: request()['containers'][0]['mem_percent'] > 60, + 'bounded worker exceeds the configured threshold') + preview = compose(LAB_FILES, 'exec', '-T', 'monit-docker', 'monit-docker', + '--name', WORKER, 'monit', '--dry-run', '--cmd-if', + 'mem_percent > 60 ? restart') + assert 'dry-run' in preview + assert started() == original_start, 'Observation or preview restarted the worker' + + ENV['DEMO_SCENARIO'] = 'demo-guard' + compose(LAB_FILES, 'up', '-d', '--no-deps', '--wait', '--wait-timeout', '180', 'monit-docker') + compose(LAB_FILES, 'restart', 'ui') + wait_for(lambda: started() != original_start, 'guard restarts the selected worker') + wait_for(lambda: request(browser=True)['containers'][0]['mem_percent'] < 60, + 'UI observes recovery after agent recreation') + journal = compose(LAB_FILES, 'exec', '-T', 'monit-docker', 'monit-docker', + '--audit-file', AUDIT_FILE, 'audit-export') + events = [json.loads(line) for line in journal.splitlines() if line.strip()] + assert any(e.get('source') == 'automatic' and e.get('result') == 'succeeded' + for e in events), 'Missing successful automatic action event' + # Consume another incident and wait out the cooldown: the persisted budget + # must prevent a second automatic restart for this same container ID. + recovered_start = started() + compose(LAB_FILES, 'kill', '--signal', 'SIGUSR1', 'demo-worker') + wait_for(lambda: request()['actions']['restart-limit'] > 0, + 'second incident is blocked by the restart budget', timeout=180) + assert started() == recovered_start + + ENV['DEMO_SCENARIO'] = 'demo-observe' + compose(LAB_FILES, 'up', '-d', '--no-deps', '--wait', '--wait-timeout', '180', 'monit-docker') + compose(LAB_FILES, 'restart', 'ui') + compose(LAB_FILES, 'kill', '--signal', 'SIGUSR2', 'demo-worker') + wait_for(lambda: request()['containers'][0]['mem_percent'] < 60, + 'worker releases memory on command') + before = compose(LAB_FILES, 'exec', '-T', 'monit-docker', 'monit-docker', + '--audit-file', AUDIT_FILE, 'audit-export') + compose(LAB_FILES, 'down') + compose(LAB_FILES, 'up', '-d', '--wait', '--wait-timeout', '180') + after = compose(LAB_FILES, 'exec', '-T', 'monit-docker', 'monit-docker', + '--audit-file', AUDIT_FILE, 'audit-export') + assert before == after, 'Observation restart changed or lost the retained journal' + print('Verified: retained journal survives down/up', flush=True) + finally: + print(compose(LAB_FILES, 'logs', '--tail=30'), flush=True) + compose(LAB_FILES, 'down', '--volumes', '--remove-orphans') + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1207a5c..a103218 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,6 +7,17 @@ permissions: contents: read jobs: + compose-quickstart: + name: Minimal Compose and tutorial acceptance + runs-on: ubuntu-latest + timeout-minutes: 12 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Exercise agent, authenticated UI, memory rule and persistence + run: python3 .github/scripts/check-quickstart-compose.py + textual: name: Optional Textual interface runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 7b838b9..f2cdb22 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ examples/monitoring/notifications.local/ examples/ui/secrets.local/ examples/ui/.env ui/test-results/ + +tutorial-actions.csv +tutorial-events.jsonl diff --git a/README.md b/README.md index 1d0dfe0..f16516f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # monit-docker project -**Current stable release: [1.1.0](https://github.com/decryptus/monit-docker/releases/tag/v1.1.0)** -— [Release notes and upgrade guidance](docs/release-1.1.0.md). +**Current package and image version: [1.1.1](https://github.com/decryptus/monit-docker/tree/v1.1.1)** +— [Release notes and upgrade guidance](docs/release-1.1.1.md).

monit-docker logo @@ -17,7 +17,7 @@ [Documentation (EN)](https://www.monit-docker.com/docs/en/) · [Live demo](https://demo.monit-docker.com/) · [Release roadmap](docs/roadmap.md) · -[1.1.0 release notes](docs/release-1.1.0.md) · +[1.1.1 release notes](docs/release-1.1.1.md) · [Logos and icons](ui/branding/README.md) The [configuration and CLI compatibility baseline](docs/config-cli-contract.md) @@ -117,6 +117,30 @@ the release workflow does not update `latest`. ## Quickstart +### Docker Compose installation + +| Installation | Files | What runs | +| --- | --- | --- | +| Minimal | `docker-compose.yml` | Agent only, observation by default | +| With web UI | Add `docker-compose.ui.yml` | Same agent plus authenticated HTTPS UI | +| Prometheus and Grafana | `examples/monitoring/compose.yaml` | Separate optional history and dashboards stack | + +```sh +git clone https://github.com/decryptus/monit-docker.git +cd monit-docker +docker compose up -d --wait +``` + +Read the [Compose quickstart](docs/compose-quickstart.md) ([français](docs/compose-quickstart-fr.md)) +for prerequisites, the UI, upgrades and stopping the stack. The +[reproducible memory tutorial](docs/tutorial-memory.md) adds a bounded test worker, +then explicitly enables one automatic restart. The default installation has no action rules. + +**Existing root Compose users:** the old cron/PHP-FPM example is replaced by +observation-only `serve`. Preserve and review your old configuration before +upgrading; the old `MONIT_DOCKER_CRONS` jobs do not run in this new entry point. + + Before running configured rules, use [`check-config`](docs/check-config.md) to validate the YAML, imports, selectors and aliases without connecting to Docker. diff --git a/docker-compose.ui.yml b/docker-compose.ui.yml new file mode 100644 index 0000000..283e0da --- /dev/null +++ b/docker-compose.ui.yml @@ -0,0 +1,14 @@ +# Add to docker-compose.yml. Reuse the existing UI and its secret preparation. +# python3 examples/ui/prepare.py --self-signed +# docker compose -f docker-compose.yml -f docker-compose.ui.yml up -d --wait +services: + ui: + extends: + file: ./examples/ui/compose.yaml + service: ui + depends_on: + monit-docker: + condition: service_healthy + +networks: + frontend: diff --git a/docker-compose.yml b/docker-compose.yml index a604c7e..ca4aed5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,35 @@ -version: '3.2' +# Default installation: observe containers, without automatic actions. +# See docs/compose-quickstart.md before replacing the old root cron example. services: monit-docker: - image: decryptus/monit-docker:latest - container_name: monit-docker - environment: - MONIT_DOCKER_CRONS: | - */2 * * * * monit-docker monit --cmd-if 'mem_usage > 200 MiB ? (kill -USR2 1)' - */2 * * * * monit-docker monit --cmd-if 'status not in (paused,running) ? restart' - */2 * * * * monit-docker monit --rsc pid + image: decryptus/monit-docker:1.1.1 + command: [monit-docker, --audit-file, /var/lib/monit-docker/audit/events.jsonl, serve, --bind, 0.0.0.0, --interval, '30'] + restart: unless-stopped + stop_grace_period: 60s + ports: + - '127.0.0.1:${MONIT_DOCKER_PORT:-9808}:9808' + networks: [agent] volumes: - - /var/run:/var/run:rw + # A read-only socket mount still grants Docker API control. + - type: bind + source: ${DOCKER_SOCKET_PATH:-/var/run/docker.sock} + target: /var/run/docker.sock + read_only: true + bind: + create_host_path: false + - state:/var/lib/monit-docker + healthcheck: + test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:9808/readyz", timeout=3).read()'] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + +networks: + agent: + # Use a normal bridge for the loopback-published API port. The standalone + # UI example instead uses an internal network and publishes no agent port. + driver: bridge + +volumes: + state: diff --git a/docs/compose-quickstart-fr.md b/docs/compose-quickstart-fr.md new file mode 100644 index 0000000..3bc84f3 --- /dev/null +++ b/docs/compose-quickstart-fr.md @@ -0,0 +1,112 @@ +# Installer avec Docker Compose + +[English version](compose-quickstart.md) + +Trois installations sont proposées. Toutes démarrent sans règle de redémarrage +automatique des conteneurs surveillés. + +| Besoin | Fichiers | Accès local | +| --- | --- | --- | +| Agent seul et API | `docker-compose.yml` | `http://127.0.0.1:9808` | +| Agent et interface web | Base et `docker-compose.ui.yml` | `https://localhost:8443` | +| Historique et graphiques | `examples/monitoring/compose.yaml` | Grafana sur le port 3000 | + +## Préparer la machine + +Utiliser Docker Engine sous Linux, son socket Unix local et une version récente +du plugin Compose avec `up --wait`. Le compte doit pouvoir accéder à Docker. Les +images publiées ciblent Linux amd64 ; une autre architecture demande une compilation +ou une émulation vérifiée séparément. Docker Desktop doit utiliser les conteneurs +Linux. Sous Windows, exécuter les commandes dans WSL avec l’intégration Docker. +La préparation de l’UI demande aussi Python 3 et OpenSSL. + +```sh +git clone https://github.com/decryptus/monit-docker.git +cd monit-docker +docker compose version +docker info +``` + +Pour Docker rootless, définir `DOCKER_SOCKET_PATH` avec le chemin absolu du socket +local. Les contextes Docker distants ne sont pas couverts par cette procédure. + +## Lancer seulement l’agent + +```sh +docker compose config --quiet +docker compose up -d --wait --wait-timeout 180 +curl -f http://127.0.0.1:9808/readyz +curl -f http://127.0.0.1:9808/v1/status +``` + +La collecte tourne toutes les 30 secondes après la fin du cycle précédent. +`readyz` renvoie HTTP 200 lorsque les mesures sont fraîches et la collecte réussie. +En cas d’erreur, consulter `docker compose ps` et +`docker compose logs --tail=100 monit-docker`. + +L’API reste accessible uniquement en local. L’agent reçoit seulement le socket +Docker, mais ce socket permet de contrôler Docker même si le montage est marqué +`read_only`. L’observation seule vient de l’absence de règles, pas du montage. + +## Ajouter l’interface web + +```sh +python3 examples/ui/prepare.py --self-signed +docker compose -f docker-compose.yml -f docker-compose.ui.yml config --quiet +docker compose -f docker-compose.yml -f docker-compose.ui.yml up -d --wait --wait-timeout 180 +``` + +Saisir un identifiant et un mot de passe d’au moins 12 caractères. Ouvrir +**https://localhost:8443**. Le certificat local est autosigné et valable sept +jours ; le navigateur signale qu’il n’est pas reconnu. Pour un vrai déploiement, +utiliser un certificat de confiance selon le [guide UI](ui.md). + +La préparation se fait une seule fois et refuse d’écraser les secrets existants. +Réutiliser `examples/ui/secrets.local` aux démarrages suivants. L’UI utilise le +même agent, sans Prometheus ni Grafana, et n’active aucun bouton de contrôle des +conteneurs. L’API de l’agent reste accessible sur la boucle locale. + +Conserver la même liste de fichiers pour arrêter cette installation : + +```sh +docker compose -f docker-compose.yml -f docker-compose.ui.yml down +``` + +Cette commande conserve le volume d’état et les secrets. `--volumes` supprimerait +les données du volume : ne pas l’ajouter pour un arrêt normal. Pour revenir à +l’agent seul, arrêter avec les deux fichiers, puis relancer le fichier de base. +Les surcharges avancées d’actions et de journal du dossier `examples/ui` ont leur +propre procédure ; ne pas les mélanger avec ces fichiers racine. + +## Régler les ports et mettre à jour + +Définir `MONIT_DOCKER_PORT`, `UI_PORT` ou `DOCKER_SOCKET_PATH` dans le shell avant +Compose si nécessaire. `UI_BIND` vaut `127.0.0.1` par défaut. Garder le même nom +de projet et le même répertoire pour réutiliser le volume. + +Les images de l’agent et de l’UI sont fixées à **1.1.1**. Pour une évolution, +choisir des versions correspondantes, préserver la configuration et les données, +puis utiliser `pull` et `up -d --wait` avec la même liste de fichiers. Un simple +`pull` ne remplace pas les conteneurs en cours d’exécution. + +## Remplacer l’ancien exemple cron + +L’ancien fichier racine utilisait `latest`, montait tout `/var/run` et exécutait +des règles cron, dont un signal PHP-FPM. Le nouveau lance uniquement `serve` en +observation. **Les anciens jobs `MONIT_DOCKER_CRONS` ne seront plus exécutés.** + +Avant la mise à jour, sauvegarder hors du dépôt l’ancien Compose, la configuration, +les états et les journaux. Arrêter l’ancienne installation avec ce fichier et son +nom de projet d’origine. Pour conserver des jobs, reprendre explicitement leurs +cibles et règles selon le [guide cron](cron.md), avec un état persistant et une +simulation préalable. Aucune conversion automatique des règles n’est effectuée. + +## Continuer avec une démonstration + +Le [tutoriel mémoire](tutorial-memory.md) ajoute un conteneur jetable et permet +d’activer explicitement un seul redémarrage automatique. La limite mémoire du +conteneur et la sélection exacte évitent de prendre un service existant comme cible. + +La [pile Prometheus et Grafana](compose.md) reste une installation distincte pour +l’historique. Arrêter la pile minimale ou modifier ses ports avant de la lancer. +Les notifications demandent une [configuration supplémentaire](notifications.md). diff --git a/docs/compose-quickstart.md b/docs/compose-quickstart.md new file mode 100644 index 0000000..1ae2085 --- /dev/null +++ b/docs/compose-quickstart.md @@ -0,0 +1,137 @@ +# Install with Docker Compose + +[Version française](compose-quickstart-fr.md) + +Choose the minimal agent, add its web UI, or use the separate Prometheus/Grafana +stack. None of these installations enables automatic container actions by default. + +| Need | Compose files | Local address | +| --- | --- | --- | +| Agent and API | `docker-compose.yml` | `http://127.0.0.1:9808` | +| Agent and web UI | Base plus `docker-compose.ui.yml` | `https://localhost:8443` | +| Metrics history and dashboards | `examples/monitoring/compose.yaml` | Grafana on `http://127.0.0.1:3000` | + +## Prerequisites + +Use Linux Docker Engine, a local Unix Docker socket and a current Docker Compose +plugin with `up --wait` support. The account needs Docker access. The published +agent/UI images target Linux amd64; other architectures need a separately tested +build or emulation. Docker Desktop must use Linux containers; for Windows commands +use WSL with Docker integration. The UI preparation also needs Python 3 and OpenSSL. + +```sh +git clone https://github.com/decryptus/monit-docker.git +cd monit-docker +docker compose version +docker info +``` + +These commands use the default local socket. For rootless Docker or another local +socket, set `DOCKER_SOCKET_PATH` to its absolute path. A missing path fails instead +of creating a directory. Remote Docker contexts are outside this quickstart. + +## Start the agent + +From the repository root: + +```sh +docker compose config --quiet +docker compose up -d --wait --wait-timeout 180 +curl -f http://127.0.0.1:9808/readyz +curl -f http://127.0.0.1:9808/v1/status +``` + +The agent observes containers every 30 seconds after each completed cycle. It +does not restart workloads. `/readyz` returns 200 only after a successful fresh +collection. `/healthz` only checks that HTTP responds. For diagnosis: + +```sh +docker compose ps +docker compose logs --tail=100 monit-docker +docker compose exec monit-docker monit-docker stats --output json +``` + +Only the agent receives the Docker socket, and the host API port is on loopback. +A socket bind marked `read_only` still grants Docker API control; it is not a +read-only authorization boundary. Keep the daemon and the agent trusted. + +## Add the web UI + +Prepare credentials once. Enter a username and a password of at least 12 +characters; the password is not printed or passed as a process argument. + +```sh +python3 examples/ui/prepare.py --self-signed +docker compose -f docker-compose.yml -f docker-compose.ui.yml config --quiet +docker compose -f docker-compose.yml -f docker-compose.ui.yml up -d --wait --wait-timeout 180 +``` + +Open **https://localhost:8443** and log in with those credentials. The generated +certificate is for local development and expires in seven days. For this local +example the browser reports it as untrusted. Use your trusted certificate for a +real deployment, following the [UI guide](ui.md). The preparation command refuses +to overwrite existing `examples/ui/secrets.local`; reuse those files on later runs. + +The overlay reuses the existing UI service and its secret paths. It adds no +Prometheus or Grafana, and enables no manual start/stop/restart controls. The +loopback-only agent API remains available. Only the UI joins the frontend network. +Do not combine the advanced `examples/ui/compose.actions.yaml` or journal overlay +with the root files: follow their standalone instructions and relative paths instead. + +Keep the same Compose file list for all UI lifecycle commands: + +```sh +docker compose -f docker-compose.yml -f docker-compose.ui.yml ps +docker compose -f docker-compose.yml -f docker-compose.ui.yml logs --tail=100 +docker compose -f docker-compose.yml -f docker-compose.ui.yml down +``` + +`down` preserves the named state volume and the credential files. Do not use +`--volumes` when you need to preserve journals, restart budgets or other state. +To return from the UI to agent-only, stop with both files, then start the base file. + +## Customize or update + +Set `MONIT_DOCKER_PORT` or `UI_PORT` in the shell before running Compose to change +host ports. The UI binds to loopback unless `UI_BIND` is explicitly changed. +Changing a host port does not change internal agent addresses. Keep the same +directory/project name so Compose reuses its named volume. + +The examples pin agent and UI to **1.1.1**. When updating versions, preserve your +configuration and state, choose matching agent/UI versions, then pull and recreate +the selected stack. `docker compose pull` alone does not change a running container. + +```sh +docker compose -f docker-compose.yml -f docker-compose.ui.yml pull +docker compose -f docker-compose.yml -f docker-compose.ui.yml up -d --wait --wait-timeout 180 +``` + +For agent-only use the same commands without `-f` arguments. See +[installation and upgrades](installation-upgrades.md) for state compatibility. + +## Existing root cron installations + +The previous root file used `latest`, mounted all of `/var/run`, and ran broad +cron rules including PHP-FPM signals. The new root file intentionally starts an +observation-only server. **It does not execute the old `MONIT_DOCKER_CRONS` jobs.** + +Before pulling this change into an existing deployment, save its Compose file, +configuration, state and audit files outside the checkout. Stop the old stack +using that saved file and its original project name. Review each old rule and its +container selection; follow the [cron guide](cron.md) to retain scheduled jobs, +including persistent state and an initial dry run. There is no automatic rule +conversion. Then start the new entry point deliberately. Do not run both old and +new action jobs over the same workloads without reviewing their coordination. + +## Try a reproducible incident + +The [memory tutorial](tutorial-memory.md) adds a disposable worker, limits its +memory, and then opts into one automatic restart after a sustained threshold. +Use it for a local walkthrough before writing rules for your applications. + +## Add history later + +Use the [Prometheus/Grafana Compose guide](compose.md) when you need time-series +history. That example is a separate stack, not another overlay on the root file. +Stop the minimal stack first, or set distinct host ports. Email/Slack delivery +requires the explicitly configured [notification stack](notifications.md). diff --git a/docs/compose.md b/docs/compose.md index 4462b73..f952803 100644 --- a/docs/compose.md +++ b/docs/compose.md @@ -1,5 +1,8 @@ # Serve, Prometheus and Grafana with Docker Compose +For the minimal agent or its optional web UI, start with the +[Compose quickstart](compose-quickstart.md). + This optional example starts the complete **serve mode** stack and loads the dashboard automatically. It observes real containers on your Docker daemon and executes no remediation rules. For a command that exits after one cycle, use @@ -137,10 +140,11 @@ backups; archive them separately if you rely on this history. ## Customize -The services are pinned to monit-docker 0.0.57, Prometheus 3.5.0 and Grafana 12.2.0. +The services are pinned to monit-docker 1.1.1, Prometheus 3.5.0 and Grafana 12.2.0. Update versions deliberately in `examples/monitoring/compose.yaml`, then run `docker compose pull` and `sh start.sh`. The repository-root `docker-compose.yml` -is the older cron example; use the file in `examples/monitoring` for this stack. +runs the minimal observation agent; use the file in `examples/monitoring` for this +separate stack. Stop the minimal stack first or change the published agent port. The dashboard is mounted directly from `examples/grafana/monit-docker.json`. Provisioned panels cannot be saved over from the Grafana UI; save a copy under a diff --git a/docs/index.rst b/docs/index.rst index cc96f6e..6753f77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,9 @@ For installation and the command reference, see the :maxdepth: 2 :caption: Serve mode + compose-quickstart + compose-quickstart-fr + tutorial-memory serve http-api-contract ui diff --git a/docs/tutorial-memory.md b/docs/tutorial-memory.md new file mode 100644 index 0000000..808a685 --- /dev/null +++ b/docs/tutorial-memory.md @@ -0,0 +1,175 @@ +# Reproduce a memory incident with Compose + +This local lab uses a disposable Python worker with a bounded cache. It simulates +retained application data, not a specific software leak. It starts in observation +mode, then explicitly enables one automatic restart. No notification service is +configured. Use a Linux amd64 lab host with the [Compose prerequisites](compose-quickstart.md). + +## Start the lab + +From the repository root, prepare the UI credentials once if not already present: + +```sh +python3 examples/ui/prepare.py --self-signed +``` + +Use this POSIX shell function in the same terminal throughout the lab: + +```sh +dc() { + docker compose -f docker-compose.yml -f docker-compose.ui.yml \ + -f examples/tutorial-memory/compose.yaml "$@" +} +export DEMO_SCENARIO=demo-observe +dc config --quiet +dc up -d --wait --wait-timeout 180 +dc exec monit-docker monit-docker -c /etc/monit-docker/tutorial.yml check-config +``` + +Open **https://localhost:8443**. The agent selects only the worker that has both +the exact demo name and a matching group label value. Existing containers are +not rule targets. The worker has 512 MiB of memory, a 0.5 CPU limit, no network +and no Docker restart policy. It starts without growing its cache. A name collision +fails creation; do not remove an unrelated existing container to make room. + +The tutorial overlay replaces the agent command and mounts its own YAML. The +normal root installation continues to observe all containers when this overlay +is absent. Stop the root stack before moving to a separate project/port to avoid +running multiple agents unintentionally. + +`dc restart ui` below refreshes the proxy after the agent is recreated. +For the lab without a browser, skip those restart commands, omit `-f docker-compose.ui.yml` from `dc` and skip +the UI preparation. Use the terminal commands below instead. + +## Observe and preview + +Record the worker's original start time, then trigger controlled growth: + +```sh +dc exec monit-docker monit-docker --name monit-docker-tutorial-memory stats --output json +docker inspect --format '{{.State.StartedAt}}' monit-docker-tutorial-memory +dc kill --signal SIGUSR1 demo-worker +``` + +`kill --signal SIGUSR1` sends a signal handled by the demo script; it does not stop +this worker. It adds 12 MiB per second, up to 360 MiB, then stays at that plateau. +The reported memory also includes the interpreter and runtime overhead. Allow +about 30 seconds for the plateau, then refresh the UI and observe real values. + +```sh +dc exec monit-docker monit-docker --name monit-docker-tutorial-memory \ + monit --dry-run --cmd-if 'mem_percent > 60 ? restart' +docker inspect --format '{{.State.StartedAt}}' monit-docker-tutorial-memory +``` + +The dry run reports the matching action without performing it. The start time +must remain unchanged. This one-shot preview demonstrates the selection/threshold; +it does not exercise the sustained timer or build durable observations. The +automatic configuration below supplies those policies. + +## Read the YAML and enable the guard + +Open `examples/tutorial-memory/monit-docker.yml`. The `tutorial` group matches +label **values**, not Docker's `key=value` filter syntax. The explicit name +intersects that group. `demo-observe` has no action rule. `demo-guard` uses: + +| Setting | Lab value | Meaning | +| --- | --- | --- | +| Memory condition | `mem_percent > 60` | Compared with the container memory limit | +| Interval | 5 seconds | Wait after each completed cycle | +| Trigger delay | 20 seconds | Require high observations spanning at least this duration | +| Maximum gap | 15 seconds | Reset the streak if observations are too far apart | +| Cooldown | 120 seconds | Space attempts of this rule | +| Restart budget | 1 | One automatic restart attempt for this container ID and state file | + +These short timings are for filming. Choose application-specific thresholds and +longer observation windows for production. A restart is only appropriate when +the workload can tolerate it; it does not diagnose or repair a memory leak. + +```sh +export DEMO_SCENARIO=demo-guard +dc up -d --no-deps --wait --wait-timeout 180 monit-docker +dc restart ui +dc logs --follow --tail=20 monit-docker +``` + +Keep the UI open and refresh the measurements. The first high sample starts the +timer. An eligible later cycle attempts the restart. Collection time affects the +delay, so do not promise an exact twenty-second response. Stop following logs +with Ctrl+C; the agent continues running in the background. + +```sh +docker inspect --format '{{.State.StartedAt}}' monit-docker-tutorial-memory +dc exec monit-docker monit-docker --name monit-docker-tutorial-memory stats --output json +dc logs --tail=12 demo-worker +``` + +The start time changes, memory drops and the worker emits new ticks. Inspect the +journal for the recorded action result. A successful Docker restart alone does +not prove a real application's health; this worker's log is only a lab liveness check. + +## Read the journal and terminal interface + +```sh +umask 077 +dc exec -T monit-docker monit-docker \ + --audit-file /var/lib/monit-docker/audit/events.jsonl \ + audit-export --format csv > tutorial-actions.csv +dc exec -T monit-docker monit-docker \ + --audit-file /var/lib/monit-docker/audit/events.jsonl \ + audit-export > tutorial-events.jsonl +``` + +Check command exit status before using an export; a write/read failure may leave +a partial file. These are event histories, not time-series metric archives. +Optional interactive view from a real terminal: + +```sh +dc exec -e TERM=xterm-256color monit-docker monit-docker \ + --name monit-docker-tutorial-memory \ + --audit-file /var/lib/monit-docker/audit/events.jsonl tui --refresh 5 +``` + +The image includes the curses interface. Use Tab for the journal, Enter for +details, and q to quit. The TUI is read-only and does not run action rules. The +[Textual interface](textual.md) is an optional Python extra, not assumed to be +installed in the standard Docker image. Do not substitute synthetic demo screens +for the lab's live measurements. + +## Stop, repeat or return to the minimal installation + +Disable the automatic scenario before another take: + +```sh +export DEMO_SCENARIO=demo-observe +dc up -d --no-deps --wait --wait-timeout 180 monit-docker +dc restart ui +dc kill --signal SIGUSR2 demo-worker +``` + +For the same worker ID, the restart budget stays consumed after recovery. To +rearm deliberately for another take, keep observation mode active, obtain the +full ID and reset only that ID: + +```sh +DEMO_CONTAINER_ID=$(docker inspect --format '{{.Id}}' monit-docker-tutorial-memory) +dc exec monit-docker monit-docker \ + --audit-file /var/lib/monit-docker/audit/events.jsonl restart-reset \ + --state-file /var/lib/monit-docker/tutorial/state.json \ + --container-id "$DEMO_CONTAINER_ID" +``` + +Run that reset only after an attempt has been recorded. It preserves cooldowns; +wait until the original 120-second cooldown has elapsed before filming a new +automatic action. Do not delete the state file to hide or bypass the budget. + +```sh +dc down +unset DEMO_SCENARIO +``` + +This removes the lab containers and networks but keeps state and UI credentials. +For agent-only, now run `docker compose up -d --wait`. If the worker is recreated +it gets a new container ID and therefore a fresh budget; the old journal remains. +Archive the exported files before another session. Prometheus/Grafana and external +notifications are optional follow-on exercises, not requirements for this lab. diff --git a/docs/ui.md b/docs/ui.md index 333ff87..baae84a 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -1,5 +1,9 @@ # Optional lightweight interface +For a minimal installation from the repository root, use the +[Compose quickstart](compose-quickstart.md#add-the-web-ui). The standalone example +below remains available for the advanced UI overlays. + `monit-docker-ui` is a separate, optional Community component. The Python package and the simple/cron mode do not include a frontend. An agent in `serve` mode supplies API v1; Nginx serves the UI and authenticates its API requests. diff --git a/examples/monitoring/compose.yaml b/examples/monitoring/compose.yaml index db440b4..1374ae3 100644 --- a/examples/monitoring/compose.yaml +++ b/examples/monitoring/compose.yaml @@ -1,6 +1,6 @@ services: monit-docker: - image: decryptus/monit-docker:0.0.57 + image: decryptus/monit-docker:1.1.1 command: [monit-docker, serve, --bind, 0.0.0.0, --interval, '30'] restart: unless-stopped stop_grace_period: 60s diff --git a/examples/tutorial-memory/compose.yaml b/examples/tutorial-memory/compose.yaml new file mode 100644 index 0000000..a3ce336 --- /dev/null +++ b/examples/tutorial-memory/compose.yaml @@ -0,0 +1,36 @@ +# Overlay for the root Compose, optionally with docker-compose.ui.yml. +# All bind paths are relative to the first Compose file, at repository root. +services: + monit-docker: + command: [monit-docker, -c, /etc/monit-docker/tutorial.yml, --audit-file, /var/lib/monit-docker/audit/events.jsonl, run, '${DEMO_SCENARIO:-demo-observe}'] + volumes: + - type: bind + source: ./examples/tutorial-memory/monit-docker.yml + target: /etc/monit-docker/tutorial.yml + read_only: true + bind: + create_host_path: false + depends_on: + demo-worker: + condition: service_started + + demo-worker: + image: python:3.12-alpine + container_name: monit-docker-tutorial-memory + command: [python, -u, /demo/worker.py] + labels: + org.decryptus.tutorial: decryptus-memory-tutorial + mem_limit: 512m + memswap_limit: 512m + cpus: 0.5 + pids_limit: 32 + read_only: true + restart: 'no' + network_mode: none + volumes: + - type: bind + source: ./examples/tutorial-memory/worker.py + target: /demo/worker.py + read_only: true + bind: + create_host_path: false diff --git a/examples/tutorial-memory/monit-docker.yml b/examples/tutorial-memory/monit-docker.yml new file mode 100644 index 0000000..5517545 --- /dev/null +++ b/examples/tutorial-memory/monit-docker.yml @@ -0,0 +1,29 @@ +ctn-groups: + tutorial: + # Label selectors match VALUES, not Docker's key=value filter syntax. + match: ['label:decryptus-memory-tutorial'] + +scenarios: + demo-observe: + description: Observe the tutorial worker without actions + mode: serve + select: + group: tutorial + name: monit-docker-tutorial-memory + bind: 0.0.0.0 + interval: 5 + + demo-guard: + description: Restart the tutorial worker once after sustained high memory + mode: serve + select: + group: tutorial + name: monit-docker-tutorial-memory + bind: 0.0.0.0 + interval: 5 + rules: ['mem_percent > 60 ? restart'] + trigger-after: 20 + max-gap: 15 + cooldown: 120 + max-restarts: 1 + state-file: /var/lib/monit-docker/tutorial/state.json diff --git a/examples/tutorial-memory/worker.py b/examples/tutorial-memory/worker.py new file mode 100644 index 0000000..a06763e --- /dev/null +++ b/examples/tutorial-memory/worker.py @@ -0,0 +1,50 @@ +"""Disposable memory demonstration, controlled explicitly with POSIX signals. + +SIGUSR1 grows a bounded in-memory cache; SIGUSR2 releases it. Restarting also +returns to the idle state. This simulates a worker retaining too much data; it +does not reproduce a particular application's memory leak. +""" + +import json +import signal +import time + + +CHUNK_BYTES = 12 * 1024 * 1024 +MAX_CHUNKS = 30 +TICK_SECONDS = 1 + + +def main(): + state = {'grow': False, 'clear': False, 'stop': False} + blocks = [] + + def grow(_signum, _frame): + state['grow'] = True + + def clear(_signum, _frame): + state.update(grow=False, clear=True) + + def stop(_signum, _frame): + state['stop'] = True + + signal.signal(signal.SIGUSR1, grow) + signal.signal(signal.SIGUSR2, clear) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + ticks = 0 + while not state['stop']: + if state['clear']: + blocks.clear() + state['clear'] = False + if state['grow'] and len(blocks) < MAX_CHUNKS: + blocks.append(bytearray(b'x') * CHUNK_BYTES) + if ticks % 5 == 0: + print(json.dumps({'tick': ticks, 'cache_mib': len(blocks) * 12, + 'growing': state['grow']}), flush=True) + ticks += 1 + time.sleep(TICK_SECONDS) + + +if __name__ == '__main__': + main() diff --git a/examples/ui/compose.yaml b/examples/ui/compose.yaml index 8cc9755..4eaab09 100644 --- a/examples/ui/compose.yaml +++ b/examples/ui/compose.yaml @@ -1,6 +1,6 @@ services: monit-docker: - image: decryptus/monit-docker:0.0.66 + image: decryptus/monit-docker:1.1.1 command: [monit-docker, serve, --bind, 0.0.0.0, --interval, '30'] restart: unless-stopped stop_grace_period: 60s @@ -15,7 +15,7 @@ services: create_host_path: false ui: - image: decryptus/monit-docker-ui:0.0.66 + image: decryptus/monit-docker-ui:1.1.1 restart: unless-stopped depends_on: [monit-docker] networks: [agent, frontend]