diff --git a/.gitignore b/.gitignore index 716447d7..336a7736 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,5 @@ commit-boost/config.toml !scripts/edit/**/test/fixtures/.charon/ !scripts/edit/**/test/fixtures/.charon/** -# Your own Engine API secret, if you provide one. The jwt-init service in docker-compose.yml generates one otherwise. +# Your own Engine API secret, if you provide one. The execution client generates one otherwise. jwt/jwt.hex* diff --git a/CLAUDE.md b/CLAUDE.md index 53a926e4..be5aaf89 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -178,7 +178,7 @@ When modifying this repository: ## Important Notes - **Never commit `.env` files** - they contain operator-specific configuration -- **JWT secret** must be shared between EL and CL clients. It is not tracked: the `jwt-init` service generates it per deployment in the `jwt` volume before the EL and CL start, or loads an operator-provided `jwt/jwt.hex`. Every EL and CL service needs `depends_on: jwt-init` (`service_completed_successfully`) and a read-only mount of the `jwt` volume +- **JWT secret** must be shared between EL and CL clients. It is not tracked: the EL generates it on start in the in-memory `jwt` volume, and it is replaced once both the EL and CL were stopped. Every EL service mounts the volume writable and every CL service read-only, and all of them run as root so they can read it. An operator-provided `jwt/jwt.hex` is mounted over it in a `docker-compose.override.yml` - **Cluster lock** in `.charon/cluster-lock.json` is critical - back it up before any edit operations - **Validator keys** in `.charon/validator_keys/` must be kept secure and never committed - **Data directory compatibility**: When switching VCs, verify the new client can handle existing key state diff --git a/README.md b/README.md index a2099dc2..de1486e4 100644 --- a/README.md +++ b/README.md @@ -211,11 +211,14 @@ docker compose up cl-grandine -d - **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 @`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo. - **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces. - **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans. -- **The Engine API secret is unique to each deployment.** On first start, the `jwt-init` service generates a random secret in the `jwt` Docker volume, which only the execution and consensus clients mount, so it is not a file in this directory. - - **Upgrading:** older releases shipped the same `jwt/jwt.hex` to every operator. After `git pull`, the next `docker compose up -d` recreates the execution and consensus clients with a new secret, and nothing else is needed. Start them together: a client still running from before the update cannot authenticate to one that has the new secret. +- **The Engine API secret is unique to each deployment and is kept in memory.** The execution client generates it on start in the `jwt` volume, which only the execution and consensus clients mount. It is not a file in this directory and is not written to disk. + - **Rotation:** the secret exists while the execution or the consensus client is running, so restarting one of them keeps it. After both were stopped, for example by `docker compose down` or a reboot, the execution client generates a new one. + - **Start-up:** the consensus client may restart a few times until the execution client has written the secret. + - **Root:** Teku and Nimbus now run as root, like the other clients, so that they can read the secret. Files they create in `data/cl-teku` and `data/cl-nimbus` are owned by root. + - **Upgrading:** older releases shipped the same `jwt/jwt.hex` to every operator, and later ones kept a generated secret in the `jwt` volume. After `git pull`, the next `docker compose up -d` recreates the execution and consensus clients with a new secret. If you ran a release with the `jwt-init` container, remove the old `jwt` volume first, otherwise its secret stays on disk and is never replaced: run `docker compose down --remove-orphans`, then `docker volume rm _jwt`, then `docker compose up -d`. - **If `git pull` stops because you changed `jwt/jwt.hex`,** run `git checkout jwt/jwt.hex` and pull again. - - **To use your own secret,** for example with an execution client outside this stack, put it in `jwt/jwt.hex` as 64 hex characters (copy it aside before the `git checkout`), then run `docker compose up -d --force-recreate`. The secret that older releases shipped is ignored. - - **To print the secret in use:** `docker compose run --rm jwt-init cat /jwt/jwt.hex`. + - **To use your own secret,** for example with an execution client outside this stack, put it in `jwt/jwt.hex` as 64 hex characters and mount it into your consensus client in a `docker-compose.override.yml`, as shown in `docker-compose.override.yml.sample`. Do not use `7074a5bf6bd6dae368fa598249d57edfcbccc67a1205b2c8d5d2fe7b800663aa`: older releases shipped this secret to everyone, so it is public. Replace it in the external client as well. + - **To print the secret in use:** `docker compose exec cat /root/jwt/jwt.hex`, for example with `el-nethermind`. - **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address. It lists anything in the old `data/grafana` that this stack does not create, such as service accounts, tokens, users, datasources, plugins, organizations, dashboards, alert rules and contact points. It also lists changed or added files under `grafana/`, which older releases mounted writable. Report findings to security@obol.tech, then delete `data/grafana`. Grafana stores secrets with a default encryption key, so if your Grafana was reachable, rotate any credentials you stored in it, such as webhook URLs or tokens used by contact points. - **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded: diff --git a/compose-cl.yml b/compose-cl.yml index 68f95fa8..a115b254 100644 --- a/compose-cl.yml +++ b/compose-cl.yml @@ -13,9 +13,6 @@ services: cl-grandine: profiles: [ cl-grandine ] image: sifrai/grandine:${CL_GRANDINE_VERSION:-${GRANDINE_VERSION:-3.0.0-rc.2}} - depends_on: - jwt-init: - condition: service_completed_successfully restart: unless-stopped labels: - "alloy-monitored=${CL_GRANDINE_ALLOY_MONITORED:-true}" @@ -51,9 +48,6 @@ services: cl-lighthouse: profiles: [ cl-lighthouse ] image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.3} - depends_on: - jwt-init: - condition: service_completed_successfully restart: unless-stopped labels: - "alloy-monitored=${CL_LIGHTHOUSE_ALLOY_MONITORED:-true}" @@ -91,9 +85,8 @@ services: cl-teku: profiles: [ cl-teku ] image: consensys/teku:${CL_TEKU_VERSION:-26.10.3} - depends_on: - jwt-init: - condition: service_completed_successfully + # The image defaults to uid 1000; root matches the execution clients, which own the Engine API secret. + user: root restart: unless-stopped labels: - "alloy-monitored=${CL_TEKU_ALLOY_MONITORED:-true}" @@ -130,9 +123,6 @@ services: cl-lodestar: profiles: [ cl-lodestar ] image: chainsafe/lodestar:${CL_LODESTAR_VERSION:-v1.50.1} - depends_on: - jwt-init: - condition: service_completed_successfully restart: unless-stopped labels: - "alloy-monitored=${CL_LODESTAR_ALLOY_MONITORED:-true}" @@ -169,9 +159,6 @@ services: cl-prysm: profiles: [ cl-prysm ] image: gcr.io/prysmaticlabs/prysm/beacon-chain:${CL_PRYSM_VERSION:-v7.3.1} - depends_on: - jwt-init: - condition: service_completed_successfully restart: unless-stopped labels: - "alloy-monitored=${CL_PRYSM_ALLOY_MONITORED:-true}" @@ -207,9 +194,8 @@ services: cl-nimbus: profiles: [ cl-nimbus ] image: statusim/nimbus-eth2:${CL_NIMBUS_VERSION:-multiarch-v26.10.2} - depends_on: - jwt-init: - condition: service_completed_successfully + # The image defaults to uid 1000; root matches the execution clients, which own the Engine API secret. + user: root restart: unless-stopped labels: - "alloy-monitored=${CL_NIMBUS_ALLOY_MONITORED:-true}" diff --git a/compose-el.yml b/compose-el.yml index 379e72e5..4236ac33 100644 --- a/compose-el.yml +++ b/compose-el.yml @@ -12,9 +12,6 @@ services: el-nethermind: profiles: [el-nethermind] image: nethermind/nethermind:${NETHERMIND_VERSION:-${EL_NETHERMIND_VERSION:-2.2.2}} - depends_on: - jwt-init: - condition: service_completed_successfully restart: unless-stopped ports: - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP @@ -42,7 +39,7 @@ services: networks: [dvnode] volumes: - ./data/nethermind:/nethermind/data - - jwt:/root/jwt:ro + - jwt:/root/jwt # _ _ # _ __ ___| |_| |__ @@ -53,9 +50,12 @@ services: el-reth: profiles: [el-reth] image: ghcr.io/paradigmxyz/reth:${RETH_VERSION:-${EL_RETH_VERSION:-v2.8.1}} - depends_on: - jwt-init: - condition: service_completed_successfully + # Reth only creates the Engine API secret at its default path, so create it here when it is missing. + entrypoint: + - sh + - -c + - '[ -s /root/jwt/jwt.hex ] || (umask 077 && openssl rand -hex 32 | tr -d "\n" > /root/jwt/jwt.hex) && exec /usr/local/bin/reth-binary "$$@"' + - -- restart: unless-stopped ports: - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP @@ -79,4 +79,4 @@ services: networks: [dvnode] volumes: - ./data/reth:/reth/data - - jwt:/root/jwt:ro + - jwt:/root/jwt diff --git a/docker-compose.override.yml.sample b/docker-compose.override.yml.sample index fef66f60..71c4a156 100644 --- a/docker-compose.override.yml.sample +++ b/docker-compose.override.yml.sample @@ -32,6 +32,12 @@ #- 127.0.0.1:5052:5052 # HTTP #- 127.0.0.1:5054:5054 # Metrics + #cl-lighthouse: + # Use your own Engine API secret, for example with an execution client outside this stack. + # Put it in jwt/jwt.hex and mount it over the generated one. The other consensus clients read it from /jwt. + #volumes: + #- ./jwt:/opt/jwt:ro + #charon: # Configure any additional env var flags in .env.charon.more #env_file: [.env.charon.more] diff --git a/docker-compose.yml b/docker-compose.yml index f32ce7bb..6b7c93f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,55 +3,6 @@ # ${VARIABLE-default} evaluates to default only if VARIABLE is unset in the environment. services: - # _ _ - # (_)_ _| |_ - # | \ \ /\ / / __| - # | |\ V V /| |_ - # _/ | \_/\_/ \__| - # |__/ - - # Creates the Engine API secret shared by the execution and consensus clients, once per deployment. - # It lives in the jwt volume, which only those clients mount, so it is not a file in this directory. - # To use your own secret, for example with an external execution client, put it in jwt/jwt.hex. - jwt-init: - image: busybox:${BUSYBOX_VERSION:-1.37.0} - network_mode: none - volumes: - - jwt:/jwt - - ./jwt:/custom:ro - command: - - sh - - -euc - - | - # One line of 64 hex characters, which every client accepts with or without 0x and a newline. - is_secret() { grep -qxE '(0x)?[0-9a-fA-F]{64}' "$$1" && [ "$$(grep -c '' "$$1")" = 1 ]; } - # Read-only for everyone: Teku and Nimbus do not run as root, the other clients do. - store() { chmod 444 /jwt/jwt.hex.tmp && mv -f /jwt/jwt.hex.tmp /jwt/jwt.hex; } - rm -f /jwt/jwt.hex.tmp - if [ -s /custom/jwt.hex ]; then - if ! is_secret /custom/jwt.hex; then - echo "jwt/jwt.hex must contain 32 bytes as 64 hex characters" >&2 - exit 1 - elif grep -qi 7074a5bf6bd6dae368fa598249d57edfcbccc67a1205b2c8d5d2fe7b800663aa /custom/jwt.hex; then - echo "Ignoring jwt/jwt.hex: older releases shipped this secret to everyone, so it is public. Delete the file." - else - if ! cmp -s /custom/jwt.hex /jwt/jwt.hex; then - cp /custom/jwt.hex /jwt/jwt.hex.tmp - store - echo "Loaded the Engine API secret from jwt/jwt.hex. If the execution or consensus client is running, recreate both." - fi - exit 0 - fi - fi - if [ -s /jwt/jwt.hex ]; then exit 0; fi - od -An -tx1 -v -N32 /dev/urandom | tr -d ' \n' > /jwt/jwt.hex.tmp - if ! is_secret /jwt/jwt.hex.tmp; then - echo "Could not generate the Engine API secret" >&2 - exit 1 - fi - store - echo "Generated a new Engine API secret" - # _ _ _ _ # _ __ ___| |_| |__ ___ _ __ _ __ ___ (_)_ __ __| | # | '_ \ / _ \ __| '_ \ / _ \ '__| '_ ` _ \| | '_ \ / _` | @@ -60,9 +11,6 @@ services: nethermind: image: nethermind/nethermind:${NETHERMIND_VERSION:-2.2.2} - depends_on: - jwt-init: - condition: service_completed_successfully profiles: [""] restart: unless-stopped ports: @@ -91,7 +39,7 @@ services: networks: [dvnode] volumes: - ./data/nethermind:/nethermind/data - - jwt:/root/jwt:ro + - jwt:/root/jwt # _ _ _ _ _ # | (_) __ _| |__ | |_| |__ ___ _ _ ___ ___ @@ -102,9 +50,6 @@ services: lighthouse: image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.3} - depends_on: - jwt-init: - condition: service_completed_successfully profiles: [""] ports: - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP @@ -265,5 +210,11 @@ networks: monitoring: volumes: + # Holds the Engine API secret. The execution client generates it on start and the consensus client reads it. + # It is kept in memory and only exists while one of them runs, so it is replaced after both were stopped. jwt: + driver_opts: + type: tmpfs + device: tmpfs + o: size=1m,mode=0700 grafana-data: