security: per deployment EL/CL JWT

.gitignore

@@ -26,3 +26,6 @@ commit-boost/config.toml
 # Allow test fixtures
 !scripts/edit/**/test/fixtures/.charon/
 !scripts/edit/**/test/fixtures/.charon/**
+
+# Your own Engine API secret, if you provide one. The jwt-init service in docker-compose.yml generates one otherwise.
+jwt/jwt.hex*

CLAUDE.md

@@ -133,7 +133,7 @@ docker compose exec charon charon alpha test <suite> --publish   # infra | peers
 
 - `.charon/`: Cluster configuration and validator keys (created by DKG or add-validators)
 - `data/`: Persistent data for all clients (execution, consensus, validator databases)
-- `jwt/`: JWT secret for execution<->consensus authentication
+- `jwt/`: optional operator-provided JWT secret for execution<->consensus authentication (generated in the `jwt` volume otherwise)
 - `grafana/`: Monitoring dashboards and configuration
 - `prometheus/`: Metrics collection configuration
 - `scripts/`: Automation scripts for cluster operations
@@ -178,7 +178,7 @@ When modifying this repository:
 ## Important Notes
 
 - **Never commit `.env` files** - they contain operator-specific configuration
-- **JWT secret** in `jwt/jwt.hex` must be shared between EL and CL clients
+- **JWT secret** must be shared between EL and CL clients. It is not tracked: the `jwt-init` service generates it per deployment in the `jwt` volume before the EL and CL start, or loads an operator-provided `jwt/jwt.hex`. Every EL and CL service needs `depends_on: jwt-init` (`service_completed_successfully`) and a read-only mount of the `jwt` volume
 - **Cluster lock** in `.charon/cluster-lock.json` is critical - back it up before any edit operations
 - **Validator keys** in `.charon/validator_keys/` must be kept secure and never committed
 - **Data directory compatibility**: When switching VCs, verify the new client can handle existing key state

README.md

@@ -211,6 +211,11 @@ docker compose up cl-grandine -d
 - **Grafana is only published on `127.0.0.1`.** To view it from another machine, use an SSH tunnel: `ssh -L 3000:127.0.0.1:3000 <user>@<node>`, then open `http://localhost:3000`. Anonymous visitors get read-only access, and password logins are disabled because dashboards and datasources are provisioned from this repo.
 - **Docker-published ports bypass host firewalls such as `ufw`.** Only bind a port to a non-loopback address (e.g. `MONITORING_IP_GRAFANA`, `EL_IP_HTTP`) if a provider or network firewall restricts who can reach it. By default only the P2P ports are published on all interfaces.
 - **Avoid descriptive hostnames.** DNS names and TLS certificates naming the network or what the node runs (e.g. `mainnet`, `validator`) make a node easy to find with internet-wide scans.
+- **The Engine API secret is unique to each deployment.** On first start, the `jwt-init` service generates a random secret in the `jwt` Docker volume, which only the execution and consensus clients mount, so it is not a file in this directory.
+  - **Upgrading:** older releases shipped the same `jwt/jwt.hex` to every operator. After `git pull`, the next `docker compose up -d` recreates the execution and consensus clients with a new secret, and nothing else is needed. Start them together: a client still running from before the update cannot authenticate to one that has the new secret.
+  - **If `git pull` stops because you changed `jwt/jwt.hex`,** run `git checkout jwt/jwt.hex` and pull again.
+  - **To use your own secret,** for example with an execution client outside this stack, put it in `jwt/jwt.hex` as 64 hex characters (copy it aside before the `git checkout`), then run `docker compose up -d --force-recreate`. The secret that older releases shipped is ignored.
+  - **To print the secret in use:** `docker compose run --rm jwt-init cat /jwt/jwt.hex`.
 - **If you ran a release before the Grafana hardening,** Grafana now starts from a fresh Docker volume and no longer uses `data/grafana`. After upgrading, run `./grafana/security-check.sh` (it may need `sudo`). It reports whether Grafana is published on a public address. It lists anything in the old `data/grafana` that this stack does not create, such as service accounts, tokens, users, datasources, plugins, organizations, dashboards, alert rules and contact points. It also lists changed or added files under `grafana/`, which older releases mounted writable. Report findings to security@obol.tech, then delete `data/grafana`. Grafana stores secrets with a default encryption key, so if your Grafana was reachable, rotate any credentials you stored in it, such as webhook URLs or tokens used by contact points.
 - **Upgrading leaves the old Grafana database behind.** Anything created in the Grafana UI, such as custom dashboards, alert rules, contact points or users, stays in the unused `data/grafana` and does not carry over to the fresh Grafana. Before upgrading, export any custom dashboards (Share > Export) and note your alert rules. Do not copy the old database across, as it may contain unauthorized changes. To set customizations up again, add dashboards as JSON files under `grafana/dashboards/`, or re-enable logins with your own admin password in a `docker-compose.override.yml`, adding `:docker-compose.override.yml` to the end of `COMPOSE_FILE` in `.env` so it is loaded:
 

compose-cl.yml

@@ -13,6 +13,9 @@ services:
   cl-grandine:
     profiles: [ cl-grandine ]
     image: sifrai/grandine:${CL_GRANDINE_VERSION:-${GRANDINE_VERSION:-3.0.0-rc.2}}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_GRANDINE_ALLOY_MONITORED:-true}"
@@ -35,7 +38,7 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/cl-grandine:/root/.grandine
-      - ./jwt:/jwt:ro
+      - jwt:/jwt:ro
     networks: [ dvnode ]
 
   #  _ _       _     _   _
@@ -48,6 +51,9 @@ services:
   cl-lighthouse:
     profiles: [ cl-lighthouse ]
     image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.3}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_LIGHTHOUSE_ALLOY_MONITORED:-true}"
@@ -73,7 +79,7 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/lighthouse:/opt/app/beacon # Keep data in lighthouse and not cl-lighthouse for backwards compatibility
-      - ./jwt:/opt/jwt
+      - jwt:/opt/jwt:ro
     networks: [ dvnode ]
 
   #  _       _
@@ -85,6 +91,9 @@ services:
   cl-teku:
     profiles: [ cl-teku ]
     image: consensys/teku:${CL_TEKU_VERSION:-26.10.3}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_TEKU_ALLOY_MONITORED:-true}"
@@ -109,7 +118,7 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/cl-teku:/opt/teku/data
-      - ./jwt:/jwt:ro
+      - jwt:/jwt:ro
     networks: [ dvnode ]
 
   #  _           _           _
@@ -121,6 +130,9 @@ services:
   cl-lodestar:
     profiles: [ cl-lodestar ]
     image: chainsafe/lodestar:${CL_LODESTAR_VERSION:-v1.50.1}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_LODESTAR_ALLOY_MONITORED:-true}"
@@ -145,7 +157,7 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/cl-lodestar:/opt/lodestar/data
-      - ./jwt:/jwt:ro
+      - jwt:/jwt:ro
     networks: [ dvnode ]
 
   #  _ __  _ __ _   _ ___ _ __ ___
@@ -157,6 +169,9 @@ services:
   cl-prysm:
     profiles: [ cl-prysm ]
     image: gcr.io/prysmaticlabs/prysm/beacon-chain:${CL_PRYSM_VERSION:-v7.3.1}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_PRYSM_ALLOY_MONITORED:-true}"
@@ -180,7 +195,7 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/cl-prysm:/data
-      - ./jwt:/jwt:ro
+      - jwt:/jwt:ro
     networks: [ dvnode ]
 
   #        _           _
@@ -192,6 +207,9 @@ services:
   cl-nimbus:
     profiles: [ cl-nimbus ]
     image: statusim/nimbus-eth2:${CL_NIMBUS_VERSION:-multiarch-v26.10.2}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     labels:
       - "alloy-monitored=${CL_NIMBUS_ALLOY_MONITORED:-true}"
@@ -219,5 +237,5 @@ services:
       - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052
     volumes:
       - ./data/cl-nimbus:/data
-      - ./jwt:/jwt:ro
+      - jwt:/jwt:ro
     networks: [ dvnode ]

compose-el.yml

@@ -12,6 +12,9 @@ services:
   el-nethermind:
     profiles: [el-nethermind]
     image: nethermind/nethermind:${NETHERMIND_VERSION:-${EL_NETHERMIND_VERSION:-2.2.2}}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     ports:
       - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP
@@ -39,7 +42,7 @@ services:
     networks: [dvnode]
     volumes:
       - ./data/nethermind:/nethermind/data
-      - ./jwt:/root/jwt
+      - jwt:/root/jwt:ro
 
   #           _   _
   #  _ __ ___| |_| |__
@@ -50,6 +53,9 @@ services:
   el-reth:
     profiles: [el-reth]
     image: ghcr.io/paradigmxyz/reth:${RETH_VERSION:-${EL_RETH_VERSION:-v2.8.1}}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     restart: unless-stopped
     ports:
       - ${EL_PORT_P2P:-30303}:30303/tcp # P2P TCP
@@ -73,4 +79,4 @@ services:
     networks: [dvnode]
     volumes:
       - ./data/reth:/reth/data
-      - ./jwt:/root/jwt
+      - jwt:/root/jwt:ro

docker-compose.yml

@@ -3,6 +3,55 @@
 # ${VARIABLE-default} evaluates to default only if VARIABLE is unset in the environment.
 
 services:
+  #    _          _
+  #   (_)_      _| |_
+  #   | \ \ /\ / / __|
+  #   | |\ V  V /| |_
+  #  _/ | \_/\_/  \__|
+  # |__/
+
+  # Creates the Engine API secret shared by the execution and consensus clients, once per deployment.
+  # It lives in the jwt volume, which only those clients mount, so it is not a file in this directory.
+  # To use your own secret, for example with an external execution client, put it in jwt/jwt.hex.
+  jwt-init:
+    image: busybox:${BUSYBOX_VERSION:-1.37.0}
+    network_mode: none
+    volumes:
+      - jwt:/jwt
+      - ./jwt:/custom:ro
+    command:
+      - sh
+      - -euc
+      - |
+        # One line of 64 hex characters, which every client accepts with or without 0x and a newline.
+        is_secret() { grep -qxE '(0x)?[0-9a-fA-F]{64}' "$$1" && [ "$$(grep -c '' "$$1")" = 1 ]; }
+        # Read-only for everyone: Teku and Nimbus do not run as root, the other clients do.
+        store() { chmod 444 /jwt/jwt.hex.tmp && mv -f /jwt/jwt.hex.tmp /jwt/jwt.hex; }
+        rm -f /jwt/jwt.hex.tmp
+        if [ -s /custom/jwt.hex ]; then
+          if ! is_secret /custom/jwt.hex; then
+            echo "jwt/jwt.hex must contain 32 bytes as 64 hex characters" >&2
+            exit 1
+          elif grep -qi 7074a5bf6bd6dae368fa598249d57edfcbccc67a1205b2c8d5d2fe7b800663aa /custom/jwt.hex; then
+            echo "Ignoring jwt/jwt.hex: older releases shipped this secret to everyone, so it is public. Delete the file."
+          else
+            if ! cmp -s /custom/jwt.hex /jwt/jwt.hex; then
+              cp /custom/jwt.hex /jwt/jwt.hex.tmp
+              store
+              echo "Loaded the Engine API secret from jwt/jwt.hex. If the execution or consensus client is running, recreate both."
+            fi
+            exit 0
+          fi
+        fi
+        if [ -s /jwt/jwt.hex ]; then exit 0; fi
+        od -An -tx1 -v -N32 /dev/urandom | tr -d ' \n' > /jwt/jwt.hex.tmp
+        if ! is_secret /jwt/jwt.hex.tmp; then
+          echo "Could not generate the Engine API secret" >&2
+          exit 1
+        fi
+        store
+        echo "Generated a new Engine API secret"
+
   #             _   _                         _           _
   #  _ __   ___| |_| |__   ___ _ __ _ __ ___ (_)_ __   __| |
   # | '_ \ / _ \ __| '_ \ / _ \ '__| '_ ` _ \| | '_ \ / _` |
@@ -11,6 +60,9 @@ services:
 
   nethermind:
     image: nethermind/nethermind:${NETHERMIND_VERSION:-2.2.2}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     profiles: [""]
     restart: unless-stopped
     ports:
@@ -39,7 +91,7 @@ services:
     networks: [dvnode]
     volumes:
       - ./data/nethermind:/nethermind/data
-      - ./jwt:/root/jwt
+      - jwt:/root/jwt:ro
 
   #  _ _       _     _   _
   # | (_) __ _| |__ | |_| |__   ___  _   _ ___  ___
@@ -50,6 +102,9 @@ services:
 
   lighthouse:
     image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.3.3}
+    depends_on:
+      jwt-init:
+        condition: service_completed_successfully
     profiles: [""]
     ports:
       - ${LIGHTHOUSE_PORT_P2P:-9000}:9000/tcp # P2P TCP
@@ -77,7 +132,7 @@ services:
     networks: [dvnode]
     volumes:
       - ./data/lighthouse:/opt/app/beacon
-      - ./jwt:/opt/jwt
+      - jwt:/opt/jwt:ro
     restart: unless-stopped
 
   #       _
@@ -210,4 +265,5 @@ networks:
   monitoring:
 
 volumes:
+  jwt:
   grafana-data: