From 1ec54ae5bf313a2ce79b0047b32c25af2bca43b4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 05:13:21 -0800 Subject: [PATCH 1/3] feat(kms)!: refuse to serve without a key handover path With the public handover off by default, other KMS nodes can only onboard through the admin listener over TLS. A KMS without either would strand them, which shows up only when a node needs to join. Before serving, the KMS now requires core.onboard.public_key_handover = true, or the admin listener enabled with [core.admin.tls] key and certs, and names the missing setting otherwise. There is no opt-out, dev auth included. The onboarding phase is not checked. The built-in kms.toml now enables the admin listener with TLS on the RPC key and certificate. The admin listener fails closed, so a KMS on the default config needs an admin token to start. A KMS deployed as a CVM with the local key provider has no secret channel for its config: the app compose is measured and public. The new core.admin.auth_token_hash takes the token's hex SHA-256, so the compose carries only the hash: - dstack-app composes and entrypoint use the ADMIN_TOKEN_HASH the deploy scripts already compute, and the scripts publish the admin listener when KMS_ADMIN_ADDR is set. - dstackup generates a KMS admin token next to the VMM one and renders its hash. - tools/dev-stack.sh generates a local token. - The full-stack e2e KMS, shared with KMS 0.5.8, keeps the admin listener off and serves the public handover. Signed-off-by: Kevin Wang --- CHANGELOG.md | 12 +++ docs/deployment.md | 2 + docs/rc-testing-runbook.md | 5 +- docs/tutorials/kms-build-configuration.md | 14 ++++ docs/tutorials/kms-cvm-deployment.md | 18 ++++ docs/upgrading-to-0.6.md | 6 ++ dstack/crates/dstack-cli-core/src/config.rs | 26 ++++++ dstack/crates/dstackup/src/install.rs | 15 +++- dstack/kms/README.md | 14 +++- dstack/kms/dstack-app/compose-dev.yaml | 13 +++ dstack/kms/dstack-app/compose-simple.yaml | 13 +++ dstack/kms/dstack-app/deploy-simple.sh | 5 ++ dstack/kms/dstack-app/deploy-to-vmm.sh | 5 ++ dstack/kms/dstack-app/entrypoint.sh | 3 + dstack/kms/kms.toml | 26 +++--- dstack/kms/src/admin_auth.rs | 84 ++++++++++++++----- dstack/kms/src/config.rs | 68 ++++++++++++++- dstack/kms/src/main.rs | 2 + .../full-stack-compose/scripts/app_compose.py | 6 ++ tools/dev-stack.sh | 15 +++- 20 files changed, 311 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b78b58b1f..f343ccc79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- KMS: a serving KMS requires a key handover path and otherwise refuses to + start, naming the missing setting: unless + `core.onboard.public_key_handover = true`, set `[core.admin] enabled = true` + and `[core.admin.tls]` with `key` and `certs`. There is no opt-out, dev auth + included. The built-in config now enables the admin listener with TLS on the + RPC key and certificate, so a KMS on it needs an admin token to start: + `core.admin.auth_token`, `DSTACK_KMS_ADMIN_TOKEN`, or the new + `core.admin.auth_token_hash` (hex SHA-256), which keeps the token out of a + public app compose. When upgrading, add the admin TLS and a token, or enable + the public handover. The `dstack-app` composes and `dstackup` generate the + token and configure its hash; the deploy scripts publish the admin listener + when `KMS_ADMIN_ADDR` is set. - KMS: `core.onboard.public_key_handover` defaults to `false`, so `KMS.GetKmsKey` is no longer served on the public port. Onboard through the source's admin listener with `source_token`; a source that still onboards diff --git a/docs/deployment.md b/docs/deployment.md index 34fd9e78c..4a6b8a7ff 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -510,6 +510,8 @@ Additional KMS instances can onboard from an existing KMS to share the same root **Configure the source KMS for key handover:** +Every serving KMS needs this, or `core.onboard.public_key_handover = true`: without a handover path it refuses to start, naming the missing setting. + `Admin.GetKmsKey` only works on an admin listener with `[core.admin.tls]`: without TLS the listener speaks plain HTTP, so the new KMS can present no RA-TLS certificate and the call fails with `No attestation provided`. The admin listener binds to `127.0.0.1` by default; bind it to an address the new KMS can reach and restrict access to the KMS nodes at the network layer. ```toml diff --git a/docs/rc-testing-runbook.md b/docs/rc-testing-runbook.md index fae031a0d..55acdd9bc 100644 --- a/docs/rc-testing-runbook.md +++ b/docs/rc-testing-runbook.md @@ -154,7 +154,10 @@ KMS first, then gateway, then the app. Each needs the previous one's identity. simplest authorization that releases keys without a chain is `[core.auth_api] type = "dev"` with `gateway_app_id = "any"` — no extra containers, unlike the on-chain compose, which runs auth-eth next to the KMS - and needs an Ethereum RPC endpoint. + and needs an Ethereum RPC endpoint. The KMS refuses to start without a + key handover path, dev auth included: give it `[core.admin]` with + `[core.admin.tls]` (`key`, `certs`) and an `auth_token_hash`, or set + `core.onboard.public_key_handover = true`. 3. **Gateway as a CVM** with `--kms`. Internal ports are RPC 8000, admin 8001, WireGuard 51820/udp, proxy 443. diff --git a/docs/tutorials/kms-build-configuration.md b/docs/tutorials/kms-build-configuration.md index 56be193fc..bb7d045fb 100644 --- a/docs/tutorials/kms-build-configuration.md +++ b/docs/tutorials/kms-build-configuration.md @@ -204,6 +204,19 @@ type = "webhook" [core.auth_api.webhook] url = "http://127.0.0.1:9200" +# Admin listener, which serves the key handover to other KMS nodes; a serving +# KMS requires it. ADMIN_TOKEN_HASH is the token's SHA-256, filled in by the +# KMS CVM deployment tutorial. +[core.admin] +enabled = true +address = "0.0.0.0" +port = 9101 +auth_token_hash = "ADMIN_TOKEN_HASH" + +[core.admin.tls] +key = "/etc/kms/certs/rpc.key" +certs = "/etc/kms/certs/rpc.crt" + # Onboarding Configuration [core.onboard] enabled = true @@ -224,6 +237,7 @@ EOF | `[core]` | `cert_dir` | Directory for certificates | | `[core]` | `pccs_url` | PCCS endpoint for quote verification | | `[core.auth_api]` | `url` | auth-eth webhook service URL | +| `[core.admin]` | `auth_token_hash` | SHA-256 of the admin token; the admin listener with `[core.admin.tls]` is required to serve | | `[core.onboard]` | `enabled` | Enable bootstrap/onboard mode | ## Step 5: Prepare the auth-eth Service diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index def92725c..5d6690670 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -135,6 +135,7 @@ services: image: registry.yourdomain.com/dstack-kms:fixed ports: - "9100:9100" + - "9101:9101" volumes: - /var/run/dstack.sock:/var/run/dstack.sock - kms-certs:/etc/kms/certs @@ -184,6 +185,18 @@ configs: [core.auth_api.webhook] url = "http://127.0.0.1:9200" + # Admin listener: serves the key handover to other KMS nodes, so a + # serving KMS requires it. Only the token's SHA-256 goes into the compose. + [core.admin] + enabled = true + address = "0.0.0.0" + port = 9101 + auth_token_hash = "ADMIN_TOKEN_HASH" + + [core.admin.tls] + key = "/etc/kms/certs/rpc.key" + certs = "/etc/kms/certs/rpc.crt" + [core.onboard] enabled = true auto_bootstrap_domain = "" @@ -203,6 +216,10 @@ sed -i 's|kms.yourdomain.com|kms.your-actual-domain.com|g' docker-compose.yaml # KMS contract address (from contract deployment tutorial) sed -i "s|YOUR_CONTRACT_ADDRESS|$(cat ~/.dstack/secrets/kms-contract-address)|g" docker-compose.yaml + +# KMS admin token: keep the token, put only its SHA-256 in the compose +(umask 077 && openssl rand -hex 32 > ~/.dstack/secrets/kms-admin-token) +sed -i "s|ADMIN_TOKEN_HASH|$(tr -d '\n' < ~/.dstack/secrets/kms-admin-token | sha256sum | cut -d' ' -f1)|g" docker-compose.yaml ``` This docker-compose uses a Docker `configs` section to inject a complete `kms.toml` into the container at `/etc/kms/kms.toml`, overriding the config baked into the image. This approach lets you change KMS configuration without rebuilding the Docker image. @@ -215,6 +232,7 @@ This docker-compose uses a Docker `configs` section to inject a complete `kms.to | `[rpc.tls]` | TLS certificate paths for HTTPS | | `[core.image]` | OS image verification — downloads images from `download.dstack.org` to compute expected TDX measurements | | `[core.auth_api]` | Authentication via auth-eth webhook on localhost:9200 | +| `[core.admin]` | Admin listener on port 9101 with TLS; other KMS nodes onboard through it with the admin token. Without it (or `core.onboard.public_key_handover = true`) the KMS does not start | | `[core.onboard]` | Bootstrap settings — `auto_bootstrap_domain` is empty so KMS enters onboard mode for manual bootstrap | > **Why manual bootstrap?** With `auto_bootstrap_domain` left empty, KMS starts in "onboard mode" — a plain HTTP server on port 9100 that waits for you to trigger bootstrap via an RPC call. This ensures `bootstrap-info.json` (containing the TDX attestation quote and public keys) is written to disk. You'll need this file later to register the KMS on-chain. diff --git a/docs/upgrading-to-0.6.md b/docs/upgrading-to-0.6.md index df4e269f9..e935f1c3e 100644 --- a/docs/upgrading-to-0.6.md +++ b/docs/upgrading-to-0.6.md @@ -236,6 +236,12 @@ Configuration changes: `KMS.GetKmsKey` from the public port. While a 0.5.x KMS still onboards from this cluster, it can only use that public entry, so set the option to `true` on the source node, and remove it once every node is upgraded. +- A KMS without a key handover path does not start. Configure the admin + listener with `[core.admin.tls]` (`key` and `certs`) and an admin token, or + set `core.onboard.public_key_handover = true`. The built-in default enables + the admin listener with TLS, so it needs `core.admin.auth_token`, + `DSTACK_KMS_ADMIN_TOKEN` or `core.admin.auth_token_hash` to start. In a KMS + app compose, which is public, use `auth_token_hash`. - Auto-bootstrap refuses to overwrite existing root keys. If you use `auth-simple`, replace it with `dstack-auth`, which reads the same diff --git a/dstack/crates/dstack-cli-core/src/config.rs b/dstack/crates/dstack-cli-core/src/config.rs index d98364580..03aaee22c 100644 --- a/dstack/crates/dstack-cli-core/src/config.rs +++ b/dstack/crates/dstack-cli-core/src/config.rs @@ -100,6 +100,9 @@ pub struct HostConfig { pub image_download_url: String, /// confidential-computing platform (selects SNP-specific KMS settings). pub platform: Platform, + /// hex SHA-256 of the KMS admin token; the KMS refuses to start without + /// an admin credential. + pub kms_admin_token_hash: String, } impl Default for HostConfig { @@ -110,6 +113,7 @@ impl Default for HostConfig { os_image_hash: String::new(), image_download_url: DEFAULT_IMAGE_DOWNLOAD_URL.to_string(), platform: Platform::Tdx, + kms_admin_token_hash: String::new(), } } } @@ -148,6 +152,17 @@ type = "webhook" [core.auth_api.webhook] url = "{webhook_url}" +# the key handover path a serving KMS requires; the token stays on the host. +[core.admin] +enabled = true +address = "127.0.0.1" +port = 8001 +auth_token_hash = "{admin_token_hash}" + +[core.admin.tls] +key = "/kms/certs/rpc.key" +certs = "/kms/certs/rpc.crt" + [core.onboard] enabled = true auto_bootstrap_domain = "{bootstrap_domain}" @@ -164,6 +179,7 @@ port = 8000 download_url = cfg.image_download_url, webhook_url = cfg.auth_webhook_url, bootstrap_domain = cfg.kms_bootstrap_domain, + admin_token_hash = cfg.kms_admin_token_hash, ) } @@ -478,10 +494,20 @@ mod tests { let cfg = HostConfig { auth_webhook_url: "http://10.0.2.2:8001".into(), kms_bootstrap_domain: "10.0.2.2".into(), + kms_admin_token_hash: "ab".repeat(32), ..Default::default() }; let toml = kms_toml(&cfg); assert!(toml.contains("enforce_self_authorization = false")); + let v: toml::Value = toml::from_str(&toml).unwrap(); + let admin = &v["core"]["admin"]; + assert_eq!(admin["enabled"].as_bool(), Some(true)); + assert_eq!( + admin["auth_token_hash"].as_str(), + Some("ab".repeat(32).as_str()) + ); + assert_eq!(admin["tls"]["key"].as_str(), Some("/kms/certs/rpc.key")); + assert_eq!(admin["tls"]["certs"].as_str(), Some("/kms/certs/rpc.crt")); assert!(toml.contains(r#"auto_bootstrap_domain = "10.0.2.2""#)); assert!(toml.contains(r#"type = "webhook""#)); assert!(toml.contains(r#"url = "http://10.0.2.2:8001""#)); diff --git a/dstack/crates/dstackup/src/install.rs b/dstack/crates/dstackup/src/install.rs index b07544830..14158c60d 100644 --- a/dstack/crates/dstackup/src/install.rs +++ b/dstack/crates/dstackup/src/install.rs @@ -14,6 +14,7 @@ use dstack_cli_core::host::Platform; use dstack_cli_core::layout::{path_string, InstallLayout}; use dstack_cli_core::vmm::Vmm; use dstack_cli_core::{host, ports, rpc}; +use sha2::{Digest, Sha256}; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -137,9 +138,16 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) // install state, so it authenticates automatically. let vmm_token = match existing_token { Some(t) => t, - None => generate_vmm_token()?, + None => generate_token()?, }; write_token_file(&token_path, &vmm_token)?; + // the KMS admin token, which the KMS config holds only as a hash. + let kms_admin_token_path = layout.config_dir.join("kms-admin-token"); + let kms_admin_token = match read_token_file(&kms_admin_token_path) { + Some(t) => t, + None => generate_token()?, + }; + write_token_file(&kms_admin_token_path, &kms_admin_token)?; // 8. resolve the key provider - run our own unless told to use an existing // one (TDX only; SNP has no SGX local provider). @@ -183,6 +191,7 @@ pub(crate) async fn cmd_install(mut o: InstallOpts, release_api_base_url: &str) auth_webhook_url: format!("http://10.0.2.2:{}", o.auth_port), os_image_hash: os_image_hash.unwrap_or_default(), platform, + kms_admin_token_hash: hex::encode(Sha256::digest(kms_admin_token.as_bytes())), ..Default::default() }; let kms = config::kms_toml(&host_cfg); @@ -1216,8 +1225,8 @@ async fn wait_ready(client_url: &str, token: Option<&str>, timeout: Duration) -> } } -/// mint a 256-bit management-API token, hex-encoded, from the OS RNG. -fn generate_vmm_token() -> Result { +/// mint a 256-bit API token, hex-encoded, from the OS RNG. +fn generate_token() -> Result { let mut buf = [0u8; 32]; let mut f = fs::File::open("/dev/urandom").context("opening /dev/urandom")?; std::io::Read::read_exact(&mut f, &mut buf).context("reading /dev/urandom")?; diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 970d7433f..e72b194eb 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -55,7 +55,9 @@ certs = "/etc/kms/certs/rpc.crt" ``` The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or -`ADMIN_API_TOKEN` environment variables instead of the config file. The admin +`ADMIN_API_TOKEN` environment variables instead of the config file, or as its +hex SHA-256 in `auth_token_hash` (`printf '%s' "$TOKEN" | sha256sum`) for a +config that is not secret, such as a KMS app compose. The admin listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) refuses to start. With `[core.admin.tls]` the listener serves HTTPS (use `http://` without it). Call it with, for example: @@ -77,6 +79,16 @@ need no client certificate. `core.onboard.public_key_handover` is off by default `true` only to also serve `KMS.GetKmsKey` on the public listener for older clients. +A KMS refuses to start serving without a handover path, so that other nodes can +always onboard from it: unless `public_key_handover = true`, it requires +`[core.admin] enabled = true` and `[core.admin.tls]` with `key` and `certs`, and +the error names the missing setting. There is no opt-out, `auth_api = "dev"` +included; the onboarding phase itself is not checked. The default `kms.toml` +enables the admin listener with TLS on the RPC key and certificate, so it +starts only once an admin token is set (`auth_token`, +`DSTACK_KMS_ADMIN_TOKEN` or `auth_token_hash`). The `dstack-app` composes and +`dstackup` set `auth_token_hash` from a token they generate. + Three settings relax authorization, each over a different scope: | Setting | Effect | diff --git a/dstack/kms/dstack-app/compose-dev.yaml b/dstack/kms/dstack-app/compose-dev.yaml index 2232a5c87..955f92687 100644 --- a/dstack/kms/dstack-app/compose-dev.yaml +++ b/dstack/kms/dstack-app/compose-dev.yaml @@ -39,6 +39,7 @@ services: - /var/run/dstack.sock:/var/run/dstack.sock ports: - 8000:8000 + - 8002:8001 depends_on: - auth-api restart: unless-stopped @@ -75,6 +76,18 @@ configs: [core.auth_api.webhook] url = "http://auth-api:8000" + [core.admin] + enabled = true + # The onboarding KMS reaches it through the CVM port 8002. + address = "0.0.0.0" + port = 8001 + # SHA-256 of the admin token, which stays with the operator. + auth_token_hash = "${ADMIN_TOKEN_HASH}" + + [core.admin.tls] + key = "/kms/certs/rpc.key" + certs = "/kms/certs/rpc.crt" + [core.onboard] enabled = true auto_bootstrap_domain = "" diff --git a/dstack/kms/dstack-app/compose-simple.yaml b/dstack/kms/dstack-app/compose-simple.yaml index 45ad2c8ea..3dbd6f2a5 100644 --- a/dstack/kms/dstack-app/compose-simple.yaml +++ b/dstack/kms/dstack-app/compose-simple.yaml @@ -14,6 +14,7 @@ services: - /var/run/dstack.sock:/var/run/dstack.sock ports: - 8000:8000 + - 8002:8001 restart: unless-stopped configs: - source: kms_config @@ -48,6 +49,18 @@ configs: [core.auth_api.webhook] url = "${AUTH_WEBHOOK_URL}" + [core.admin] + enabled = true + # The onboarding KMS reaches it through the CVM port 8002. + address = "0.0.0.0" + port = 8001 + # SHA-256 of the admin token, which stays with the operator. + auth_token_hash = "${ADMIN_TOKEN_HASH}" + + [core.admin.tls] + key = "/kms/certs/rpc.key" + certs = "/kms/certs/rpc.crt" + [core.onboard] enabled = true auto_bootstrap_domain = "" diff --git a/dstack/kms/dstack-app/deploy-simple.sh b/dstack/kms/dstack-app/deploy-simple.sh index 857eadc99..3e36b17a3 100755 --- a/dstack/kms/dstack-app/deploy-simple.sh +++ b/dstack/kms/dstack-app/deploy-simple.sh @@ -37,6 +37,10 @@ else # Required: The address of the KMS service listening on Host machine # KMS_RPC_ADDR=0.0.0.0:9201 +# Optional: the address of the KMS admin listener on the Host machine, which +# other KMS nodes onboard from with ADMIN_TOKEN; firewall it to KMS nodes only +# KMS_ADMIN_ADDR=127.0.0.1:9203 + # Required: The address of the guest agent service listening on Host machine # GUEST_AGENT_ADDR=127.0.0.1:9205 @@ -130,6 +134,7 @@ echo "Deploying KMS to dstack-vmm..." --compose .app-compose.json \ --image "$OS_IMAGE" \ --port "tcp:$KMS_RPC_ADDR:8000" \ + ${KMS_ADMIN_ADDR:+--port "tcp:$KMS_ADMIN_ADDR:8002"} \ --port "tcp:$GUEST_AGENT_ADDR:8090" \ --vcpu 8 \ --memory 8G \ diff --git a/dstack/kms/dstack-app/deploy-to-vmm.sh b/dstack/kms/dstack-app/deploy-to-vmm.sh index 96ed530fd..be598fb6f 100755 --- a/dstack/kms/dstack-app/deploy-to-vmm.sh +++ b/dstack/kms/dstack-app/deploy-to-vmm.sh @@ -32,6 +32,10 @@ else # The address of the KMS service listening on Host machine # KMS_RPC_ADDR=0.0.0.0:9201 +# Optional: the address of the KMS admin listener on the Host machine, which +# other KMS nodes onboard from with ADMIN_TOKEN; firewall it to KMS nodes only +# KMS_ADMIN_ADDR=127.0.0.1:9203 + # The address of the guest agent service listening on Host machine # GUEST_AGENT_ADDR=127.0.0.1:9205 @@ -152,6 +156,7 @@ $CLI deploy \ --compose .app-compose.json \ --image "$OS_IMAGE" \ --port tcp:"$KMS_RPC_ADDR":8000 \ + ${KMS_ADMIN_ADDR:+--port "tcp:$KMS_ADMIN_ADDR:8002"} \ --port tcp:"$AUTH_API_RPC_ADDR":8001 \ --port tcp:"$GUEST_AGENT_ADDR":8090 \ --vcpu 8 \ diff --git a/dstack/kms/dstack-app/entrypoint.sh b/dstack/kms/dstack-app/entrypoint.sh index 049e4acc2..de1ae7dfb 100755 --- a/dstack/kms/dstack-app/entrypoint.sh +++ b/dstack/kms/dstack-app/entrypoint.sh @@ -12,6 +12,9 @@ cat < ./kms.toml cache_dir = "./images" download_url = "${IMAGE_DOWNLOAD_URL}" download_timeout = "2m" + +[core.admin] +auth_token_hash = "${ADMIN_TOKEN_HASH}" EOF exec dstack-kms -c ./kms.toml diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 7c169d338..0dad6a10f 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -82,11 +82,14 @@ cache_dir = "/usr/share/dstack/images" download_url = "http://localhost:8000/{OS_IMAGE_HASH}.tar.gz" download_timeout = "2m" -# Admin API: serves operator RPCs (e.g. ClearImageCache) on a dedicated -# listener behind the shared HTTP authenticator. Disabled by default; when -# enabled it fails closed unless auth_token or htpasswd_file is set. +# Admin API: serves operator RPCs (e.g. ClearImageCache) and the key handover +# to onboarding KMS nodes on a dedicated listener behind the shared HTTP +# authenticator. It fails closed: the KMS does not start until auth_token (or +# DSTACK_KMS_ADMIN_TOKEN), auth_token_hash or htpasswd_file is set. Unless +# core.onboard.public_key_handover = true, a serving KMS also requires this +# listener and its TLS below, so that other KMS nodes can onboard. [core.admin] -enabled = false +enabled = true # Bind address/port for the admin listener (keep it off the public network). address = "127.0.0.1" port = 8001 @@ -95,19 +98,22 @@ port = 8001 # `Authorization: Bearer ` or the `X-Admin-Token: ` header. # Required unless insecure_no_auth = true. Generate with: openssl rand -hex 32 auth_token = "" +# Or the token's hex SHA-256, for a config that is not secret (an app compose): +# printf '%s' "$TOKEN" | sha256sum +auth_token_hash = "" # Optional Apache bcrypt htpasswd file (create with `htpasswd -B -c file admin`). htpasswd_file = "" # Development/testing escape hatch only. Never enable this on an admin # interface that is reachable from the network. insecure_no_auth = false -# Optional TLS for the admin listener. When configured, clients may present an -# RA-TLS certificate. GetKmsKey requires it; other admin RPCs do not. To serve -# KMS onboarding, enable this and bind `address` to one the new KMS can reach. +# TLS for the admin listener, so that clients may present an RA-TLS +# certificate. GetKmsKey requires it; other admin RPCs do not. To serve KMS +# onboarding, also bind `address` to one the new KMS can reach. # No `[core.admin.tls.mutual]` is needed, as for `[rpc.tls]` above. -# [core.admin.tls] -# key = "/etc/kms/certs/rpc.key" -# certs = "/etc/kms/certs/rpc.crt" +[core.admin.tls] +key = "/etc/kms/certs/rpc.key" +certs = "/etc/kms/certs/rpc.crt" [core.metrics] # Expose unauthenticated Prometheus metrics on /metrics. diff --git a/dstack/kms/src/admin_auth.rs b/dstack/kms/src/admin_auth.rs index 22fe33bd6..540b50aa1 100644 --- a/dstack/kms/src/admin_auth.rs +++ b/dstack/kms/src/admin_auth.rs @@ -8,11 +8,13 @@ //! attestation), the admin API is reached by operators/tooling, so it uses the //! same shared-token/htpasswd HTTP mechanism as the VMM and gateway. The token //! is configured via `core.admin.auth_token`, or the `DSTACK_KMS_ADMIN_TOKEN` / -//! `ADMIN_API_TOKEN` environment variables. +//! `ADMIN_API_TOKEN` environment variables, or as its SHA-256 in +//! `core.admin.auth_token_hash`. -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use dstack_api_auth::{Authenticator, HttpAuthConfig, HttpAuthFairing}; use rocket::Route; +use sha2::{Digest, Sha256}; use crate::config::AdminConfig; @@ -29,28 +31,47 @@ impl AdminAuthFairing { http_config(), ))); } - let token = if !config.auth_token.is_empty() { - config.auth_token.trim().to_owned() - } else { - std::env::var(ENV_ADMIN_TOKEN) - .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT)) - .unwrap_or_default() - .trim() - .to_owned() - }; - if token.is_empty() && config.htpasswd_file.as_os_str().is_empty() { - bail!( - "admin API is enabled but neither auth_token nor htpasswd_file is configured; \ - set core.admin.auth_token, {ENV_ADMIN_TOKEN}, {ENV_ADMIN_TOKEN_COMPAT}, \ - core.admin.htpasswd_file, or insecure_no_auth = true (testing only)" - ); - } - let mut auth = Authenticator::from_tokens([token]); - if !config.htpasswd_file.as_os_str().is_empty() { - auth = auth.with_htpasswd_file(&config.htpasswd_file)?; - } - Ok(Self(HttpAuthFairing::new(auth, http_config()))) + Ok(Self(HttpAuthFairing::new( + authenticator(config)?, + http_config(), + ))) + } +} + +fn authenticator(config: &AdminConfig) -> Result { + let token = if !config.auth_token.is_empty() { + config.auth_token.trim().to_owned() + } else { + std::env::var(ENV_ADMIN_TOKEN) + .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT)) + .unwrap_or_default() + .trim() + .to_owned() + }; + let mut hashes = Vec::new(); + if !token.is_empty() { + hashes.push(Sha256::digest(token.as_bytes()).into()); + } + let hash = config.auth_token_hash.trim(); + if !hash.is_empty() { + let hash = hex::decode(hash) + .ok() + .and_then(|hash| <[u8; 32]>::try_from(hash).ok()) + .context("core.admin.auth_token_hash must be a hex SHA-256 digest")?; + hashes.push(hash); + } + if hashes.is_empty() && config.htpasswd_file.as_os_str().is_empty() { + bail!( + "admin API is enabled but no credential is configured; set core.admin.auth_token, \ + {ENV_ADMIN_TOKEN}, {ENV_ADMIN_TOKEN_COMPAT}, core.admin.auth_token_hash, \ + core.admin.htpasswd_file, or insecure_no_auth = true (testing only)" + ); + } + let mut auth = Authenticator::from_token_hashes(hashes); + if !config.htpasswd_file.as_os_str().is_empty() { + auth = auth.with_htpasswd_file(&config.htpasswd_file)?; } + Ok(auth) } fn http_config() -> HttpAuthConfig { @@ -102,6 +123,23 @@ mod tests { assert!(AdminAuthFairing::from_config(&cfg).is_ok()); } + #[test] + fn a_token_hash_admits_its_token() { + let cfg = AdminConfig { + enabled: true, + auth_token_hash: hex::encode(Sha256::digest(b"secret")), + ..Default::default() + }; + let auth = authenticator(&cfg).unwrap(); + assert!(auth.verify_token("secret")); + assert!(!auth.verify_token("other")); + let bad = AdminConfig { + auth_token_hash: "00".into(), + ..cfg + }; + assert!(authenticator(&bad).is_err()); + } + #[test] fn configured_token_builds() { let cfg = AdminConfig { diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 079ecb2e3..2819eae76 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 +use anyhow::{bail, Result}; use dstack_attest::attestation::AttestationVerifierConfig; use load_config::load_config; use rocket::figment::Figment; @@ -110,6 +111,10 @@ pub(crate) struct AdminConfig { /// `insecure_no_auth = true`. #[serde(default)] pub auth_token: String, + /// Hex SHA-256 of the admin token, for configs that are not secret, such + /// as an app compose, so the token itself stays out of them. + #[serde(default)] + pub auth_token_hash: String, /// Optional Apache bcrypt htpasswd file, accepted in addition to the token. #[serde(default)] pub htpasswd_file: PathBuf, @@ -123,6 +128,37 @@ fn default_true() -> bool { true } +/// Without the public handover, other KMS nodes can get the keys only from the +/// admin listener over TLS. A serving KMS without either would strand them, so +/// it refuses to start. +pub(crate) fn check_key_handover(config: &KmsConfig, figment: &Figment) -> Result<()> { + if config.onboard.public_key_handover { + return Ok(()); + } + let set = |key: &str| { + figment + .find_value(key) + .ok() + .and_then(|value| value.into_string()) + .is_some_and(|value| !value.is_empty()) + }; + let missing = if !config.admin.enabled { + "[core.admin] enabled = true" + } else if !set("core.admin.tls.key") { + "[core.admin.tls] key" + } else if !set("core.admin.tls.certs") { + "[core.admin.tls] certs" + } else { + return Ok(()); + }; + bail!( + "no key handover path for onboarding other KMS nodes: {missing} is not set. \ + Configure the admin listener with TLS ([core.admin] enabled = true, an auth_token \ + or DSTACK_KMS_ADMIN_TOKEN, and [core.admin.tls] key and certs), or set \ + core.onboard.public_key_handover = true" + ) +} + impl KmsConfig { /// The root keys are the last files a bootstrap writes, so both existing /// marks a completed one. Anything short of that is left over from an @@ -252,13 +288,14 @@ mod tests { use rocket::figment::providers::{Format, Toml}; #[test] - fn default_config_parses_with_admin_disabled_and_no_hash() { + fn default_config_serves_the_handover_on_the_admin_listener() { let figment = load_config_figment(None); let config: KmsConfig = figment .focus("core") .extract() .expect("kms.toml must parse into KmsConfig"); - assert!(!config.admin.enabled, "admin must be off by default"); + assert!(config.admin.enabled); + check_key_handover(&config, &figment).expect("the default config has a handover path"); assert!( config.admin.auth_token.is_empty(), "default admin token must be empty (fail-closed)" @@ -270,6 +307,33 @@ mod tests { ); } + fn handover_check(toml: &str) -> Result<()> { + let figment = load_config_figment(None).merge(Toml::string(toml)); + let config: KmsConfig = figment.focus("core").extract().unwrap(); + check_key_handover(&config, &figment) + } + + #[test] + fn a_serving_kms_needs_a_key_handover_path() { + let public = "[core.onboard]\npublic_key_handover = true\n"; + let no_admin = "[core.admin]\nenabled = false\n"; + let no_key = "[core.admin.tls]\nkey = \"\"\n"; + let no_certs = "[core.admin.tls]\ncerts = \"\"\n"; + assert!(handover_check("").is_ok()); + for missing in [no_admin, no_key, no_certs] { + assert!(handover_check(&format!("{public}{missing}")).is_ok()); + } + for (toml, setting) in [ + (no_admin, "[core.admin] enabled = true"), + (no_key, "[core.admin.tls] key"), + (no_certs, "[core.admin.tls] certs"), + ] { + let err = handover_check(toml).unwrap_err().to_string(); + assert!(err.contains(setting), "{err}"); + assert!(err.contains("public_key_handover = true"), "{err}"); + } + } + #[test] fn rpc_cert_is_attested_by_default() { let figment = load_config_figment(None); diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index 205790d79..3db1da263 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -173,6 +173,8 @@ async fn main() -> Result<()> { } } + config::check_key_handover(&config, &figment)?; + let keyring = main_service::Keyring::load(&config).context("Failed to load the root keys")?; let rpc_epoch = main_service::wait_for_servable_epoch(&config, &keyring).await; info!("Issuing RPC certificate under root epoch {rpc_epoch}"); diff --git a/test-suites/full-stack-compose/scripts/app_compose.py b/test-suites/full-stack-compose/scripts/app_compose.py index f1c3f25bb..0ff376ae2 100755 --- a/test-suites/full-stack-compose/scripts/app_compose.py +++ b/test-suites/full-stack-compose/scripts/app_compose.py @@ -85,8 +85,14 @@ def kms(args: argparse.Namespace) -> None: auto_bootstrap_domain = "" # Required by KMS 0.5.8. Current KMS ignores this retired field. quote_enabled = true +# Current KMS needs a handover path to serve; this test keeps the admin +# listener off and serves the public one. KMS 0.5.8 ignores both settings. +public_key_handover = true address = "0.0.0.0" port = 8000 + +[core.admin] +enabled = false """ docker_compose = f"""services: kms: diff --git a/tools/dev-stack.sh b/tools/dev-stack.sh index ebf98c872..27d56384a 100755 --- a/tools/dev-stack.sh +++ b/tools/dev-stack.sh @@ -158,7 +158,10 @@ build_cfg() { fi GATEWAY_WG_PUBKEY=$(echo $GATEWAY_WG_KEY | wg pubkey) - # kms + # kms: other KMS nodes onboard through the admin listener + if [ ! -f kms-admin-token ]; then + (umask 077 && openssl rand -hex 32 >kms-admin-token) + fi cat <kms.toml log_level = "info" @@ -181,6 +184,16 @@ enabled = $VMM_ENABLE_GPU [core.auth_api] type = "dev" +[core.admin] +enabled = true +address = "127.0.0.1" +port = $(($BASE_PORT + 5)) +auth_token = "$(cat kms-admin-token)" + +[core.admin.tls] +key = "$CERTS_DIR/rpc.key" +certs = "$CERTS_DIR/rpc.crt" + [core.onboard] address = "127.0.0.1" port = $KMS_RPC_LISTEN_PORT From 97ca19b6749e6ce51364d92ca6b2dee4a1b47871 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 05:22:40 -0800 Subject: [PATCH 2/3] fix(kms): check the key handover path before onboarding or bootstrap Checked only before serving, a misconfigured node completed onboarding or bootstrap, getting or generating root keys, and only then failed. Check at startup instead, in every mode. Also document that an auth_token_hash, public in an app compose, needs a high-entropy random token: a weak one can be brute-forced offline from its hash. Signed-off-by: Kevin Wang --- CHANGELOG.md | 7 ++++--- docs/deployment.md | 2 +- docs/tutorials/kms-cvm-deployment.md | 2 +- docs/upgrading-to-0.6.md | 3 ++- dstack/kms/README.md | 9 ++++++--- dstack/kms/kms.toml | 6 ++++-- dstack/kms/src/config.rs | 8 ++++++-- dstack/kms/src/main.rs | 6 ++++-- 8 files changed, 28 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f343ccc79..2e4b38e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,15 +33,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- KMS: a serving KMS requires a key handover path and otherwise refuses to - start, naming the missing setting: unless +- KMS: a KMS requires a key handover path and otherwise refuses to start, + before onboarding or bootstrap, naming the missing setting: unless `core.onboard.public_key_handover = true`, set `[core.admin] enabled = true` and `[core.admin.tls]` with `key` and `certs`. There is no opt-out, dev auth included. The built-in config now enables the admin listener with TLS on the RPC key and certificate, so a KMS on it needs an admin token to start: `core.admin.auth_token`, `DSTACK_KMS_ADMIN_TOKEN`, or the new `core.admin.auth_token_hash` (hex SHA-256), which keeps the token out of a - public app compose. When upgrading, add the admin TLS and a token, or enable + public app compose; the hash is then public, so use a high-entropy random + token such as `openssl rand -hex 32`. When upgrading, add the admin TLS and a token, or enable the public handover. The `dstack-app` composes and `dstackup` generate the token and configure its hash; the deploy scripts publish the admin listener when `KMS_ADMIN_ADDR` is set. diff --git a/docs/deployment.md b/docs/deployment.md index 4a6b8a7ff..9095496dd 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -510,7 +510,7 @@ Additional KMS instances can onboard from an existing KMS to share the same root **Configure the source KMS for key handover:** -Every serving KMS needs this, or `core.onboard.public_key_handover = true`: without a handover path it refuses to start, naming the missing setting. +Every KMS needs this, or `core.onboard.public_key_handover = true`: without a handover path it refuses to start, before onboarding or bootstrap, naming the missing setting. In a public config such as an app compose, set `auth_token_hash` (the token's SHA-256) instead of `auth_token`, with a high-entropy random token (`openssl rand -hex 32`), since the hash can be brute-forced offline. `Admin.GetKmsKey` only works on an admin listener with `[core.admin.tls]`: without TLS the listener speaks plain HTTP, so the new KMS can present no RA-TLS certificate and the call fails with `No attestation provided`. The admin listener binds to `127.0.0.1` by default; bind it to an address the new KMS can reach and restrict access to the KMS nodes at the network layer. diff --git a/docs/tutorials/kms-cvm-deployment.md b/docs/tutorials/kms-cvm-deployment.md index 5d6690670..a8a16ad53 100644 --- a/docs/tutorials/kms-cvm-deployment.md +++ b/docs/tutorials/kms-cvm-deployment.md @@ -186,7 +186,7 @@ configs: url = "http://127.0.0.1:9200" # Admin listener: serves the key handover to other KMS nodes, so a - # serving KMS requires it. Only the token's SHA-256 goes into the compose. + # KMS requires it. Only the token's SHA-256 goes into the compose. [core.admin] enabled = true address = "0.0.0.0" diff --git a/docs/upgrading-to-0.6.md b/docs/upgrading-to-0.6.md index e935f1c3e..ce5c0aeae 100644 --- a/docs/upgrading-to-0.6.md +++ b/docs/upgrading-to-0.6.md @@ -241,7 +241,8 @@ Configuration changes: set `core.onboard.public_key_handover = true`. The built-in default enables the admin listener with TLS, so it needs `core.admin.auth_token`, `DSTACK_KMS_ADMIN_TOKEN` or `core.admin.auth_token_hash` to start. In a KMS - app compose, which is public, use `auth_token_hash`. + app compose, which is public, use `auth_token_hash` with a high-entropy random + token (`openssl rand -hex 32`), since its hash can be brute-forced offline. - Auto-bootstrap refuses to overwrite existing root keys. If you use `auth-simple`, replace it with `dstack-auth`, which reads the same diff --git a/dstack/kms/README.md b/dstack/kms/README.md index e72b194eb..1c89cc615 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -57,7 +57,9 @@ certs = "/etc/kms/certs/rpc.crt" The token can also be supplied via the `DSTACK_KMS_ADMIN_TOKEN` or `ADMIN_API_TOKEN` environment variables instead of the config file, or as its hex SHA-256 in `auth_token_hash` (`printf '%s' "$TOKEN" | sha256sum`) for a -config that is not secret, such as a KMS app compose. The admin +config that is not secret, such as a KMS app compose. The hash is then public, +so the token must be high-entropy random, such as `openssl rand -hex 32`: a +weak token can be brute-forced offline from its hash. The admin listener fails closed: enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) refuses to start. With `[core.admin.tls]` the listener serves HTTPS (use `http://` without it). Call it with, for example: @@ -82,8 +84,9 @@ clients. A KMS refuses to start serving without a handover path, so that other nodes can always onboard from it: unless `public_key_handover = true`, it requires `[core.admin] enabled = true` and `[core.admin.tls]` with `key` and `certs`, and -the error names the missing setting. There is no opt-out, `auth_api = "dev"` -included; the onboarding phase itself is not checked. The default `kms.toml` +the error names the missing setting. The check runs at startup, before +onboarding or bootstrap, so a misconfigured node fails before it gets or +generates root keys. There is no opt-out, `auth_api = "dev"` included. The default `kms.toml` enables the admin listener with TLS on the RPC key and certificate, so it starts only once an admin token is set (`auth_token`, `DSTACK_KMS_ADMIN_TOKEN` or `auth_token_hash`). The `dstack-app` composes and diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 0dad6a10f..eb98cd9c6 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -86,7 +86,7 @@ download_timeout = "2m" # to onboarding KMS nodes on a dedicated listener behind the shared HTTP # authenticator. It fails closed: the KMS does not start until auth_token (or # DSTACK_KMS_ADMIN_TOKEN), auth_token_hash or htpasswd_file is set. Unless -# core.onboard.public_key_handover = true, a serving KMS also requires this +# core.onboard.public_key_handover = true, the KMS also requires this # listener and its TLS below, so that other KMS nodes can onboard. [core.admin] enabled = true @@ -98,7 +98,9 @@ port = 8001 # `Authorization: Bearer ` or the `X-Admin-Token: ` header. # Required unless insecure_no_auth = true. Generate with: openssl rand -hex 32 auth_token = "" -# Or the token's hex SHA-256, for a config that is not secret (an app compose): +# Or the token's hex SHA-256, for a config that is not secret (an app compose). +# The hash is public there, so the token must be high-entropy random, or it can +# be brute-forced offline: TOKEN=$(openssl rand -hex 32) # printf '%s' "$TOKEN" | sha256sum auth_token_hash = "" # Optional Apache bcrypt htpasswd file (create with `htpasswd -B -c file admin`). diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 2819eae76..7e95e8c45 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -129,8 +129,8 @@ fn default_true() -> bool { } /// Without the public handover, other KMS nodes can get the keys only from the -/// admin listener over TLS. A serving KMS without either would strand them, so -/// it refuses to start. +/// admin listener over TLS. A KMS without either would strand them, so it +/// refuses to start, in any mode. pub(crate) fn check_key_handover(config: &KmsConfig, figment: &Figment) -> Result<()> { if config.onboard.public_key_handover { return Ok(()); @@ -320,6 +320,10 @@ mod tests { let no_key = "[core.admin.tls]\nkey = \"\"\n"; let no_certs = "[core.admin.tls]\ncerts = \"\"\n"; assert!(handover_check("").is_ok()); + // onboarding and bootstrap modes are checked alike. + let onboarding = "[core.onboard]\nenabled = true\nauto_bootstrap_domain = \"kms.test\"\n"; + assert!(handover_check(onboarding).is_ok()); + assert!(handover_check(&format!("{onboarding}{no_admin}")).is_err()); for missing in [no_admin, no_key, no_certs] { assert!(handover_check(&format!("{public}{missing}")).is_ok()); } diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index 3db1da263..32d3b9a0d 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -165,6 +165,10 @@ async fn main() -> Result<()> { ); } + // before onboarding or bootstrap, so a misconfigured node fails before it + // gets or generates root keys. + config::check_key_handover(&config, &figment)?; + if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); run_onboard_service(config.clone(), figment.clone()).await?; @@ -173,8 +177,6 @@ async fn main() -> Result<()> { } } - config::check_key_handover(&config, &figment)?; - let keyring = main_service::Keyring::load(&config).context("Failed to load the root keys")?; let rpc_epoch = main_service::wait_for_servable_epoch(&config, &keyring).await; info!("Issuing RPC certificate under root epoch {rpc_epoch}"); From df79d1ae78e0dbd6713ecfb0ef0d63c533eb216f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 05:30:41 -0800 Subject: [PATCH 3/3] fix(kms): validate the admin credential before onboarding or bootstrap With the default config and no admin token, the handover check passed (admin enabled, TLS paths set) but the missing credential surfaced only when the admin listener was built, after onboarding or bootstrap. Build the admin listener setup right after the handover check, so such a node also fails before it gets or generates root keys. Signed-off-by: Kevin Wang --- dstack/kms/src/main.rs | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/dstack/kms/src/main.rs b/dstack/kms/src/main.rs index 32d3b9a0d..ce730abfc 100644 --- a/dstack/kms/src/main.rs +++ b/dstack/kms/src/main.rs @@ -166,8 +166,21 @@ async fn main() -> Result<()> { } // before onboarding or bootstrap, so a misconfigured node fails before it - // gets or generates root keys. + // gets or generates root keys: the handover path, and the admin listener + // serving it (enabled with no credential fails here). config::check_key_handover(&config, &figment)?; + let admin_config = config.admin.clone(); + let admin_setup = if admin_config.enabled { + let admin_value = figment + .find_value("core.admin") + .context("core.admin section not found")?; + let admin_figment = + Figment::from(rocket::Config::default()).merge(Serialized::defaults(admin_value)); + let admin_fairing = admin_auth::AdminAuthFairing::from_config(&admin_config)?; + Some((admin_figment, admin_fairing)) + } else { + None + }; if config.onboard.enabled && !config.keys_exists() { info!("Onboarding"); @@ -199,21 +212,6 @@ async fn main() -> Result<()> { } let metrics_enabled = config.metrics.enabled; - let admin_config = config.admin.clone(); - // build the admin listener figment from `[core.admin]` before `config` is - // moved into the state; the fairing is built now so a misconfigured admin - // (enabled but no credential) fails fast before we start serving. - let admin_setup = if admin_config.enabled { - let admin_value = figment - .find_value("core.admin") - .context("core.admin section not found")?; - let admin_figment = - Figment::from(rocket::Config::default()).merge(Serialized::defaults(admin_value)); - let admin_fairing = admin_auth::AdminAuthFairing::from_config(&admin_config)?; - Some((admin_figment, admin_fairing)) - } else { - None - }; // Reload certificates after renewal before exposing them through RPCs. let rpc_ca_cert = fs_err::read_to_string(config.root_key_paths(rpc_epoch, keyring.latest()).ca_cert)?;