From 34dcfbd257a04b0f1c501fc37a951035b3dcb8be Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:12:36 -0800 Subject: [PATCH 01/12] fix(dstack-auth): allow key handover at a registered pending root epoch Signed-off-by: Kevin Wang --- docs/dstack-auth-operations.md | 2 +- dstack/crates/dstack-auth/src/main.rs | 59 ++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/docs/dstack-auth-operations.md b/docs/dstack-auth-operations.md index d7a4f2826..54b203580 100644 --- a/docs/dstack-auth-operations.md +++ b/docs/dstack-auth-operations.md @@ -112,7 +112,7 @@ Hex values are compared case-insensitively, with or without a `0x` prefix. | `apps..allowAnyDevice` | No | If true, skip the device ID check for this app | | `apps..currentAppEpoch` | No | Current app key epoch (default `0`), served by `GET /app/`. A non-zero epoch denies the legacy `/bootAuth/app` route. | | `rootEpochs` | No | KMS root public keys of each root epoch, indexed by epoch, `[{"caPubkey": "0x...", "k256Pubkey": "0x..."}]`. Epoch 0 is the original roots. Append only. `GET /` reports the current epoch as `rootEpoch` with its chain hash as `rootEpochHash`; the last entry is reported as `latestRootEpoch`, and entries beyond `currentRootEpoch` are pending activation. | -| `currentRootEpoch` | No | Root epoch KMS instances serve (default 0). Raise it only once every KMS node holds that epoch. | +| `currentRootEpoch` | No | Root epoch KMS instances serve (default 0). Key-issuing operations (`kms_serve`, `app_get_key`, `app_get_cert`) may request up to this epoch; KMS key handover (`kms_export_keys`, `kms_import_keys`) may request any registered epoch, so the other KMS nodes can onboard from the rotated one while the new epoch is pending. Raise it only once every KMS node holds that epoch. | ### Validation Logic diff --git a/dstack/crates/dstack-auth/src/main.rs b/dstack/crates/dstack-auth/src/main.rs index 2496d69e1..06efdb8bb 100644 --- a/dstack/crates/dstack-auth/src/main.rs +++ b/dstack/crates/dstack-auth/src/main.rs @@ -386,13 +386,26 @@ fn authorize( }, }; let info = &request.boot_info; + // key handover may carry a registered but not yet active epoch, so the + // keyring can be distributed before activation; key-issuing operations + // stay on active epochs. + let handover = matches!( + request.operation, + Operation::KmsExportKeys | Operation::KmsImportKeys + ); result.rejection_reason = if request.operation.is_kms() != is_kms { "invalid authorization operation".into() } else if current.is_none() && request.operation != Operation::KmsBootstrap { "root epoch not registered".into() } else if request .requested_root_epoch - .is_some_and(|epoch| epoch as usize > result.current_root_epoch) + .is_some_and(|epoch| epoch as usize > result.latest_root_epoch) + { + "requested root epoch is not registered".into() + } else if !handover + && request + .requested_root_epoch + .is_some_and(|epoch| epoch as usize > result.current_root_epoch) { "requested root epoch is not active".into() } else if request @@ -1128,6 +1141,50 @@ mod tests { ); } + #[test] + fn a_pending_root_epoch_is_authorized_for_kms_key_handover() { + // epoch 1 is registered (latest) but not current: pending + let al: Allowlist = serde_json::from_value(json!({ + "kmsContractAddr": "0x1111111111111111111111111111111111111111", + "kms": {"mrAggregated": ["bb"], "allowAnyDevice": true}, + "rootEpochs": [ + {"caPubkey": "aa", "k256Pubkey": "bb"}, + {"caPubkey": "cc", "k256Pubkey": "dd"} + ], + "currentRootEpoch": 0 + })) + .unwrap(); + let ask = |operation: &str, requested: u32| { + let request: AuthorizationRequest = serde_json::from_value(json!({ + "operation": operation, + "appId": "11".repeat(20), + "mrAggregated": "bb", + "requestedRootEpoch": requested, + })) + .unwrap(); + authorize(&request, true, &al).unwrap() + }; + + let current = ask("kms_export_keys", 0); + assert!(current.authorized, "{}", current.rejection_reason); + assert_eq!(current.current_root_epoch, 0); + assert_eq!(current.latest_root_epoch, 1); + + for operation in ["kms_import_keys", "kms_export_keys"] { + let pending = ask(operation, 1); + assert!( + pending.authorized, + "{operation} at the pending epoch 1: {}", + pending.rejection_reason + ); + assert!(!ask(operation, 2).authorized); + } + + let serve = ask("kms_serve", 1); + assert!(!serve.authorized); + assert_eq!(serve.rejection_reason, "requested root epoch is not active"); + } + #[test] fn allows_only_a_kms_serving_the_current_root_epoch() { // Shared with the DstackKms contract tests. From 296aacb69d6123710e9522581ca1a7c46e94d913 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:21:00 -0800 Subject: [PATCH 02/12] fix(kms): reject anonymous callers before the self check and audit cert issuance Report a backend rejection before the epoch fields are checked, require attestation in GetAppKey and the key handover before the KMS self check reaches the auth backend, and add audit lines for SignCert and GetTempCaCert. Signed-off-by: Kevin Wang --- dstack/kms/src/main_service.rs | 56 +++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 876fdf796..2d237b821 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -199,10 +199,18 @@ pub(crate) enum KeyRelease { RpcHandover, AdminHandover, AppKey, + AppCert, + TempCa, } impl KeyRelease { - const ALL: [Self; 3] = [Self::RpcHandover, Self::AdminHandover, Self::AppKey]; + const ALL: [Self; 5] = [ + Self::RpcHandover, + Self::AdminHandover, + Self::AppKey, + Self::AppCert, + Self::TempCa, + ]; /// The listener and operation labels. fn labels(self) -> (&'static str, &'static str) { @@ -210,6 +218,8 @@ impl KeyRelease { Self::RpcHandover => ("rpc", "kms_export_keys"), Self::AdminHandover => ("admin", "kms_export_keys"), Self::AppKey => ("rpc", "app_get_key"), + Self::AppCert => ("rpc", "app_get_cert"), + Self::TempCa => ("rpc", "temp_ca"), } } } @@ -362,6 +372,9 @@ impl KmsState { result: &AuthorizationResult, requested: Option, ) -> Result { + if !result.authorized { + bail!("Authorization denied: {}", result.rejection_reason); + } let current = if self.config.auth_api.is_dev() { self.keyring.latest() } else { @@ -371,9 +384,6 @@ impl KmsState { self.keyring .servable_epoch(RootEpochRegistry::Current(current))? }; - if !result.authorized { - bail!("Authorization denied: {}", result.rejection_reason); - } let epoch = requested.unwrap_or(current); if epoch > current { bail!("root epoch {epoch} is not active; the current one is {current}"); @@ -496,25 +506,28 @@ impl RpcHandler { } /// Count a key release and log it, regardless of the metrics setting. + /// `boot_info` is the authorized caller, if the operation identifies one. fn audit_key_release( &self, release: KeyRelease, - boot_info: &BootInfo, + boot_info: Option<&BootInfo>, requested: &str, returned: &str, cached: bool, ) { self.state.metrics().record_key_release(release); - let instance_id = match &boot_info.instance_id[..] { + let hex_or_unknown = |bytes: &[u8]| match bytes { [] => "UNKNOWN".to_string(), - id => hex::encode(id), + bytes => hex::encode(bytes), }; + let app_id = hex_or_unknown(boot_info.map_or(&[], |info| &info.app_id[..])); + let instance_id = hex_or_unknown(boot_info.map_or(&[], |info| &info.instance_id[..])); let (listener, operation) = release.labels(); info!( target: "dstack_kms::audit", listener, operation, - app_id = %hex::encode(&boot_info.app_id), + app_id = %app_id, instance_id = %instance_id, remote = ?self.remote_endpoint, requested_epoch = requested, @@ -533,6 +546,8 @@ impl RpcHandler { release: KeyRelease, request: GetKmsKeyRequest, ) -> Result { + // reject anonymous callers before the self check reaches the backend + self.ensure_attested()?; let cached = self.self_allowed_cached(Operation::KmsExportKeys); self.ensure_self_allowed(Operation::KmsExportKeys) .await @@ -540,7 +555,7 @@ impl RpcHandler { let info = self.ensure_kms_allowed(&request.vm_config).await?; ensure_key_release_allowed(&info, (&self.state.config).into())?; let returned = format!("root 0..={}", self.state.keyring.latest()); - self.audit_key_release(release, &info, "all", &returned, cached); + self.audit_key_release(release, Some(&info), "all", &returned, cached); Ok(KmsKeyResponse { temp_ca_key: self.state.temp_ca.key.serialize_pem(), keys: self @@ -774,6 +789,8 @@ impl KmsRpc for RpcHandler { if request.api_version > 1 { bail!("Unsupported API version: {}", request.api_version); } + // reject anonymous callers before the self check reaches the backend + self.ensure_attested()?; let cached = self.self_allowed_cached(Operation::KmsServe); self.ensure_self_allowed(Operation::KmsServe) .await @@ -799,7 +816,7 @@ impl KmsRpc for RpcHandler { let epoch = |epoch: Option| epoch.map_or("current".into(), |e| e.to_string()); self.audit_key_release( KeyRelease::AppKey, - &boot_info, + Some(&boot_info), &format!( "root {} app {}", epoch(request.root_epoch), @@ -943,6 +960,7 @@ impl KmsRpc for RpcHandler { } async fn get_temp_ca_cert(self) -> Result { + let cached = self.self_allowed_cached(Operation::KmsServe); let epoch = self.ensure_self_allowed(Operation::KmsServe).await?; if self.state.config.enforce_self_authorization { let boot_info = self @@ -953,14 +971,23 @@ impl KmsRpc for RpcHandler { .context("missing authorized KMS boot info")?; ensure_key_release_allowed(&boot_info, (&self.state.config).into())?; } + let ca_cert = self.state.epoch_root(epoch)?.ca.pem_cert.clone(); + self.audit_key_release( + KeyRelease::TempCa, + None, + "current", + &format!("root {epoch}"), + cached, + ); Ok(GetTempCaCertResponse { temp_ca_cert: self.state.temp_ca.pem_cert.clone(), temp_ca_key: self.state.temp_ca.key.serialize_pem(), - ca_cert: self.state.epoch_root(epoch)?.ca.pem_cert.clone(), + ca_cert, }) } async fn sign_cert(self, request: SignCertRequest) -> Result { + let cached = self.self_allowed_cached(Operation::KmsServe); self.ensure_self_allowed(Operation::KmsServe) .await .context("KMS self authorization failed")?; @@ -1009,6 +1036,13 @@ impl KmsRpc for RpcHandler { let cert = app_ca .sign_csr(&csr, Some(&cert_app_info), "app:custom") .context("Failed to sign certificate")?; + self.audit_key_release( + KeyRelease::AppCert, + Some(&app_info.boot_info), + "current", + &format!("root {} app {}", app_info.root_epoch, app_info.app_epoch), + cached, + ); Ok(SignCertResponse { certificate_chain: vec![ cert.pem(), From b3e6ab429a6f8fbe2f88314c8888db442b6987a5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:21:36 -0800 Subject: [PATCH 03/12] test(kms): cover GetAppKey, SignCert and env public keys after rotation Signed-off-by: Kevin Wang --- dstack/Cargo.lock | 1 + dstack/kms/Cargo.toml | 1 + dstack/kms/src/main_service.rs | 348 ++++++++++++++++++++++++++++++++- 3 files changed, 342 insertions(+), 8 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 39936e3ae..ca00bfe48 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2215,6 +2215,7 @@ dependencies = [ "http-client", "k256", "load_config", + "mock-attestation", "moka", "p256 0.13.2", "parity-scale-codec", diff --git a/dstack/kms/Cargo.toml b/dstack/kms/Cargo.toml index 2979b4230..b60339120 100644 --- a/dstack/kms/Cargo.toml +++ b/dstack/kms/Cargo.toml @@ -53,6 +53,7 @@ moka = { workspace = true, features = ["future"] } [dev-dependencies] cc-eventlog.workspace = true +mock-attestation = { path = "../crates/mock-attestation" } [features] default = [] diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 2d237b821..2b4f4b2db 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -1813,11 +1813,9 @@ mod tests { ); } - #[rocket::async_test] - async fn authorization_selects_epochs_without_a_serving_epoch_snapshot() { - let dir = tempfile::tempdir().unwrap(); - let initial = kms_state(dir.path(), "http://127.0.0.1:1"); - let mut keyring = Keyring::load(&initial.config).unwrap(); + /// The keyring in `config.cert_dir` with a second root epoch appended. + fn rotated_keyring(config: &KmsConfig) -> Keyring { + let mut keyring = Keyring::load(config).unwrap(); let key = KeyPair::generate_for(&ra_tls::rcgen::PKCS_ECDSA_P256_SHA256).unwrap(); let cert = CertRequest::builder() .key(&key) @@ -1832,6 +1830,14 @@ mod tests { }); keyring.chain_hashes = root_chain_hashes(keyring.roots.iter().map(|root| (&root.ca.key, &root.k256))); + keyring + } + + #[rocket::async_test] + async fn authorization_selects_epochs_without_a_serving_epoch_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let initial = kms_state(dir.path(), "http://127.0.0.1:1"); + let keyring = rotated_keyring(&initial.config); let state = KmsState::new( initial.config.clone(), keyring, @@ -1876,6 +1882,10 @@ mod tests { rejected.current_root_epoch_hash = hash; assert!(state.authorized_epoch(&rejected, None).is_err()); } + // A rejection is reported even when the epoch fields do not match. + rejected.authorized = false; + let err = state.authorized_epoch(&rejected, None).unwrap_err(); + assert!(err.to_string().contains("Policy denied"), "{err}"); let mut missing = result(1); missing.current_root_epoch = u32::MAX; assert!(state.authorized_epoch(&missing, Some(0)).is_err()); @@ -1955,11 +1965,16 @@ mod tests { } fn auth_answer(state: &KmsState, allowed: bool) -> String { - let hash = hex::encode(state.keyring.chain_hashes[0]); + epoch_answer(&state.keyring, allowed, 0, 0) + } + + fn epoch_answer(keyring: &Keyring, allowed: bool, root_epoch: u32, app_epoch: u32) -> String { + let hash = hex::encode(keyring.chain_hashes[root_epoch as usize]); serde_json::json!({ "authorized": allowed, "rejectionReason": "denied", "gatewayAppId": "", - "currentRootEpoch": 0, "currentAppEpoch": 0, "currentRootEpochHash": hash, - "latestRootEpoch": 0, "latestRootEpochHash": hash, + "currentRootEpoch": root_epoch, "currentAppEpoch": app_epoch, + "currentRootEpochHash": hash, + "latestRootEpoch": root_epoch, "latestRootEpochHash": hash, }) .to_string() } @@ -2137,6 +2152,272 @@ mod tests { } } + type AuthRequests = std::thread::JoinHandle>; + + /// A KMS holding root epochs 0 and 1 and serving 1, with self authorization + /// enforced. Its auth API reports root epoch 1 and app epoch 1 as current + /// and answers `allowed` to each request in turn (self check first). + async fn rotated_kms( + dir: &Path, + allowed: &[bool], + configure: impl FnOnce(&mut KmsConfig), + ) -> (KmsState, AuthRequests) { + use upgrade_authority::tests::{boot_info, serve, webhook}; + let initial = kms_state(dir, "http://127.0.0.1:1"); + let keyring = rotated_keyring(&initial.config); + let answers = allowed + .iter() + .map(|allowed| epoch_answer(&keyring, *allowed, 1, 1)) + .collect(); + let (url, server) = serve(answers); + let mut config = initial.config.clone(); + config.enforce_self_authorization = true; + config.auth_api = webhook(url); + configure(&mut config); + let rpc_ca = keyring.roots[1].ca.pem_cert.clone(); + let state = KmsState::new(config, keyring, rpc_ca).unwrap(); + state.self_boot_info.insert((), boot_info(1)).await; + (state, server) + } + + fn released(metrics: &str, operation: &str, count: u64) -> bool { + metrics.contains(&format!( + "dstack_kms_key_releases_total{{listener=\"rpc\",operation=\"{operation}\"}} {count}\n" + )) + } + + /// Request an app key from a [`rotated_kms`] as an SNP caller, or anonymously. + async fn rotated_app_key( + allowed: &[bool], + attested: bool, + root_epoch: Option, + app_epoch: Option, + ) -> ( + Result, + Vec<(String, serde_json::Value)>, + KmsState, + BootInfo, + ) { + let dir = tempfile::tempdir().unwrap(); + let (state, server) = rotated_kms(dir.path(), allowed, |config| { + config.sev_snp_key_release = true + }) + .await; + let (attestation, vm_config) = snp_caller(); + let boot_info = build_boot_info_for_attestation(&attestation, false, &vm_config).unwrap(); + let handler = RpcHandler { + state: state.clone(), + attestation: attested.then_some(attestation), + remote_endpoint: None, + }; + let result = handler + .get_app_key(GetAppKeyRequest { + api_version: 1, + vm_config, + root_epoch, + app_epoch, + }) + .await; + (result, auth_requests(server), state, boot_info) + } + + #[rocket::async_test] + async fn get_app_key_derives_from_the_requested_epochs_after_rotation() { + for (requested_root, requested_app, root_epoch, app_epoch) in [ + (None, None, 1, 1), + (Some(0), Some(0), 0, 0), + (Some(0), Some(1), 0, 1), + (Some(1), Some(0), 1, 0), + ] { + let (result, requests, state, caller) = + rotated_app_key(&[true, true], true, requested_root, requested_app).await; + let response = result.unwrap(); + assert_eq!( + (response.root_epoch, response.app_epoch), + (root_epoch, app_epoch) + ); + let root = state.epoch_root(root_epoch).unwrap(); + assert_eq!(response.ca_cert, root.ca.pem_cert); + let expected = + derive_app_keys(root, &caller.app_id, &caller.instance_id, app_epoch).unwrap(); + assert_eq!(response.disk_crypt_key, expected.disk_crypt_key); + assert_eq!(response.sealed_env_key, expected.sealed_env_key); + assert_eq!(response.k256_key, expected.k256_key); + + let [(_, self_check), (route, check)] = &requests[..] else { + panic!("unexpected auth API requests: {requests:?}"); + }; + assert_eq!(self_check["operation"], "kms_serve"); + assert_eq!(route, "/authorizeApp"); + assert_eq!(check["operation"], "app_get_key"); + assert_eq!( + check.get("requestedRootEpoch"), + requested_root.map(Into::into).as_ref() + ); + assert_eq!( + check.get("requestedAppEpoch"), + requested_app.map(Into::into).as_ref() + ); + let metrics = state.metrics().render_prometheus(); + assert!(released(&metrics, "app_get_key", 1), "{metrics}"); + } + } + + #[rocket::async_test] + async fn get_app_key_fails_unless_the_request_is_authorized() { + let (result, requests, state, _) = rotated_app_key(&[true, false], true, None, None).await; + let err = format!("{:#}", result.unwrap_err()); + assert!( + err.contains("App not allowed") && err.contains("denied"), + "{err}" + ); + assert_eq!(requests.len(), 2); + let metrics = state.metrics().render_prometheus(); + assert!(released(&metrics, "app_get_key", 0), "{metrics}"); + + // A backend that authorizes an epoch it reports as inactive is not trusted. + let (result, _, _, _) = rotated_app_key(&[true, true], true, None, Some(2)).await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("requested app epoch is not active"), "{err}"); + let (result, _, _, _) = rotated_app_key(&[true, true], true, Some(2), None).await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("root epoch 2 is not active"), "{err}"); + + // An anonymous caller is refused before the KMS reaches its backend. + let (result, requests, _, _) = rotated_app_key(&[], false, None, None).await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("No attestation provided"), "{err}"); + assert!(requests.is_empty()); + } + + fn pem_public_key(pem: &str) -> Vec { + let (_, pem) = x509_parser::pem::parse_x509_pem(pem.as_bytes()).unwrap(); + pem.parse_x509().unwrap().public_key().raw.to_vec() + } + + /// Sign a certificate on a [`rotated_kms`] for a mock nitro enclave app. + async fn rotated_sign_cert( + allowed: &[bool], + ) -> ( + Result, + Vec<(String, serde_json::Value)>, + KmsState, + ) { + use ra_tls::{ + attestation::{ + Attestation, AttestationQuote, AttestationVerifierConfig, DstackNitroQuote, + QuoteContentType, RootCaPaths, + }, + cert::{CertConfigV2, Csr}, + }; + let dir = tempfile::tempdir().unwrap(); + let attester = mock_attestation::nsm::NsmGenerator::new().unwrap(); + let attestation_root = dir.path().join("nitro-root.pem"); + fs::write(&attestation_root, attester.root_ca_pem()).unwrap(); + let (state, server) = rotated_kms(dir.path(), allowed, |config| { + config.nitro_enclave_key_release = true; + config.attestation = AttestationVerifierConfig { + insecure_allow_external_trust_anchors: true, + root_ca: RootCaPaths { + aws_nitro_enclave: Some(attestation_root), + ..Default::default() + }, + ..Default::default() + }; + }) + .await; + + let key = KeyPair::generate_for(&ra_tls::rcgen::PKCS_ECDSA_P256_SHA256).unwrap(); + let pubkey = key.public_key_der(); + let report_data = QuoteContentType::RaTlsCert.to_report_data(&pubkey); + let pcrs: BTreeMap> = [0, 1, 2, 4] + .into_iter() + .map(|i| (i, vec![i as u8 + 1; 48])) + .collect(); + let os_image_hash = sha256_chunks(&[&pcrs[&0], &pcrs[&1], &pcrs[&2]]); + let attestation = Attestation { + quote: AttestationQuote::DstackNitroEnclave(DstackNitroQuote { + nsm_quote: attester.attest_with_pcrs(&report_data, pcrs).unwrap(), + }), + runtime_events: vec![runtime_event("app-id", vec![0x22; 20])], + report_data, + config: String::new(), + report: (), + } + .into_versioned(); + let csr = CertSigningRequestV2 { + confirm: "please sign cert:".into(), + pubkey, + config: CertConfigV2 { + org_name: None, + subject: "app.test".into(), + subject_alt_names: vec![], + usage_server_auth: true, + usage_client_auth: false, + ext_quote: false, + ext_app_info: false, + not_before: None, + not_after: None, + }, + attestation, + }; + let request = SignCertRequest { + api_version: 2, + signature: csr.signed_by(&key).unwrap(), + csr: csr.to_vec(), + vm_config: serde_json::json!({ "os_image_hash": hex::encode(os_image_hash) }) + .to_string(), + }; + let result = RpcHandler { + state: state.clone(), + attestation: None, + remote_endpoint: None, + } + .sign_cert(request) + .await; + (result, auth_requests(server), state) + } + + #[rocket::async_test] + async fn sign_cert_issues_from_the_current_epochs_after_rotation() { + let (result, requests, state) = rotated_sign_cert(&[true, true]).await; + let chain = result.unwrap().certificate_chain; + let root = state.epoch_root(1).unwrap(); + assert_eq!(chain[2], root.ca.pem_cert); + let app_ca_key = kdf::derive_p256_key_pair( + &root.ca.key, + &[&[0x22; 20][..], b"app-ca", &app_epoch_context(1)], + ) + .unwrap(); + assert_eq!(pem_public_key(&chain[1]), app_ca_key.public_key_der()); + + let [(_, self_check), (route, check)] = &requests[..] else { + panic!("unexpected auth API requests: {requests:?}"); + }; + assert_eq!(self_check["operation"], "kms_serve"); + assert_eq!(route, "/authorizeApp"); + assert_eq!(check["operation"], "app_get_cert"); + assert!(check.get("requestedRootEpoch").is_none()); + assert!(check.get("requestedAppEpoch").is_none()); + let metrics = state.metrics().render_prometheus(); + assert!(released(&metrics, "app_get_cert", 1), "{metrics}"); + } + + #[rocket::async_test] + async fn sign_cert_fails_when_the_backend_rejects_the_app() { + let (result, requests, state) = rotated_sign_cert(&[true, false]).await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("Authorization denied: denied"), "{err}"); + assert_eq!(requests.len(), 2); + let metrics = state.metrics().render_prometheus(); + assert!(released(&metrics, "app_get_cert", 0), "{metrics}"); + + let (result, requests, _) = rotated_sign_cert(&[false]).await; + let err = format!("{:#}", result.unwrap_err()); + assert!(err.contains("KMS self authorization failed"), "{err}"); + assert_eq!(requests.len(), 1); + } + /// Fetch an env public key while the auth API answers the app epoch query /// with `status` and `body`. async fn env_pub_key(sealed: bool, status: u16, body: &'static str) -> Result<()> { @@ -2179,6 +2460,57 @@ mod tests { } } + #[rocket::async_test] + async fn env_public_keys_at_a_rotated_app_epoch() { + let err = env_pub_key(false, 200, r#"{"currentAppEpoch":1}"#) + .await + .unwrap_err(); + assert!( + format!("{err:#}").contains("must use GetSealedEnvPubKey"), + "{err:#}" + ); + + let (url, server) = + upgrade_authority::tests::serve(vec![AUTH_API_INFO, r#"{"currentAppEpoch":1}"#]); + let dir = tempfile::tempdir().unwrap(); + let state = kms_state(dir.path(), &url); + let app_id = [0x11; 20]; + let response = RpcHandler { + state: state.clone(), + attestation: None, + remote_endpoint: None, + } + .get_sealed_env_pub_key(AppId { + app_id: app_id.to_vec(), + }) + .await + .unwrap(); + auth_requests(server); + assert_eq!((response.root_epoch, response.app_epoch), (0, 1)); + let root = state.epoch_root(0).unwrap(); + let secret = derive_sealed_env_key(root, &app_id, 1).unwrap(); + assert_ne!(secret, derive_sealed_env_key(root, &app_id, 0).unwrap()); + let public_key = + x25519_dalek::PublicKey::from(&x25519_dalek::StaticSecret::from(secret)).to_bytes(); + assert_eq!(response.public_key, public_key); + let message = [ + &0u32.to_be_bytes()[..], + &1u32.to_be_bytes(), + &public_key, + &response.timestamp.to_be_bytes(), + ] + .concat(); + let signature = sign_message( + &root.k256, + b"dstack-sealed-env-pubkey-v1", + &app_id, + &message, + ) + .unwrap(); + // The recoverable signature is deterministic (RFC 6979). + assert_eq!(response.signature, signature); + } + #[rocket::async_test] async fn legacy_temp_ca_rpc_returns_a_separate_ca_and_the_served_root() { assert!(dstack_kms_rpc::kms_server::KMSMethod::from_str("GetTempCaCert").is_some()); From 83013e6fe1e681f6d6eb3a77ca2eb16c62db3732 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:24:15 -0800 Subject: [PATCH 04/12] test(dstack-util): drop sealed env tamper cases that never reach the checked logic The root_epoch and app_epoch tamper cases failed while generating the RA-TLS certificate for the historical key request, so they passed even without the epoch checks. Rename the test to what it checks, and report both epochs when a sealed env is ahead of the current ones; the error printed the root epoch even when the app epoch was the one exceeding. Signed-off-by: Kevin Wang --- dstack/dstack-util/src/system_setup.rs | 44 ++++++++++---------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 78aa51c05..6b224cdf8 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -3327,16 +3327,16 @@ impl Stage1<'_> { bail!("sealed env encapsulated key must be 32 bytes"); } let payload = [envelope.enc, envelope.ciphertext].concat(); - let current = self.keys.root_epoch(); - if envelope.root_epoch > current || envelope.app_epoch > self.keys.app_epoch() { + let (current_root, current_app) = (self.keys.root_epoch(), self.keys.app_epoch()); + if envelope.root_epoch > current_root || envelope.app_epoch > current_app { bail!( - "sealed env epoch {} exceeds current epoch {current}", - envelope.root_epoch + "sealed env epoch (root {}, app {}) exceeds current epoch (root {current_root}, app {current_app})", + envelope.root_epoch, + envelope.app_epoch ); } let historical; - let secret = if envelope.root_epoch == current - && envelope.app_epoch == self.keys.app_epoch() + let secret = if envelope.root_epoch == current_root && envelope.app_epoch == current_app { current_secret } else { @@ -3351,17 +3351,10 @@ impl Stage1<'_> { &sealed_env_info_at_epochs(envelope.root_epoch, envelope.app_epoch), &payload, )?; - if envelope.root_epoch == self.keys.root_epoch() - && envelope.app_epoch == self.keys.app_epoch() - { + if envelope.root_epoch == current_root && envelope.app_epoch == current_app { return Ok((plaintext, Vec::new())); } - let sealed = seal_env_at_epochs( - current_secret, - self.keys.root_epoch(), - self.keys.app_epoch(), - &plaintext, - )?; + let sealed = seal_env_at_epochs(current_secret, current_root, current_app, &plaintext)?; return Ok((plaintext, sealed)); } let ciphertext = &self.shared.encrypted_env; @@ -4539,7 +4532,7 @@ mod sealed_env_file_tests { } #[tokio::test] - async fn sealed_env_binds_both_epochs_and_rejects_mislabeled_versions() { + async fn sealed_env_at_the_current_epochs_opens_and_binds_both_epochs() { let dir = tempfile::tempdir().unwrap(); let args = SetupArgs { work_dir: dir.path().into(), @@ -4555,17 +4548,14 @@ mod sealed_env_file_tests { let (opened, update) = current.decrypt_env().await.unwrap(); assert_eq!(opened, plaintext); assert!(update.is_empty()); - for (field, value) in [ - ("version", 2), - ("root_epoch", 2), - ("app_epoch", 1), - ("app_epoch", 3), - ] { - let mut tampered: Value = serde_json::from_slice(&sealed).unwrap(); - tampered[field] = value.into(); - current.shared.sealed_env = Some(serde_json::to_vec(&tampered).unwrap()); - assert!(current.decrypt_env().await.is_err()); - } + let mut tampered: Value = serde_json::from_slice(&sealed).unwrap(); + tampered["version"] = 2.into(); + current.shared.sealed_env = Some(serde_json::to_vec(&tampered).unwrap()); + let err = current.decrypt_env().await.unwrap_err(); + assert!( + err.to_string().contains("unsupported sealed env version"), + "{err}" + ); let envelope: SealedEnv = serde_json::from_slice(&sealed).unwrap(); assert_eq!(envelope.version, 1); let payload = [envelope.enc.clone(), envelope.ciphertext.clone()].concat(); From 3c3ab0e349458f77130670113d6ecf3435831a4a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:09 -0800 Subject: [PATCH 05/12] docs: dstack-auth accepts pending root epochs for KMS handover Signed-off-by: Kevin Wang --- docs/app-key-epochs.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/app-key-epochs.md b/docs/app-key-epochs.md index ca1bf697a..fd73df504 100644 --- a/docs/app-key-epochs.md +++ b/docs/app-key-epochs.md @@ -131,8 +131,9 @@ Direct legacy app checks use the current app epoch; legacy KMS app authorization still rejects nonzero app epochs. These material matrices are implemented by the Solidity policies. Config-file auth -backends such as dstack-auth retain their own policy model and may reject handover -of pending epochs. +backends such as dstack-auth retain their own policy model; dstack-auth applies the +same epoch limits, accepting a registered pending root epoch for KMS handover and +only active epochs for key-issuing operations. ## Derivation and lifetime From c241e4201c0be00b03dfd6103e15697b6f3966fa Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:09 -0800 Subject: [PATCH 06/12] test(sdk): cover the GetKey epoch mismatch check in all four SDKs Signed-off-by: Kevin Wang --- sdk/go/dstack/client_v1_test.go | 35 ++++++++++++++++++++++ sdk/js/src/__tests__/index-v1.test.ts | 41 +++++++++++++++++--------- sdk/python/tests/test_client_v1.py | 23 +++++++++++++++ sdk/rust/tests/test_client_v1.rs | 42 +++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 14 deletions(-) diff --git a/sdk/go/dstack/client_v1_test.go b/sdk/go/dstack/client_v1_test.go index 49b78a0ce..bc9a23c13 100644 --- a/sdk/go/dstack/client_v1_test.go +++ b/sdk/go/dstack/client_v1_test.go @@ -68,6 +68,41 @@ func TestV1GetKey(t *testing.T) { } } +// A key from another epoch than the one asked for must not pass as the requested one. +func TestV1GetKeyRejectsUnrequestedEpochs(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "key": strings.Repeat("aa", 32), + "public_key": strings.Repeat("bb", 33), + "signature_chain": []string{}, + "root_epoch": 1, + "app_epoch": 2, + }) + })) + defer server.Close() + client := dstack.NewDstackClientV1(dstack.WithEndpoint(server.URL)) + epoch := func(e uint32) *uint32 { return &e } + + for _, tc := range []struct{ root, app *uint32 }{ + {epoch(0), nil}, + {nil, epoch(1)}, + {epoch(1), epoch(3)}, + } { + _, err := client.GetKeyAtEpochs(context.Background(), "d", "secp256k1", tc.root, tc.app) + if err == nil || !strings.Contains(err.Error(), "different epochs than requested") { + t.Errorf("root %v app %v: expected an epoch mismatch error, got %v", tc.root, tc.app, err) + } + } + resp, err := client.GetKeyAtEpochs(context.Background(), "d", "secp256k1", epoch(1), epoch(2)) + if err != nil { + t.Fatal(err) + } + if resp.RootEpoch != 1 || resp.AppEpoch != 2 { + t.Errorf("unexpected epochs: root %d app %d", resp.RootEpoch, resp.AppEpoch) + } +} + // v1 binds the algorithm into the derivation, so the two curves no longer share // one 32-byte secret the way they did on the frozen surface. func TestV1GetKeyIsDomainSeparated(t *testing.T) { diff --git a/sdk/js/src/__tests__/index-v1.test.ts b/sdk/js/src/__tests__/index-v1.test.ts index 38005dc2d..fac85fbe4 100644 --- a/sdk/js/src/__tests__/index-v1.test.ts +++ b/sdk/js/src/__tests__/index-v1.test.ts @@ -8,6 +8,21 @@ import type { AddressInfo } from 'net' import { DstackClient, DstackClientV0, DstackClientV1 } from '../index' import type { GpuEvidenceBundleV1 } from '../index' +/** Run `fn` against a stub agent that answers every request with `body`. */ +async function withAgentAnswering(body: unknown, fn: (client: DstackClientV1) => Promise) { + const server = http.createServer((_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }) + res.end(JSON.stringify(body)) + }) + await new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve())) + try { + const { port } = server.address() as AddressInfo + await fn(new DstackClientV1(`http://127.0.0.1:${port}`)) + } finally { + await new Promise(resolve => server.close(() => resolve())) + } +} + describe('DstackClientV1', () => { it('should be what the unsuffixed DstackClient names, as value and as type', () => { expect(DstackClient).toBe(DstackClientV1) @@ -139,6 +154,18 @@ describe('DstackClientV1', () => { expect(result.key.length).toBe(32) }) + it('should reject a key from other epochs than the requested ones', async () => { + const key = { key: 'aa'.repeat(32), public_key: 'bb'.repeat(33), signature_chain: [], root_epoch: 1, app_epoch: 2 } + await withAgentAnswering(key, async client => { + for (const [rootEpoch, appEpoch] of [[0, undefined], [undefined, 1], [1, 3]]) { + await expect(client.getKey('d', 'secp256k1', rootEpoch, appEpoch)) + .rejects.toThrow(/different epochs than requested/) + } + const response = await client.getKey('d', 'secp256k1', 1, 2) + expect([response.root_epoch, response.app_epoch]).toEqual([1, 2]) + }) + }) + it('should reject an empty algorithm without a round trip', async () => { const client = new DstackClientV1() await expect(() => client.getKey('storage-encryption', '')).rejects.toThrow('algorithm is required') @@ -296,20 +323,6 @@ describe('DstackClientV1', () => { // signature chain quietly one link short is a verification failure nobody can // trace back to its cause. describe('malformed hex from the agent', () => { - async function withAgentAnswering(body: unknown, fn: (client: DstackClientV1) => Promise) { - const server = http.createServer((_req, res) => { - res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify(body)) - }) - await new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve())) - try { - const { port } = server.address() as AddressInfo - await fn(new DstackClientV1(`http://127.0.0.1:${port}`)) - } finally { - await new Promise(resolve => server.close(() => resolve())) - } - } - const identity = { app_id: 'aa'.repeat(20), compose_hash: 'bb'.repeat(32), diff --git a/sdk/python/tests/test_client_v1.py b/sdk/python/tests/test_client_v1.py index 941e325c9..9363242e8 100644 --- a/sdk/python/tests/test_client_v1.py +++ b/sdk/python/tests/test_client_v1.py @@ -158,6 +158,29 @@ async def test_async_v1_get_key_rejects_the_v0_k256_alias(): assert "k256" in str(excinfo.value) +@pytest.mark.asyncio +async def test_async_v1_get_key_rejects_epochs_other_than_requested(monkeypatch): + """A key from another epoch must not pass as the requested one.""" + + async def fake_send(self, method, payload): + return { + "key": "aa" * 32, + "public_key": "bb" * 33, + "signature_chain": [], + "root_epoch": 1, + "app_epoch": 2, + } + + monkeypatch.setenv("DSTACK_SIMULATOR_ENDPOINT", "http://localhost:0") + monkeypatch.setattr(AsyncDstackClientV1, "_send_rpc_request", fake_send) + client = AsyncDstackClientV1() + for root_epoch, app_epoch in [(0, None), (None, 1), (1, 3)]: + with pytest.raises(ValueError, match="different epochs than requested"): + await client.get_key("d", "secp256k1", root_epoch, app_epoch) + key = await client.get_key("d", "secp256k1", 1, 2) + assert (key.root_epoch, key.app_epoch) == (1, 2) + + def test_v1_keys_differ_from_v0_keys(): """No compatibility mode: the same name yields different key material.""" v0 = DstackClientV0().get_key("storage-encryption", "") diff --git a/sdk/rust/tests/test_client_v1.rs b/sdk/rust/tests/test_client_v1.rs index 76834a177..c1ebde0ba 100644 --- a/sdk/rust/tests/test_client_v1.rs +++ b/sdk/rust/tests/test_client_v1.rs @@ -34,6 +34,48 @@ async fn get_key_returns_a_key_public_key_and_two_link_chain() { } } +/// A key from another epoch than the one asked for must not pass as the +/// requested one. A stub agent stands in for the simulator, answering every +/// request with root epoch 1 and app epoch 2. +#[tokio::test] +async fn get_key_rejects_epochs_other_than_the_requested_ones() { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let body = format!( + r#"{{"key":"{}","public_key":"{}","signature_chain":[],"root_epoch":1,"app_epoch":2}}"#, + "aa".repeat(32), + "bb".repeat(33) + ); + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let _ = stream.read(&mut [0; 4096]); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } + }); + let client = DstackClientV1::new(Some(&endpoint)); + for (root, app) in [(Some(0), None), (None, Some(1)), (Some(1), Some(3))] { + let err = client + .get_key_at_epochs("d", "secp256k1", root, app) + .await + .unwrap_err(); + assert!( + err.to_string().contains("different epochs than requested"), + "{err:#}" + ); + } + let key = client + .get_key_at_epochs("d", "secp256k1", Some(1), Some(2)) + .await + .unwrap(); + assert_eq!((key.root_epoch, key.app_epoch), (1, 2)); +} + #[tokio::test] async fn get_key_public_key_lengths_are_the_specified_ones() { let secp = client() From 8cfc8f3312440ce0202d2381b500d2a1ba000f8c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:09 -0800 Subject: [PATCH 07/12] chore(auth-eth): drop a debug log from GET / Signed-off-by: Kevin Wang --- dstack/kms/auth-eth/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/dstack/kms/auth-eth/index.ts b/dstack/kms/auth-eth/index.ts index 0413269ae..380f88805 100644 --- a/dstack/kms/auth-eth/index.ts +++ b/dstack/kms/auth-eth/index.ts @@ -465,7 +465,6 @@ app.get('/', async (c) => { ethereum.getAppImplementation(block), ethereum.getRootEpoch(block), ]); - console.log('batch', batch); return c.json({ status: 'ok', From 908e7046a32f6954294269b4346cf419f7a1006e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:18 -0800 Subject: [PATCH 08/12] docs(auth-eth): document reorg rollback with latest or safe block tags Signed-off-by: Kevin Wang --- docs/deployment.md | 9 +++++++++ dstack/kms/auth-eth/README.md | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 36f07addb..bf6504f44 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -190,6 +190,15 @@ For adding Gateway, apps, and other config fields, see [dstack-auth Operations G For decentralized governance via smart contracts, see [On-Chain Governance](./onchain-governance.md). +auth-eth reads policy at the block `ETH_BLOCK_TAG` names, `finalized` by +default. Only `finalized` protects policy from reorgs. With `latest` (required +for an OP-stack chain read through Helios) or `safe`, a reorg can revert both +activations and revocations: a removed image or compose hash is authorized +again, and a rolled-back root epoch activation makes every KMS restart onto the +previous epoch. `ETH_FINALITY_CONFIRMATIONS=N` hides reorgs shallower than N +blocks but delays every policy change, emergency revocations included, by N +blocks. See [reorgs and the block tag](../dstack/kms/auth-eth/README.md#reorgs-and-the-block-tag). + #### Getting OS Image Hash The OS image hash is in the `digest.txt` file inside the guest image tarball: diff --git a/dstack/kms/auth-eth/README.md b/dstack/kms/auth-eth/README.md index dcb6c647c..5a905ceb2 100644 --- a/dstack/kms/auth-eth/README.md +++ b/dstack/kms/auth-eth/README.md @@ -53,9 +53,33 @@ The Bun server sets `idleTimeout` to 255s (Bun's maximum). Bun's default of 10s - `KMS_CONTRACT_ADDR` - KMS contract address (default: 0x0000000000000000000000000000000000000000) - `PORT` - server port (default: 3000) - `ETH_CHAIN_ID` - reject every decision if the RPC reports a different chain ID (default: unset, no check) -- `ETH_BLOCK_TAG` - block policy is read at: `latest`, `safe` or `finalized` (default: `finalized`). each request resolves the tag to one block and reads all state there. the server resolves the tag once at startup and refuses to start if the RPC cannot serve it. a later tag survives reorgs but delays every policy change, including emergency revocations, until that block is reached (about 13 minutes for `finalized` on Ethereum mainnet). use `finalized` on L1. set `latest` explicitly for an OP-stack chain read through the Helios light client, which tracks only the sequencer-signed unsafe head and serves neither `safe` nor `finalized`, and for local development chains such as Anvil, which finalizes 64 blocks behind its head +- `ETH_BLOCK_TAG` - block policy is read at: `latest`, `safe` or `finalized` (default: `finalized`). each request resolves the tag to one block and reads all state there. the server resolves the tag once at startup and refuses to start if the RPC cannot serve it. a later tag delays every policy change, including emergency revocations, until that block is reached (about 13 minutes for `finalized` on Ethereum mainnet). use `finalized` on L1. set `latest` explicitly for an OP-stack chain read through the Helios light client, which tracks only the sequencer-signed unsafe head and serves neither `safe` nor `finalized`, and for local development chains such as Anvil, which finalizes 64 blocks behind its head. see [reorgs and the block tag](#reorgs-and-the-block-tag) before using `latest` or `safe` - `ETH_FINALITY_CONFIRMATIONS` - read this many blocks further behind the block `ETH_BLOCK_TAG` resolves to (default: 0) +### Reorgs and the block tag + +Only `finalized` protects policy from reorgs. With `latest` or `safe`, a reorg +can revert any policy transaction the server has already acted on, in both +directions: + +- **Revocations roll back.** A removed OS image, compose hash or device, or a + revoked KMS measurement, is authorized again until the revocation is + re-included and the read block reaches it. +- **Activations roll back.** A root epoch activation or an app epoch increase + can disappear. Neither this server nor the KMS keeps a high-water mark: the + KMS only checks the reported epoch against its keyring's chain hash, so after + a root epoch rollback it restarts once and serves the previous epoch as + current, and apps again receive keys of their previous app epoch. + +Keys released in that window stay released. `ETH_FINALITY_CONFIRMATIONS=N` +reads N blocks behind the tagged block, so a reorg shallower than N blocks is +never observed, at the cost of delaying every policy change, revocations +included, by N more blocks. A reorg deeper than N still rolls back, so this +narrows the risk with `latest` or `safe` but does not remove it. Through Helios +on an OP-stack chain, `latest` is the sequencer's unsafe head: choose N from +how deep a reorg of that head you are willing to accept, and send emergency +revocations with that delay in mind. + ### Container image Tagged releases are published by the `KMS auth-eth` workflow From 9ca70c2fe6c0fde217c95440b36b513d0ada6eb3 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:18 -0800 Subject: [PATCH 09/12] docs(kms): the source admin token is required for onboarding Signed-off-by: Kevin Wang --- docs/deployment.md | 2 +- docs/kms-emergency-key-rotation-protocol.md | 10 +++++----- dstack/kms/README.md | 3 ++- dstack/kms/src/www/onboard.html | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index bf6504f44..34fd9e78c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -559,7 +559,7 @@ curl http://:9203/finish > **Admin authentication.** The KMS *admin* RPCs (for example `ClearImageCache`) are served on a dedicated `[core.admin]` listener behind the shared HTTP authenticator, just like the VMM and gateway: set `[core.admin] enabled = true` with an `auth_token` (or the `DSTACK_KMS_ADMIN_TOKEN` / `ADMIN_API_TOKEN` env vars), and clients send `Authorization: Bearer ` or `X-Admin-Token`. Enabled with neither `auth_token` nor `htpasswd_file` (and `insecure_no_auth = false`) fails closed — the KMS refuses to start. See [dstack-kms admin authentication](../dstack/kms/README.md#admin-api-authentication). > -> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL (`https://`) and its token (`source_token`) in the onboarding request. The legacy public `KMS.GetKmsKey` is off by default; set `core.onboard.public_key_handover = true` on the source only when an onboarding KMS cannot use the admin listener, such as a 0.5.x KMS or an onboarding request without `source_token`. +> **Key handover.** It is served as `Admin.GetKmsKey` on the `[core.admin]` listener, which needs `[core.admin.tls]` so it can verify the onboarding KMS's RA-TLS certificate. Pass the source admin URL (`https://`) and its token (`source_token`) in the onboarding request; the token is required unless the source enables the public handover. The legacy public `KMS.GetKmsKey` is off by default; set `core.onboard.public_key_handover = true` on the source only when an onboarding KMS cannot use the admin listener, such as a 0.5.x KMS or an onboarding request without `source_token`. #### Rotating KMS Root Keys diff --git a/docs/kms-emergency-key-rotation-protocol.md b/docs/kms-emergency-key-rotation-protocol.md index b76e06971..2ae993d1c 100644 --- a/docs/kms-emergency-key-rotation-protocol.md +++ b/docs/kms-emergency-key-rotation-protocol.md @@ -261,11 +261,11 @@ between the current epoch and L is refused, since its tail cannot be checked. Wi Every handover requires the recipient's KMS attestation: the source checks the attested RA-TLS client certificate on both `Admin.GetKmsKey` and the public -`KMS.GetKmsKey`. Admin authorization (`source_token`) is recommended for joins -and rotations alike, but not required: a rotation only produces a pending epoch -N+1, which no node accepts until governance registers and activates it. Where -possible, disable public legacy handover and reject incompatible legacy -receivers without reducing the payload. The source's certificate must carry its +`KMS.GetKmsKey`. Admin authorization (`source_token`) is required for joins +and rotations alike, because the public `KMS.GetKmsKey` is off by default +(`core.onboard.public_key_handover = false`). Enable it on a source only for +receivers that cannot use the admin listener, such as 0.5.x KMS nodes, and +reject incompatible legacy receivers without reducing the payload. The source's certificate must carry its OS image hash. Deployments that never registered epochs can still join at epoch 0. diff --git a/dstack/kms/README.md b/dstack/kms/README.md index e4d0c5625..970d7433f 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -390,7 +390,8 @@ still bind the root identity; a valid quote alone is not a KMS policy decision. `--root-ca` additionally pins the CLI connection before sending the request. Onboarding with an admin token uses a self-issued client certificate. Tokenless -onboarding obtains the source temp CA first so pre-0.6 sources remain usable. +onboarding, which needs a source with `core.onboard.public_key_handover = true`, +obtains the source temp CA first so pre-0.6 sources remain usable. Both paths check the source attestation captured from the key-carrying connection against local KMS policy before importing anything. Retries and reconnects repeat verification on the response connection; no assumption relies on one TCP session diff --git a/dstack/kms/src/www/onboard.html b/dstack/kms/src/www/onboard.html index b5ee3d42a..a5d428d0b 100644 --- a/dstack/kms/src/www/onboard.html +++ b/dstack/kms/src/www/onboard.html @@ -257,7 +257,7 @@

Onboard from an Existing KMS Instance

placeholder="RPC URL of the existing KMS">
- + From aa5ec816bb06178f1ad2978c5d0a4c3886746179 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:31:18 -0800 Subject: [PATCH 10/12] docs: correct stale key rotation and rc testing statements Signed-off-by: Kevin Wang --- docs/kms-emergency-key-rotation-protocol.md | 4 ++-- docs/rc-testing-runbook.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/kms-emergency-key-rotation-protocol.md b/docs/kms-emergency-key-rotation-protocol.md index 2ae993d1c..fe628d37c 100644 --- a/docs/kms-emergency-key-rotation-protocol.md +++ b/docs/kms-emergency-key-rotation-protocol.md @@ -1,6 +1,6 @@ # KMS emergency key rotation protocol -**Status:** Design draft; not implemented. Proposed fields require protobuf definitions. +**Status:** Implemented. ## 1. Scope and assumptions @@ -60,7 +60,7 @@ new `n`, whose OS image, KMS MR and device rows start empty because policy rows are keyed by the epoch's chain hash. Prepare them again before activating. A current epoch is never popped. `kmsInfo` follows the current epoch: registering epoch 0 and each activation copy its keys there and clear the legacy -quote and event log, and `setKmsInfo` is refused once epochs exist. Epoch 0 must be the `kmsInfo` keys +quote and event log; the legacy `setKmsInfo`, `setKmsQuote` and `setKmsEventlog` setters are removed. Epoch 0 must be the `kmsInfo` keys when those are set; a legacy `kmsInfo` quote and event log are converted off chain into the standard attestation encoding (`Attestation::from_tdx_quote`) before publishing. The KMS sends the epoch it serves and its chain hash (`rootEpoch`, diff --git a/docs/rc-testing-runbook.md b/docs/rc-testing-runbook.md index 53808e6a9..fae031a0d 100644 --- a/docs/rc-testing-runbook.md +++ b/docs/rc-testing-runbook.md @@ -153,7 +153,8 @@ KMS first, then gateway, then the app. Each needs the previous one's identity. gramine-sealing-key-provider on the host, default `127.0.0.1:3443`). The simplest authorization that releases keys without a chain is `[core.auth_api] type = "dev"` with `gateway_app_id = "any"` — no extra - containers, unlike the on-chain compose which needs a Helios light client. + containers, unlike the on-chain compose, which runs auth-eth next to the KMS + and needs an Ethereum RPC endpoint. 3. **Gateway as a CVM** with `--kms`. Internal ports are RPC 8000, admin 8001, WireGuard 51820/udp, proxy 443. From f1cdfc14ac6d93e74f69d9f9823c0073991e8737 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 10 Nov 2026 21:38:04 -0800 Subject: [PATCH 11/12] docs(changelog): record the key epoch upgrade notes and policy changes Add the deployment order, the GET /app/:appId backend requirement, the per-epoch setter semantics from #1561, the removed DstackKms setters and legacy KMS info setters, two-step ownership for v0.5.9 proxies, dstack-auth pending-epoch handover, and the reorg caveat of non-finalized block tags. Signed-off-by: Kevin Wang --- CHANGELOG.md | 50 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f991acf5..013bb4199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Upgrade notes + +- **Deployment order for key epochs:** upgrade the `DstackKms` and `DstackApp` + proxies and the auth backend, register the existing roots as root epoch 0 + (`registerRootEpoch`), then roll out the KMS nodes, then guests. Keep root and + app epochs at 0 until every KMS and guest is upgraded. See + `dstack/kms/README.md` and `docs/deployment.md`. +- **Auth backends must serve `GET /app/:appId`**, which reports an app's current + app epoch. The KMS reads it for `GetAppEnvEncryptPubKey` and + `GetSealedEnvPubKey`. `auth-eth` and `dstack-auth` serve it; custom webhook + backends must add it. + ### Added - KMS root and application key epochs, epoch-scoped authorization, SealedEnv @@ -36,12 +48,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 denies a KMS that reports no root epoch, as `/bootAuth/app` already did, so KMS binaries predating root epochs stop serving at activation. Upgrade every KMS before activating a new root epoch. -- auth-eth, auth-eth-bun: policy is read at the `finalized` block by default - instead of `latest`, so on-chain changes, including revocations, take effect - only once finalized. Set `ETH_BLOCK_TAG=latest` (or `safe`) to override; an - OP-stack chain read through the Helios light client requires `latest`. The - server refuses to start if the RPC cannot serve the configured tag. In - auth-eth-bun, `ETH_FINALITY_CONFIRMATIONS` now counts back from that block. +- auth-eth: policy is read at the `finalized` block by default instead of + `latest`, so on-chain changes, including revocations, take effect only once + finalized. Set `ETH_BLOCK_TAG=latest` (or `safe`) to override; an OP-stack + chain read through the Helios light client requires `latest`. Only + `finalized` protects policy from reorgs: with `latest` or `safe`, a reorg can + roll back an activation or a revocation the KMS has already acted on. The + server refuses to start if the RPC cannot serve the configured tag. + `ETH_FINALITY_CONFIRMATIONS` now counts back from that block. +- DstackKms, DstackApp: material approvals are per epoch. A request is decided + on the row of the epoch it selects, so a value is revoked only once every row + allowing it is cleared. `DstackKms` material setters name their rows: + `*AtEpoch(epoch, value)` writes one root epoch, `*AtEpochs(from, to, value)` a + range, and `remove*AtEpochs(0, latest, value)` revokes a value everywhere. + `DstackApp`'s `addComposeHash`, `removeComposeHash`, `addDevice`, + `removeDevice` and `setAllowAnyDevice` keep their ABI but write every app epoch + row from 0 through the next one, so a removal is a full revocation. The + `Manage.s.sol` KMS scripts take `ROOT_EPOCH`, or `FROM_ROOT_EPOCH` and + `TO_ROOT_EPOCH`. +- DstackKms, DstackApp: ownership transfers take two steps for proxies upgraded + from v0.5.9: `transferOwnership` nominates, and the new owner must call + `acceptOwnership`. +- dstack-auth: `kms_export_keys` and `kms_import_keys` accept a registered + pending root epoch, so KMS nodes can onboard a new epoch before + `currentRootEpoch` is raised, as with the contract backend. Key-issuing + operations stay limited to the current epoch. - KMS: retain `GetTempCaCert`, temporary CA files and legacy handover fields so KMS nodes can be upgraded before existing OS images. @@ -65,6 +96,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- DstackKms: the epoch-less `addOsImageHash`, `removeOsImageHash`, + `addKmsAggregatedMr`, `removeKmsAggregatedMr`, `addKmsDevice` and + `removeKmsDevice`. Use the `*AtEpoch` or `*AtEpochs` forms; before any + rotation, `addOsImageHashAtEpoch(0, hash)` is the old call. +- DstackKms: `setKmsInfo`, `setKmsQuote` and `setKmsEventlog`. Root keys are + set with `registerRootEpoch`, and each epoch's attestation lives at its + `evidenceUrl`. - KMS: the `auth-mock` webhook backend. For development, use the built-in `[core.auth_api] type = "dev"`, which allows everything. - `auth-simple` (`dstack/kms/auth-simple`). Use `dstack-auth`, which reads the From e8e8ff2a6881e3cb579d6bddb549f348689e862a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 11 Nov 2026 00:43:43 -0800 Subject: [PATCH 12/12] docs(changelog): record the removed DstackApp setters and legacy app support Follow #1561: DstackApp loses its epoch-less setters and IAppAuthBasicManagement, reports IAppEpochManagement, and app policies predating app epochs keep working at app epoch 0. Signed-off-by: Kevin Wang --- CHANGELOG.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013bb4199..b78b58b1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,14 +58,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `ETH_FINALITY_CONFIRMATIONS` now counts back from that block. - DstackKms, DstackApp: material approvals are per epoch. A request is decided on the row of the epoch it selects, so a value is revoked only once every row - allowing it is cleared. `DstackKms` material setters name their rows: - `*AtEpoch(epoch, value)` writes one root epoch, `*AtEpochs(from, to, value)` a - range, and `remove*AtEpochs(0, latest, value)` revokes a value everywhere. - `DstackApp`'s `addComposeHash`, `removeComposeHash`, `addDevice`, - `removeDevice` and `setAllowAnyDevice` keep their ABI but write every app epoch - row from 0 through the next one, so a removal is a full revocation. The - `Manage.s.sol` KMS scripts take `ROOT_EPOCH`, or `FROM_ROOT_EPOCH` and - `TO_ROOT_EPOCH`. + allowing it is cleared. Material setters name their rows: + `*AtEpoch(epoch, value)` writes one epoch, `*AtEpochs(from, to, value)` a + range; `remove*AtEpochs(0, latest root epoch, value)` in `DstackKms` and + `remove*AtEpochs(0, currentAppEpoch, value)` in `DstackApp` revoke a value + everywhere. `DstackApp` reports its setters as the new ERC-165 interface + `IAppEpochManagement` (`0x853d2b79`). The `Manage.s.sol` scripts require + `ROOT_EPOCH` or `APP_EPOCH`, or a `FROM_*`/`TO_*` range. +- App contracts deployed before app epochs (v0.5.x, v0.6.0) keep working with + the upgraded `DstackKms` and KMS, which authorize them for app epoch 0 only. + `Manage.s.sol` detects them by interface ID and calls their epoch-less + setters. Upgrading one to the current `DstackApp` keeps its compose hashes, + devices and `allowAnyDevice` as app epoch 0. - DstackKms, DstackApp: ownership transfers take two steps for proxies upgraded from v0.5.9: `transferOwnership` nominates, and the new owner must call `acceptOwnership`. @@ -100,6 +104,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `addKmsAggregatedMr`, `removeKmsAggregatedMr`, `addKmsDevice` and `removeKmsDevice`. Use the `*AtEpoch` or `*AtEpochs` forms; before any rotation, `addOsImageHashAtEpoch(0, hash)` is the old call. +- **Breaking:** DstackApp: the epoch-less `addComposeHash`, `removeComposeHash`, + `addDevice`, `removeDevice` and `setAllowAnyDevice`, and with them + `IAppAuthBasicManagement`: an upgraded app no longer reports interface ID + `0x8fd37527`, and calls to those functions revert. Use the `*AtEpoch` or + `*AtEpochs` forms (`IAppEpochManagement`); at app epoch 0, + `addComposeHashAtEpoch(0, hash)` is the old call. - DstackKms: `setKmsInfo`, `setKmsQuote` and `setKmsEventlog`. Root keys are set with `registerRootEpoch`, and each epoch's attestation lives at its `evidenceUrl`.