diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 34e3f36094..748ce2d48b 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -107,8 +107,53 @@ OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET=e2e-regcred \ mise run e2e:kubernetes:shared-supervisor ``` +`mise run e2e:kubernetes:shared-supervisor-spiffe` installs pinned SPIRE charts +(`ci/values-spire-broker-stack.yaml`) and the `openshell.gateway.spiffe_broker` +overlay (`ci/values-supervisor-spiffe.yaml`). It exercises provider exchange and +abrupt supervisor takeover over TLS, using Sandbox CR UID registrations and +enforced Kubernetes authorization. Workload generation, Pod, process, and stable +SPIFFE identity survive; the new owner acquires a fresh provider token and the +broker rejects the old bearer. The fixture's 30-second workload X.509-SVIDs +exercise acquisition after credential rotation. The gateway Pod must pass +Restricted Pod Security admission. The task owns its SPIRE install and cluster. + +The task creates three nodes (`HELM_K3S_AGENT_COUNT=2` adds two k3d agents) and +requires three SPIRE agents. Its cluster-scoped `server_spiffe_id_prefix` accepts +the concrete node identities, and a node alias distributes Sandbox registrations +to all eligible agents. The test selects each Broker backend in turn and forces +a new generation's acquisition, then replaces one agent and checks acquisition +without re-registering the sandbox. Exact-ID mode remains available for a fixed +endpoint. The installed SPIRE version and operator must expose the experimental +Broker API. + +For remote clusters, set `OPENSHELL_E2E_HOST_GATEWAY_IP` to a host reachable +from Pods. Optional `OPENSHELL_E2E_SPIFFE_STS_PORT` and +`OPENSHELL_E2E_SPIFFE_TARGET_PORT` fix the test's host listener ports so an +operator can forward them through a reverse tunnel. Keep the tunnel available +through the test and remove it afterward. Use an explicit kubeconfig and context +to avoid repository environment defaults targeting a local cluster. +`OPENSHELL_E2E_SPIFFE_DRIVER_CONFIG_JSON` can pass a driver override to test +sandbox creation, such as a node selector for a Landlock-capable pool. Enable +`allow_driver_config` in the test gateway when using this override. GKE COS +nodes may lack Landlock; use compatible Ubuntu nodes for sandbox workloads. +For remote abrupt-failover coverage, set +`OPENSHELL_E2E_SUPERVISOR_PROCESS_CONTROL=1`. Before creating sandboxes, the test +adds an unprivileged process-control sidecar and enables the supervisor Pod's +shared process namespace. The sidecar sends SIGKILL to the supervisor as the +same UID; it needs no host PID namespace, capabilities, or runtime socket. +The default k3d/kind test continues to kill through the node container runtime. + ### Shared supervisor high availability +`mise run e2e:kubernetes:shared-supervisor-spiffe-node-local` validates direct +node-local Broker TCP routing. The chart injects `OPENSHELL_NODE_IP` from +`status.hostIP` for `spiffe_broker.node_local_port`; it creates no Broker Service. +The task uses an external PostgreSQL fixture and a gateway Deployment, moves +the gateway across the SPIRE nodes, restricts Broker egress to its current node, +and forces fresh token acquisition on each node and after local agent replacement. +The shared-supervisor failover, credential rotation, and identity fencing checks +also run. Keep gateway/supervisor/sandbox images from the same branch. + The shared supervisor is always a pool of `supervisor.replicas` members (default 1), or an HPA-managed pool when `supervisor.autoscaling.enabled=true`. `ci/values-supervisor-autoscaling.yaml` enables the memory HPA (default `512Mi` diff --git a/.config/nextest.toml b/.config/nextest.toml index e650db838b..c5a25293b4 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -32,7 +32,7 @@ test-group = "kubernetes-ha" # they kill members or gateways, or scale the pool. Never run them at once. # Test names, not binary(), for the reason above. [[profile.e2e-kubernetes.overrides]] -filter = "test(/^(losing_a_supervisor_pod_moves_its_sandboxes_without_restarting_them|scaling_a_supervisor_pool_up_and_down_keeps_its_sandboxes_running|pool_sessions_follow_their_supervisors_gateway|supervisors_spread_evenly_across_gateways)$/)" +filter = "test(/^(losing_a_supervisor_pod_moves_its_sandboxes_without_restarting_them|scaling_a_supervisor_pool_up_and_down_keeps_its_sandboxes_running|pool_sessions_follow_their_supervisors_gateway|shared_supervisor_acts_as_each_sandboxs_stable_spiffe_identity|supervisors_spread_evenly_across_gateways)$/)" test-group = "kubernetes-shared-supervisor-pool" # Scaling the pool runs two bounded Helm upgrades (up to 6 minutes each), waits @@ -47,6 +47,12 @@ slow-timeout = { period = "60s", terminate-after = 20 } filter = "test(/^pool_sessions_follow_their_supervisors_gateway$/)" slow-timeout = { period = "60s", terminate-after = 20 } +[[profile.e2e-kubernetes.overrides]] +filter = "test(/^shared_supervisor_acts_as_each_sandboxs_stable_spiffe_identity$/)" +# Fresh acquisitions through every node and a replacement agent precede +# supervisor failover and credential rotation. +slow-timeout = { period = "60s", terminate-after = 15 } + # One bounded rollout per supervisor and one for the gateway, each followed by # a settle round. [[profile.e2e-kubernetes.overrides]] diff --git a/AGENTS.md b/AGENTS.md index 0ed5902932..dfd769c7e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,8 @@ Do not rely on this file for a full inventory. The detailed public and contribut | `crates/openshell-gateway/` | Gateway binary composition | Links selected first-party compute drivers into the backend-agnostic server registry | | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` | | `crates/openshell-providers/` | Provider management | Credential provider backends | +| `crates/openshell-spiffe/` | SPIFFE identity types | Dependency-light SPIFFE ID mapping, claim parsing, and the sandbox identity provider interface | +| `crates/openshell-spire/` | SPIRE adapter | Gateway-owned SPIFFE Broker API client that issues and verifies stable sandbox JWT-SVIDs | | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring | | `crates/openshell-driver-kubernetes-secrets/` | Kubernetes Secrets credential driver | In-process `CredentialDriver` backend for OpenShell-managed K8s Secret storage | | `crates/openshell-driver-vault/` | Vault credential driver | In-process `CredentialDriver` backend for Vault-compatible KV storage | diff --git a/Cargo.lock b/Cargo.lock index 7c9eb08165..b66b3662f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,6 +1250,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-oid" version = "0.10.2" @@ -1719,13 +1725,22 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", +] + [[package]] name = "der" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid", + "const-oid 0.10.2", "pem-rfc7468", "zeroize", ] @@ -1860,7 +1875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" dependencies = [ "block-buffer 0.12.0", - "const-oid", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", "zeroize", @@ -1910,12 +1925,12 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der", + "der 0.8.2", "digest 0.11.2", "elliptic-curve", "rfc6979", "signature 3.0.0", - "spki", + "spki 0.8.1", "zeroize", ] @@ -1925,7 +1940,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8", + "pkcs8 0.11.0", "signature 3.0.0", ] @@ -1981,7 +1996,7 @@ dependencies = [ "hkdf", "hybrid-array", "pem-rfc7468", - "pkcs8", + "pkcs8 0.11.0", "rand_core 0.10.1", "sec1", "subtle", @@ -3987,7 +4002,7 @@ dependencies = [ "hybrid-array", "kem", "module-lattice", - "pkcs8", + "pkcs8 0.11.0", "rand_core 0.10.1", "sha3 0.11.0", ] @@ -4495,6 +4510,7 @@ dependencies = [ "openshell-extension-core", "openshell-otel", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -5216,6 +5232,8 @@ dependencies = [ "openshell-policy", "openshell-prover", "openshell-providers", + "openshell-spiffe", + "openshell-spire", "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", @@ -5266,6 +5284,38 @@ dependencies = [ "x509-parser 0.16.0", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + +[[package]] +name = "openshell-spire" +version = "0.0.0" +dependencies = [ + "async-trait", + "http 1.4.0", + "hyper-util", + "openshell-core", + "openshell-crypto", + "openshell-spiffe", + "prost", + "rustls", + "spiffe", + "tokio", + "tokio-rustls", + "tonic", + "tonic-prost", + "tower", + "uuid", +] + [[package]] name = "openshell-supervisor" version = "0.0.0" @@ -5403,6 +5453,7 @@ dependencies = [ "openshell-policy", "openshell-policy-schema", "openshell-sandbox-backend", + "openshell-spiffe", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", "opentelemetry", @@ -6009,8 +6060,8 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6250dd7321adbe93d12b750aee0ced159fd13e77f625edcdf1cb6f83cd99f650" dependencies = [ - "der", - "spki", + "der 0.8.2", + "spki 0.8.1", ] [[package]] @@ -6022,12 +6073,22 @@ dependencies = [ "aes", "aes-gcm", "cbc", - "der", + "der 0.8.2", "pbkdf2", "rand_core 0.10.1", "scrypt", "sha2 0.11.0", - "spki", + "spki 0.8.1", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der 0.7.10", + "spki 0.7.3", ] [[package]] @@ -6036,10 +6097,10 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der", + "der 0.8.2", "pkcs5", "rand_core 0.10.1", - "spki", + "spki 0.8.1", ] [[package]] @@ -6886,18 +6947,18 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23e77fd77e0dea28b3011fd62d15445b7de7959333b0276d930e03be0f755699" dependencies = [ - "const-oid", + "const-oid 0.10.2", "crypto-bigint", "crypto-common 0.2.2", "crypto-primes", "digest 0.11.2", "hmac 0.13.0", "pkcs1", - "pkcs8", + "pkcs8 0.11.0", "rand_core 0.10.1", "sha2 0.11.0", "signature 3.0.0", - "spki", + "spki 0.8.1", "zeroize", ] @@ -6920,7 +6981,7 @@ dependencies = [ "curve25519-dalek", "data-encoding", "delegate", - "der", + "der 0.8.2", "digest 0.11.2", "ecdsa", "ed25519-dalek", @@ -6947,7 +7008,7 @@ dependencies = [ "pbkdf2", "pkcs1", "pkcs5", - "pkcs8", + "pkcs8 0.11.0", "polyval", "rand 0.10.2", "rand_core 0.10.1", @@ -6961,7 +7022,7 @@ dependencies = [ "sha2 0.11.0", "sha3 0.12.0", "signature 3.0.0", - "spki", + "spki 0.8.1", "ssh-encoding", "ssh-key", "subtle", @@ -7287,7 +7348,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ "base16ct", "ctutils", - "der", + "der 0.8.2", "hybrid-array", "subtle", "zeroize", @@ -7711,6 +7772,7 @@ dependencies = [ "hyper-util", "jsonwebtoken", "log", + "pkcs8 0.10.2", "prost", "prost-types", "serde", @@ -7724,6 +7786,7 @@ dependencies = [ "tower", "tracing", "url", + "x509-parser 0.18.1", "zeroize", ] @@ -7736,6 +7799,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "der 0.7.10", +] + [[package]] name = "spki" version = "0.8.1" @@ -7743,7 +7815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" dependencies = [ "base64ct", - "der", + "der 0.8.2", ] [[package]] diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs index be3e4f2e49..8066c99f5d 100644 --- a/crates/openshell-cli/tests/ensure_providers_integration.rs +++ b/crates/openshell-cli/tests/ensure_providers_integration.rs @@ -80,6 +80,14 @@ impl TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs index 1017712035..f659ffe8e2 100644 --- a/crates/openshell-cli/tests/mtls_integration.rs +++ b/crates/openshell-cli/tests/mtls_integration.rs @@ -32,6 +32,14 @@ struct TestOpenShell; #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index 89247ba718..1821749316 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -235,6 +235,14 @@ impl TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 8e7c0b190f..39414f1f7f 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -102,6 +102,14 @@ struct TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs index 772a26e58e..3d56cecf99 100644 --- a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs +++ b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs @@ -163,7 +163,6 @@ macro_rules! mock_gateway { ) => { #[tonic::async_trait] impl OpenShell for MockGateway { - $(async fn $method(&self, _: Request<$request>) -> Result, Status> { Err(Status::unimplemented("unused test RPC")) })* @@ -272,6 +271,7 @@ mock_gateway! { report_sandbox_configuration(proto::ReportSandboxConfigurationRequest) -> proto::ReportSandboxConfigurationResponse; get_sandbox_provider_environment(proto::GetSandboxProviderEnvironmentRequest) -> proto::GetSandboxProviderEnvironmentResponse; exchange_provider_subject_token(proto::ExchangeProviderSubjectTokenRequest) -> proto::ExchangeProviderSubjectTokenResponse; + get_sandbox_provider_access_token(proto::GetSandboxProviderAccessTokenRequest) -> proto::GetSandboxProviderAccessTokenResponse; get_sandbox_logs(proto::GetSandboxLogsRequest) -> proto::GetSandboxLogsResponse; report_main_process_exit(proto::ReportMainProcessExitRequest) -> proto::ReportMainProcessExitResponse; finalize_main_process_exit(proto::FinalizeMainProcessExitRequest) -> proto::FinalizeMainProcessExitResponse; diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 497f3ce778..7f31b1c4aa 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -47,6 +47,14 @@ struct TestOpenShell { #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index aa83e10f86..55339e09b5 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -15,6 +15,7 @@ openshell-crypto = { workspace = true, features = ["tls-tonic"] } async-trait = "0.1" openshell-extension-core = { path = "../openshell-extension-core" } openshell-policy-schema = { path = "../openshell-policy-schema" } +openshell-spiffe = { path = "../openshell-spiffe" } glob = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 6ffeaf1dce..0589d42a90 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -250,6 +250,91 @@ pub fn normalize_compute_driver_name(value: &str) -> Result { Ok(value.to_ascii_lowercase()) } +/// Gateway-owned SPIFFE Broker API issuance for Kubernetes sandbox identities. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SpiffeBrokerConfig { + pub spiffe_id_prefix: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub endpoint: String, + /// Connect to the gateway's node IP supplied through the Downward API. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub node_local_port: Option, + #[serde(default)] + pub server_spiffe_id: Option, + #[serde(default)] + pub server_spiffe_id_prefix: Option, + pub workload_api_socket_path: String, +} + +impl SpiffeBrokerConfig { + /// Resolve the selected transport without making Kubernetes API calls. + /// Node-local mode requires the Pod's trusted Downward API node address. + pub fn endpoint_for_node(&self, node_ip: Option<&str>) -> Result { + match (self.endpoint.is_empty(), self.node_local_port) { + (false, None) => Ok(self.endpoint.clone()), + (true, Some(port)) => { + let ip: std::net::IpAddr = node_ip + .ok_or("spiffe_broker.node_local_port requires OPENSHELL_NODE_IP from status.hostIP")? + .parse() + .map_err(|_| "OPENSHELL_NODE_IP must be an IPv4 or IPv6 address")?; + Ok(format!("https://{}", SocketAddr::new(ip, port.get()))) + } + _ => Err("spiffe_broker requires exactly one of endpoint and node_local_port".into()), + } + } + + /// Validate the identity and authenticated transport before use. + pub fn validate(&self, gateway_id: &str) -> Result<(), String> { + openshell_spiffe::SandboxSpiffeIdPrefix::parse(&self.spiffe_id_prefix)? + .sandbox_spiffe_id(gateway_id, "sandbox")?; + // Validate the mode independently of the startup environment. The + // adapter requires and resolves the actual node IP before serving. + let endpoint = self.endpoint_for_node(Some("127.0.0.1"))?; + let endpoint = url::Url::parse(&endpoint) + .map_err(|_| "spiffe_broker.endpoint must be an HTTPS endpoint")?; + let socket = std::path::Path::new(&self.workload_api_socket_path); + let valid_socket = socket.is_absolute() + && !self.workload_api_socket_path.ends_with('/') + && socket.file_name().is_some() + && socket + .parent() + .is_some_and(|parent| parent != std::path::Path::new("/")) + && self.workload_api_socket_path.trim() == self.workload_api_socket_path + && !self + .workload_api_socket_path + .split('/') + .any(|segment| matches!(segment, "." | "..")); + if endpoint.scheme() != "https" + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.path() != "/" + || endpoint.query().is_some() + || endpoint.fragment().is_some() + || !valid_socket + { + return Err( + "spiffe_broker requires an HTTPS endpoint and absolute Workload API socket path" + .into(), + ); + } + match (&self.server_spiffe_id, &self.server_spiffe_id_prefix) { + (Some(id), None) => { + openshell_spiffe::SandboxSpiffeIdPrefix::parse(id)?; + } + (None, Some(prefix)) => { + openshell_spiffe::SandboxSpiffeIdPrefix::parse(prefix)?; + if !prefix.trim_start_matches("spiffe://").contains('/') { + return Err("spiffe_broker.server_spiffe_id_prefix requires an agent path, not a whole trust domain".into()); + } + } + _ => return Err("spiffe_broker requires exactly one of server_spiffe_id and server_spiffe_id_prefix".into()), + } + Ok(()) + } +} + /// Server configuration. /// /// Built programmatically in [`crate::Config::new`] and the gateway CLI from @@ -326,6 +411,8 @@ pub struct Config { /// JWTs as `Principal::Sandbox`. Required for the per-sandbox identity /// flow (issue #1354). pub gateway_jwt: Option, + /// Gateway-only access to SPIRE; shared supervisors receive access tokens. + pub spiffe_broker: Option, /// Database URL for persistence. pub database_url: String, @@ -1115,6 +1202,7 @@ impl Config { provider_profile_sources: vec![GatewayProviderProfileSourceConfig::User], mtls_auth: MtlsAuthConfig::default(), gateway_jwt: None, + spiffe_broker: None, database_url: String::new(), database_auth: DatabaseAuthMode::default(), database_max_connections: None, @@ -2097,4 +2185,78 @@ mod tests { "compiled:default" ); } + #[test] + fn broker_configuration_rejects_ambiguous_transport() { + let mut config = super::SpiffeBrokerConfig { + spiffe_id_prefix: "spiffe://example.org/openshell".into(), + endpoint: "https://spire-broker.spire.svc:8443".into(), + node_local_port: None, + server_spiffe_id: Some("spiffe://example.org/spire/agent/node".into()), + server_spiffe_id_prefix: None, + workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock".into(), + }; + config.validate("gw").unwrap(); + let mut local = config.clone(); + local.endpoint.clear(); + local.node_local_port = std::num::NonZeroU16::new(8443); + local.validate("gw").unwrap(); + assert_eq!( + local.endpoint_for_node(Some("10.20.0.2")).unwrap(), + "https://10.20.0.2:8443" + ); + assert_eq!( + local.endpoint_for_node(Some("2001:db8::2")).unwrap(), + "https://[2001:db8::2]:8443" + ); + for address in [ + None, + Some(""), + Some("agent.example.org"), + Some("10.20.0.2:8443"), + ] { + assert!(local.endpoint_for_node(address).is_err()); + } + local.endpoint.clone_from(&config.endpoint); + assert!(local.validate("gw").is_err()); + local.endpoint.clear(); + local.node_local_port = None; + assert!(local.validate("gw").is_err()); + let encoded = serde_json::to_value(&config).unwrap(); + for port in [0, 65536] { + let mut invalid = encoded.clone(); + invalid["node_local_port"] = port.into(); + assert!(serde_json::from_value::(invalid).is_err()); + } + config.server_spiffe_id = None; + config.server_spiffe_id_prefix = + Some("spiffe://example.org/spire/agent/k8s_psat/cluster".into()); + config.validate("gw").unwrap(); + config.server_spiffe_id_prefix = Some("spiffe://example.org".into()); + assert!(config.validate("gw").is_err()); + config.server_spiffe_id = Some("spiffe://example.org/spire/agent/node".into()); + assert!(config.validate("gw").is_err()); + config.server_spiffe_id_prefix = None; + for endpoint in [ + "http://broker:8443", + "https://user:password@broker", + "https://broker/path", + "https://broker/?query", + "https://broker/#fragment", + ] { + config.endpoint = endpoint.into(); + assert!(config.validate("gw").is_err()); + } + config.endpoint = "https://broker:8443".into(); + for socket in [ + "tcp:broker:1234", + "/", + "/agent.sock", + "/run/../agent.sock", + "/run/spire/", + " /run/spire/agent.sock", + ] { + config.workload_api_socket_path = socket.into(); + assert!(config.validate("gw").is_err(), "{socket}"); + } + } } diff --git a/crates/openshell-core/src/dynamic_credential_key.rs b/crates/openshell-core/src/dynamic_credential_key.rs index 8211b6267c..2dbc2bc939 100644 --- a/crates/openshell-core/src/dynamic_credential_key.rs +++ b/crates/openshell-core/src/dynamic_credential_key.rs @@ -109,6 +109,27 @@ pub fn credential_identity(key: &str) -> Option<&str> { Some(decode(key)?.1) } +/// The key a gateway encoded, recovered from an owned key the supervisor may +/// have scoped with [`ProviderCredentialSnapshot::scoped_key`]. +/// +/// Only the revision and installation are removed, so their spelling never +/// distinguishes two keys. Keys without an endpoint selector or owner, and +/// malformed keys, return `None`. The result still names a configured +/// credential only if it equals one of the gateway's keys exactly. +#[must_use] +pub fn unscoped_owned_key(key: &str) -> Option { + decode(key)?.0?; + let fields = key.split('\t').collect::>(); + match fields.as_slice() { + [selector @ .., owner, identity] | [selector @ .., owner, _, _, identity] + if selector.len() == 3 => + { + Some(format!("{}\t{owner}\t{identity}", selector.join("\t"))) + } + _ => None, + } +} + fn decode(key: &str) -> Option<(Option>, &str)> { // A stack buffer bounds parsing and avoids allocating on each request. let mut fields = [""; 7]; @@ -314,6 +335,22 @@ mod tests { Some("/v1/**") ); assert_eq!(credential_identity(&scoped), Some("provider:access_token")); + // Only the generation is removed; its spelling cannot select another key. + assert_eq!(unscoped_owned_key(&scoped), Some(key())); + assert_eq!(unscoped_owned_key(&key()), Some(key())); + assert_eq!( + unscoped_owned_key(&scoped.replace("rev:7", "rev:007")), + Some(key()) + ); + for unowned in [ + "provider:access_token", + "rev:7\tinstallation:install\tprovider:access_token", + "api.example.com\t443\t/v1/**\tprovider:access_token", + "api.example.com\t443\t/v1/**\trev:7\tinstallation:install\tprovider:access_token", + "api.example.com\t443\t/v1/**\towner-a\trev:x\tinstallation:install\tprovider:access_token", + ] { + assert_eq!(unscoped_owned_key(unowned), None, "{unowned:?}"); + } assert_eq!( snapshot.scoped_key("provider:access_token"), "rev:7\tinstallation:install\tprovider:access_token" diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 64268a7866..acc3e9a10a 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -388,7 +388,7 @@ fn configure_gateway_tls( .wrap_err("failed to configure TLS") } -/// Build the plain (un-intercepted) gRPC channel. +/// Configure the plain (un-intercepted) gRPC endpoint. /// /// When the endpoint uses `https://`, server-authenticated TLS is configured /// using this env var: @@ -396,7 +396,7 @@ fn configure_gateway_tls( /// /// When the endpoint uses `http://`, a plaintext connection is used (for /// deployments where TLS is disabled, e.g. behind a Cloudflare Tunnel). -async fn build_plain_channel(endpoint: &str) -> Result { +fn build_plain_endpoint(endpoint: &str) -> Result { let mut ep = Endpoint::from_shared(endpoint.to_string()) .into_diagnostic() .wrap_err("invalid gRPC endpoint")? @@ -431,7 +431,12 @@ async fn build_plain_channel(endpoint: &str) -> Result { ep = configure_gateway_tls(ep, ca_pem, server_name.as_deref())?; } - ep.connect() + Ok(ep) +} + +async fn build_plain_channel(endpoint: &str) -> Result { + build_plain_endpoint(endpoint)? + .connect() .await .into_diagnostic() .wrap_err("failed to connect to OpenShell server") @@ -1702,6 +1707,106 @@ pub async fn exchange_provider_subject_token( )) } +/// Final provider access tokens for one sandbox instance over one reused +/// gateway channel. +/// +/// The lazy channel connects when used, so initialization never waits on a +/// network connection. Concurrent requests share it without holding a +/// lock across the call, and tonic reconnects it after transport loss. Each +/// request reads the instance's current bearer, so renewal and revocation +/// apply without reconnecting, and the gateway authorizes every request. +pub struct SandboxProviderTokenClient { + // The validated endpoint shares the instance's rotating credentials. + gateway: std::result::Result<(url::Url, GatewayEndpoint), TokenGrantFailure>, + client: tokio::sync::OnceCell>, + deadline: Duration, +} + +impl SandboxProviderTokenClient { + #[must_use] + pub fn new(gateway: GatewayEndpoint) -> Self { + Self::with_deadline( + gateway, + crate::token_grant_failure::BROKERED_TOKEN_CLIENT_DEADLINE, + ) + } + + fn with_deadline(gateway: GatewayEndpoint, deadline: Duration) -> Self { + Self { + gateway: token_grant_gateway_endpoint(gateway.url()).map(|endpoint| { + let gateway = gateway.with_url(endpoint.as_str()); + (endpoint, gateway) + }), + client: tokio::sync::OnceCell::new(), + deadline, + } + } + + /// Obtain only the final access token; the gateway selects and verifies + /// the sandbox identity and resolves all grant parameters from its + /// configuration. + /// + /// One deadline bounds connection readiness and the call. Network failures + /// do not serialize channel initialization for concurrent requests. + #[tracing::instrument(name = "supervisor.gateway.get_provider_access_token", skip_all)] + pub async fn get_access_token( + &self, + credential_key: &str, + request_id: Option<&str>, + ) -> std::result::Result { + let (endpoint, gateway) = self.gateway.as_ref().map_err(|failure| *failure)?; + // Like connect(), refuse locally revoked sandbox instance credentials + // before the call; the response boundary rejects them again. + let unavailable = || gateway_grant_failure(TokenGrantFailureKind::Unavailable); + if gateway.credentials().is_revoked() { + return Err(unavailable()); + } + let key = crate::dynamic_credential_key::unscoped_owned_key(credential_key) + .ok_or_else(|| gateway_grant_failure(TokenGrantFailureKind::Configuration))?; + let mut request = tonic::Request::new(crate::proto::GetSandboxProviderAccessTokenRequest { + credential_key: key, + }); + if let Some(value) = request_id.and_then(|value| value.parse().ok()) { + request.metadata_mut().insert("x-request-id", value); + } + let response = tokio::time::timeout(self.deadline, async { + let mut client = self + .client + .get_or_try_init(|| async { + let channel = build_plain_endpoint(gateway.url())?.connect_lazy(); + let intercepted = InterceptedService::new( + channel.clone(), + AuthInterceptor::new(gateway.credentials.inner.bearer.clone()), + ); + gateway + .credentials + .start_renewal(&intercepted, &channel, gateway.url()); + Ok::<_, miette::Report>(OpenShellClient::new(intercepted)) + }) + .await + .map_err(|_| unavailable())? + .clone(); + client + .get_sandbox_provider_access_token(request) + .await + .map_err(|status| provider_subject_token_exchange_status(&status, endpoint)) + }) + .await + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))?? + .into_inner(); + if gateway.credentials().is_revoked() { + return Err(gateway_grant_failure( + TokenGrantFailureKind::WorkloadIdentityRejected, + )); + } + provider_subject_token_exchange_result(crate::proto::ExchangeProviderSubjectTokenResponse { + access_token: response.access_token, + expires_after: response.expires_after, + token_type: response.token_type, + }) + } +} + /// Attach the exchange's request ID as `x-request-id` gRPC metadata. /// /// A value that is not valid metadata is omitted, never rewritten. @@ -1828,6 +1933,53 @@ mod token_grant_exchange_tests { assert!(token_grant_gateway_endpoint("https://localhost:8443").is_ok()); } + #[tokio::test] + async fn access_token_deadline_bounds_connection_and_call_for_each_request() { + // The gateway accepts connections but never answers. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let held = tokio::spawn(async move { + let (_connection, _) = listener.accept().await.unwrap(); + std::future::pending::<()>().await; + }); + let credentials = GatewayCredentials::new( + AsciiMetadataValue::from_static("Bearer test"), + TokenSource::Env, + ); + let deadline = Duration::from_millis(300); + let client = SandboxProviderTokenClient::with_deadline( + GatewayEndpoint::new(format!("http://{address}"), credentials), + deadline, + ); + let started = tokio::time::Instant::now(); + let results: [_; 2] = tokio::join!( + client.get_access_token("api.test\t443\t/\towner\tprovider:token", None), + client.get_access_token("api.test\t443\t/\towner\tprovider:token", None), + ) + .into(); + for result in results { + let failure = result.err().expect("unanswered gateway times out"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Timeout); + } + assert!( + client.client.get().is_some(), + "an unresponsive gateway must not prevent channel initialization" + ); + // Concurrent requests share one budget rather than queueing behind it. + assert!(started.elapsed() < deadline * 3); + held.abort(); + + let invalid = SandboxProviderTokenClient::new(GatewayEndpoint::new( + "https://localhost/SECRET", + GatewayCredentials::new( + AsciiMetadataValue::from_static("Bearer test"), + TokenSource::Env, + ), + )); + let failure = invalid.get_access_token("key", None).await.err().unwrap(); + assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); + } + #[test] fn gateway_response_keeps_expiration_and_rejects_invalid_duration() { let response = |seconds, nanos| crate::proto::ExchangeProviderSubjectTokenResponse { diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 12473a702c..5e775ca86d 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -65,7 +65,6 @@ pub mod sandbox_update; pub mod secrets; pub mod settings; pub mod shell; -pub mod spiffe; pub mod telemetry; pub mod time; pub mod token_grant_failure; diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index dcf56cf5c6..8e31239d97 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -15,7 +15,8 @@ use std::time::Duration; use serde::Deserialize; use crate::token_grant_failure::{ - REQUEST_ID_HEADER, TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + REQUEST_ID_HEADER, TOKEN_SERVICE_REQUEST_TIMEOUT, TokenGrantFailure, TokenGrantFailureKind, + TokenGrantStage, }; pub const DEFAULT_CLIENT_ASSERTION_TYPE: &str = @@ -24,6 +25,9 @@ pub const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_tok const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; const ERROR_CONTRACT: &str = "openshell.token-grant.v1"; const MAX_TOKEN_RESPONSE_BYTES: usize = 64 * 1024; +pub const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; +pub const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; +pub const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; /// HTTP client that authenticates HTTPS endpoints and never redirects assertions. /// @@ -54,8 +58,8 @@ impl TokenGrantClient { // idempotent for this caller and never relaxes peer authentication. openshell_crypto::tls::ensure_default_provider(); reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(30)) + .timeout(TOKEN_SERVICE_REQUEST_TIMEOUT) + .connect_timeout(TOKEN_SERVICE_REQUEST_TIMEOUT) .no_proxy() .redirect(reqwest::redirect::Policy::none()) } @@ -263,6 +267,28 @@ pub fn effective_token_type(token_type: &str) -> &str { } } +/// Cache lifetime for a granted access token. +/// +/// A configured override is used exactly, including zero to disable caching. +/// Otherwise the response lifetime (capped, or a default when omitted) is +/// shortened by a fixed expiry skew, with a one-second floor. +pub fn token_cache_ttl(cache_ttl_override: Option, expires_in: i64) -> Duration { + if let Some(cache_ttl_override) = cache_ttl_override { + return cache_ttl_override; + } + + let ttl = if expires_in > 0 { + expires_in.min(MAX_TOKEN_EXPIRES_IN_SECONDS) + } else { + DEFAULT_TOKEN_CACHE_TTL_SECONDS + }; + + Duration::from_secs( + u64::try_from(ttl.saturating_sub(TOKEN_CACHE_EXPIRY_SKEW_SECONDS).max(1)) + .unwrap_or(u64::MAX), + ) +} + /// Validate the configured endpoint before acquiring any workload credentials. /// /// HTTPS is accepted; plain HTTP is limited to loopback and Kubernetes service diff --git a/crates/openshell-core/src/spiffe.rs b/crates/openshell-core/src/spiffe.rs deleted file mode 100644 index c288dfd81a..0000000000 --- a/crates/openshell-core/src/spiffe.rs +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Shared SPIFFE helpers used by the gateway and sandbox supervisor. - -use std::path::Path; - -use base64::Engine as _; -use serde::Deserialize; - -/// SPIFFE JWT-SVID claims used by `OpenShell` token exchange flows. -#[derive(Debug, Clone, Deserialize)] -pub struct SpiffeJwtClaims { - pub iss: String, - pub sub: String, - pub aud: AudienceClaim, - #[serde(default)] - pub exp: i64, -} - -/// JWT `aud` claim representation accepted by SPIFFE JWT-SVIDs. -#[derive(Debug, Clone, Deserialize)] -#[serde(untagged)] -pub enum AudienceClaim { - One(String), - Many(Vec), -} - -impl AudienceClaim { - pub fn contains(&self, expected: &str) -> bool { - match self { - Self::One(value) => value == expected, - Self::Many(values) => values.iter().any(|value| value == expected), - } - } -} - -#[derive(Debug, thiserror::Error)] -pub enum JwtSvidParseError { - #[error("invalid JWT-SVID format")] - Format, - #[error("invalid JWT-SVID payload encoding")] - PayloadEncoding, - #[error("invalid JWT-SVID payload")] - Payload, -} - -/// Convert a path to a SPIFFE Workload API endpoint URL. -/// -/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. -/// Otherwise, assume it is a Unix socket path and prepend `unix:`. -pub fn workload_api_endpoint(path: &Path) -> String { - let path = path.to_string_lossy(); - if path.starts_with("unix:") || path.starts_with("tcp:") { - path.into_owned() - } else { - format!("unix:{path}") - } -} - -/// # Security -/// -/// This function decodes the JWT payload without verifying the signature. -/// Callers must separately verify the JWT signature before trusting the claims. -pub fn parse_unverified_jwt_svid_claims(token: &str) -> Result { - let segments = token.split('.').collect::>(); - if segments.len() != 3 || segments.iter().any(|segment| segment.is_empty()) { - return Err(JwtSvidParseError::Format); - } - let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD - .decode(segments[1]) - .map_err(|_| JwtSvidParseError::PayloadEncoding)?; - serde_json::from_slice::(&decoded).map_err(|_| JwtSvidParseError::Payload) -} - -pub fn trust_domain(subject: &str) -> Option<&str> { - let rest = subject.strip_prefix("spiffe://")?; - let (trust_domain, _) = rest.split_once('/').unwrap_or((rest, "")); - (!trust_domain.is_empty()).then_some(trust_domain) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn unsigned_svid_fixture(issuer: &str, subject: &str, audience: serde_json::Value) -> String { - let header = serde_json::json!({ "alg": "RS256", "kid": "test-key" }); - let payload = serde_json::json!({ - "iss": issuer, - "sub": subject, - "aud": audience, - "exp": 4_102_444_800_i64 - }); - let encoded_header = base64::engine::general_purpose::URL_SAFE_NO_PAD - .encode(serde_json::to_vec(&header).expect("serialize header")); - let encoded_payload = base64::engine::general_purpose::URL_SAFE_NO_PAD - .encode(serde_json::to_vec(&payload).expect("serialize payload")); - format!("{encoded_header}.{encoded_payload}.signature") - } - - #[test] - fn workload_api_endpoint_preserves_explicit_scheme() { - assert_eq!( - workload_api_endpoint(Path::new("unix:/run/spire/agent.sock")), - "unix:/run/spire/agent.sock" - ); - assert_eq!( - workload_api_endpoint(Path::new("tcp:127.0.0.1:8081")), - "tcp:127.0.0.1:8081" - ); - } - - #[test] - fn workload_api_endpoint_defaults_to_unix_socket() { - assert_eq!( - workload_api_endpoint(Path::new("/run/spire/agent.sock")), - "unix:/run/spire/agent.sock" - ); - } - - #[test] - fn parse_unverified_jwt_svid_claims_accepts_string_audience() { - let token = unsigned_svid_fixture( - "https://spiffe.example.test", - "spiffe://openshell/openshell/sandbox/sb-a", - serde_json::json!("https://auth.example.com"), - ); - - let claims = parse_unverified_jwt_svid_claims(&token).expect("valid claims"); - - assert_eq!(claims.iss, "https://spiffe.example.test"); - assert_eq!(claims.sub, "spiffe://openshell/openshell/sandbox/sb-a"); - assert!(claims.aud.contains("https://auth.example.com")); - assert!(!claims.aud.contains("https://other.example.com")); - } - - #[test] - fn parse_unverified_jwt_svid_claims_accepts_array_audience() { - let token = unsigned_svid_fixture( - "https://spiffe.example.test", - "spiffe://openshell/openshell/sandbox/sb-a", - serde_json::json!(["https://auth.example.com", "https://other.example.com"]), - ); - - let claims = parse_unverified_jwt_svid_claims(&token).expect("valid claims"); - - assert!(claims.aud.contains("https://auth.example.com")); - assert!(claims.aud.contains("https://other.example.com")); - } - - #[test] - fn parse_unverified_jwt_svid_claims_rejects_truncated_jwt() { - assert!(matches!( - parse_unverified_jwt_svid_claims("header.payload"), - Err(JwtSvidParseError::Format) - )); - } - - #[test] - fn parse_unverified_jwt_svid_claims_rejects_empty_jwt_segments() { - assert!(matches!( - parse_unverified_jwt_svid_claims("header..signature"), - Err(JwtSvidParseError::Format) - )); - } - - #[test] - fn trust_domain_extracts_domain_from_spiffe_id() { - assert_eq!( - trust_domain("spiffe://openshell/openshell/sandbox/sb-a"), - Some("openshell") - ); - assert_eq!(trust_domain("spiffe://openshell"), Some("openshell")); - assert_eq!(trust_domain("not-a-spiffe-id"), None); - assert_eq!(trust_domain("spiffe:///empty"), None); - } -} diff --git a/crates/openshell-core/src/token_grant_failure.rs b/crates/openshell-core/src/token_grant_failure.rs index d469047a75..44d279cd1a 100644 --- a/crates/openshell-core/src/token_grant_failure.rs +++ b/crates/openshell-core/src/token_grant_failure.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::fmt; +use std::time::Duration; use prost::Message; use tonic::{Code, Status}; @@ -22,6 +23,32 @@ use crate::rpc_error::{ERROR_DOMAIN, ErrorDetails, StatusExt}; /// random ID and grants nothing. pub const REQUEST_ID_HEADER: &str = "x-openshell-request-id"; +// Time limits of a grant. Exceeding one is a `Timeout` failure. + +/// One token-service request, including its connection. +pub const TOKEN_SERVICE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// One SPIFFE Workload API or SPIRE agent call, including its connection. +pub const WORKLOAD_IDENTITY_CALL_TIMEOUT: Duration = Duration::from_secs(10); + +/// One gateway-brokered provider access token request on the gateway, +/// including waiting for a concurrent acquisition of the same token. +/// +/// A miss bounds the complete sandbox identity acquisition at ten seconds, +/// then may fetch the gateway's own SVID and make two token-service requests. +/// The remaining forty seconds cover admission, authorization, and store reads. +pub const BROKERED_TOKEN_GATEWAY_DEADLINE: Duration = Duration::from_secs( + 2 * WORKLOAD_IDENTITY_CALL_TIMEOUT.as_secs() + 2 * TOKEN_SERVICE_REQUEST_TIMEOUT.as_secs() + 40, +); + +/// Deadline for one brokered token request on the supervisor. +/// +/// Covers waiting for and connecting the gateway channel (bounded at 10 +/// seconds), then the gateway's deadline, so its classified timeout normally +/// arrives first. +pub const BROKERED_TOKEN_CLIENT_DEADLINE: Duration = + Duration::from_secs(BROKERED_TOKEN_GATEWAY_DEADLINE.as_secs() + 15); + const FAILURE_MESSAGE: &str = "dynamic token grant failed"; const FAILURE_REASON: &str = "TOKEN_GRANT_FAILURE"; const MAX_DETAILS_BYTES: usize = 1024; diff --git a/crates/openshell-crypto/src/tls.rs b/crates/openshell-crypto/src/tls.rs index 1be43b5d75..c08be4df4f 100644 --- a/crates/openshell-crypto/src/tls.rs +++ b/crates/openshell-crypto/src/tls.rs @@ -4,6 +4,8 @@ //! Rustls adapter. Explicit configurations use this provider. Library callers //! may retain an embedder's process default; that default is not attested here. +pub mod spiffe; + use rustls::crypto::{CryptoProvider, WebPkiSupportedAlgorithms}; use std::sync::Arc; diff --git a/crates/openshell-crypto/src/tls/spiffe.rs b/crates/openshell-crypto/src/tls/spiffe.rs new file mode 100644 index 0000000000..66d8ccdebc --- /dev/null +++ b/crates/openshell-crypto/src/tls/spiffe.rs @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! TLS verification of a peer authorized by SPIFFE URI identity. + +use rustls::{ + DigitallySignedStruct, RootCertStore, SignatureScheme, + client::{ + danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + verify_server_cert_signed_by_trust_anchor, + }, + pki_types::{CertificateDer, ServerName, UnixTime}, + server::ParsedCertificate, +}; +use std::sync::Arc; +use x509_parser::{extensions::GeneralName, prelude::FromDer}; + +/// Authorized SPIFFE server identities. Prefixes authorize descendants of one +/// SPIFFE path, never sibling paths or another trust domain. +#[derive(Clone, Debug)] +pub enum ServerIdentity { + Exact(String), + PathPrefix(String), +} + +impl ServerIdentity { + fn matches(&self, uri: &str) -> bool { + match self { + Self::Exact(expected) => uri == expected, + Self::PathPrefix(prefix) => { + let (Some((domain, path)), Some((expected_domain, expected_path))) = + (spiffe_path(uri), spiffe_path(prefix)) + else { + return false; + }; + domain == expected_domain + && path + .strip_prefix(expected_path) + .is_some_and(|suffix| suffix.starts_with('/') && suffix.len() > 1) + } + } + } +} + +fn spiffe_path(uri: &str) -> Option<(&str, &str)> { + let (domain, path) = uri.strip_prefix("spiffe://")?.split_once('/')?; + (uri.len() <= 2048 + && !domain.is_empty() + && domain + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b".-_".contains(&b)) + && path.split('/').all(|segment| { + !segment.is_empty() + && segment != "." + && segment != ".." + && segment + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-_".contains(&b)) + })) + .then_some((domain, path)) +} + +/// Verify the chain and one URI SAN against the configured identity policy. +/// +/// DNS names are not used for SPIFFE authentication. Revocation checking is +/// not enabled; deployments rely on short-lived SVIDs. +#[must_use] +pub fn server_verifier_with_identity( + roots: RootCertStore, + expected: ServerIdentity, +) -> Arc { + Arc::new(SpiffeServerVerifier { roots, expected }) +} + +#[derive(Debug)] +struct SpiffeServerVerifier { + roots: RootCertStore, + expected: ServerIdentity, +} +impl ServerCertVerifier for SpiffeServerVerifier { + fn verify_server_cert( + &self, + leaf: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + _: &ServerName<'_>, + _: &[u8], + now: UnixTime, + ) -> Result { + let algorithms = super::signature_verification_algorithms(); + verify_server_cert_signed_by_trust_anchor( + &ParsedCertificate::try_from(leaf)?, + &self.roots, + intermediates, + now, + algorithms.all, + )?; + let (_, cert) = x509_parser::certificate::X509Certificate::from_der(leaf.as_ref()) + .map_err(|_| rustls::Error::General("invalid agent certificate".into()))?; + let san = cert + .subject_alternative_name() + .map_err(|_| rustls::Error::General("invalid agent SAN".into()))? + .ok_or_else(|| rustls::Error::General("missing agent SAN".into()))?; + let uris: Vec<_> = san + .value + .general_names + .iter() + .filter_map(|name| match name { + GeneralName::URI(uri) => Some(*uri), + _ => None, + }) + .collect(); + if !matches!(uris.as_slice(), [uri] if self.expected.matches(uri)) { + return Err(rustls::Error::General( + "unexpected agent SPIFFE identity".into(), + )); + } + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + signature, + &super::signature_verification_algorithms(), + ) + } + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + signature: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + signature, + &super::signature_verification_algorithms(), + ) + } + fn supported_verify_schemes(&self) -> Vec { + super::signature_verification_algorithms().supported_schemes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PREFIX: &str = "spiffe://example.org/spire/agent/k8s_psat/production"; + + #[test] + fn peer_prefix_is_scoped_to_valid_descendants_in_one_cluster() { + let policy = ServerIdentity::PathPrefix(PREFIX.into()); + for node in ["node-a", "node-b", "new-node"] { + assert!(policy.matches(&format!("{PREFIX}/{node}"))); + } + for id in [ + PREFIX.to_string(), + format!("{PREFIX}-other/node"), + format!("{PREFIX}/../other/node"), + format!("{PREFIX}//node"), + format!("{PREFIX}/node?extra"), + "spiffe://other.org/spire/agent/k8s_psat/production/node".into(), + ] { + assert!(!policy.matches(&id), "{id}"); + } + for prefix in [ + "", + "spiffe://example.org", + "spiffe://example.org/", + "https://example.org/agent", + ] { + assert!(!ServerIdentity::PathPrefix(prefix.into()).matches(&format!("{PREFIX}/node"))); + } + } + + #[test] + #[cfg(any(feature = "aws-lc", feature = "openssl"))] + fn certificate_verification_requires_trust_validity_and_one_authorized_uri() { + use crate::pki; + + #[cfg(all(feature = "openssl", not(feature = "aws-lc")))] + crate::install_default_context(crate::CryptoContext::new(Box::new(crate::OpenSsl))) + .unwrap(); + + let ca_key = pki::generate_keypair().unwrap(); + let mut ca_params = rcgen::CertificateParams::default(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params.key_usages = vec![rcgen::KeyUsagePurpose::KeyCertSign]; + let ca = pki::self_signed(ca_params, &ca_key).unwrap(); + let mut roots = RootCertStore::empty(); + roots.add(ca.der().clone()).unwrap(); + let expected = format!("{PREFIX}/node-a"); + let now = UnixTime::since_unix_epoch(std::time::Duration::from_hours(500_000)); + let server_name = ServerName::try_from("broker.invalid").unwrap(); + for (name, uris, trusted, expired, prefix, succeeds) in [ + ("exact", vec![expected.clone()], true, false, false, true), + ( + "cluster", + vec![format!("{PREFIX}/node-b")], + true, + false, + true, + true, + ), + ( + "wrong node", + vec![format!("{PREFIX}/node-b")], + true, + false, + false, + false, + ), + ( + "wrong cluster", + vec![format!("{PREFIX}-other/node")], + true, + false, + true, + false, + ), + ( + "extra URI", + vec![expected.clone(), format!("{PREFIX}/node-b")], + true, + false, + true, + false, + ), + ("missing URI", vec![], true, false, true, false), + ( + "untrusted", + vec![expected.clone()], + false, + false, + true, + false, + ), + ("expired", vec![expected.clone()], true, true, true, false), + ] { + let key = pki::generate_keypair().unwrap(); + let mut params = rcgen::CertificateParams::default(); + params.subject_alt_names = uris + .into_iter() + .map(|uri| rcgen::SanType::URI(uri.try_into().unwrap())) + .collect(); + params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ServerAuth]; + if expired { + params.not_before = rcgen::date_time_ymd(2019, 1, 1); + params.not_after = rcgen::date_time_ymd(2020, 1, 1); + } + let leaf = pki::signed_by(params, &key, &ca, &ca_key).unwrap(); + let verifier = server_verifier_with_identity( + if trusted { + roots.clone() + } else { + RootCertStore::empty() + }, + if prefix { + ServerIdentity::PathPrefix(PREFIX.into()) + } else { + ServerIdentity::Exact(expected.clone()) + }, + ); + assert_eq!( + verifier + .verify_server_cert(leaf.der(), &[], &server_name, &[], now) + .is_ok(), + succeeds, + "{name}" + ); + } + } +} diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index d1db98a0d1..bb2c4e49fb 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -261,8 +261,12 @@ fails closed. when the token that opened it expires. Sandboxes cannot reach the stream, and the routing preface only selects which generation-pinned TLS identity the supervisor demands. +- **Workload identity.** The gateway owns SPIFFE Broker API identity acquisition and provider-token + exchange when `openshell.gateway.spiffe_broker` is configured. + Shared supervisors receive final access tokens through their authenticated + sessions. No SPIFFE socket is mounted in a shared supervisor or workload Pod. -The topology does not support upstream proxies or provider SPIFFE identity. +The topology does not support upstream proxies. With a live replacement supervisor and a takeover-capable runtime, the gateway can move a paused sandbox generation to the replacement using a signed takeover grant. Recovery preserves workload processes and memory, but proxied connections diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a18b51e1de..adc98a5f9b 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -610,9 +610,6 @@ impl KubernetesComputeConfig { { return Err("supervisor.app_listener.port conflicts with shared_supervisor.runtime_port".into()); } - if self.provider_spiffe_enabled() { - return Err("supervisor_topology=gateway does not support provider_spiffe_workload_api_socket_path; a shared supervisor has no per-sandbox workload identity".into()); - } if self.https_proxy.is_some() || self.proxy_auth_secret_name.is_some() || self.proxy_ca_bundle.is_some() diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index f98a180eea..9779d965e8 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -245,6 +245,13 @@ fn workload_template_proto(name: &str, workspace: &str) -> proto::SandboxWorkloa #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 9d7f9299db..5bd37c8314 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -22,6 +22,8 @@ openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } openshell-prover = { path = "../openshell-prover" } openshell-providers = { path = "../openshell-providers" } +openshell-spiffe = { path = "../openshell-spiffe" } +openshell-spire = { path = "../openshell-spire" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } diff --git a/crates/openshell-server/README.md b/crates/openshell-server/README.md index 6d846227ca..c9185f5ba9 100644 --- a/crates/openshell-server/README.md +++ b/crates/openshell-server/README.md @@ -268,6 +268,51 @@ runs under `mise run test:rust:postgres` (see `TESTING.md`) and still accepts It exercises independent sessions without the local mutex. Do not point this fixture at an existing deployment database. +## Sandbox identity broker + +`openshell.gateway.spiffe_broker` enables gateway-owned SPIFFE Broker API +acquisition over mTLS for the Kubernetes shared supervisor. The gateway obtains +its own X.509-SVID through the CSI Workload API socket. `GetSandboxProviderAccessToken` accepts an +endpoint-bound configured credential key; sandbox identity comes from the +verified, currently authorized session JWT. The key must equal one of the +gateway's configured keys, including its endpoint owner, once the supervisor's +revision and installation are removed. Provider attachment and profile +configuration determine the endpoint, audience, and scopes. The gateway performs +RFC 7523 or both RFC 8693 exchanges and returns only the final access token. +Its bounded cache is isolated by sandbox generation, credential epoch, provider +revision, the matched configured key and grant, and the subject token, which is +resolved before every lookup so its revocation or rotation also applies to +cached tokens. Concurrent misses for one cache key wait for a single +acquisition, then use its cached token. A token is served until the usual skew +before the earliest of its `expires_in`, its JWT `exp`, and the session, SVID, +and subject expiries, and no longer than the default cache lifetime; a +configured cache TTL can only shorten that. The response reports the remaining +lifetime. The session is authorized again around each SPIRE and +token-service call. A cache hit and a fresh token then pass the same final +check: the grant configuration still matches and the session is still +authorized. A fresh token is cached only after that check. Startup validates +the Broker API settings, and gateway state construction builds the +[`openshell-spire`](../openshell-spire/README.md) adapter once and returns a +configuration error when they are invalid. The broker depends only on +[`openshell-spiffe`](../openshell-spiffe/README.md)'s `SandboxIdentityProvider` +with `TokenGrantFailure` as its error, which returns a JWT-SVID +verified for the requested gateway, sandbox, and audience; each cache miss +fetches a new SVID and the agent's current JWT bundle. SPIRE attests the referenced Sandbox resource UID and enforces the gateway's +`impersonate-via-spire` authorization for that object. + +`openshell_core::token_grant_failure` defines the time limits: SPIRE and +Workload API calls, token-service requests, the gateway's deadline for one +request (including waiting for a concurrent acquisition), and the longer +supervisor deadline that covers connecting to the gateway. + +Deployments must enable the SPIFFE Broker endpoint, authorize the gateway +SPIFFE ID and Kubernetes object-reference type, and set the Kubernetes workload +attestor's `access_policy` to `enforced`. The gateway's persisted driver runtime +identity supplies the Sandbox CR UID. SPIRE resolves that reference and matches +its attested selectors against registrations distributed to each selected agent. +The adapter fetches current gateway X.509 credentials and bundles on every +acquisition, allowing rotation without restarting the gateway. + ## Application route installation records Supervisor installation reports arrive only through the authenticated sandbox diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index e3c8b27186..d3661f75bc 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -807,6 +807,30 @@ fn prepare_server_config_with_drivers( // the generated TLS bundle so upgrades pick up sandbox auth without a // user-authored config file. config.gateway_jwt = gateway_jwt; + config.spiffe_broker = file + .as_ref() + .and_then(|file| file.openshell.gateway.spiffe_broker.clone()); + + if let Some(kubernetes) = file + .as_ref() + .and_then(|file| file.openshell.drivers.get("kubernetes")) + && kubernetes + .get("supervisor_topology") + .and_then(toml::Value::as_str) + == Some("gateway") + && kubernetes + .get("provider_spiffe_workload_api_socket_path") + .and_then(toml::Value::as_str) + .is_some_and(|socket| !socket.is_empty()) + && config.spiffe_broker.is_none() + { + return Err(miette::miette!( + "shared supervisor SPIFFE grants require spiffe_broker; remove provider_spiffe_workload_api_socket_path from the shared supervisor" + )); + } + + // Reject invalid broker before startup; gateway state parses it later. + crate::spiffe_broker::validate_config(&config).map_err(|error| miette::miette!("{error}"))?; if let Some(normalized) = normalized_network_trust.as_ref() { // Use the same resolver as runtime construction, with the explicit diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index bd04a38eef..c0082c75f9 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -227,6 +227,8 @@ pub struct GatewayFileSection { #[serde(default)] pub gateway_jwt: Option, #[serde(default)] + pub spiffe_broker: Option, + #[serde(default)] pub otlp: Option, #[serde(default)] pub ocsf_log: Option, diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 3cfae86954..59e3c59ed0 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -846,6 +846,14 @@ impl OpenShell for OpenShellService { policy::handle_get_sandbox_provider_environment(&self.state, request).await } + async fn get_sandbox_provider_access_token( + &self, + request: Request, + ) -> Result, Status> + { + provider::access_token::handle(&self.state, request).await + } + async fn exchange_provider_subject_token( &self, request: Request, diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 5ede6d062f..c7a5782541 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -5,6 +5,8 @@ #![allow(clippy::result_large_err)] // gRPC handlers return Result, Status> +pub(super) mod access_token; + use crate::compute::MutationScope; #[cfg(test)] use crate::credentials::RefreshMaterialScope; @@ -2669,15 +2671,15 @@ use openshell_core::proto::{ ProviderResponse, RotateProviderCredentialRequest, RotateProviderCredentialResponse, UpdateProviderProfilesRequest, UpdateProviderProfilesResponse, UpdateProviderRequest, }; -use openshell_core::spiffe::{ - SpiffeJwtClaims, parse_unverified_jwt_svid_claims, trust_domain as spiffe_trust_domain, - workload_api_endpoint, -}; use openshell_providers::{ CredentialRefreshProfile, ProfileValidationDiagnostic, ProviderTypeProfile, normalize_profile_id, strategy_output_env_key, strategy_output_spec, strategy_primary_env_key, validate_profile_set, }; +use openshell_spiffe::{ + SpiffeJwtClaims, parse_unverified_jwt_svid_claims, trust_domain as spiffe_trust_domain, + workload_api_endpoint, +}; use std::sync::{Arc, LazyLock, RwLock}; use tonic::{Request, Response}; @@ -2691,7 +2693,7 @@ use openshell_core::oauth::{ effective_token_type, }; use openshell_core::token_grant_failure::{ - TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, WORKLOAD_IDENTITY_CALL_TIMEOUT, }; const DEFAULT_INTERMEDIATE_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const MAX_INTERMEDIATE_TOKEN_CACHE_TTL_SECONDS: i64 = 3600; @@ -2728,17 +2730,23 @@ impl IntermediateTokenCache { } fn get(&self, key: &str) -> Option { + self.get_until(key).map(|(token, _)| token) + } + + /// The unexpired token with the time its entry expires, in milliseconds. + fn get_until(&self, key: &str) -> Option<(oauth::OAuthTokenResponse, i64)> { let now_ms = crate::persistence::current_time_ms(); let tokens = self.tokens.read().ok()?; let cached = tokens.get(key)?; if cached.expires_at_ms <= now_ms { return None; } - Some(oauth::OAuthTokenResponse { + let token = oauth::OAuthTokenResponse { access_token: cached.access_token.clone(), expires_in: cached.expires_at_ms.saturating_sub(now_ms) / 1000, token_type: cached.token_type.clone(), - }) + }; + Some((token, cached.expires_at_ms)) } fn set(&self, key: String, token: &oauth::OAuthTokenResponse, expires_at_ms: i64) { @@ -4768,18 +4776,44 @@ fn jwt_exp_ms(token: &str) -> Option { } async fn fetch_gateway_jwt_svid(audience: &str) -> Result { - let socket_path = std::env::var(GATEWAY_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|value| !value.trim().is_empty()) + fetch_gateway_jwt_svid_from(audience, None).await +} + +async fn fetch_gateway_jwt_svid_from( + audience: &str, + broker_socket: Option<&str>, +) -> Result { + let socket_path = broker_socket + .map(str::to_owned) + .or_else(|| { + std::env::var(GATEWAY_SPIFFE_WORKLOAD_API_SOCKET) + .ok() + .filter(|value| !value.trim().is_empty()) + }) .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let endpoint = workload_api_endpoint(std::path::Path::new(&socket_path)); - let client = spiffe::WorkloadApiClient::connect_to(&endpoint) - .await - .map_err(|error| token_grant_workload_api_status(error, tonic::Code::FailedPrecondition))?; - client - .fetch_jwt_token([audience], None) - .await - .map_err(|error| token_grant_workload_api_status(error, tonic::Code::FailedPrecondition)) + // A wedged Workload API socket must not hold the grant past its deadline. + tokio::time::timeout(WORKLOAD_IDENTITY_CALL_TIMEOUT, async { + let client = spiffe::WorkloadApiClient::connect_to(&endpoint) + .await + .map_err(|error| { + token_grant_workload_api_status(error, tonic::Code::FailedPrecondition) + })?; + client + .fetch_jwt_token([audience], None) + .await + .map_err(|error| { + token_grant_workload_api_status(error, tonic::Code::FailedPrecondition) + }) + }) + .await + .map_err(|_| { + TokenGrantFailure::new( + TokenGrantFailureKind::Timeout, + TokenGrantStage::WorkloadIdentity, + ) + .to_status() + })? } fn validate_gateway_jwt_svid_claims( @@ -5825,6 +5859,54 @@ mod tests { assert!(!logs.contains(SECRET), "{logs}"); } + #[tokio::test] + async fn broker_failure_log_names_authenticated_sandbox() { + let state = test_server_state_without_provider_profiles().await; + let mut request = Request::new( + openshell_core::proto::GetSandboxProviderAccessTokenRequest { + credential_key: SECRET.to_string(), + }, + ); + request + .extensions_mut() + .insert(Principal::Sandbox(SandboxPrincipal { + sandbox_id: "broker-sandbox".to_string(), + source: SandboxIdentitySource::BootstrapJwt { + issuer: SECRET.to_string(), + auth_epoch: None, + }, + trust_domain: None, + })); + request + .metadata_mut() + .insert("x-request-id", REQUEST_ID.parse().unwrap()); + + let output = GrantTraceBuffer::default(); + let writer = output.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .without_time() + .with_writer(move || writer.clone()) + .finish(); + let capture = crate::otel_tracing::test_exporter::install_scoped(subscriber); + let error = access_token::handle(&state, request).await.unwrap_err(); + drop(capture); + + let failure = assert_generic(&error, Code::Unauthenticated); + let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + assert!( + logs.contains(&format!( + "dynamic token grant failed request_id={REQUEST_ID} code={} stage={}", + failure.code(), + failure.stage().code(), + )), + "{logs}" + ); + assert!(logs.contains("sandbox_id=broker-sandbox"), "{logs}"); + assert!(!logs.contains(SECRET), "{logs}"); + } + #[tokio::test] async fn missing_expired_or_unresolvable_subject_credential_is_generic() { let config = openshell_core::Config::new(None).with_credential_drivers(["test-static"]); @@ -5953,7 +6035,7 @@ mod tests { let claims = SpiffeJwtClaims { iss: SECRET.to_string(), sub: SECRET.to_string(), - aud: openshell_core::spiffe::AudienceClaim::One(SECRET.to_string()), + aud: openshell_spiffe::AudienceClaim::One(SECRET.to_string()), exp: 0, }; let gateway = @@ -8822,17 +8904,20 @@ mod tests { // Model a second gateway replica: it shares the durable database and // credential backend, but owns an independent sandbox synchronization // guard, so process-local serialization cannot hide this race. - let second_state = Arc::new(ServerState::new_with_credentials( - first_state.config.clone(), - Arc::clone(&first_state.store), - crate::compute::new_test_runtime(Arc::clone(&first_state.store)).await, - crate::sandbox_index::SandboxIndex::new(), - crate::sandbox_watch::SandboxWatchBus::new(), - crate::tracing_bus::TracingLogBus::new(), - Arc::new(crate::supervisor_session::SupervisorSessionRegistry::new()), - None, - first_state.credentials.clone(), - )); + let second_state = Arc::new( + ServerState::new_with_credentials( + first_state.config.clone(), + Arc::clone(&first_state.store), + crate::compute::new_test_runtime(Arc::clone(&first_state.store)).await, + crate::sandbox_index::SandboxIndex::new(), + crate::sandbox_watch::SandboxWatchBus::new(), + crate::tracing_bus::TracingLogBus::new(), + Arc::new(crate::supervisor_session::SupervisorSessionRegistry::new()), + None, + first_state.credentials.clone(), + ) + .unwrap(), + ); let request = |client_secret: &str| ConfigureProviderRefreshRequest { request_id: String::new(), provider: "configure-create-race".to_string(), diff --git a/crates/openshell-server/src/grpc/provider/access_token.rs b/crates/openshell-server/src/grpc/provider/access_token.rs new file mode 100644 index 0000000000..2936803b0a --- /dev/null +++ b/crates/openshell-server/src/grpc/provider/access_token.rs @@ -0,0 +1,872 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-owned sandbox identity and provider-token acquisition. +//! The caller selects an attached configured credential, never grant parameters. +use super::{ + Arc, HashMap, IntermediateTokenCache, LazyLock, Message, ObjectId, ObjectWorkspace, Principal, + Provider, ProviderCredentialTokenGrantType, Request, Response, Sandbox, ServerState, Status, + TokenExchangeParams, TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + canonical_request_id, effective_jwt_svid_audience, fetch_gateway_jwt_svid_from, + get_provider_type_profile_for_scope, insert_dynamic_credentials_for_profile, jwt_exp_ms, + log_token_grant_failure, oauth, perform_intermediate_token_exchange, + provider_credential_expiration_ms, resolve_subject_token_credential, + sanitize_token_grant_status, token_exchange_http_client, token_grant_configuration_status, + token_grant_dependency_status, token_grant_identity_rejected, token_grant_store_status, +}; +use openshell_core::jwt::AuthenticatedSandboxSession; +use openshell_core::proto::{ + GetSandboxProviderAccessTokenRequest, GetSandboxProviderAccessTokenResponse, + ProviderCredentialTokenGrant, +}; +use openshell_core::token_grant_failure::BROKERED_TOKEN_GATEWAY_DEADLINE; +use openshell_spiffe::SandboxIdentityProvider; +use std::time::Duration; + +static ACCESS_TOKENS: LazyLock = LazyLock::new(IntermediateTokenCache::new); +static ACQUISITIONS: LazyLock = LazyLock::new(Acquisitions::default); +static ADMISSION: LazyLock = LazyLock::new(Admission::default); + +pub(in crate::grpc) async fn handle( + state: &Arc, + request: Request, +) -> Result, Status> { + let request_id = canonical_request_id(request.metadata()); + // Failures go to the authenticated sandbox's log stream; other callers + // are logged on the gateway only. + let log_sandbox_id = match request.extensions().get::() { + Some(Principal::Sandbox(principal)) => Some(principal.sandbox_id.clone()), + _ => None, + }; + // The deadline also bounds waiting for a concurrent acquisition. + let result = tokio::time::timeout( + BROKERED_TOKEN_GATEWAY_DEADLINE, + obtain(state, request, request_id.as_deref()), + ) + .await + .unwrap_or_else(|_| { + Err( + TokenGrantFailure::new(TokenGrantFailureKind::Timeout, TokenGrantStage::GatewayRpc) + .to_status(), + ) + }); + result.map_err(|status| { + let status = sanitize_token_grant_status(status); + log_token_grant_failure( + log_sandbox_id.as_deref(), + request_id.as_deref(), + TokenGrantFailure::from_status(&status), + ); + status + }) +} + +/// Verify the original bearer again after asynchronous work, including expiry +/// and the persisted generation, epoch, active phase, and current token lineage. +async fn authorize( + state: &ServerState, + bearer: &str, +) -> Result { + let authority = state + .sandbox_session_jwt_authority + .as_ref() + .ok_or_else(|| token_grant_identity_rejected(tonic::Code::Unauthenticated))?; + let session = authority + .verify_gateway_token(bearer) + .map_err(|error| error.status)?; + if session.expires_at.saturating_mul(1000) <= crate::persistence::current_time_ms() { + return Err(token_grant_identity_rejected(tonic::Code::Unauthenticated)); + } + crate::auth::sandbox_session::authorize_persisted(&state.store, &session) + .await + .map_err(|error| error.status)?; + Ok(session) +} + +/// One configured credential of an attached provider. +struct ConfiguredGrant { + provider: Provider, + grant: ProviderCredentialTokenGrant, + /// The gateway's exact key for the credential, including its endpoint owner. + key: String, +} + +/// Resolve the exact configured credential from the gateway's approved profile. +/// Audience and scopes (including endpoint overrides) never come from the caller. +/// +/// The caller's key must name a configured key exactly, apart from the +/// generation a supervisor adds; a different or missing owner matches nothing. +async fn grant_configuration( + state: &ServerState, + sandbox_id: &str, + key: &str, +) -> Result { + let invalid = || token_grant_configuration_status(tonic::Code::PermissionDenied); + let key = + openshell_core::dynamic_credential_key::unscoped_owned_key(key).ok_or_else(invalid)?; + let identity = + openshell_core::dynamic_credential_key::credential_identity(&key).ok_or_else(invalid)?; + let (provider_name, _) = identity.split_once(':').ok_or_else(invalid)?; + let sandbox = state + .store + .get_message::(sandbox_id) + .await + .map_err(|_| token_grant_store_status())? + .ok_or_else(invalid)?; + let spec = sandbox.spec.as_ref().ok_or_else(invalid)?; + if !spec.providers.iter().any(|name| name == provider_name) { + return Err(invalid()); + } + let provider = state + .store + .get_message_by_name::(sandbox.object_workspace(), provider_name) + .await + .map_err(|_| token_grant_store_status())? + .ok_or_else(invalid)?; + let catalog = state + .provider_profile_sources + .profile_catalog(&state.store, sandbox.object_workspace(), &provider.r#type) + .await + .map_err(token_grant_dependency_status)?; + let profile = get_provider_type_profile_for_scope( + &catalog, + &provider.r#type, + &provider.profile_workspace, + ) + .ok_or_else(invalid)?; + let mut configured = HashMap::new(); + insert_dynamic_credentials_for_profile( + &mut configured, + &profile, + provider_name, + provider.object_id(), + )?; + let credential = configured.remove(&key).ok_or_else(invalid)?; + let grant = credential.token_grant.ok_or_else(invalid)?; + oauth::parse_token_endpoint_url(&grant.token_endpoint).map_err(TokenGrantFailure::to_status)?; + Ok(ConfiguredGrant { + provider, + grant, + key, + }) +} + +/// Bind a cached token to its runtime, configured credential, and subject. +/// +/// Include the provider revision and resolved subject token, so an in-place +/// rotation of a handle-backed subject never serves a token minted from the +/// previous one. The digest is an in-memory map key and is never exposed. +fn cache_key( + session: &AuthenticatedSandboxSession, + configured: &ConfiguredGrant, + subject: Option<&str>, + gateway_id: &str, +) -> Result { + let resource_version = configured + .provider + .metadata + .as_ref() + .map_or(0, |metadata| metadata.resource_version); + let fields: [&[u8]; 11] = [ + b"openshell-provider-access-token-v1", + gateway_id.as_bytes(), + session.sandbox_id.as_str().as_bytes(), + session.runtime_generation.as_str().as_bytes(), + &session.auth_epoch.get().to_be_bytes(), + configured.provider.object_id().as_bytes(), + &resource_version.to_be_bytes(), + configured.key.as_bytes(), + &Message::encode_to_vec(&configured.grant), + &[u8::from(subject.is_some())], + subject.map_or(&[], str::as_bytes), + ]; + digest_fields(&fields) + .map(hex::encode) + .map_err(|_| token_grant_configuration_status(tonic::Code::Internal)) +} + +/// SHA-256 of length-prefixed fields, so no two field lists share an encoding. +fn digest_fields(fields: &[&[u8]]) -> Result<[u8; 32], openshell_crypto::CryptoError> { + let mut digest = openshell_crypto::sha256_digest()?; + for field in fields { + digest.update(&u64::try_from(field.len()).unwrap_or(u64::MAX).to_be_bytes())?; + digest.update(field)?; + } + digest.finish() +} + +/// The subject token of a token exchange grant, resolved for this request. +struct Subject { + value: String, + token_type: String, + expires_at_ms: Option, +} + +#[allow(clippy::too_many_lines)] +async fn obtain( + state: &Arc, + request: Request, + request_id: Option<&str>, +) -> Result, Status> { + let Some(Principal::Sandbox(principal)) = request.extensions().get::() else { + return Err(token_grant_identity_rejected(tonic::Code::PermissionDenied)); + }; + let mut values = request.metadata().get_all("authorization").iter(); + let bearer = values + .next() + .and_then(|value| value.to_str().ok()) + .and_then(openshell_core::auth::strip_bearer_scheme) + .ok_or_else(|| token_grant_identity_rejected(tonic::Code::Unauthenticated))? + .to_string(); + if values.next().is_some() { + return Err(token_grant_identity_rejected(tonic::Code::Unauthenticated)); + } + let session = authorize(state, &bearer).await?; + if principal.sandbox_id != session.sandbox_id.as_str() { + return Err(token_grant_identity_rejected(tonic::Code::PermissionDenied)); + } + // Bound outstanding work after authentication, independently of caching. + // A busy sandbox cannot occupy the gateway's entire acquisition budget. + let _admission = ADMISSION.enter(session.sandbox_id.as_str())?; + // Startup enables an identity provider only with gateway session JWTs. + let (Some(identity), Some(gateway_jwt)) = ( + state.sandbox_identity.as_deref(), + state.config.gateway_jwt.as_ref(), + ) else { + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, + )); + }; + if state.compute.configured_driver_name() != "kubernetes" { + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, + )); + } + let gateway_id = gateway_jwt.gateway_id.as_str(); + let key = &request.get_ref().credential_key; + let configured = grant_configuration(state, session.sandbox_id.as_str(), key).await?; + let grant = &configured.grant; + let ttl_override = grant + .cache_ttl + .as_ref() + .map(openshell_core::time::duration_to_std) + .transpose() + .map_err(|_| token_grant_configuration_status(tonic::Code::InvalidArgument))?; + // Millisecond truncation and final authorization reads would consume tiny + // cache lifetimes before they could be returned. Zero remains no-cache. + if ttl_override.is_some_and(|ttl| !ttl.is_zero() && ttl < Duration::from_secs(1)) { + return Err(token_grant_configuration_status( + tonic::Code::InvalidArgument, + )); + } + let grant_type = ProviderCredentialTokenGrantType::try_from(grant.grant_type) + .map_err(|_| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; + // Resolve the subject before any cache lookup, so its revocation or + // expiry also denies cached tokens. + let subject = if grant_type == ProviderCredentialTokenGrantType::TokenExchange { + let subject = grant + .subject_token + .as_ref() + .filter(|subject| subject.source == "provider_credential") + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; + Some(Subject { + expires_at_ms: provider_credential_expiration_ms( + &configured.provider, + &subject.credential, + )?, + value: resolve_subject_token_credential( + &state.credentials, + &configured.provider, + &subject.credential, + ) + .await?, + token_type: subject.subject_token_type.clone(), + }) + } else { + None + }; + let subject_value = subject.as_ref().map(|subject| subject.value.as_str()); + let fingerprint = cache_key(&session, &configured, subject_value, gateway_id)?; + // A zero override disables caching, and with it shared acquisition. + let caching = ttl_override != Some(Duration::ZERO); + let mut refresh = false; + loop { + let cached = || { + (caching && !refresh) + .then(|| ACCESS_TOKENS.get_until(&fingerprint)) + .flatten() + .filter(|(_, until)| { + *until > crate::persistence::current_time_ms().saturating_add(1000) + }) + }; + // A miss holds its fingerprint's turn until its token is cached, so + // concurrent misses wait and then find it. A miss is cached only after + // the final validation below, which a hit passes as well. + let mut turn = None; + let (response, serve_until_ms, acquired) = if let Some((token, until)) = cached() { + (token, until, false) + } else { + let waited = if caching { + Some(ACQUISITIONS.turn(&fingerprint).await?) + } else { + None + }; + if let Some((token, until)) = cached() { + (token, until, false) + } else { + turn = waited; + let (token, until) = acquire( + state, + &bearer, + &session, + identity, + gateway_id, + grant, + subject.as_ref(), + ttl_override, + request_id, + ) + .await + .inspect_err(|status| { + if let Some(turn) = turn.as_mut() { + *turn.failure = Some(status.clone()); + } + })?; + (token, until, true) + } + }; + // Re-read attachment, provider revision, and profile, then the session, + // before serving or caching any token. + let current = grant_configuration(state, session.sandbox_id.as_str(), key).await?; + if cache_key(&session, ¤t, subject_value, gateway_id)? != fingerprint { + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, + )); + } + authorize(state, &bearer).await?; + // The lifetime runs from when the token service answered, so time spent + // in the final validation counts against it. + let remaining_ms = serve_until_ms.saturating_sub(crate::persistence::current_time_ms()); + if remaining_ms <= 0 && !acquired { + // A hit can expire during the final asynchronous checks. Refresh once; + // a fresh acquisition still must pass every check before it is served. + refresh = true; + continue; + } + if remaining_ms <= 0 { + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::Timeout, + TokenGrantStage::GatewayRpc, + ) + .to_status()); + } + if acquired && caching { + ACCESS_TOKENS.set(fingerprint, &response, serve_until_ms); + } + drop(turn); + return Ok(Response::new(GetSandboxProviderAccessTokenResponse { + access_token: response.access_token, + token_type: response.token_type, + expires_after: openshell_core::time::duration_from_std(Duration::from_millis( + u64::try_from(remaining_ms).unwrap_or_default(), + )) + .ok(), + })); + } +} + +/// Acquire a final token, re-authorizing the session between network calls. +/// Returns the token and when to stop serving it. +#[allow(clippy::too_many_arguments)] +async fn acquire( + state: &ServerState, + bearer: &str, + session: &AuthenticatedSandboxSession, + identity: &dyn SandboxIdentityProvider, + gateway_id: &str, + grant: &ProviderCredentialTokenGrant, + subject: Option<&Subject>, + ttl_override: Option, + request_id: Option<&str>, +) -> Result<(oauth::OAuthTokenResponse, i64), Status> { + authorize(state, bearer).await?; + let audience = effective_jwt_svid_audience(&grant.token_endpoint, &grant.jwt_svid_audience); + let sandbox = state + .store + .get_message::(session.sandbox_id.as_str()) + .await + .map_err(|_| token_grant_store_status())? + .ok_or_else(|| token_grant_identity_rejected(tonic::Code::Unauthenticated))?; + let runtime_identity = sandbox + .metadata + .as_ref() + .and_then(|metadata| { + metadata + .annotations + .get(crate::compute::COMPUTE_RUNTIME_IDENTITY_ANNOTATION) + }) + .map_or("", String::as_str); + let svid = identity + .fetch_jwt_svid( + gateway_id, + session.sandbox_id.as_str(), + &audience, + runtime_identity, + ) + .await + .map_err(TokenGrantFailure::to_status)?; + authorize(state, bearer).await?; + let client = token_exchange_http_client()?; + let subject_expiry = subject.and_then(|subject| subject.expires_at_ms); + let token = if let Some(subject) = subject { + let component_svid = fetch_gateway_jwt_svid_from( + &audience, + state + .config + .spiffe_broker + .as_ref() + .map(|broker| broker.workload_api_socket_path.as_str()), + ) + .await?; + let intermediate = perform_intermediate_token_exchange( + &grant.token_endpoint, + &TokenExchangeParams { + client_assertion: &component_svid, + client_assertion_type: &grant.client_assertion_type, + subject_token: &subject.value, + subject_token_type: &subject.token_type, + audience: &svid.spiffe_id, + scopes: &[], + requested_token_type: &grant.requested_token_type, + }, + request_id, + ) + .await?; + authorize(state, bearer).await?; + oauth::post_oauth_token_exchange( + client, + &grant.token_endpoint, + &TokenExchangeParams { + client_assertion: &svid.token, + client_assertion_type: &grant.client_assertion_type, + subject_token: &intermediate.access_token, + subject_token_type: oauth::ACCESS_TOKEN_TYPE, + audience: &grant.audience, + scopes: &grant.scopes, + requested_token_type: &grant.requested_token_type, + }, + request_id, + ) + .await + .map_err(TokenGrantFailure::to_status)? + } else { + oauth::post_oauth_token_grant( + client, + &grant.token_endpoint, + &oauth::TokenGrantParams { + client_assertion: &svid.token, + client_assertion_type: &grant.client_assertion_type, + audience: &grant.audience, + scopes: &grant.scopes, + }, + request_id, + ) + .await + .map_err(TokenGrantFailure::to_status)? + }; + let obtained_at_ms = crate::persistence::current_time_ms(); + let identity_expires_at_ms = session + .expires_at + .saturating_mul(1000) + .min(svid.expires_at.saturating_mul(1000)) + .min(subject_expiry.unwrap_or(i64::MAX)); + let serve_until_ms = + serve_until_ms(&token, ttl_override, obtained_at_ms, identity_expires_at_ms).ok_or_else( + || { + TokenGrantFailure::new( + TokenGrantFailureKind::Unclassified, + TokenGrantStage::TokenEndpoint, + ) + .to_status() + }, + )?; + authorize(state, bearer).await?; + Ok((token, serve_until_ms)) +} + +/// When to stop serving a token the token service issued at `obtained_at_ms`, +/// or `None` when it is already expired. +/// +/// The issuer's `expires_in`, a parseable JWT `exp` of the access token, and +/// the session, SVID, and subject expiries each bound the token; `exp` can +/// only shorten it. Serving stops the usual skew before the earliest bound, +/// after at least one second, and never later than the default cache lifetime +/// (whose fallback applies when the issuer omits `expires_in`). A configured +/// cache TTL may shorten that lifetime but, unlike a dedicated supervisor's +/// cache, never extends it. +fn serve_until_ms( + token: &oauth::OAuthTokenResponse, + ttl_override: Option, + obtained_at_ms: i64, + identity_expires_at_ms: i64, +) -> Option { + let issuer_expires_at_ms = (token.expires_in > 0) + .then(|| obtained_at_ms.saturating_add(token.expires_in.saturating_mul(1000))); + let expires_at_ms = [issuer_expires_at_ms, jwt_exp_ms(&token.access_token)] + .into_iter() + .flatten() + .fold(identity_expires_at_ms, i64::min); + if expires_at_ms <= obtained_at_ms { + return None; + } + let default_ttl = oauth::token_cache_ttl(None, token.expires_in); + let ttl = ttl_override + .filter(|ttl| !ttl.is_zero()) + .map_or(default_ttl, |ttl| ttl.min(default_ttl)); + let skewed_expiry_ms = expires_at_ms + .saturating_sub(oauth::TOKEN_CACHE_EXPIRY_SKEW_SECONDS.saturating_mul(1000)) + .max(obtained_at_ms.saturating_add(1000).min(expires_at_ms)); + Some( + obtained_at_ms + .saturating_add(i64::try_from(ttl.as_millis()).unwrap_or(i64::MAX)) + .min(skewed_expiry_ms), + ) +} + +type AcquisitionSlots = HashMap>>, usize)>; + +#[derive(Default)] +struct Admission { + active: std::sync::Mutex>, +} + +impl Admission { + fn enter(&self, sandbox: &str) -> Result, Status> { + let mut active = self + .active + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if active.values().sum::() >= 64 + || active.get(sandbox).copied().unwrap_or_default() >= 4 + { + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(tonic::Code::ResourceExhausted)); + } + *active.entry(sandbox.to_string()).or_default() += 1; + Ok(AdmissionGuard { + admission: self, + sandbox: sandbox.to_string(), + }) + } +} + +struct AdmissionGuard<'a> { + admission: &'a Admission, + sandbox: String, +} + +impl Drop for AdmissionGuard<'_> { + fn drop(&mut self) { + let mut active = self + .admission + .active + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(count) = active.get_mut(&self.sandbox) { + *count -= 1; + if *count == 0 { + active.remove(&self.sandbox); + } + } + } +} + +/// Coordinate token acquisition separately for each cache fingerprint. +/// +/// Concurrent misses share a single acquisition without locking unrelated +/// tokens. Current waiters receive the holder's acquisition failure; a new +/// group can retry after they leave. Entries exist only while held or awaited. +#[derive(Default)] +struct Acquisitions { + slots: std::sync::Mutex, +} + +impl Acquisitions { + async fn turn(&self, key: &str) -> Result, Status> { + let registration = { + let mut slots = self.lock(); + let (slot, users) = slots.entry(key.to_string()).or_default(); + *users += 1; + Registration { + acquisitions: self, + key: key.to_string(), + slot: slot.clone(), + } + }; + let failure = registration.slot.clone().lock_owned().await; + if let Some(status) = failure.as_ref() { + return Err(status.clone()); + } + Ok(AcquisitionTurn { + failure, + _registration: registration, + }) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, AcquisitionSlots> { + self.slots + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +struct Registration<'a> { + acquisitions: &'a Acquisitions, + key: String, + slot: Arc>>, +} + +impl Drop for Registration<'_> { + fn drop(&mut self) { + let mut slots = self.acquisitions.lock(); + if let Some((_, users)) = slots.get_mut(&self.key) { + *users -= 1; + if *users == 0 { + slots.remove(&self.key); + } + } + } +} + +/// Fields drop in order: the turn is released before its registration. +struct AcquisitionTurn<'a> { + failure: tokio::sync::OwnedMutexGuard>, + _registration: Registration<'a>, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::principal::{SandboxIdentitySource, SandboxPrincipal}; + + #[test] + fn admission_is_bounded_per_sandbox_and_releases_cancelled_work() { + let admission = Admission::default(); + let guards: Vec<_> = (0..4).map(|_| admission.enter("busy").unwrap()).collect(); + assert_eq!( + admission.enter("busy").err().unwrap().code(), + tonic::Code::ResourceExhausted + ); + let other = admission.enter("other").unwrap(); + drop(guards); + drop(admission.enter("busy").unwrap()); + drop(other); + assert!(admission.active.lock().unwrap().is_empty()); + let guards: Vec<_> = (0..64) + .map(|i| admission.enter(&i.to_string()).unwrap()) + .collect(); + assert!(admission.enter("overflow").is_err()); + drop(guards); + assert!(admission.active.lock().unwrap().is_empty()); + } + #[tokio::test] + async fn broker_requires_a_signed_session_even_with_a_sandbox_principal() { + let state = crate::grpc::test_support::test_server_state().await; + for principal in [ + Principal::Anonymous, + Principal::Sandbox(SandboxPrincipal { + sandbox_id: uuid::Uuid::new_v4().to_string(), + source: SandboxIdentitySource::BootstrapJwt { + issuer: "launch-session".into(), + auth_epoch: None, + }, + trust_domain: Some("openshell".into()), + }), + ] { + let mut request = Request::new(GetSandboxProviderAccessTokenRequest { + credential_key: "untrusted".into(), + }); + request.extensions_mut().insert(principal); + let error = handle(&state, request).await.unwrap_err(); + assert!(matches!( + error.code(), + tonic::Code::Unauthenticated | tonic::Code::PermissionDenied + )); + } + } + + #[test] + fn fingerprint_binds_the_runtime_configured_key_and_resolved_subject() { + use openshell_core::jwt::{CredentialEpoch, SandboxId}; + let session = AuthenticatedSandboxSession { + sandbox_id: SandboxId::parse(uuid::Uuid::new_v4().to_string()).unwrap(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "g1", + ) + .unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + token_id: uuid::Uuid::new_v4(), + issued_at: 1, + expires_at: 2, + }; + let configured = |key: &str| ConfiguredGrant { + provider: Provider::default(), + grant: ProviderCredentialTokenGrant::default(), + key: key.to_string(), + }; + let owned = "api.example.com\t443\t/v1/**\towner\tprovider:access"; + let fingerprint = |session, key, subject| { + cache_key(session, &configured(key), subject, "gateway").unwrap() + }; + let base = fingerprint(&session, owned, Some("subject-a")); + // A renewed session token of the same runtime shares cached tokens. + let renewed = AuthenticatedSandboxSession { + token_id: uuid::Uuid::new_v4(), + expires_at: 3, + ..session.clone() + }; + assert_eq!(fingerprint(&renewed, owned, Some("subject-a")), base); + // A rotated subject, another credential, or another epoch does not. + assert_ne!(fingerprint(&session, owned, Some("subject-b")), base); + let spoofed = owned.replace("owner", "other"); + assert_ne!(fingerprint(&session, &spoofed, Some("subject-a")), base); + let rotated = AuthenticatedSandboxSession { + auth_epoch: CredentialEpoch::new(2).unwrap(), + ..session.clone() + }; + assert_ne!(fingerprint(&rotated, owned, Some("subject-a")), base); + assert_ne!( + fingerprint(&session, owned, None), + fingerprint(&session, owned, Some("")) + ); + } + + #[test] + fn served_lifetime_never_exceeds_the_token_or_identity() { + const NOW: i64 = 1_700_000_000_000; + let token = |expires_in, access_token: &str| oauth::OAuthTokenResponse { + access_token: access_token.to_string(), + expires_in, + token_type: String::new(), + }; + let jwt = |exp_ms: i64| { + use base64::Engine as _; + let claims = format!(r#"{{"exp":{}}}"#, exp_ms / 1000); + let claims = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(claims); + format!("e30.{claims}.sig") + }; + let hour = Some(Duration::from_hours(1)); + let identity = NOW + 86_400_000; + let serve = |token, ttl, identity| serve_until_ms(&token, ttl, NOW, identity); + // A longer cache override is capped by the issuer's lifetime and skew. + assert_eq!( + serve(token(300, "opaque"), hour, identity), + Some(NOW + 270_000) + ); + // A shorter override still applies. + let minute = Some(Duration::from_mins(1)); + assert_eq!( + serve(token(300, "opaque"), minute, identity), + Some(NOW + 60_000) + ); + // A JWT exp only shortens the lifetime. + let short = jwt(NOW + 120_000); + assert_eq!( + serve(token(300, &short), hour, identity), + Some(NOW + 90_000) + ); + let long = jwt(NOW + 7_200_000); + assert_eq!( + serve(token(300, &long), None, identity), + Some(NOW + 270_000) + ); + // An omitted expires_in keeps the default fallback, even with an override. + assert_eq!( + serve(token(0, "opaque"), hour, identity), + Some(NOW + 270_000) + ); + // Zero disables caching, not the bound on serving. + let zero = Some(Duration::ZERO); + assert_eq!( + serve(token(300, "opaque"), zero, identity), + Some(NOW + 270_000) + ); + // Identity expiry caps the lifetime with skew but still serves briefly. + assert_eq!( + serve(token(300, "opaque"), None, NOW + 60_000), + Some(NOW + 30_000) + ); + assert_eq!( + serve(token(300, "opaque"), None, NOW + 10_000), + Some(NOW + 1_000) + ); + assert_eq!( + serve(token(300, "opaque"), None, NOW + 500), + Some(NOW + 500) + ); + // An already expired acquisition is never served. + assert_eq!(serve(token(300, &jwt(NOW)), None, identity), None); + assert_eq!(serve(token(300, "opaque"), None, NOW), None); + } + + #[tokio::test] + async fn concurrent_misses_take_turns_per_fingerprint_and_leave_no_entries() { + let acquisitions: &'static Acquisitions = Box::leak(Box::default()); + let first = acquisitions.turn("a").await.unwrap(); + // Another fingerprint is independent. + drop(acquisitions.turn("b").await.unwrap()); + // A waiter for the same fingerprint blocks; cancelling it leaves only + // the holder's entry. + let waiting = tokio::time::timeout(Duration::from_millis(50), acquisitions.turn("a")); + assert!(waiting.await.is_err()); + assert_eq!( + acquisitions.lock().get("a").map(|(_, users)| *users), + Some(1) + ); + assert_eq!(acquisitions.lock().len(), 1); + let waiter = tokio::spawn(async move { drop(acquisitions.turn("a").await.unwrap()) }); + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + drop(first); + waiter.await.unwrap(); + assert!(acquisitions.lock().is_empty()); + } + + #[tokio::test] + async fn failed_acquisition_is_shared_with_waiters_but_not_later_requests() { + let acquisitions: &'static Acquisitions = Box::leak(Box::default()); + let mut holder = acquisitions.turn("grant").await.unwrap(); + let waiters: Vec<_> = (0..3) + .map(|_| tokio::spawn(async move { acquisitions.turn("grant").await })) + .collect(); + tokio::time::timeout(Duration::from_secs(5), async { + while acquisitions.lock().get("grant").map(|(_, users)| *users) != Some(4) { + tokio::task::yield_now().await; + } + }) + .await + .expect("all requests should join the in-flight acquisition"); + let expected = TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::TokenEndpoint, + ); + *holder.failure = Some(expected.to_status()); + drop(holder); + for waiter in waiters { + let result = tokio::time::timeout(Duration::from_secs(5), waiter) + .await + .expect("waiters should fail without another acquisition") + .unwrap(); + let Err(status) = result else { + panic!("a waiter started another acquisition after the holder failed"); + }; + assert_eq!(TokenGrantFailure::from_status(&status), expected); + } + assert!(acquisitions.lock().is_empty()); + drop(acquisitions.turn("grant").await.unwrap()); + assert!(acquisitions.lock().is_empty()); + } +} diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 3ce36b14bb..10a0f0d409 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -48,6 +48,7 @@ mod refresh_store; mod sandbox_index; mod sandbox_watch; mod service_routing; +mod spiffe_broker; mod ssh_identity; mod ssh_sessions; mod storage_proto; @@ -74,11 +75,13 @@ use hyper_util::{ }; use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::telemetry::TelemetryComputeDriver; +use openshell_core::token_grant_failure::TokenGrantFailure; use openshell_core::{Config, Error, NetworkSupervisorTrustBundle, ObjectLabels, Result}; use openshell_extension_core::{ BearerTokenSlot, ExtensionAudience, ExtensionCallerKind, ExtensionKind, MAX_EXTENSION_TOKEN_TTL, }; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId}; +use openshell_spiffe::SandboxIdentityProvider; use openshell_supervisor_middleware::MiddlewareRegistry; use std::collections::{BTreeMap, HashMap}; use std::io::ErrorKind; @@ -430,6 +433,7 @@ pub struct ServerState { /// Gateway-wide gRPC request rate limiter shared by every multiplex path. pub(crate) grpc_rate_limiter: Option, + pub(crate) provider_token_rate_limiter: Option, /// Immutable public reflection index and its per-query rate limiter. pub(crate) reflection_service: reflection::GatewayReflectionServer, @@ -446,6 +450,11 @@ pub struct ServerState { /// on demand when the user source is configured. pub(crate) provider_profile_sources: provider_profile_sources::ProviderProfileSources, + /// Source of sandbox JWT-SVIDs for brokered provider tokens, built once + /// from the SPIRE broker settings. `None` when disabled. + pub(crate) sandbox_identity: + Option>>, + /// OIDC admin role name for workspace-level authorization. /// Empty when OIDC is not configured — `authorize_workspace()` treats /// every authenticated user as Platform Admin in that case. @@ -595,6 +604,12 @@ fn is_benign_connection_close(error: &(dyn std::error::Error + 'static)) -> bool impl ServerState { /// Create new server state. + /// + /// # Panics + /// + /// Panics when `config` holds credential-driver or SPIRE broker + /// settings that startup validation would reject. Gateway startup uses + /// [`Self::new_with_credentials`], which reports those as errors. #[must_use] #[allow(clippy::too_many_arguments)] pub fn new( @@ -621,10 +636,18 @@ impl ServerState { oidc_cache, credentials, ) + .expect("server config should be validated before ServerState::new") } /// Create new server state with an already-initialized credential runtime. - #[must_use] + /// + /// Parses the SPIRE broker settings into the state that serves every + /// provider access-token request. + /// + /// # Errors + /// + /// Returns a configuration error when the SPIRE broker settings are + /// invalid. #[allow(clippy::too_many_arguments)] pub fn new_with_credentials( config: Config, @@ -636,7 +659,9 @@ impl ServerState { supervisor_sessions: Arc, oidc_cache: Option>, credentials: credentials::CredentialRuntime, - ) -> Self { + ) -> Result { + let sandbox_identity = + spiffe_broker::sandbox_identity_provider(&config).map_err(Error::config)?; let replica_id = compute::lease::replica_id(); let compute = compute.with_sandbox_updates(config.enable_sandbox_updates); compute.configure_ssh_identities(credentials.clone()); @@ -646,6 +671,7 @@ impl ServerState { compute.configure_service_route_index(Arc::clone(&service_route_index)); let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); + let provider_token_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); let reflection_service = reflection::build_gateway_reflection_service(&config) .expect("compiled public gateway descriptors must be valid"); let admin_role = config @@ -655,7 +681,7 @@ impl ServerState { let config_delivery = config_delivery::ConfigDelivery::for_db_connections(store.max_connections()); let refresh = provider_refresh::runtime(&store, Some(&credentials), Some(&compute)); - Self { + Ok(Self { config, store, compute, @@ -690,12 +716,14 @@ impl ServerState { peer_authenticator: None, logons: auth::audit::Logons::default(), grpc_rate_limiter, + provider_token_rate_limiter, reflection_service, gateway_interceptors: None, provider_profile_sources: provider_profile_sources::ProviderProfileSources::with_default_sources(), + sandbox_identity, admin_role, - } + }) } } @@ -987,7 +1015,7 @@ pub(crate) async fn run_server( supervisor_sessions, oidc_cache, credentials, - ); + )?; state.middleware_registry = middleware_registry; state.gateway_interceptors = gateway_interceptors; state.provider_profile_sources = provider_profile_sources; diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 9b947859f2..0479e7a443 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -346,8 +346,9 @@ impl MultiplexService { .with_operator(operator) .with_connection_source(source) .with_logons(self.state.logons.clone()); - let grpc_service = + let mut grpc_service = GrpcRateLimitService::new(grpc_service, self.state.grpc_rate_limiter.clone()); + grpc_service.provider_token_limiter = self.state.provider_token_rate_limiter.clone(); let http_service = http_router(self.state.clone()); let grpc_service = request_id_middleware!(grpc_service); @@ -928,6 +929,7 @@ impl GrpcRateLimiter { struct GrpcRateLimitService { inner: S, limiter: Option, + provider_token_limiter: Option, /// Set by `poll_ready` when it reports synthetic readiness for an /// exhausted limiter without polling the inner service. The paired `call` /// must then reject with `RESOURCE_EXHAUSTED` instead of forwarding to an @@ -942,6 +944,7 @@ impl GrpcRateLimitService { Self { inner, limiter, + provider_token_limiter: None, rate_limited: false, } } @@ -972,6 +975,10 @@ where .limiter .as_ref() .is_some_and(|limiter| !limiter.has_capacity()) + && self + .provider_token_limiter + .as_ref() + .is_none_or(|limiter| !limiter.has_capacity()) { self.rate_limited = true; return Poll::Ready(Ok(())); @@ -991,11 +998,15 @@ where tonic::Status::resource_exhausted("gRPC rate limit exceeded").into_http(); return Box::pin(async move { Ok(response) }); } - if self - .limiter - .as_ref() - .is_some_and(|limiter| !limiter.allow()) + let limiter = if req.uri().path() == "/openshell.v1.OpenShell/GetSandboxProviderAccessToken" { + self.provider_token_limiter + .as_ref() + .or(self.limiter.as_ref()) + } else { + self.limiter.as_ref() + }; + if limiter.is_some_and(|limiter| !limiter.allow()) { let response = tonic::Status::resource_exhausted("gRPC rate limit exceeded").into_http(); return Box::pin(async move { Ok(response) }); @@ -2425,6 +2436,55 @@ mod tests { ); } + #[tokio::test] + async fn provider_token_budget_is_shared_across_connections_and_isolated_from_control() { + let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60)); + let control = GrpcRateLimiter::from_config(&config); + let tokens = GrpcRateLimiter::from_config(&config); + let calls = Arc::new(AtomicUsize::new(0)); + let mut service = GrpcRateLimitService::new(CountingGrpcService { calls }, control); + service.provider_token_limiter = tokens; + let mut other_connection = service.clone(); + let token_request = || { + Request::builder() + .uri("/openshell.v1.OpenShell/GetSandboxProviderAccessToken") + .body(()) + .unwrap() + }; + let first = service + .ready() + .await + .unwrap() + .call(token_request()) + .await + .unwrap(); + assert_eq!(grpc_status_from_response(&first), "0"); + let exhausted = other_connection + .ready() + .await + .unwrap() + .call(token_request()) + .await + .unwrap(); + assert_eq!(grpc_status_from_response(&exhausted), "8"); + let control = service + .ready() + .await + .unwrap() + .call(Request::new(())) + .await + .unwrap(); + assert_eq!(grpc_status_from_response(&control), "0"); + let exhausted = service + .ready() + .await + .unwrap() + .call(Request::new(())) + .await + .unwrap(); + assert_eq!(grpc_status_from_response(&exhausted), "8"); + } + #[tokio::test] async fn grpc_rate_limit_returns_resource_exhausted_after_limit() { let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60)); diff --git a/crates/openshell-server/src/provider_profile_sources.rs b/crates/openshell-server/src/provider_profile_sources.rs index 77230e5cce..0c5715def0 100644 --- a/crates/openshell-server/src/provider_profile_sources.rs +++ b/crates/openshell-server/src/provider_profile_sources.rs @@ -76,6 +76,29 @@ pub trait ProviderProfileSource: Send + Sync + std::fmt::Debug { store: &Store, workspace: &str, ) -> Result; + + /// Read one profile without rebuilding unrelated user-managed profiles. + async fn snapshot_profile( + &self, + store: &Store, + workspace: &str, + id: &str, + ) -> Result { + let mut snapshot = self.snapshot(store, workspace).await?; + // Uncached sources must still validate their complete contract. Cached + // sources validate on refresh and override this with an indexed read. + build_effective_profiles(vec![CollectedProviderProfileSnapshot { + source_id: self.source_id().to_string(), + revision: snapshot.revision.clone(), + profiles: snapshot.profiles.clone(), + user_managed: self.user_managed(), + allow_empty: self.allow_empty(), + }])?; + snapshot + .profiles + .retain(|profile| normalize_profile_id(&profile.profile.id).as_deref() == Some(id)); + Ok(snapshot) + } } #[derive(Debug, Clone, Default)] @@ -95,6 +118,44 @@ impl ProviderProfileSource for UserProviderProfileSource { true } + async fn snapshot_profile( + &self, + store: &Store, + workspace: &str, + id: &str, + ) -> Result { + let mut profiles = Vec::with_capacity(2); + for (namespace, scope) in [ + ("", ProfileScope::Platform), + (workspace, ProfileScope::Workspace), + ] { + if namespace.is_empty() && scope == ProfileScope::Workspace { + continue; + } + if let Some(stored) = store + .get_message_by_name::(namespace, id) + .await + .map_err(|error| { + Status::internal(format!("fetch provider profile failed: {error}")) + })? + { + let version = stored_profile_resource_version(&stored); + if let Some(profile) = stored.profile { + let mut profile = profile_response_payload(profile, version); + normalize_provider_profile_mcp_fields(&mut profile); + profiles.push(ScopedSnapshotProfile { scope, profile }); + } + } + } + let revision = profile_snapshot_revision( + &profiles + .iter() + .map(|entry| entry.profile.clone()) + .collect::>(), + ); + Ok(ProviderProfileSnapshot { revision, profiles }) + } + async fn snapshot( &self, store: &Store, @@ -207,6 +268,8 @@ struct CachedProviderProfileSource { struct CachedSnapshot { snapshot: ProviderProfileSnapshot, fetched_at: Instant, + profiles_by_id: BTreeMap>, + validation: Result<(), Status>, } impl CachedProviderProfileSource { @@ -219,10 +282,26 @@ impl CachedProviderProfileSource { } /// Fetch a new snapshot. Returns whether its revision differs from the - /// cached one; a failure keeps serving the cached snapshot until it ages - /// out. + /// cached one. Transport failures keep the cache until it ages out; + /// invalid snapshots revoke it immediately. async fn refresh(&self, store: &Store) -> Result { let snapshot = self.inner.snapshot(store, "").await?; + // Validate the complete source once when it changes, not on every + // provider-token request. Invalid snapshots revoke cached authorization. + let validation = build_effective_profiles(vec![CollectedProviderProfileSnapshot { + source_id: self.source_id().to_string(), + revision: snapshot.revision.clone(), + profiles: snapshot.profiles.clone(), + user_managed: self.user_managed(), + allow_empty: self.allow_empty(), + }]) + .map(drop); + let mut profiles_by_id: BTreeMap> = BTreeMap::new(); + for profile in &snapshot.profiles { + if let Some(id) = normalize_profile_id(&profile.profile.id) { + profiles_by_id.entry(id).or_default().push(profile.clone()); + } + } let changed = self .cached .load() @@ -231,8 +310,10 @@ impl CachedProviderProfileSource { self.cached.store(Some(Arc::new(CachedSnapshot { snapshot, fetched_at: Instant::now(), + profiles_by_id, + validation: validation.clone(), }))); - Ok(changed) + validation.map(|()| changed) } fn age(&self) -> Option { @@ -257,6 +338,33 @@ impl ProviderProfileSource for CachedProviderProfileSource { self.inner.allow_empty() } + async fn snapshot_profile( + &self, + _store: &Store, + _workspace: &str, + id: &str, + ) -> Result { + let cached = self.cached.load(); + let cached = cached + .as_ref() + .filter(|cached| cached.fetched_at.elapsed() <= self.max_age) + .ok_or_else(|| { + openshell_core::rpc_error::unavailable( + "PROFILE_SOURCE_UNAVAILABLE", + format!( + "provider profile source '{}' has no current snapshot", + self.source_id() + ), + Duration::from_secs(1), + ) + })?; + cached.validation.clone()?; + Ok(ProviderProfileSnapshot { + revision: cached.snapshot.revision.clone(), + profiles: cached.profiles_by_id.get(id).cloned().unwrap_or_default(), + }) + } + async fn snapshot( &self, _store: &Store, @@ -505,6 +613,29 @@ impl ProviderProfileSources { Ok(catalog) } + pub(crate) async fn profile_catalog( + &self, + store: &Store, + workspace: &str, + id: &str, + ) -> Result { + let id = normalize_profile_id(id) + .ok_or_else(|| Status::failed_precondition("invalid provider profile id"))?; + let mut snapshots = Vec::with_capacity(self.sources.len()); + for source in &self.sources { + let snapshot = source.snapshot_profile(store, workspace, &id).await?; + snapshots.push(CollectedProviderProfileSnapshot { + source_id: source.source_id().to_string(), + revision: snapshot.revision, + profiles: snapshot.profiles, + user_managed: source.user_managed(), + // A valid non-empty source may not contain this particular ID. + allow_empty: true, + }); + } + build_effective_profiles(snapshots) + } + async fn snapshots( &self, store: &Store, @@ -919,7 +1050,6 @@ fn format_diagnostic(diagnostic: ProfileValidationDiagnostic) -> String { } } -#[cfg(test)] fn profile_snapshot_revision(profiles: &[ProviderProfile]) -> String { let mut profiles = profiles.to_vec(); profiles @@ -1957,6 +2087,90 @@ mod tests { ); } + #[tokio::test] + async fn point_lookup_preserves_scopes_and_excludes_unrelated_profiles() { + let store = crate::persistence::test_store().await; + for (id, workspace) in [ + ("selected", ""), + ("selected", "alpha"), + ("unrelated", "alpha"), + ("beta-only", "beta"), + ] { + store + .put_message(&stored_profile_in_workspace(id, workspace)) + .await + .unwrap(); + } + let sources = ProviderProfileSources::with_default_sources(); + let full = sources.snapshot_catalog(&store, "alpha").await.unwrap(); + let selected = sources + .profile_catalog(&store, "alpha", "selected") + .await + .unwrap(); + for scope in ["", "alpha"] { + assert_eq!( + selected.get_type_profile_for_scope("selected", scope), + full.get_type_profile_for_scope("selected", scope) + ); + } + assert_eq!(selected.profiles.len(), 1); + assert!( + sources + .profile_catalog(&store, "alpha", "beta-only") + .await + .unwrap() + .get_profile("beta-only") + .is_none() + ); + } + + #[tokio::test] + async fn point_lookup_preserves_source_authority_and_collision_checks() { + let store = crate::persistence::test_store().await; + store + .put_message(&stored_profile_in_workspace("user-only", "alpha")) + .await + .unwrap(); + let sources = + ProviderProfileSources::from_test_profiles_with_user_source(vec![profile("external")]); + assert!( + sources + .profile_catalog(&store, "alpha", "user-only") + .await + .unwrap() + .get_profile("user-only") + .is_some() + ); + assert!( + sources + .profile_catalog(&store, "alpha", "external") + .await + .unwrap() + .get_profile("external") + .is_some() + ); + let invalid_source = ProviderProfileSources::from_test_profiles_with_user_source(vec![ + profile("external"), + profile(""), + ]); + assert!( + invalid_source + .profile_catalog(&store, "alpha", "external") + .await + .is_err() + ); + store + .put_message(&stored_profile_in_workspace("external", "alpha")) + .await + .unwrap(); + assert!( + sources + .profile_catalog(&store, "alpha", "external") + .await + .is_err() + ); + } + #[tokio::test] async fn cross_workspace_duplicate_profile_ids_do_not_collide() { let store = crate::persistence::test_store().await; @@ -2519,6 +2733,12 @@ mod tests { let error = cached.snapshot(&store, "default").await.unwrap_err(); assert_eq!(error.code(), tonic::Code::Unavailable); + assert!( + cached + .snapshot_profile(&store, "default", "github") + .await + .is_err() + ); assert_eq!(inner.fetches.load(Ordering::SeqCst), 0); } @@ -2534,6 +2754,23 @@ mod tests { cached.snapshot(&store, "default").await.unwrap().revision, "r1" ); + assert_eq!( + cached + .snapshot_profile(&store, "default", "github") + .await + .unwrap() + .profiles + .len(), + 1 + ); + assert!( + cached + .snapshot_profile(&store, "default", "missing") + .await + .unwrap() + .profiles + .is_empty() + ); } assert_eq!(inner.fetches.load(Ordering::SeqCst), 1); @@ -2565,5 +2802,11 @@ mod tests { tokio::time::advance(Duration::from_secs(11)).await; let error = cached.snapshot(&store, "default").await.unwrap_err(); assert_eq!(error.code(), tonic::Code::Unavailable); + assert!( + cached + .snapshot_profile(&store, "default", "github") + .await + .is_err() + ); } } diff --git a/crates/openshell-server/src/spiffe_broker.rs b/crates/openshell-server/src/spiffe_broker.rs new file mode 100644 index 0000000000..886b2b4902 --- /dev/null +++ b/crates/openshell-server/src/spiffe_broker.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Startup selection of the gateway's sandbox identity provider. +//! +//! The provider broker depends only on +//! [`SandboxIdentityProvider`]; this module chooses the SPIRE adapter when +//! `[openshell.gateway.spiffe_broker]` is configured. +use std::sync::Arc; + +use openshell_core::config::{Config, SpiffeBrokerConfig}; +use openshell_core::token_grant_failure::TokenGrantFailure; +use openshell_spiffe::SandboxIdentityProvider; + +/// Check the broker settings `config` enables without building them. +/// +/// # Errors +/// +/// Rejects broker without gateway session JWTs and a positive session +/// TTL, and settings [`SpiffeBrokerConfig::validate`] rejects. +pub fn validate_config(config: &Config) -> Result<(), String> { + enabled(config).map(drop) +} + +/// The sandbox identity provider `config` enables, if any. +/// +/// # Errors +/// +/// Rejects what [`validate_config`] rejects. +pub fn sandbox_identity_provider( + config: &Config, +) -> Result>>, String> { + let Some(broker) = enabled(config)? else { + return Ok(None); + }; + let provider = openshell_spire::SpireSandboxIdentity::new(broker)?; + Ok(Some(Arc::new(provider))) +} + +/// Validated broker settings, when broker is enabled. +fn enabled(config: &Config) -> Result, String> { + let Some(broker) = &config.spiffe_broker else { + return Ok(None); + }; + let jwt = config + .gateway_jwt + .as_ref() + .ok_or("spiffe_broker requires gateway session JWTs")?; + broker.validate(&jwt.gateway_id)?; + if jwt.ttl_secs.is_none() { + return Err("spiffe_broker requires a positive gateway_jwt.ttl_secs".into()); + } + Ok(Some(broker)) +} diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 723cbed928..ab273786e5 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -179,7 +179,7 @@ mod tests { // updates use a separate capability/service and add one classified RPC. // Discovery RPCs move to ServiceDiscovery; message and durable closures are unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "daa132037ee47c7a46b08dfab9bab379961bc06ff3e9ebbb329809b724f2a4f6"; + "711a31c11baa7547d967a9a46d0932d92f08121d4195b6f29fb192d7c1ec25a7"; const DURABLE_SCHEMA_SHA256: &str = "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -655,7 +655,7 @@ mod tests { } assert_eq!( compiled_method_count, - 113 + PROVIDER_READINESS_RPC_SIGNATURES.len() + 114 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len() + PEER_CONFIG_RPC_SIGNATURES.len() + isolation_methods.len() @@ -665,7 +665,7 @@ mod tests { ); assert_eq!( methods.len(), - 83 + PROVIDER_READINESS_RPC_SIGNATURES.len() + 84 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len() + PEER_CONFIG_RPC_SIGNATURES.len(), "inventory every public gateway RPC" @@ -675,7 +675,7 @@ mod tests { .iter() .filter(|method| method.starts_with("openshell.v1.OpenShell/")) .count(), - 80 + PROVIDER_READINESS_RPC_SIGNATURES.len() + 81 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len() + PEER_CONFIG_RPC_SIGNATURES.len() ); @@ -726,7 +726,7 @@ mod tests { overlap_hash.as_str(), ), ( - (344, 31), + (346, 31), (100, 22), (86, 22), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-server/tests/common/mod.rs b/crates/openshell-server/tests/common/mod.rs index dd91664d39..235c300141 100644 --- a/crates/openshell-server/tests/common/mod.rs +++ b/crates/openshell-server/tests/common/mod.rs @@ -52,6 +52,14 @@ pub struct TestOpenShell; #[tonic::async_trait] impl OpenShell for TestOpenShell { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-server/tests/supervisor_relay_integration.rs b/crates/openshell-server/tests/supervisor_relay_integration.rs index ceb2fa6bf8..dfe68af972 100644 --- a/crates/openshell-server/tests/supervisor_relay_integration.rs +++ b/crates/openshell-server/tests/supervisor_relay_integration.rs @@ -50,6 +50,14 @@ struct RelayGateway { #[tonic::async_trait] impl OpenShell for RelayGateway { + async fn get_sandbox_provider_access_token( + &self, + _request: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("not used by this test server")) + } + async fn peer_notify_config_update( &self, _request: tonic::Request, diff --git a/crates/openshell-spiffe/Cargo.toml b/crates/openshell-spiffe/Cargo.toml new file mode 100644 index 0000000000..258e062841 --- /dev/null +++ b/crates/openshell-spiffe/Cargo.toml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-spiffe" +description = "SPIFFE identity types, parsing, and sandbox identity provider interface for OpenShell" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +async-trait = "0.1" +base64 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-spiffe/README.md b/crates/openshell-spiffe/README.md new file mode 100644 index 0000000000..ac13366591 --- /dev/null +++ b/crates/openshell-spiffe/README.md @@ -0,0 +1,25 @@ +# openshell-spiffe + +Dependency-light SPIFFE types shared by the gateway, `openshell-core` +configuration validation, and the supervisor. It has no cryptographic +implementation, SPIRE client, or `openshell-core` dependency, so core can +depend on it without a cycle. + +## Contents + +- `SandboxSpiffeIdPrefix` maps a gateway and sandbox to the stable identity + `/gateway//sandbox/`. Both IDs must already + be SPIFFE path segments; nothing is encoded, so distinct IDs always map to + distinct SPIFFE IDs. `SpiffeBrokerConfig::validate` in `openshell-core` + uses it. +- `SandboxIdentityProvider` is the gateway's interface for obtaining a + `SandboxJwtSvid` verified for that stable identity and one audience, using + the gateway's persisted driver runtime identity as the workload reference. The + error is an associated type: the SPIRE adapter in `openshell-spire` uses the + sanitized `TokenGrantFailure`. `SandboxJwtSvid` omits the token from `Debug` + output. +- `workload_api_endpoint`, `trust_domain`, and `is_spiffe_path_segment` handle + Workload API endpoints and SPIFFE ID parts. +- `parse_unverified_jwt_svid_claims` decodes JWT-SVID claims without checking + the signature. Callers must verify the token separately before trusting + them. diff --git a/crates/openshell-spiffe/src/lib.rs b/crates/openshell-spiffe/src/lib.rs new file mode 100644 index 0000000000..07ea123e43 --- /dev/null +++ b/crates/openshell-spiffe/src/lib.rs @@ -0,0 +1,338 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared SPIFFE helpers used by the gateway and sandbox supervisor. +//! +//! Protocol types, unverified claim parsing, stable sandbox identity mapping, +//! and the sandbox identity provider interface. Signature verification and +//! SPIRE access belong to the crates that implement the interface. + +use std::fmt; +use std::path::Path; + +use base64::Engine as _; +use serde::Deserialize; + +/// SPIFFE JWT-SVID claims used by `OpenShell` token exchange flows. +#[derive(Debug, Clone, Deserialize)] +pub struct SpiffeJwtClaims { + pub iss: String, + pub sub: String, + pub aud: AudienceClaim, + #[serde(default)] + pub exp: i64, +} + +/// JWT `aud` claim representation accepted by SPIFFE JWT-SVIDs. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum AudienceClaim { + One(String), + Many(Vec), +} + +impl AudienceClaim { + pub fn contains(&self, expected: &str) -> bool { + match self { + Self::One(value) => value == expected, + Self::Many(values) => values.iter().any(|value| value == expected), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum JwtSvidParseError { + #[error("invalid JWT-SVID format")] + Format, + #[error("invalid JWT-SVID payload encoding")] + PayloadEncoding, + #[error("invalid JWT-SVID payload")] + Payload, +} + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} + +/// # Security +/// +/// This function decodes the JWT payload without verifying the signature. +/// Callers must separately verify the JWT signature before trusting the claims. +pub fn parse_unverified_jwt_svid_claims(token: &str) -> Result { + let segments = token.split('.').collect::>(); + if segments.len() != 3 || segments.iter().any(|segment| segment.is_empty()) { + return Err(JwtSvidParseError::Format); + } + let decoded = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(segments[1]) + .map_err(|_| JwtSvidParseError::PayloadEncoding)?; + serde_json::from_slice::(&decoded).map_err(|_| JwtSvidParseError::Payload) +} + +pub fn trust_domain(subject: &str) -> Option<&str> { + let rest = subject.strip_prefix("spiffe://")?; + let (trust_domain, _) = rest.split_once('/').unwrap_or((rest, "")); + (!trust_domain.is_empty()).then_some(trust_domain) +} + +/// Longest SPIFFE ID the SPIFFE specification permits. +const MAX_SPIFFE_ID_BYTES: usize = 2048; + +/// Operator-configured prefix of the stable SPIFFE IDs the gateway requests +/// for sandboxes using shared supervision. +/// +/// A sandbox's ID is `/gateway//sandbox/`. +/// Both IDs must already be valid SPIFFE path segments; nothing is encoded, +/// so distinct gateway and sandbox IDs always produce distinct SPIFFE IDs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SandboxSpiffeIdPrefix(String); + +impl SandboxSpiffeIdPrefix { + /// Accept `spiffe://` optionally followed by path segments, + /// with no trailing slash, query, fragment, or percent-encoding. + pub fn parse(value: &str) -> Result { + let invalid = || { + format!( + "SPIFFE ID prefix must be spiffe://[/...] with lowercase trust domain characters [a-z0-9.-_] and path segment characters [A-Za-z0-9.-_]: {value:?}" + ) + }; + let rest = value.strip_prefix("spiffe://").ok_or_else(invalid)?; + let (domain, path) = rest + .split_once('/') + .map_or((rest, None), |(domain, path)| (domain, Some(path))); + if domain.is_empty() + || !domain.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b".-_".contains(&byte) + }) + { + return Err(invalid()); + } + if let Some(path) = path + && !path.split('/').all(is_spiffe_path_segment) + { + return Err(invalid()); + } + Ok(Self(value.to_string())) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Stable SPIFFE ID of `sandbox_id` under `gateway_id`. + pub fn sandbox_spiffe_id(&self, gateway_id: &str, sandbox_id: &str) -> Result { + if !is_spiffe_path_segment(gateway_id) || !is_spiffe_path_segment(sandbox_id) { + return Err( + "gateway and sandbox IDs must be SPIFFE path segments [A-Za-z0-9.-_]".to_string(), + ); + } + let id = format!("{}/gateway/{gateway_id}/sandbox/{sandbox_id}", self.0); + if id.len() > MAX_SPIFFE_ID_BYTES { + return Err("sandbox SPIFFE ID exceeds 2048 bytes".to_string()); + } + Ok(id) + } +} + +/// A JWT-SVID verified for a sandbox's stable SPIFFE ID and one audience. +#[derive(Clone)] +pub struct SandboxJwtSvid { + /// The stable SPIFFE ID the token was issued and verified for. + pub spiffe_id: String, + /// The signed token. It is a credential and never printed. + pub token: String, + /// The verified `exp` claim, in Unix seconds. + pub expires_at: i64, +} + +impl fmt::Debug for SandboxJwtSvid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SandboxJwtSvid") + .field("spiffe_id", &self.spiffe_id) + .field("token", &"[REDACTED]") + .field("expires_at", &self.expires_at) + .finish() + } +} + +/// Source of verified sandbox JWT-SVIDs for the gateway's provider broker. +/// +/// The gateway authorizes the sandbox session and the provider credential +/// before each request; implementations own issuance and verification only. +#[async_trait::async_trait] +pub trait SandboxIdentityProvider: fmt::Debug + Send + Sync { + /// Failure the implementation reports. It must not carry credentials or + /// upstream diagnostics. + type Error; + + /// Obtain a JWT-SVID for `audience` whose identity is exactly the stable + /// SPIFFE ID of `sandbox_id` under `gateway_id`. + /// `runtime_identity` is the compute driver's immutable runtime reference + /// persisted by the gateway. It must never come from a workload request; + /// adapters use it to bind issuance to the attested backing object. + /// + /// # Errors + /// + /// Fails when issuance or verification fails, or when the IDs cannot + /// form a stable SPIFFE ID. + async fn fetch_jwt_svid( + &self, + gateway_id: &str, + sandbox_id: &str, + audience: &str, + runtime_identity: &str, + ) -> Result; +} + +/// A non-empty SPIFFE path segment other than `.` and `..`. +#[must_use] +pub fn is_spiffe_path_segment(segment: &str) -> bool { + !segment.is_empty() + && segment != "." + && segment != ".." + && segment + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b".-_".contains(&byte)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn unsigned_svid_fixture(issuer: &str, subject: &str, audience: serde_json::Value) -> String { + let header = serde_json::json!({ "alg": "RS256", "kid": "test-key" }); + let payload = serde_json::json!({ + "iss": issuer, + "sub": subject, + "aud": audience, + "exp": 4_102_444_800_i64 + }); + let encoded_header = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_vec(&header).expect("serialize header")); + let encoded_payload = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(serde_json::to_vec(&payload).expect("serialize payload")); + format!("{encoded_header}.{encoded_payload}.signature") + } + + #[test] + fn workload_api_endpoint_preserves_explicit_scheme() { + assert_eq!( + workload_api_endpoint(Path::new("unix:/run/spire/agent.sock")), + "unix:/run/spire/agent.sock" + ); + assert_eq!( + workload_api_endpoint(Path::new("tcp:127.0.0.1:8081")), + "tcp:127.0.0.1:8081" + ); + } + + #[test] + fn workload_api_endpoint_defaults_to_unix_socket() { + assert_eq!( + workload_api_endpoint(Path::new("/run/spire/agent.sock")), + "unix:/run/spire/agent.sock" + ); + } + + #[test] + fn parse_unverified_jwt_svid_claims_accepts_string_audience() { + let token = unsigned_svid_fixture( + "https://spiffe.example.test", + "spiffe://openshell/openshell/sandbox/sb-a", + serde_json::json!("https://auth.example.com"), + ); + + let claims = parse_unverified_jwt_svid_claims(&token).expect("valid claims"); + + assert_eq!(claims.iss, "https://spiffe.example.test"); + assert_eq!(claims.sub, "spiffe://openshell/openshell/sandbox/sb-a"); + assert!(claims.aud.contains("https://auth.example.com")); + assert!(!claims.aud.contains("https://other.example.com")); + } + + #[test] + fn parse_unverified_jwt_svid_claims_accepts_array_audience() { + let token = unsigned_svid_fixture( + "https://spiffe.example.test", + "spiffe://openshell/openshell/sandbox/sb-a", + serde_json::json!(["https://auth.example.com", "https://other.example.com"]), + ); + + let claims = parse_unverified_jwt_svid_claims(&token).expect("valid claims"); + + assert!(claims.aud.contains("https://auth.example.com")); + assert!(claims.aud.contains("https://other.example.com")); + } + + #[test] + fn parse_unverified_jwt_svid_claims_rejects_truncated_jwt() { + assert!(matches!( + parse_unverified_jwt_svid_claims("header.payload"), + Err(JwtSvidParseError::Format) + )); + } + + #[test] + fn parse_unverified_jwt_svid_claims_rejects_empty_jwt_segments() { + assert!(matches!( + parse_unverified_jwt_svid_claims("header..signature"), + Err(JwtSvidParseError::Format) + )); + } + + #[test] + fn sandbox_spiffe_ids_are_unambiguous_and_reject_unsafe_input() { + let prefix = SandboxSpiffeIdPrefix::parse("spiffe://example.org/openshell").unwrap(); + assert_eq!( + prefix.sandbox_spiffe_id("gw", "4f1c2b3a-9d8e").unwrap(), + "spiffe://example.org/openshell/gateway/gw/sandbox/4f1c2b3a-9d8e" + ); + for invalid in [ + "spiffe://Example.org", + "spiffe://example.org/", + "spiffe://example.org//a", + "spiffe://example.org/a/../b", + "spiffe://example.org/a%2Fb", + "spiffe://example.org:8443/a", + "https://example.org/a", + ] { + assert!(SandboxSpiffeIdPrefix::parse(invalid).is_err(), "{invalid}"); + } + for (gateway, sandbox) in [("gw/x", "a"), ("gw", "a/b"), ("gw", ".."), ("", "a")] { + assert!(prefix.sandbox_spiffe_id(gateway, sandbox).is_err()); + } + } + + #[test] + fn sandbox_jwt_svid_debug_redacts_the_token() { + let svid = SandboxJwtSvid { + spiffe_id: "spiffe://example.org/openshell/gateway/gw/sandbox/sb".into(), + token: "secret-token".into(), + expires_at: 1, + }; + let printed = format!("{svid:?}"); + assert!(!printed.contains("secret-token")); + assert!(printed.contains("sandbox/sb")); + } + + #[test] + fn trust_domain_extracts_domain_from_spiffe_id() { + assert_eq!( + trust_domain("spiffe://openshell/openshell/sandbox/sb-a"), + Some("openshell") + ); + assert_eq!(trust_domain("spiffe://openshell"), Some("openshell")); + assert_eq!(trust_domain("not-a-spiffe-id"), None); + assert_eq!(trust_domain("spiffe:///empty"), None); + } +} diff --git a/crates/openshell-spire/Cargo.toml b/crates/openshell-spire/Cargo.toml new file mode 100644 index 0000000000..dab47ff483 --- /dev/null +++ b/crates/openshell-spire/Cargo.toml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-spire" +description = "SPIFFE Broker API adapter for gateway-owned sandbox JWT-SVIDs" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-spiffe = { path = "../openshell-spiffe" } +openshell-crypto = { workspace = true } +async-trait = "0.1" +http = { workspace = true } +prost = { workspace = true } +spiffe = { workspace = true, features = ["workload-api-x509"] } +tokio = { workspace = true } +tonic = { workspace = true, features = ["channel"] } +tonic-prost = { workspace = true } + +hyper-util = { workspace = true } +rustls = { workspace = true } +tokio-rustls = { workspace = true } +tower = { workspace = true } +uuid = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-spire/README.md b/crates/openshell-spire/README.md new file mode 100644 index 0000000000..e6f73f67df --- /dev/null +++ b/crates/openshell-spire/README.md @@ -0,0 +1,67 @@ +# openshell-spire + +SPIFFE Broker API adapter for gateway-owned sandbox identities. Only the +gateway obtains sandbox JWT-SVIDs; supervisors receive final provider tokens. + +`SpireSandboxIdentity` implements `SandboxIdentityProvider` using +`openshell_core::token_grant_failure::TokenGrantFailure` as its error type. +Startup selects it from `[openshell.gateway.spiffe_broker]`. The provider-token +broker owns session authorization, provider configuration, exchange, and caching. + +For each acquisition the adapter: + +1. Extracts the Sandbox CR UID from the gateway's persisted Kubernetes runtime + identity. Supervisors cannot supply a reference or selectors. +2. Fetches current gateway X.509-SVID credentials and trust bundles through the + local CSI Workload API socket. +3. Connects to the selected Broker endpoint over mTLS, verifying the agent certificate + chain and configured exact or cluster-scoped SPIFFE URI policy through `openshell-crypto`. +4. Requests a JWT-SVID for a `KubernetesObjectReference` containing the Sandbox + resource type and UID. SPIRE resolves and attests the object and enforces the + gateway's Kubernetes authorization. +5. Fetches the JWT bundle and verifies the signature, audience, expiry, and + exact expected subject `/gateway//sandbox/`. + +Every RPC includes `broker.spiffe.io: true`. Acquisitions fetch fresh credentials +and bundles so Workload API rotation does not require a gateway restart. The +whole operation is bounded by `WORKLOAD_IDENTITY_CALL_TIMEOUT`. Errors retain +only bounded classifications; credentials and upstream diagnostics are not logged. + +The operator enables SPIRE's experimental Broker endpoint, authorizes the +gateway SPIFFE ID and `KubernetesObjectReference` over TCP, and configures +`access_policy = "enforced"`. The agent needs Sandbox lookup and +SubjectAccessReview permissions. UID-only references require cluster-wide +`get`/`list` access to Sandbox resources for the SPIRE agents; metadata-only +responses do not narrow Kubernetes RBAC permissions. The gateway SPIFFE ID needs +`impersonate-via-spire` only for its permitted Sandbox resources. Registrations +match attested Sandbox UIDs and must reach each selected agent. The configured +peer policy must authorize the endpoint's agent certificate. Configure exactly +one of `server_spiffe_id` (exact identity) and `server_spiffe_id_prefix` +(descendants of one cluster-scoped SPIFFE path). Use a node alias as the parent +of registrations served by multiple agents; the alias does not replace their +concrete certificate identities. + +Set `node_local_port` to connect directly to the gateway's node-local agent. +The gateway resolves `OPENSHELL_NODE_IP` from the Pod's Downward API at startup; +missing or invalid addresses fail startup. IPv4 and IPv6 addresses are supported. +The Broker listener must be reachable on that node address. Alternatively, set +an explicit HTTPS `endpoint`. These settings are mutually exclusive. Node-local +mode uses no Broker Service; gateway movement requires registrations on every +eligible agent, even when the sandbox and supervisor remain on other nodes. + +## Verification + +```shell +mise run e2e:kubernetes:shared-supervisor-spiffe +mise run e2e:kubernetes:shared-supervisor-spiffe-node-local +``` + +The test installs pinned SPIRE charts, exercises the full provider exchange, +forces acquisition through each agent on a three-node cluster and a replacement +agent without re-registering sandboxes, checks supervisor +takeover and stale-session fencing, and forces acquisition +after the fixture's 30-second workload X.509-SVID expires. It also checks that +the gateway Pod passes Restricted Pod Security admission. +The node-local variant uses external PostgreSQL to move the gateway across +nodes, restricts Broker egress to its current node, and verifies fresh acquisition +without a Broker Service, including after its local agent is replaced. diff --git a/crates/openshell-spire/src/broker.rs b/crates/openshell-spire/src/broker.rs new file mode 100644 index 0000000000..e58d9cf14e --- /dev/null +++ b/crates/openshell-spire/src/broker.rs @@ -0,0 +1,388 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! SPIFFE Broker API JWT profile with attested Kubernetes object references. + +use crate::{configuration_failure, identity_rejected}; +use hyper_util::rt::TokioIo; +use openshell_core::config::SpiffeBrokerConfig; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, WORKLOAD_IDENTITY_CALL_TIMEOUT, +}; +use openshell_spiffe::SandboxJwtSvid; +use prost::Message; +use rustls::{ + RootCertStore, + pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}, +}; +use std::{collections::HashMap, sync::Arc}; +use tonic::{ + Request, + transport::{Channel, Endpoint}, +}; + +// JWT profile and KubernetesObjectReference fields from +// spiffe/spiffe standards/brokerapi.proto. +#[derive(Clone, PartialEq, Message)] +struct AnyReference { + #[prost(string, tag = "1")] + type_url: String, + #[prost(bytes = "vec", tag = "2")] + value: Vec, +} +#[derive(Clone, PartialEq, Message)] +struct WorkloadReference { + #[prost(message, optional, tag = "1")] + reference: Option, +} +#[derive(Clone, PartialEq, Message)] +struct ObjectType { + #[prost(string, tag = "1")] + plural: String, + #[prost(string, tag = "2")] + group: String, +} +#[derive(Clone, PartialEq, Message)] +struct ObjectKey { + #[prost(string, tag = "1")] + namespace: String, + #[prost(string, tag = "2")] + name: String, +} +#[derive(Clone, PartialEq, Message)] +struct ObjectReference { + #[prost(message, optional, tag = "1")] + r#type: Option, + #[prost(message, optional, tag = "2")] + key: Option, + #[prost(string, tag = "3")] + uid: String, +} +#[derive(Clone, PartialEq, Message)] +struct FetchRequest { + #[prost(message, optional, tag = "1")] + reference: Option, + #[prost(string, repeated, tag = "2")] + audience: Vec, + #[prost(string, tag = "3")] + spiffe_id: String, +} +#[derive(Clone, PartialEq, Message)] +struct JwtSvid { + #[prost(string, tag = "1")] + spiffe_id: String, + #[prost(string, tag = "2")] + svid: String, + #[prost(string, tag = "3")] + hint: String, +} +#[derive(Clone, PartialEq, Message)] +struct FetchResponse { + #[prost(message, repeated, tag = "1")] + svids: Vec, +} +#[derive(Clone, PartialEq, Message)] +struct BundlesRequest { + #[prost(message, optional, tag = "1")] + reference: Option, +} +#[derive(Clone, PartialEq, Message)] +struct BundlesResponse { + #[prost(map = "string, bytes", tag = "1")] + bundles: HashMap>, +} + +fn failure(kind: TokenGrantFailureKind) -> TokenGrantFailure { + TokenGrantFailure::new(kind, TokenGrantStage::WorkloadIdentity) +} + +fn workload_failure(error: spiffe::WorkloadApiError) -> TokenGrantFailure { + use spiffe::WorkloadApiError; + failure(match error { + WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, + WorkloadApiError::NoIdentityIssued | WorkloadApiError::PermissionDenied(_) => { + TokenGrantFailureKind::WorkloadIdentityRejected + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Status(status)) => { + return status_failure(status); + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Tonic(_)) => { + TokenGrantFailureKind::Unavailable + } + _ => TokenGrantFailureKind::Unclassified, + }) +} + +fn transport_failure(error: &(dyn std::error::Error + 'static)) -> TokenGrantFailure { + let mut cause = Some(error); + while let Some(error) = cause { + if error.downcast_ref::().is_some() { + return identity_rejected(); + } + // io::Error does not expose its contained error through source on + // every platform. Inspect it explicitly for TLS handshake failures. + if let Some(inner) = error + .downcast_ref::() + .and_then(std::io::Error::get_ref) + && inner.downcast_ref::().is_some() + { + return identity_rejected(); + } + cause = error.source(); + } + failure(TokenGrantFailureKind::Unavailable) +} +fn status_failure(status: tonic::Status) -> TokenGrantFailure { + failure(match status.code() { + tonic::Code::PermissionDenied | tonic::Code::Unauthenticated => { + TokenGrantFailureKind::WorkloadIdentityRejected + } + tonic::Code::DeadlineExceeded => TokenGrantFailureKind::Timeout, + tonic::Code::Unavailable => TokenGrantFailureKind::Unavailable, + tonic::Code::InvalidArgument | tonic::Code::Unimplemented => { + TokenGrantFailureKind::Configuration + } + _ => TokenGrantFailureKind::Unclassified, + }) +} +fn request(message: T) -> Request { + let mut request = Request::new(message); + request + .metadata_mut() + .insert("broker.spiffe.io", "true".parse().expect("static metadata")); + request +} + +/// The reference is derived only from the driver's immutable identity retained +/// by the gateway, not mutable workload labels or caller-provided selectors. +fn reference(runtime: &str) -> Result { + let parts: Vec<_> = runtime + .strip_prefix("kubernetes://") + .ok_or_else(configuration_failure)? + .split('/') + .collect(); + let [namespace, uid, component] = parts.as_slice() else { + return Err(configuration_failure()); + }; + if namespace.is_empty() || component.is_empty() || uid.parse::().is_err() { + return Err(configuration_failure()); + } + Ok(WorkloadReference { + reference: Some(AnyReference { + type_url: "type.googleapis.com/spiffe.broker.KubernetesObjectReference".into(), + value: ObjectReference { + r#type: Some(ObjectType { + plural: "sandboxes".into(), + group: "agents.x-k8s.io".into(), + }), + // Kubernetes UIDs are cluster-unique. SPIRE resolves this UID itself; + // an object recreated with the same name cannot satisfy the reference. + key: None, + uid: (*uid).to_string(), + } + .encode_to_vec(), + }), + }) +} + +async fn channel( + config: &SpiffeBrokerConfig, + endpoint: &str, +) -> Result { + use openshell_crypto::tls::spiffe::ServerIdentity; + let (server_id, identity) = match (&config.server_spiffe_id, &config.server_spiffe_id_prefix) { + (Some(id), None) => (id, ServerIdentity::Exact(id.clone())), + (None, Some(prefix)) => (prefix, ServerIdentity::PathPrefix(prefix.clone())), + _ => return Err(configuration_failure()), + }; + let workload_endpoint = format!("unix:{}", config.workload_api_socket_path); + let workload = spiffe::WorkloadApiClient::connect_to(workload_endpoint) + .await + .map_err(workload_failure)?; + // Fetch afresh at every acquisition. Neither client credentials nor trust + // bundles are pinned across their Workload API rotation. + let context = workload + .fetch_x509_context() + .await + .map_err(workload_failure)?; + let svid = context.default_svid().ok_or_else(identity_rejected)?; + let domain = spiffe::TrustDomain::new( + openshell_spiffe::trust_domain(server_id).ok_or_else(configuration_failure)?, + ) + .map_err(|_| configuration_failure())?; + let bundle = context + .bundle_set() + .get(&domain) + .ok_or_else(identity_rejected)?; + let mut roots = RootCertStore::empty(); + for cert in bundle.authorities() { + roots + .add(CertificateDer::from(cert.as_ref().to_vec())) + .map_err(|_| identity_rejected())?; + } + let chain = svid + .cert_chain() + .iter() + .map(|cert| CertificateDer::from(cert.as_ref().to_vec())) + .collect(); + let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + svid.private_key().as_ref().to_vec(), + )); + let mut tls = openshell_crypto::tls::try_client_builder() + .map_err(|_| configuration_failure())? + .dangerous() + .with_custom_certificate_verifier( + openshell_crypto::tls::spiffe::server_verifier_with_identity(roots, identity), + ) + .with_client_auth_cert(chain, key) + .map_err(|_| configuration_failure())?; + tls.alpn_protocols = vec![b"h2".to_vec()]; + let connector = tokio_rustls::TlsConnector::from(Arc::new(tls)); + let uri: http::Uri = endpoint.parse().map_err(|_| configuration_failure())?; + let host = uri + .host() + .ok_or_else(configuration_failure)? + .trim_matches(['[', ']']) + .to_string(); + let port = uri.port_u16().unwrap_or(443); + Endpoint::from_static("http://broker.invalid") + .connect_with_connector(tower::service_fn(move |_| { + let connector = connector.clone(); + let host = host.clone(); + async move { + let tcp = + openshell_core::net::connect_tcp_nodelay_best_effort((host.as_str(), port)) + .await?; + let stream = connector + .connect( + ServerName::try_from("broker.invalid").expect("static name"), + tcp, + ) + .await?; + Ok::<_, std::io::Error>(TokioIo::new(stream)) + } + })) + .await + .map_err(|error| transport_failure(&error)) +} + +pub async fn fetch( + config: &SpiffeBrokerConfig, + endpoint: &str, + runtime: &str, + expected: &str, + audience: &str, +) -> Result { + let reference = reference(runtime)?; + tokio::time::timeout(WORKLOAD_IDENTITY_CALL_TIMEOUT, async { + let mut grpc = tonic::client::Grpc::new(channel(config, endpoint).await?); + grpc.ready() + .await + .map_err(|_| failure(TokenGrantFailureKind::Unavailable))?; + let response: tonic::Response = grpc + .unary( + request(FetchRequest { + reference: Some(reference.clone()), + audience: vec![audience.to_string()], + spiffe_id: expected.to_string(), + }), + http::uri::PathAndQuery::from_static("/spiffe.broker.API/FetchJWTSVID"), + tonic_prost::ProstCodec::default(), + ) + .await + .map_err(status_failure)?; + let response = response.into_inner(); + let [svid] = response.svids.as_slice() else { + return Err(identity_rejected()); + }; + if svid.spiffe_id != expected { + return Err(identity_rejected()); + } + grpc.ready() + .await + .map_err(|_| failure(TokenGrantFailureKind::Unavailable))?; + let mut stream: tonic::Streaming = grpc + .server_streaming( + request(BundlesRequest { + reference: Some(reference), + }), + http::uri::PathAndQuery::from_static("/spiffe.broker.API/SubscribeToJWTBundles"), + tonic_prost::ProstCodec::default(), + ) + .await + .map_err(status_failure)? + .into_inner(); + let bundles = stream + .message() + .await + .map_err(status_failure)? + .ok_or_else(identity_rejected)?; + let domain = spiffe::TrustDomain::new( + openshell_spiffe::trust_domain(expected).ok_or_else(configuration_failure)?, + ) + .map_err(|_| configuration_failure())?; + let bundle = spiffe::JwtBundle::from_jwt_authorities( + domain.clone(), + bundles + .bundles + .get(&domain.id_string()) + .ok_or_else(identity_rejected)?, + ) + .map_err(|_| identity_rejected())?; + let mut set = spiffe::JwtBundleSet::new(); + set.add_bundle(bundle); + openshell_crypto::install_jwt_provider(); + let verified = spiffe::JwtSvid::parse_and_validate(&svid.svid, &set, &[audience]) + .map_err(|_| identity_rejected())?; + if verified.spiffe_id().to_string() != expected { + return Err(identity_rejected()); + } + Ok(SandboxJwtSvid { + spiffe_id: expected.to_string(), + token: svid.svid.clone(), + expires_at: verified.claims().exp(), + }) + }) + .await + .map_err(|_| failure(TokenGrantFailureKind::Timeout))? +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn permanent_identity_failures_are_distinct_from_connection_failures() { + assert_eq!( + workload_failure(spiffe::WorkloadApiError::NoIdentityIssued).kind(), + TokenGrantFailureKind::WorkloadIdentityRejected + ); + let tls = std::io::Error::other(rustls::Error::General("test rejection".into())); + assert_eq!( + transport_failure(&tls).kind(), + TokenGrantFailureKind::WorkloadIdentityRejected + ); + let unavailable = std::io::Error::from(std::io::ErrorKind::ConnectionRefused); + assert_eq!( + transport_failure(&unavailable).kind(), + TokenGrantFailureKind::Unavailable + ); + } + #[test] + fn only_driver_authenticated_kubernetes_uid_references_are_accepted() { + for runtime in [ + "", + "docker://container", + "kubernetes://ns/not-a-uid/component", + "kubernetes://ns/3d185166-01ad-4696-8d7c-ac315d4c01e3/", + ] { + assert!(reference(runtime).is_err()); + } + let reference = reference("kubernetes://ns/3d185166-01ad-4696-8d7c-ac315d4c01e3/component") + .unwrap() + .reference + .unwrap(); + let object = ObjectReference::decode(reference.value.as_slice()).unwrap(); + assert_eq!(object.uid, "3d185166-01ad-4696-8d7c-ac315d4c01e3"); + assert!(object.key.is_none()); + } +} diff --git a/crates/openshell-spire/src/lib.rs b/crates/openshell-spire/src/lib.rs new file mode 100644 index 0000000000..c19f8d430e --- /dev/null +++ b/crates/openshell-spire/src/lib.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-owned sandbox identities obtained through the SPIFFE Broker API. + +mod broker; + +use openshell_core::config::SpiffeBrokerConfig; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, +}; +use openshell_spiffe::{SandboxIdentityProvider, SandboxJwtSvid, SandboxSpiffeIdPrefix}; + +/// SPIRE Broker API adapter. Each acquisition gets current Workload API +/// credentials and verifies an exact sandbox identity and audience. +#[derive(Debug)] +pub struct SpireSandboxIdentity { + config: SpiffeBrokerConfig, + prefix: SandboxSpiffeIdPrefix, + endpoint: String, +} + +impl SpireSandboxIdentity { + /// Parse the stable identity prefix; startup also validates the transport. + /// + /// # Errors + /// Rejects an invalid SPIFFE ID prefix or missing/invalid node address. + pub fn new(config: &SpiffeBrokerConfig) -> Result { + Ok(Self { + config: config.clone(), + prefix: SandboxSpiffeIdPrefix::parse(&config.spiffe_id_prefix)?, + endpoint: config + .endpoint_for_node(std::env::var("OPENSHELL_NODE_IP").ok().as_deref())?, + }) + } +} + +#[async_trait::async_trait] +impl SandboxIdentityProvider for SpireSandboxIdentity { + type Error = TokenGrantFailure; + + async fn fetch_jwt_svid( + &self, + gateway_id: &str, + sandbox_id: &str, + audience: &str, + runtime_identity: &str, + ) -> Result { + let expected = self + .prefix + .sandbox_spiffe_id(gateway_id, sandbox_id) + .map_err(|_| configuration_failure())?; + broker::fetch( + &self.config, + &self.endpoint, + runtime_identity, + &expected, + audience, + ) + .await + } +} + +fn configuration_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ) +} +fn identity_rejected() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantStage::WorkloadIdentity, + ) +} diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 8f000adb7c..475ae64f3e 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -20,6 +20,7 @@ openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-policy-schema = { path = "../openshell-policy-schema" } +openshell-spiffe = { path = "../openshell-spiffe" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } async-trait = "0.1" diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 4ea0b76013..591b7274fb 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -22,14 +22,29 @@ Before the supervisor forwards a connection it cannot inspect (`tls: skip`, tran ## Sandbox instance inputs `run_networking` takes a `SandboxInstanceNetwork` with the sandbox identity, -the sandbox instance's `GatewayEndpoint`, the SPIFFE Workload API socket for +the sandbox instance's `GatewayEndpoint`, the `ProviderIdentitySource` for provider token grants, the `policy.local` denial source, and proxy CA files. None of these are read from the process environment, so one process can run -several sandbox instances. Each instance owns its provider token cache and -HTTP client; a token acquired for one sandbox is never served to another. -Without a Workload API socket, token grants fail closed. A process hosting -several instances serves `/v1/denials` from a per-instance -`SandboxInstanceDenialLog` instead of the shared shorthand log files. +several sandbox instances. `run_networking` selects the instance's token grant +resolver once from its identity source; a token acquired for one sandbox is +never served to another. + +A dedicated supervisor uses `SpiffeTokenGrantResolver` with its SPIFFE Workload +API socket. That resolver owns the instance's token cache and HTTP client and +uses the gateway only for the intermediate exchange of RFC 8693 grants. +Without a socket, its grants fail closed. Shared sandbox instances use +`GatewayTokenGrantResolver`, which holds only the instance's authenticated +gateway channel, connected on the first grant and reused. Each admitted +operation forwards a configured dynamic credential key and request ID; the +gateway derives the sandbox identity, +validates current authorization, resolves grant parameters, and obtains the +final access token, which the resolver validates. No SPIRE client, SVID, +intermediate token, or grant cache is used in the shared supervisor. +`mise run e2e:kubernetes:shared-supervisor-spiffe` exercises the gateway's +SPIFFE Broker API flow against a real server and agent. A process hosting +several instances serves +`/v1/denials` from a per-instance `SandboxInstanceDenialLog` instead of the shared +shorthand log files. ## Policy DNS expiry diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index d24a77ec34..2b76265970 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -54,42 +54,6 @@ pub trait TokenGrantResolver: Send + Sync { ) -> Pin> + Send + 'a>>; } -/// Resolves provider tokens through one sandbox instance's authenticated grant flow. -pub struct SpiffeTokenGrantResolver { - grants: crate::token_grant::SandboxInstanceTokenGrants, -} - -impl SpiffeTokenGrantResolver { - #[must_use] - pub fn new(grants: crate::token_grant::SandboxInstanceTokenGrants) -> Self { - Self { grants } - } -} - -impl TokenGrantResolver for SpiffeTokenGrantResolver { - fn obtain<'a>( - &'a self, - request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.grants - .obtain_provider_token(crate::token_grant::ObtainProviderTokenRequest { - provider_name: request.provider_key, - token_endpoint: request.token_endpoint, - jwt_svid_audience: request.jwt_svid_audience, - client_assertion_type: request.client_assertion_type, - audience: request.audience, - scopes: request.scopes, - cache_ttl_override: request.cache_ttl, - grant_type: request.grant_type, - requested_token_type: request.requested_token_type, - request_id: request.request_id, - }) - .await - }) - } -} - /// Token grant headers reapplied after streamed middleware mutations. #[derive(Default)] pub struct InjectedHeaders(Vec<(String, String)>); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 169add5c5e..9eb744da2b 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -35,11 +35,12 @@ use crate::l7::tls::{ CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, read_system_ca_bundle, write_ca_files, }; -use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; +use crate::l7::token_grant_injection::TokenGrantResolver; use crate::opa::OpaEngine; use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; -use crate::token_grant::SandboxInstanceTokenGrants; +pub use crate::token_grant::ProviderIdentitySource; +use crate::token_grant::{GatewayTokenGrantResolver, SpiffeTokenGrantResolver}; use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::grpc_client::GatewayEndpoint; @@ -203,9 +204,8 @@ pub struct SandboxInstanceNetwork { pub sandbox_name: Option, /// Gateway access with this instance's credentials. pub gateway: Option, - /// SPIFFE Workload API socket for provider token grants. Without one, - /// grants fail closed. - pub provider_workload_api_socket: Option, + /// How this instance obtains provider access tokens. + pub provider_identity: ProviderIdentitySource, /// Source served by `policy.local` `/v1/denials`. pub denial_log: DenialLogSource, /// Proxy interception CA and trust-file directory. @@ -305,15 +305,26 @@ pub async fn run_networking( sandbox_id, sandbox_name, gateway, - provider_workload_api_socket, + provider_identity, denial_log, proxy_tls, } = instance; - let token_grants = SpiffeTokenGrantResolver::new(SandboxInstanceTokenGrants::new( - gateway.clone(), - sandbox_id.clone(), - provider_workload_api_socket, - )); + // Choose how this instance obtains provider tokens once, for every grant. + let token_grants: Arc = match (provider_identity, gateway.clone()) { + (ProviderIdentitySource::Gateway, Some(gateway)) => { + Arc::new(GatewayTokenGrantResolver::new(gateway)) + } + // Nothing can issue this sandbox's tokens without a gateway; a + // resolver without a workload identity fails every grant closed. + (ProviderIdentitySource::Gateway, None) => Arc::new(SpiffeTokenGrantResolver::new( + None, + sandbox_id.clone(), + None, + )), + (ProviderIdentitySource::WorkloadApi(socket), gateway) => Arc::new( + SpiffeTokenGrantResolver::new(gateway, sandbox_id.clone(), socket), + ), + }; let policy_local_ctx = Arc::new( PolicyLocalContext::new( retained_proto.cloned(), @@ -323,7 +334,7 @@ pub async fn run_networking( workspace_rx, ) .with_denial_log(denial_log) - .with_token_grants(Arc::new(token_grants)), + .with_token_grants(token_grants), ); // Readiness signal for the proxy accept loop: the proxy binds the TCP diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index 83d36d36b3..df2a25a559 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -29,21 +29,28 @@ //! //! ## Sandbox instance scope //! -//! Each sandbox instance owns a [`SandboxInstanceTokenGrants`]: its token cache, -//! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API -//! socket. A token is never served to another sandbox. Without a configured -//! Workload API socket, grants fail closed with a configuration failure. +//! Each sandbox instance selects one resolver when its networking starts, from +//! its [`ProviderIdentitySource`]. A token is never served to another sandbox. +//! +//! A dedicated supervisor uses a [`SpiffeTokenGrantResolver`], which presents +//! the supervisor's own identity from a SPIFFE Workload API socket and owns +//! the instance's token cache and HTTP client. Without a socket, its grants +//! fail closed with a configuration failure. A shared supervisor uses a +//! [`GatewayTokenGrantResolver`], which requests only the final provider access +//! token from the gateway; the gateway owns SPIRE acquisition, exchange, and +//! caching. use std::collections::HashMap; use std::future::Future; use std::path::PathBuf; +use std::pin::Pin; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use openshell_core::grpc_client::GatewayEndpoint; +use openshell_core::grpc_client::{GatewayEndpoint, SandboxProviderTokenClient}; use openshell_core::oauth::{ self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantClient, - TokenGrantParams, effective_client_assertion_type, effective_token_type, + TokenGrantParams, effective_client_assertion_type, effective_token_type, token_cache_ttl, }; use openshell_core::proto::ProviderCredentialTokenGrantType; use openshell_core::token_grant_failure::{ @@ -51,9 +58,7 @@ use openshell_core::token_grant_failure::{ }; use spiffe::{WorkloadApiClient, WorkloadApiError}; -const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; -const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; -const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; +use crate::l7::token_grant_injection::{TokenGrantRequest, TokenGrantResolver}; /// Cached access token with expiration metadata. #[derive(Clone)] @@ -100,12 +105,31 @@ impl TokenCache { } } -/// Provider token grants owned by one supervised sandbox. +/// How a sandbox instance obtains provider access tokens. +#[derive(Debug)] +pub enum ProviderIdentitySource { + /// Acquire tokens with this supervisor's own identity from a SPIFFE + /// Workload API socket, for a supervisor dedicated to one sandbox. + /// Without a socket, every grant fails closed. + WorkloadApi(Option), + /// Request final tokens from the gateway, which owns the sandbox's + /// workload identity. + Gateway, +} + +impl Default for ProviderIdentitySource { + fn default() -> Self { + Self::WorkloadApi(None) + } +} + +/// Provider token grants acquired with this supervisor's SPIFFE identity. /// /// The cache, HTTP connection pool, gateway credentials, sandbox ID, and -/// workload identity socket all belong to this sandbox instance, so a token acquired -/// for one sandbox is never served to another. -pub struct SandboxInstanceTokenGrants { +/// Workload API socket all belong to one sandbox instance, so a token +/// acquired for one sandbox is never served to another. The gateway is used +/// only for the intermediate exchange of RFC 8693 token exchange grants. +pub struct SpiffeTokenGrantResolver { cache: TokenCache, // Initialization failures stay bounded and typed; acquisition must not panic. http_client: Result, @@ -114,9 +138,9 @@ pub struct SandboxInstanceTokenGrants { workload_api_socket: Option, } -impl SandboxInstanceTokenGrants { - /// Grants authenticated as `sandbox_id` through `gateway`. A missing - /// Workload API socket makes every grant fail closed. +impl SpiffeTokenGrantResolver { + /// Grants authenticated as `sandbox_id` through `gateway`. A missing or + /// empty socket makes every grant fail closed. #[must_use] pub fn new( gateway: Option, @@ -132,6 +156,14 @@ impl SandboxInstanceTokenGrants { } } + async fn fetch_jwt_svid(&self, audience: &str) -> Result { + let socket_path = self + .workload_api_socket + .as_deref() + .ok_or_else(configuration_failure)?; + fetch_jwt_svid_from_socket(socket_path, audience).await + } + /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. /// /// This fetches the sandbox's SPIFFE JWT-SVID from the instance's Workload @@ -150,22 +182,22 @@ impl SandboxInstanceTokenGrants { /// - JWT-SVID fetch fails /// - Token service request fails /// - Token response is invalid - pub async fn obtain_provider_token( + async fn obtain_provider_token( &self, - request: ObtainProviderTokenRequest<'_>, + request: TokenGrantRequest<'_>, ) -> Result { let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) .map_err(|_| configuration_failure())?; - obtain_provider_token_with_grant( + let token = obtain_provider_token_with_grant( ObtainProviderTokenInput { cache: &self.cache, - provider_name: request.provider_name, + provider_name: request.provider_key, token_endpoint: request.token_endpoint, jwt_svid_audience: request.jwt_svid_audience, client_assertion_type: request.client_assertion_type, audience: request.audience, scopes: request.scopes, - cache_ttl_override: request.cache_ttl_override, + cache_ttl_override: request.cache_ttl, grant_type, requested_token_type: request.requested_token_type, }, @@ -176,7 +208,7 @@ impl SandboxInstanceTokenGrants { | ProviderCredentialTokenGrantType::Unspecified => None, ProviderCredentialTokenGrantType::TokenExchange => { let (provider, credential_key) = - parse_provider_credential_key(request.provider_name)?; + parse_provider_credential_key(request.provider_key)?; let gateway = self.gateway.as_ref().ok_or_else(configuration_failure)?; openshell_core::grpc_client::token_grant_gateway_endpoint(gateway.url())?; let sandbox_id = self @@ -187,15 +219,11 @@ impl SandboxInstanceTokenGrants { } }; let http_client = self.http_client.as_ref().map_err(|failure| *failure)?; - let socket_path = self - .workload_api_socket - .as_deref() - .ok_or_else(configuration_failure)?; // Fetch JWT-SVID with authorization server as audience // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm - let jwt_svid = fetch_jwt_svid_from_socket(socket_path, &jwt_audience).await?; + let jwt_svid = self.fetch_jwt_svid(&jwt_audience).await?; - match exchange_context { + let response = match exchange_context { None => { // Perform OAuth2 JWT client assertion grant. The audience // parameter in the token request specifies the resource server. @@ -240,27 +268,58 @@ impl SandboxInstanceTokenGrants { ) .await } - } + }?; + Ok(response) }, ) - .await + .await?; + Ok(token) + } +} + +impl TokenGrantResolver for SpiffeTokenGrantResolver { + fn obtain<'a>( + &'a self, + request: TokenGrantRequest<'a>, + ) -> Pin> + Send + 'a>> { + Box::pin(self.obtain_provider_token(request)) + } +} + +/// Final provider access tokens from the gateway, for a sandbox instance whose +/// supervisor holds no workload identity of its own. +/// +/// It forwards only the credential key and request ID over the instance's +/// reused authenticated gateway connection and validates the returned token. +/// The gateway authorizes, acquires, exchanges, and caches on every request; +/// this resolver holds no cache, SVID, intermediate token, or HTTP client. +pub struct GatewayTokenGrantResolver { + gateway: SandboxProviderTokenClient, +} + +impl GatewayTokenGrantResolver { + #[must_use] + pub fn new(gateway: GatewayEndpoint) -> Self { + Self { + gateway: SandboxProviderTokenClient::new(gateway), + } } } -/// One provider token request; see [`SandboxInstanceTokenGrants::obtain_provider_token`]. -pub struct ObtainProviderTokenRequest<'a> { - pub provider_name: &'a str, - pub token_endpoint: &'a str, - pub jwt_svid_audience: &'a str, - pub client_assertion_type: &'a str, - pub audience: &'a str, - pub scopes: &'a [String], - pub cache_ttl_override: Option, - pub grant_type: i32, - pub requested_token_type: &'a str, - /// Supervisor-minted ID of the HTTP exchange that needs the token. It is - /// sent to the gateway and token service; a cache hit sends nothing. - pub request_id: &'a str, +impl TokenGrantResolver for GatewayTokenGrantResolver { + fn obtain<'a>( + &'a self, + request: TokenGrantRequest<'a>, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + let token = self + .gateway + .get_access_token(request.provider_key, Some(request.request_id)) + .await?; + validate_access_token(&token.access_token)?; + Ok(token.access_token) + }) + } } struct ObtainProviderTokenInput<'a> { @@ -333,7 +392,7 @@ async fn fetch_jwt_svid_from_socket( socket_path: &std::path::Path, audience: &str, ) -> Result { - let endpoint = openshell_core::spiffe::workload_api_endpoint(socket_path); + let endpoint = openshell_spiffe::workload_api_endpoint(socket_path); // Connect to SPIRE agent let client = WorkloadApiClient::connect_to(&endpoint) @@ -413,23 +472,6 @@ async fn perform_token_exchange( pub use oauth::validate_access_token; -fn token_cache_ttl(cache_ttl_override: Option, expires_in: i64) -> Duration { - if let Some(cache_ttl_override) = cache_ttl_override { - return cache_ttl_override; - } - - let ttl = if expires_in > 0 { - expires_in.min(MAX_TOKEN_EXPIRES_IN_SECONDS) - } else { - DEFAULT_TOKEN_CACHE_TTL_SECONDS - }; - - Duration::from_secs( - u64::try_from(ttl.saturating_sub(TOKEN_CACHE_EXPIRY_SKEW_SECONDS).max(1)) - .unwrap_or(u64::MAX), - ) -} - /// Derive the issuer/realm URL from a token endpoint URL. /// /// For Keycloak token endpoints like: @@ -512,9 +554,8 @@ pub mod test_support { //! socket or OAuth service is contacted, and no process-wide cache is used. use super::*; - use crate::l7::token_grant_injection::{TokenGrantRequest, TokenGrantResolver}; + use openshell_core::oauth::MAX_TOKEN_EXPIRES_IN_SECONDS; use std::collections::HashSet; - use std::pin::Pin; use std::sync::Mutex; /// One actual acquisition, excluding requests served from the cache. @@ -652,7 +693,10 @@ pub mod test_support { #[cfg(test)] mod tests { use super::*; - use openshell_core::oauth::{ACCESS_TOKEN_TYPE, DEFAULT_CLIENT_ASSERTION_TYPE}; + use openshell_core::oauth::{ + ACCESS_TOKEN_TYPE, DEFAULT_CLIENT_ASSERTION_TYPE, DEFAULT_TOKEN_CACHE_TTL_SECONDS, + MAX_TOKEN_EXPIRES_IN_SECONDS, TOKEN_CACHE_EXPIRY_SKEW_SECONDS, + }; use std::collections::HashMap; use std::sync::{ Arc, diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index da6afbf271..f19eb68301 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -79,8 +79,9 @@ is a single path component, then, under the registry lock, accepts the same launch fingerprint for an occupied key again, rejects a different one, and rejects any admission while the key is being removed. A new instance gets an empty private directory under the host root, and `sandbox_instance_launch` -prepares its bindings and body there. A failed admission leaves no entry and -no files. +prepares its bindings and body there. A failed admission retains no entry; +its directory is deleted on a best-effort basis, and the next admission of the +key must delete any stale files before reusing it. Removing an instance stops its workload, shuts down only its runtime, which drops its runtime route, and revokes its gateway credentials; only then is its @@ -263,10 +264,13 @@ control listener: it dials the gateway and receives its assignments from it. leaving, waits up to 40 seconds for the gateway to move its instances, and relinquishes what remains. -Sandbox instances get no upstream proxy, extra CA bundle, or provider workload identity: -the Kubernetes driver refuses the topology when an upstream proxy or provider -SPIFFE identity is configured, so a workload never silently loses them. The -host also initializes no OTLP trace or OCSF exporters, because one process +Sandbox instances get no upstream proxy or extra CA bundle: the Kubernetes driver +refuses the topology when an upstream proxy is configured, so a workload never +silently loses it. The process's own SPIFFE identity never stands in for a +sandbox. Shared sandbox instances obtain final provider access tokens through the gateway's +`GetSandboxProviderAccessToken` RPC. They hold no SPIRE socket, SVID, or +intermediate token. The gateway owns identity authorization, acquisition, +exchange, and caching. The host also initializes no OTLP trace or OCSF exporters, because one process cannot export each instance's records with that instance's collector headers and identity. Its console logs and each instance's gateway log push remain, and the Kubernetes driver warns at startup when the gateway configures supervisor diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 822cf1ae26..36c8060633 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -857,7 +857,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { main_exit_marker: Some(marker.clone()), attach_request_timeout: Duration::from_mins(5), trace_context: None, - provider_workload_api_socket: None, + provider_identity: openshell_supervisor_network::run::ProviderIdentitySource::default(), // The binary loads the proxy trust directory from its environment. proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), takeover: None, @@ -1075,7 +1075,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { main_exit_marker: Some(marker.clone()), attach_request_timeout: Duration::from_mins(5), trace_context: None, - provider_workload_api_socket: None, + provider_identity: openshell_supervisor_network::run::ProviderIdentitySource::default(), // The binary loads the proxy trust directory from its environment. proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), takeover: None, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index a28e0cf4bf..e3af1d51bf 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -313,7 +313,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl main_exit_marker: None, attach_request_timeout: Duration::from_mins(5), trace_context: None, - provider_workload_api_socket: None, + provider_identity: openshell_supervisor_network::run::ProviderIdentitySource::default(), proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), takeover: None, supervisor_recovery_timeout_secs: None, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index fba41bb4e7..fe1cbec6d9 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -937,8 +937,8 @@ pub struct SandboxInstanceConfig { pub attach_request_timeout: Duration, /// Parent of the startup span. pub trace_context: Option, - /// SPIFFE Workload API socket for provider token grants; absent fails closed. - pub provider_workload_api_socket: Option, + /// How this instance obtains provider access tokens. + pub provider_identity: openshell_supervisor_network::run::ProviderIdentitySource, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, /// Gateway-granted authority to replace the generation's current owner. /// The instance then adopts the running workload instead of starting it. @@ -1112,7 +1112,7 @@ pub async fn run_sandbox_instance( main_exit_marker, attach_request_timeout, trace_context, - provider_workload_api_socket, + provider_identity, proxy_tls, takeover, supervisor_recovery_timeout_secs, @@ -1476,7 +1476,7 @@ pub async fn run_sandbox_instance( sandbox_id: sandbox_id.clone(), sandbox_name: sandbox_name_for_agg.clone(), gateway: gateway.clone(), - provider_workload_api_socket, + provider_identity, denial_log, proxy_tls, }, @@ -4793,7 +4793,7 @@ mod tests { main_exit_marker: None, attach_request_timeout: Duration::from_mins(5), trace_context: None, - provider_workload_api_socket: None, + provider_identity: openshell_supervisor_network::run::ProviderIdentitySource::default(), proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), takeover: None, supervisor_recovery_timeout_secs: None, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 6c54c03ee9..28b0e131c6 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -675,11 +675,14 @@ fn main() -> Result<()> { args.attach_request_timeout_seconds, ), trace_context: openshell_supervisor::trace_context_from_environment(), - provider_workload_api_socket: std::env::var_os( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ) - .filter(|path| !path.is_empty()) - .map(PathBuf::from), + provider_identity: + openshell_supervisor_network::run::ProviderIdentitySource::WorkloadApi( + std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(PathBuf::from), + ), proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), takeover: None, supervisor_recovery_timeout_secs: None, diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 3da50aa3d3..ba8a9f9265 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -263,7 +263,9 @@ impl SandboxInstanceHost { /// /// Rejects an occupied key with different launch material or one still /// being removed, and fails when the instance cannot be prepared or - /// started. A failed admission leaves no instance and no private files. + /// started. A failed admission retains no instance; deleting its private + /// directory is best effort, and the next admission of the key deletes + /// any stale files before reuse. pub fn admit>( &self, admission: SandboxInstanceAdmission, diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 3785deaa8d..c7f11a57b5 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -811,7 +811,7 @@ impl SharedHost { ssh_socket_path: Some(ssh_socket.to_string_lossy().into_owned()), shared_ssh_socket: false, // The Kubernetes driver refuses the gateway topology when an - // upstream proxy or provider workload identity is configured, + // upstream proxy is configured, // and it configures no extra CA for dedicated supervisors. upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), @@ -822,9 +822,9 @@ impl SharedHost { main_exit_marker: None, attach_request_timeout: self.attach_request_timeout, trace_context: registration.trace_context, - // One process cannot hold a per-sandbox workload identity, so - // providers that need one fail closed. - provider_workload_api_socket: None, + // The gateway owns each sandbox's workload identity and returns + // only final provider access tokens. + provider_identity: openshell_supervisor_network::run::ProviderIdentitySource::Gateway, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles { ca_certificate: Some(ca_certificate), ca_private_key: Some(ca_private_key), diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 08d8e6d754..71fe4a7af1 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -532,7 +532,7 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` -to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +to let dedicated sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs and passes the configured socket path to the Kubernetes driver. @@ -549,6 +549,17 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this path does not require access to the SPIRE OIDC discovery endpoint or its TLS CA. +For `supervisor_topology: gateway`, configure +`gatewayConfig.openshell.gateway.spiffe_broker` instead. Only the gateway mounts +the CSI Workload API socket and obtains stable sandbox identities through the +Broker API; shared supervisors receive final access tokens. +Set `node_local_port: 8443` instead of `endpoint` to use the gateway's local +agent directly. The chart injects its node address from `status.hostIP`; +the agent must expose its Broker port on that address. This path needs no +Broker Service. See +[gateway configuration](https://docs.nvidia.com/openshell/how-it-works/gateways/configuration#sandbox-identity-through-the-spiffe-broker-api) +for peer identity policies, multi-node registrations, and authorization. + ## Values | Key | Type | Default | Description | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 1d9f8c58dd..7c7a246bf6 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -533,7 +533,7 @@ JWT signing Secret. ## SPIFFE/SPIRE provider token grants Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` -to let sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token +to let dedicated sandbox supervisors use SPIFFE JWT-SVIDs for dynamic provider token grants. The chart keeps supervisor-to-gateway authentication on gateway-minted sandbox JWTs and passes the configured socket path to the Kubernetes driver. @@ -550,5 +550,16 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this path does not require access to the SPIRE OIDC discovery endpoint or its TLS CA. +For `supervisor_topology: gateway`, configure +`gatewayConfig.openshell.gateway.spiffe_broker` instead. Only the gateway mounts +the CSI Workload API socket and obtains stable sandbox identities through the +Broker API; shared supervisors receive final access tokens. +Set `node_local_port: 8443` instead of `endpoint` to use the gateway's local +agent directly. The chart injects its node address from `status.hostIP`; +the agent must expose its Broker port on that address. This path needs no +Broker Service. See +[gateway configuration](https://docs.nvidia.com/openshell/how-it-works/gateways/configuration#sandbox-identity-through-the-spiffe-broker-api) +for peer identity policies, multi-node registrations, and authorization. + {{ template "chart.valuesSection" . }} {{ template "helm-docs.versionFooter" . }} diff --git a/deploy/helm/openshell/ci/values-spire-broker-stack.yaml b/deploy/helm/openshell/ci/values-spire-broker-stack.yaml new file mode 100644 index 0000000000..c131de2cbd --- /dev/null +++ b/deploy/helm/openshell/ci/values-spire-broker-stack.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# SPIRE 1.15.3 / hardened chart 0.30.3 Broker API fixture. +global: + spire: + clusterName: openshell-e2e + jwtIssuer: https://spire.openshell.local + trustDomain: openshell.local +spire-server: + defaultJwtSvidTTL: 5m + defaultX509SvidTTL: 30s + agentTTL: 1h +spire-agent: + brokerAPI: + tcp: + enabled: true + bindAddress: 0.0.0.0:8443 + brokers: + openshell: + enabled: true + idTemplate: spiffe://{{ .TrustDomain }}/ns/openshell/sa/openshell + allowedReferenceTypes: + - typeURL: type.googleapis.com/spiffe.broker.KubernetesObjectReference + allowOverTCP: true + workloadAttestors: + k8s: + brokerAPI: + accessPolicy: enforced + brokers: + openshell: + enabled: true + podReferenceScope: cluster + impersonation: + clusterWidePodsOnly: false +spiffe-oidc-discovery-provider: + enabled: false diff --git a/deploy/helm/openshell/ci/values-supervisor-spiffe-node-local.yaml b/deploy/helm/openshell/ci/values-supervisor-spiffe-node-local.yaml new file mode 100644 index 0000000000..9a84ea7a9c --- /dev/null +++ b/deploy/helm/openshell/ci/values-supervisor-spiffe-node-local.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# The test supplies external PostgreSQL so the gateway can move across nodes. +workload: + kind: deployment +server: + externalDbSecret: e2e-spiffe-postgres +supervisor: + replicas: 2 + recoveryTimeoutSeconds: 60 +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://openshell.local/openshell + node_local_port: 8443 + server_spiffe_id_prefix: spiffe://openshell.local/spire/agent/k8s_psat/openshell-e2e + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock diff --git a/deploy/helm/openshell/ci/values-supervisor-spiffe.yaml b/deploy/helm/openshell/ci/values-supervisor-spiffe.yaml new file mode 100644 index 0000000000..ed3f25dc7d --- /dev/null +++ b/deploy/helm/openshell/ci/values-supervisor-spiffe.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# The peer policy covers all agents in this fixture's attested cluster. +supervisor: + replicas: 2 + recoveryTimeoutSeconds: 60 +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://openshell.local/openshell + endpoint: https://spire-broker.spire.svc:8443 + server_spiffe_id_prefix: spiffe://openshell.local/spire/agent/k8s_psat/openshell-e2e + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock diff --git a/deploy/helm/openshell/templates/NOTES.txt b/deploy/helm/openshell/templates/NOTES.txt new file mode 100644 index 0000000000..c25b465aff --- /dev/null +++ b/deploy/helm/openshell/templates/NOTES.txt @@ -0,0 +1,22 @@ +{{- /* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ -}} +{{- $broker := get (.Values.gatewayConfig | default dict) "openshell.gateway.spiffe_broker" | default dict -}} +{{- if $broker }} +SPIFFE Broker API (openshell.gateway.spiffe_broker) is enabled. +The gateway uses mTLS with X.509-SVIDs from its CSI Workload API socket. +Enable the experimental Broker endpoint on SPIRE, allowlist the gateway SPIFFE +ID and KubernetesObjectReference over TCP, and configure enforced Kubernetes +SubjectAccessReview authorization (impersonate-via-spire) for Sandbox resources. +Registrations must match the attested Sandbox resource UID on every selected +agent, for example through a cluster-scoped SPIRE node alias. +{{- if hasKey $broker "node_local_port" }} +The gateway connects directly to its node's Broker port using status.hostIP; +no Broker Service is used. Expose the agent listener on that node address. +Registrations must cover every node where a gateway replica may run. +{{- end }} +Use server_spiffe_id_prefix for agents in one attested cluster, +or server_spiffe_id for one exact identity. The alias is a registration parent, +not a replacement for the agent's certificate identity. +{{- end }} diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index fed49a498c..a478e25ac2 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -9,7 +9,8 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. {{- $oidcRuntimeConfig := get $gatewayConfig "openshell.gateway.oidc" | default dict -}} {{- if not (get $oidcRuntimeConfig "issuer") -}}{{- $oidcRuntimeConfig = .Values.server.oidc | default dict -}}{{- end -}} {{- $kubernetesRuntimeConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} -{{- $spiffeSocketPath := get $kubernetesRuntimeConfig "provider_spiffe_workload_api_socket_path" -}} +{{- $broker := get $gatewayConfig "openshell.gateway.spiffe_broker" | default dict -}} +{{- $spiffeSocketPath := get $broker "workload_api_socket_path" | default (get $kubernetesRuntimeConfig "provider_spiffe_workload_api_socket_path") -}} {{- $hasExternalCredentialDriver := or (eq (include "openshell.credentialDriverEnabled" (list . "kubernetes-secrets")) "true") (eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true") -}} {{- $vaultCredentialDriverEnabled := eq (include "openshell.credentialDriverEnabled" (list . "vault")) "true" -}} {{- $credentialDrivers := .Values.credentialDrivers | default dict -}} @@ -66,6 +67,12 @@ spec: - {{ .Values.server.dbUrl | quote }} {{- end }} env: + {{- if hasKey $broker "node_local_port" }} + - name: OPENSHELL_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- end }} - name: OPENSHELL_REPLICA_ID valueFrom: fieldRef: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index a4af52cea4..3ae617c353 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -393,6 +393,44 @@ fullname, so the name never collapses into another release resource's name. {{- printf "%s%s" $base $suffix -}} {{- end }} +{{/* Validate gateway-owned SPIFFE Broker API settings. */}} +{{- define "openshell.validateSpiffeBroker" -}} +{{- $broker := .broker -}} +{{- $nodeLocal := hasKey $broker "node_local_port" -}} +{{- if eq $nodeLocal (not (empty (get $broker "endpoint"))) -}} +{{- fail "spiffe_broker requires exactly one of endpoint and node_local_port" -}} +{{- end -}} +{{- if $nodeLocal -}} +{{- $port := get $broker "node_local_port" -}} +{{- if or (kindIs "string" $port) (not (regexMatch "^[0-9]+$" (toString $port))) (lt (int $port) 1) (gt (int $port) 65535) -}} +{{- fail "spiffe_broker.node_local_port must be between 1 and 65535" -}} +{{- end -}} +{{- else -}} +{{- $url := urlParse (get $broker "endpoint" | default "") -}} +{{- if or (ne $url.scheme "https") (empty $url.host) $url.userinfo $url.query $url.fragment (not (has $url.path (list "" "/"))) -}} +{{- fail "spiffe_broker.endpoint must be an HTTPS endpoint" -}} +{{- end -}} +{{- end -}} +{{- $socket := get $broker "workload_api_socket_path" | default "" -}} +{{- if or (not (hasPrefix "/" $socket)) (ne (trim $socket) $socket) (eq (dir $socket) "/") (hasSuffix "/" $socket) (regexMatch "(^|/)\\.\\.?(/|$)" $socket) -}} +{{- fail "spiffe_broker.workload_api_socket_path must be an absolute UNIX socket path" -}} +{{- end -}} +{{- $exact := get $broker "server_spiffe_id" | default "" -}} +{{- $prefix := get $broker "server_spiffe_id_prefix" | default "" -}} +{{- if eq (empty $exact) (empty $prefix) -}} +{{- fail "spiffe_broker requires exactly one of server_spiffe_id and server_spiffe_id_prefix" -}} +{{- end -}} +{{- $peerField := ternary "server_spiffe_id" "server_spiffe_id_prefix" (not (empty $exact)) -}} +{{- range $field := list $peerField "spiffe_id_prefix" -}} +{{- if not (regexMatch "^spiffe://[a-z0-9._-]+(/[A-Za-z0-9._-]+)*$" (get $broker $field | default "")) -}} +{{- fail (printf "spiffe_broker.%s must be a SPIFFE ID" $field) -}} +{{- end -}} +{{- end -}} +{{- if and $prefix (not (regexMatch "^spiffe://[^/]+/.+" $prefix)) -}} +{{- fail "spiffe_broker.server_spiffe_id_prefix requires an agent path" -}} +{{- end -}} +{{- end }} + {{/* Pod labels of the shared supervisor; its Service selects exactly these. */}} @@ -757,8 +795,8 @@ Validate chart values that Helm would otherwise accept silently. {{- if ne (get $gateway "compute_driver" | default "kubernetes") "kubernetes" -}} {{- fail "supervisor_topology requires the kubernetes compute driver" -}} {{- end -}} -{{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") (get $kubernetes "provider_spiffe_workload_api_socket_path") -}} -{{- fail "supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity" -}} +{{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") -}} +{{- fail "supervisor_topology=gateway does not support upstream proxy settings" -}} {{- end -}} {{- if .Values.server.disableTls -}} {{- fail "supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS" -}} @@ -767,6 +805,16 @@ Validate chart values that Helm would otherwise accept silently. {{- fail "supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value" -}} {{- end -}} {{- end -}} +{{- $broker := get (.Values.gatewayConfig | default dict) "openshell.gateway.spiffe_broker" | default dict -}} +{{- if and (eq (get $kubernetes "supervisor_topology" | default "") "gateway") (get $kubernetes "provider_spiffe_workload_api_socket_path") (empty $broker) -}} +{{- fail "shared supervisor SPIFFE grants require spiffe_broker; remove provider_spiffe_workload_api_socket_path from the shared supervisor" -}} +{{- end -}} +{{- if $broker -}} +{{- if ne (get $kubernetes "supervisor_topology" | default "") "gateway" -}} +{{- fail "spiffe_broker requires supervisor_topology=gateway" -}} +{{- end -}} +{{- include "openshell.validateSpiffeBroker" (dict "broker" $broker) -}} +{{- end -}} {{- if hasKey (get (.Values.gatewayConfig | default dict) "openshell.drivers.kubernetes" | default dict) "shared_supervisor" -}} {{- fail "shared_supervisor is rendered by the chart; set only supervisor_topology" -}} {{- end -}} diff --git a/deploy/helm/openshell/tests/notes_test.yaml b/deploy/helm/openshell/tests/notes_test.yaml new file mode 100644 index 0000000000..c644cab5b9 --- /dev/null +++ b/deploy/helm/openshell/tests/notes_test.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: install notes +templates: + - templates/NOTES.txt +release: + name: openshell + namespace: gateway +tests: + - it: describes enforced Broker API authorization + set: + gatewayConfig: + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://example.org/openshell + endpoint: https://spire-broker.spire.svc:8443 + server_spiffe_id: spiffe://example.org/spire/agent/node + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - matchRegexRaw: + pattern: "gateway uses mTLS" + - matchRegexRaw: + pattern: "impersonate-via-spire" diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index a9f124c5f5..e0b9cf4467 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -807,7 +807,7 @@ tests: supervisor_topology: gateway asserts: - failedTemplate: - errorMessage: supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity + errorMessage: supervisor_topology=gateway does not support upstream proxy settings - it: rejects workspace until implemented template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml index b37c16e295..a9d757697f 100644 --- a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml @@ -10,6 +10,40 @@ release: name: openshell namespace: gateway tests: + - it: accepts a cluster-scoped Broker peer prefix + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://example.org/openshell + endpoint: https://spire-broker.spire.svc:8443 + server_spiffe_id_prefix: spiffe://example.org/spire/agent/k8s_psat/cluster + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: spiffe-workload-api + csi: + driver: csi.spiffe.io + readOnly: true + - it: refuses ambiguous Broker peer policies + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://example.org/openshell + endpoint: https://spire-broker.spire.svc:8443 + server_spiffe_id: spiffe://example.org/agent/node + server_spiffe_id_prefix: spiffe://example.org/agent + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - failedTemplate: + errorMessage: spiffe_broker requires exactly one of server_spiffe_id and server_spiffe_id_prefix - it: issues no extra PKI for the shared supervisor template: templates/certgen.yaml set: @@ -104,3 +138,91 @@ tests: name: supervisor-token secret: secretName: openshell-supervisor-token + - it: rejects shared supervisor SPIFFE configuration without a broker + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + provider_spiffe_workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - failedTemplate: + errorMessage: "shared supervisor SPIFFE grants require spiffe_broker; remove provider_spiffe_workload_api_socket_path from the shared supervisor" + - it: uses only the Workload API CSI mount for broker identity + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://example.org/openshell + endpoint: https://spire-broker.spire.svc:8443 + server_spiffe_id: spiffe://example.org/spire/agent/node + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - notExists: + path: spec.template.spec.securityContext.supplementalGroups + - notExists: + path: spec.template.spec.volumes[*].hostPath + - contains: + path: spec.template.spec.volumes + content: + name: spiffe-workload-api + csi: + driver: csi.spiffe.io + readOnly: true + - it: injects the node address for node-local Broker transport + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + spiffe_id_prefix: spiffe://example.org/openshell + node_local_port: 8443 + server_spiffe_id_prefix: spiffe://example.org/spire/agent/k8s_psat/cluster + workload_api_socket_path: /spiffe-workload-api/spire-agent.sock + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - notExists: + path: spec.template.spec.volumes[*].hostPath + - it: rejects ambiguous Broker endpoint selection + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + endpoint: https://broker:8443 + node_local_port: 8443 + asserts: + - failedTemplate: + errorMessage: spiffe_broker requires exactly one of endpoint and node_local_port + - it: rejects an invalid node-local Broker port + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + node_local_port: 0 + asserts: + - failedTemplate: + errorMessage: spiffe_broker.node_local_port must be between 1 and 65535 + - it: rejects a quoted node-local Broker port before gateway startup + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.gateway.spiffe_broker: + node_local_port: "8443" + asserts: + - failedTemplate: + errorMessage: spiffe_broker.node_local_port must be between 1 and 65535 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 6d60da3a28..7129244d59 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -411,6 +411,10 @@ topologySpreadConstraints: [] gatewayConfig: openshell: version: 2 + # openshell.gateway.spiffe_broker enables gateway-owned sandbox JWT-SVID + # acquisition over mTLS. Configure node_local_port or endpoint, an exact or + # cluster-scoped peer identity, the Workload API socket, and the stable sandbox + # SPIFFE ID prefix. SPIRE must authorize the gateway and attest its Sandbox references. # Server configuration server: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index c86c419ab1..c1468455ef 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1533,7 +1533,7 @@ In managed workspace mode, the Kubernetes driver copies each explicitly named namespace on sandbox creation. Shared and operator modes require the Secret to already exist in the sandbox namespace. -For token-exchange provider profiles, the gateway also needs access to its own +For dedicated-supervisor token-exchange profiles, the gateway also needs access to its own SPIFFE Workload API socket. In Helm deployments, set `server.providerTokenGrants.spiffe.enabled=true`; the chart mounts the socket into the gateway pod and sets `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`. @@ -1541,6 +1541,78 @@ The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so this validation path does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. +### Sandbox identity through the SPIFFE Broker API + +For shared-supervisor Kubernetes sandboxes, the gateway can obtain sandbox +JWT-SVIDs through SPIRE's experimental Broker API. This uses the gateway's own +X.509-SVID over mTLS. + +```toml +[openshell.gateway.spiffe_broker] +spiffe_id_prefix = "spiffe://example.org/openshell" +node_local_port = 8443 +server_spiffe_id_prefix = "spiffe://example.org/spire/agent/k8s_psat/cluster" +workload_api_socket_path = "/spiffe-workload-api/spire-agent.sock" +``` + +`node_local_port` connects directly to the SPIRE agent on the gateway Pod's +node. Helm injects `OPENSHELL_NODE_IP` from the Downward API's `status.hostIP`; +the gateway formats the IPv4 or IPv6 endpoint at startup without Kubernetes API +calls or additional RBAC. A missing or invalid node IP prevents startup. The +agent must expose its mTLS Broker listener on that node address through host +networking or a host port. Broker traffic does not traverse a Kubernetes Service. +The gateway does not fall back to a remote agent when its local agent is unavailable. + +Configure exactly one transport: `node_local_port`, or an explicit HTTPS +`endpoint` for a fixed address or Service. Outside Helm, node-local deployments +must supply `OPENSHELL_NODE_IP` themselves. Both modes retain the same mTLS, +SPIFFE peer verification, broker allowlist, and Kubernetes authorization checks. + +Gateway session JWTs with a positive TTL are required. +When enabling the broker on a gateway that omitted `gateway_jwt.ttl_secs`, set +a positive value and restart existing sandboxes to issue expiring sessions; +existing non-expiring sessions cannot request brokered tokens. +The gateway fetches fresh X.509 credentials and trust bundles for each token +acquisition, so Workload API rotation does not require a gateway restart. +The same `workload_api_socket_path` supplies the gateway's JWT-SVID for +`token_exchange`; no separate socket environment variable is needed. Omit +the driver's `provider_spiffe_workload_api_socket_path` for shared supervisors. + +The operator must enable SPIRE's Broker endpoint, authorize the gateway SPIFFE +ID for `KubernetesObjectReference` over TCP, and configure the Kubernetes +attestor with `access_policy = "enforced"`. Give the agent permission to look +up Sandbox resources and create SubjectAccessReviews. UID-only references +require cluster-wide Sandbox `get`/`list` permission for the agents, even +though SPIRE requests metadata-only responses. Grant the gateway's +SPIFFE ID the `impersonate-via-spire` verb on Sandbox resources only in its +permitted namespaces. + +Register each sandbox's stable SPIFFE ID against its attested Sandbox resource +UID. The gateway supplies the UID from its persisted compute-runtime identity; +workload Pod replacement does not change it. Registrations must reach every +eligible agent. In node-local mode this includes every node where a gateway +replica may run, so gateway rescheduling does not require re-registration. +Use a SPIRE node alias as the +registration parent to cover agents attested with the intended cluster, +namespace, and ServiceAccount selectors, including newly added nodes. The +alias does not change the agents' certificate identities. + +Set exactly one peer policy. `server_spiffe_id_prefix` accepts descendants of +the configured SPIFFE path in the same trust domain. The example accepts all +node IDs in one attested cluster and rejects sibling cluster paths. For an +endpoint serving one fixed identity, use `server_spiffe_id` instead. Certificate +chain verification and the requirement for exactly one URI SAN apply to both +modes. Do not use a trust-domain-wide peer policy. The installed SPIRE version and operator +must support the experimental API; this configuration alone does not enable it. + +If grants fail after rescheduling or adding nodes, check that the eligible +agents match the peer prefix and node alias, and that Sandbox registrations +have reached each agent. Identity or configuration rejections require fixing +registration, trust, or authorization; connection failures can be retried. +Provider-token requests use a separate instance of the configured gRPC rate +budget. Authenticated work is also capped at four concurrent requests per +sandbox and 64 per gateway, including when token caching is disabled. + ### MXC The MXC driver runs native Windows workloads through `wxc-exec`. Use this diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 13be91fd24..9b1346ec5e 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -703,15 +703,17 @@ Token grant fields: | `jwt_svid_audience` | No | Audience used when requesting the JWT-SVID. When omitted, OpenShell derives an issuer-style audience from Keycloak token endpoint paths or falls back to the full token endpoint URL. | | `client_assertion_type` | No | OAuth2 `client_assertion_type` form value. Defaults to RFC 7523 `urn:ietf:params:oauth:client-assertion-type:jwt-bearer`. Set `urn:ietf:params:oauth:client-assertion-type:jwt-spiffe` when the token issuer expects the SPIFFE assertion type. | | `scopes` | No | OAuth2 scopes sent as a space-separated `scope` parameter. | -| `cache_ttl` | No | Protobuf-duration cache override, such as `300s` or `0.500s`. Omission uses the token response expiry; `0s` disables caching. The legacy `cache_ttl_seconds` input remains accepted. | +| `cache_ttl` | No | Protobuf-duration cache override, such as `300s`. Omission uses the token response expiry; `0s` disables caching. Shared supervisor grants require a positive override to be at least `1s` and cap it at the issuer and identity expiries. The legacy `cache_ttl_seconds` input remains accepted. | | `requested_token_type` | No | RFC 8693 `requested_token_type` sent during token exchange. Defaults to `urn:ietf:params:oauth:token-type:access_token`. | | `subject_token` | Required for `token_exchange` | Subject-token source used for the gateway-brokered intermediate exchange. Phase one supports `source: provider_credential`, where `credential` names another credential declared in the same profile. That referenced credential is broker-only and cannot declare workload injection metadata such as env vars, static placement, refresh, or token grant metadata. | | `subject_token.subject_token_type` | No | RFC 8693 `subject_token_type` for the stored subject token. Defaults to `urn:ietf:params:oauth:token-type:access_token`. | | `audience_overrides` | No | Endpoint-specific final-exchange `audience` and `scopes` overrides selected by host, port, and path. These overrides do not affect the gateway intermediate exchange. | -Token grants require the sandbox supervisor to have access to a SPIFFE Workload API socket. `token_exchange` also requires the gateway to have its own Workload API socket so it can present a gateway JWT-SVID during the intermediate exchange. They apply to HTTP traffic that the proxy can inspect. Token-grant profiles reject endpoints with `protocol: sql` or `tls: skip`, including when `allow_uninspected_credentials` is enabled, because those modes cannot inject the required HTTP credential. The token service must return a token value that is safe for HTTP header placement; malformed values are rejected before caching or header injection. +Dedicated supervisors use a local SPIFFE Workload API socket for the exchanges above. Shared supervisors use the gateway's [SPIFFE Broker API integration](/how-it-works/gateways/configuration#sandbox-identity-through-the-spiffe-broker-api): the gateway obtains the stable sandbox JWT-SVID and performs both exchanges, then returns only the final access token. The intermediate audience is the stable sandbox SPIFFE ID. Only the gateway mounts the Workload API socket, including for its own JWT-SVID in `token_exchange`. -The gateway only brokers an intermediate token for a sandbox principal, and only when the requested provider is attached to that sandbox. It verifies the supervisor JWT-SVID issuer, audience, signature, and SPIFFE trust domain against the gateway's own JWT-SVID, then uses the verified supervisor SVID subject as the intermediate-token audience. +Token grants apply to HTTP traffic that the proxy can inspect. Token-grant profiles reject endpoints with `protocol: sql` or `tls: skip`, including when `allow_uninspected_credentials` is enabled, because those modes cannot inject the required HTTP credential. The token service must return a token value that is safe for HTTP header placement; malformed values are rejected before caching or header injection. + +The gateway authorizes the sandbox session and provider attachment on every request. In the dedicated path it also verifies the supervisor JWT-SVID issuer, audience, signature, and SPIFFE trust domain. In the shared path it resolves the configured credential itself and rechecks authorization before returning even a cached token. ### Token Grant Failures diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index eaacec3671..50cd1da2c6 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -34,9 +34,9 @@ the authenticated sandbox ID and runtime identity. The gateway requires the driver and runtime identity to match its stored sandbox record before issuing a gateway JWT. -Dynamic provider token grants can use SPIFFE without changing supervisor-to-gateway authentication. Set `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` to mount the SPIFFE CSI Workload API socket into gateway and sandbox pods while keeping gateway-issued session JWTs for sandbox-specific RPCs. +Dynamic provider token grants use SPIFFE while keeping gateway-issued session JWTs for sandbox-specific RPCs. Dedicated supervisors use `gatewayConfig.openshell.drivers.kubernetes.provider_spiffe_workload_api_socket_path` to mount the CSI Workload API socket into gateway and sandbox pods. Shared supervisors instead use the gateway's [SPIFFE Broker API integration](/how-it-works/gateways/configuration#sandbox-identity-through-the-spiffe-broker-api); only the gateway mounts the socket. -Provider token grants require a SPIFFE implementation such as SPIRE and identities for the gateway and sandbox pods. The repository's local SPIRE overlay assigns sandbox IDs from the pod's `openshell.ai/sandbox-id` annotation, but the gateway validation path only requires the supervisor SVID to be valid and in the same SPIFFE trust domain as the gateway SVID. Provider profiles with `token_grant` metadata cause the sandbox supervisor to request JWT-SVIDs and exchange them for upstream OAuth2 access tokens. Token-exchange profiles also require a gateway SPIFFE identity because the gateway brokers the intermediate token exchange with its own JWT-SVID. +Provider token grants require a SPIFFE implementation such as SPIRE. Dedicated supervisors obtain Pod identities in the gateway's trust domain. For shared supervisors, the gateway obtains the stable sandbox identity from the attested Sandbox CR UID, authorizes the session and attached provider, and performs the exchanges. SPIRE enforces the gateway SPIFFE ID's permission to access that Sandbox resource. Token-exchange profiles also use the gateway's own SPIFFE identity for the intermediate exchange. The gateway verifies supervisor JWT-SVIDs with JWT bundles fetched from the SPIFFE Workload API, so intermediate token exchange does not require gateway access to the SPIRE OIDC discovery endpoint or its TLS CA. diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 287d2c65ec..30897d9dee 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e567177890eb1617b1f774005b66b26b2377afd138a2ca37aae7d8f0c81429d4" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -54,7 +63,7 @@ checksum = "0258346f92c78dbc5b336b9a5b27c64447aa10643bc64e651db2edb145361564" dependencies = [ "anstyle", "memchr", - "unicode-width", + "unicode-width 0.2.3", ] [[package]] @@ -100,7 +109,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -258,6 +267,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f4220217f3288537b83684130f42fad43320b1c3b805f1910d683d52ded24f" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.22.1" @@ -418,6 +451,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.2" @@ -879,6 +918,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5905ec755cc8386d7928353e4d356d2be3e89e81e5ca01fe630a1ca4d08dc175" + [[package]] name = "flate2" version = "1.1.10" @@ -918,6 +963,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1095,6 +1146,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "gimli" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033caf0b349c518623b5396bfb2cf0bddf44f0306d543a250e5743297aafd10" + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -1136,6 +1199,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1144,7 +1216,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1155,7 +1227,7 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1390,9 +1462,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -1404,22 +1476,43 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406e9ae5b95d8aa37db6293405931b766a6172d1848daee22d70eb44aa78a43c" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3632ebbd0c5c3637ba1a40bdf8288f2cb9d2268e73441a69d5fc73d7cfbaa87c" + [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1431,16 +1524,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1451,18 +1545,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1470,6 +1566,29 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b195dc605f81aa181bec9b3bd14e26cc71b903e4644f6af578ca9772c7a710" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913a7d294ad056474749152107309fdee3ec17d2482d1ab70fdc61a81c64b9a3" + [[package]] name = "ident_case" version = "1.0.1" @@ -1509,6 +1628,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "itertools" version = "0.14.0" @@ -1970,6 +2101,36 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "mime" version = "0.3.17" @@ -2003,12 +2164,30 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "multiversion_no_op" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2151,6 +2330,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.8.1" @@ -2166,6 +2354,43 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openshell-core" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "glob", + "ipnet", + "jsonwebtoken", + "libc", + "miette", + "nix", + "openshell-crypto", + "openshell-extension-core", + "openshell-policy-schema", + "openshell-spiffe", + "prost", + "prost-types", + "protoc-bin-vendored", + "rustix", + "rustls", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tonic-types", + "tracing", + "url", + "uuid", + "zeroize", +] + [[package]] name = "openshell-crypto" version = "0.0.0" @@ -2198,6 +2423,7 @@ dependencies = [ "hyper-util", "jsonwebtoken", "noyalib", + "openshell-core", "openshell-crypto", "openshell-ocsf-validate", "openshell-otel-test-support", @@ -2220,6 +2446,18 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-extension-core" +version = "0.0.0" +dependencies = [ + "hyper-util", + "serde", + "thiserror", + "tokio", + "tonic", + "tower", +] + [[package]] name = "openshell-ocsf" version = "0.0.0" @@ -2296,6 +2534,29 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "openshell-policy-schema" +version = "0.0.0" +dependencies = [ + "miette", + "noyalib", + "serde", + "serde_json", + "serde_path_to_error", + "thiserror", +] + +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2394,6 +2655,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "owo-colors" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" + [[package]] name = "parking" version = "2.2.1" @@ -2491,6 +2758,17 @@ dependencies = [ "pest", ] +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -2544,6 +2822,8 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -2562,6 +2842,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2581,6 +2871,27 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.14.4" @@ -2603,6 +2914,90 @@ dependencies = [ "prost", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8760a25b6ff9c620324822737e468478fa092234190d2e449760344354896ed9" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fa2624782ca04cd44f51554566717377acd240e4c0016d757dd74fccc9324f" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2417e9817fa237dab803ad4dda7357a111656e242959cc6b8f9a1a583367d42" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d189c34636356a46a7ed3188233dc8a88c431278cc54d4a19b096a2d270e985" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171e39f1e846e5f322ced1ac3b8d4cd3a3833ca24b6e5d58b3632574fe6204fa" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873cdcc097593432086661aa432b8078f1cd87bfb02847c332e98ae2c119e966" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb72df001783b8297847fe8f5f874ee400fd742c843d60583e8c23d96977c7f" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04652167eca899dda05f32f5481adeaf25c623a98ce2fc146a001cc59a2add7" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263a3f48f01e7309e857138bd47f785585b4a005e8e56c6d2824ce91195999c3" + +[[package]] +name = "pulldown-cmark" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b" +dependencies = [ + "bitflags 2.13.1", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quote" version = "1.0.47" @@ -2791,6 +3186,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3083,6 +3484,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -3499,6 +3911,27 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "2.0.119" @@ -3538,6 +3971,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -3545,12 +3989,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" +dependencies = [ + "icu_segmenter", + "unicode-width 0.2.3", +] + [[package]] name = "thiserror" version = "2.0.19" @@ -3612,11 +4076,12 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -3722,6 +4187,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "tonic-prost" version = "0.14.6" @@ -3733,6 +4210,22 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-prost-build" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea216ddc448e08e7839ab09e20e60b06b6a1c32745ccb212c0ebd161c199f1b" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + [[package]] name = "tonic-types" version = "0.14.6" @@ -3920,6 +4413,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -3953,6 +4452,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.3" @@ -4343,9 +4848,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-parser" @@ -4377,9 +4882,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4388,14 +4893,14 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", - "synstructure", + "syn 3.0.3", + "synstructure 0.14.0", ] [[package]] @@ -4436,7 +4941,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -4461,21 +4966,23 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -4483,13 +4990,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index c35f4ffea6..2efe8ed842 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -41,6 +41,8 @@ e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] e2e-kubernetes-tracing = ["e2e-kubernetes"] # One supervisor process shared by the gateway's sandboxes. e2e-kubernetes-shared-supervisor = ["e2e-kubernetes", "e2e-host-gateway"] +# Gateway-owned sandbox identities through the SPIFFE Broker API. +e2e-kubernetes-shared-supervisor-spiffe = ["e2e-kubernetes-shared-supervisor", "dep:openshell-core"] # A high-availability pool of shared supervisors. e2e-kubernetes-shared-supervisor-ha = ["e2e-kubernetes", "e2e-host-gateway"] # Scaling a one-member high-availability pool of shared supervisors to two @@ -257,6 +259,7 @@ path = "tests/otlp_relay.rs" required-features = ["e2e-local-container-driver"] [dependencies] +openshell-core = { path = "../../crates/openshell-core", default-features = false, features = ["jwt"], optional = true } openshell-crypto = { path = "../../crates/openshell-crypto" } openshell-otel-test-support = { path = "../../crates/openshell-otel-test-support" } opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "trace"] } diff --git a/e2e/rust/tests/kubernetes_shared_supervisor_ha.rs b/e2e/rust/tests/kubernetes_shared_supervisor_ha.rs index 1024abceeb..5ab9255062 100644 --- a/e2e/rust/tests/kubernetes_shared_supervisor_ha.rs +++ b/e2e/rust/tests/kubernetes_shared_supervisor_ha.rs @@ -28,8 +28,8 @@ use std::time::Duration; use openshell_e2e::harness::sandbox::SandboxGuard; use shared_supervisor::{ EXEC_PROBE_TIMEOUT, GATEWAY_SELECTOR, HostEndpoint, SUPERVISOR_CONTAINER, call_host, - counts_past, create, gateway_log_lines, generation, is_takeover, kubectl, kubectl_command, - kubectl_json, main_process, owner_of, policy, release_namespace, supervisor_pods, + counts_past, create, gateway_log_lines, generation, is_takeover, kill_abruptly, kubectl, + kubectl_command, main_process, owner_of, policy, release_namespace, supervisor_pods, wait_for_owner, wait_until_attached, }; use tokio::io::AsyncBufReadExt as _; @@ -50,116 +50,6 @@ const EXPIRY_EVIDENCE: Duration = Duration::from_secs(10); /// What a supervisor logs as it begins to leave the pool. const LEAVING: &str = "pool member is leaving"; -/// Run `command` in a cluster node's container; k3d and kind name node -/// containers after their nodes. -async fn node_exec(node: &str, command: &[&str]) -> Result { - let output = tokio::process::Command::new("docker") - .arg("exec") - .arg(node) - .args(command) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("docker exec {node} {command:?}: {error}"))?; - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } else { - Err(format!( - "docker exec {node} {command:?}: {}", - String::from_utf8_lossy(&output.stderr).trim() - )) - } -} - -/// SIGKILL the supervisor container of `pod` through its node's container -/// runtime, so it cannot announce that it is leaving, then force-delete the -/// Pod so its member does not come back in place. Kills the task with -/// `ctr`, or failing that the container's process with `kill -KILL`; panics -/// when neither succeeds. Returns how the container was killed. -async fn kill_abruptly(pod: &str) -> String { - let namespace = release_namespace(); - let description = kubectl_json(&["-n", &namespace, "get", "pod", pod, "-o", "json"]).await; - let node = description["spec"]["nodeName"] - .as_str() - .unwrap_or_else(|| panic!("{pod} has no node: {description}")) - .to_string(); - let container = description["status"]["containerStatuses"] - .as_array() - .into_iter() - .flatten() - .find(|status| status["name"] == SUPERVISOR_CONTAINER) - .and_then(|status| status["containerID"].as_str()) - .and_then(|id| id.split_once("://").map(|(_, id)| id.to_string())) - .unwrap_or_else(|| panic!("{pod} has no running {SUPERVISOR_CONTAINER} container")); - let mut failures = Vec::new(); - let mut killed = None; - // k3s ships `ctr` as a multi-call alias; kind ships it on its own. - for ctr in [&["ctr"][..], &["k3s", "ctr"][..]] { - let kill = [ - ctr, - &[ - "-n", "k8s.io", "tasks", "kill", "--signal", "SIGKILL", &container, - ], - ] - .concat(); - match node_exec(&node, &kill).await { - Ok(_) => { - killed = Some(format!("ctr tasks kill --signal SIGKILL on {node}")); - break; - } - Err(error) => failures.push(error), - } - } - if killed.is_none() { - for ctr in [&["ctr"][..], &["k3s", "ctr"][..]] { - let list = [ctr, &["-n", "k8s.io", "tasks", "ls"]].concat(); - let pid = match node_exec(&node, &list).await { - Ok(tasks) => tasks.lines().find_map(|line| { - let mut fields = line.split_whitespace(); - (fields.next() == Some(container.as_str())) - .then(|| fields.next()) - .flatten() - .filter(|pid| pid.parse::().is_ok_and(|pid| pid > 1)) - .map(str::to_string) - }), - Err(error) => { - failures.push(error); - continue; - } - }; - let Some(pid) = pid else { - failures.push(format!("no running task {container} on {node}")); - continue; - }; - match node_exec(&node, &["kill", "-KILL", &pid]).await { - Ok(_) => { - killed = Some(format!("kill -KILL {pid} on {node}")); - break; - } - Err(error) => failures.push(error), - } - } - } - let killed = killed.unwrap_or_else(|| { - panic!( - "could not hard-kill supervisor {pod}; a graceful stop proves nothing: {failures:#?}" - ) - }); - kubectl(&[ - "-n", - &namespace, - "delete", - "pod", - pod, - "--grace-period=0", - "--force", - "--wait=false", - ]) - .await; - killed -} - /// Lines a supervisor Pod logs from its start until its container ends. struct FollowedLogs { lines: std::sync::Arc>>, diff --git a/e2e/rust/tests/kubernetes_shared_supervisor_spiffe.rs b/e2e/rust/tests/kubernetes_shared_supervisor_spiffe.rs new file mode 100644 index 0000000000..b89f60fd90 --- /dev/null +++ b/e2e/rust/tests/kubernetes_shared_supervisor_spiffe.rs @@ -0,0 +1,1374 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-shared-supervisor-spiffe")] + +//! Gateway-owned SPIFFE identities and RFC 8693 provider tokens for shared supervisors. +//! +//! Covers sandbox isolation, multi-node agent routing and replacement, credential +//! rotation, and supervisor failover with stale-session fencing against real SPIRE. + +#[path = "kubernetes_shared_supervisor_spiffe/node_local.rs"] +mod node_local; +mod shared_supervisor; + +use std::collections::{BTreeSet, HashMap}; +use std::io::Write as _; +use std::process::Stdio; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use base64::Engine as _; +use openshell_core::jwt::{SecretJwt, SupervisorAuthBundle}; +use openshell_core::proto::{ + GetSandboxProviderAccessTokenRequest, GetSandboxProviderEnvironmentRequest, + open_shell_client::OpenShellClient, +}; +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; +use shared_supervisor::{ + COUNTER_MAIN, counts_past, gateway_log_lines, generation, is_takeover, kill_abruptly, + main_process, release_namespace, supervisor_pods, wait_for_owner, wait_until_attached, +}; +use tempfile::NamedTempFile; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint}; + +const TEST_HOST: &str = "host.openshell.internal"; +/// Matches `ci/values-spire-broker-stack.yaml`. +const TRUST_DOMAIN: &str = "openshell.local"; +/// Matches `ci/values-supervisor-spiffe.yaml`. +const ID_PREFIX: &str = "spiffe://openshell.local/openshell"; +/// Label on the registration entries this run creates, naming its SPIRE +/// fixture so cleanup never touches another run's entries. +const ENTRY_LABEL: &str = "openshell.ai/e2e-spire-fixture"; +const STS_SERVICE: &str = "openshell-e2e-sts"; +const CLIENT_ASSERTION_TYPE: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe"; +const TOKEN_TYPE_ACCESS_TOKEN: &str = "urn:ietf:params:oauth:token-type:access_token"; +const TOPOLOGY_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-topology"; +/// Sandbox instance route annotation. +const INSTANCE_ROUTE_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; + +/// Fixed listener ports let a remote cluster reach these host fixtures through +/// an operator-provided reverse tunnel. Local runs keep ephemeral ports. +fn fixture_port(variable: &str) -> u16 { + std::env::var(variable).map_or(0, |value| { + value.parse().expect("fixture port must be a u16") + }) +} + +fn kubectl_command() -> tokio::process::Command { + shared_supervisor::kubectl_command() +} + +async fn kubectl_with_input(args: &[&str], input: Option<&str>) -> String { + let mut child = kubectl_command() + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|error| panic!("spawn kubectl {args:?}: {error}")); + let mut stdin = child.stdin.take().expect("kubectl stdin"); + if let Some(input) = input { + stdin + .write_all(input.as_bytes()) + .await + .expect("write kubectl input"); + } + drop(stdin); + let output = child.wait_with_output().await.expect("wait for kubectl"); + assert!( + output.status.success(), + "kubectl {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +async fn kubectl(args: &[&str]) -> String { + kubectl_with_input(args, None).await +} + +async fn cli(args: &[&str]) -> (bool, String) { + let output = openshell_cmd() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("spawn openshell CLI"); + ( + output.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + ) +} + +fn temp_yaml(contents: &str) -> NamedTempFile { + let mut file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .expect("create temp file"); + file.write_all(contents.as_bytes()) + .expect("write temp file"); + file.flush().expect("flush temp file"); + file +} + +fn spiffe_id(gateway_id: &str, sandbox_id: &str) -> String { + format!("{ID_PREFIX}/gateway/{gateway_id}/sandbox/{sandbox_id}") +} + +/// Claims of a JWT, decoded for observation only. +fn claims(token: &str) -> serde_json::Value { + let payload = token.split('.').nth(1).unwrap_or_default(); + base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(payload) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() +} + +/// One exchange the token service answered. +#[derive(Clone, Debug)] +struct Exchange { + /// Subject of the client assertion. + assertion_subject: String, + /// `audience` parameter of the request. + audience: String, + /// Whether the subject token was the stored user token (the gateway's + /// intermediate exchange) rather than an intermediate token. + intermediate: bool, +} + +/// RFC 8693 token service that binds each intermediate token to the SPIFFE ID +/// it was issued for, and issues a final token only to that identity. +struct TokenService { + port: u16, + state: Arc>, + task: tokio::task::JoinHandle<()>, +} + +#[derive(Default)] +struct TokenState { + exchanges: Vec, + /// Intermediate token -> the SPIFFE ID it was issued for. + intermediates: HashMap, +} + +impl TokenService { + async fn start(stored_token: String) -> Self { + let listener = + TcpListener::bind(("0.0.0.0", fixture_port("OPENSHELL_E2E_SPIFFE_STS_PORT"))) + .await + .expect("bind token service"); + let port = listener.local_addr().expect("token service address").port(); + let state = Arc::new(Mutex::new(TokenState::default())); + let (shared, stored) = (state.clone(), Arc::new(stored_token)); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let (state, stored) = (shared.clone(), stored.clone()); + tokio::spawn(async move { + let _ = answer_token_request(stream, &state, &stored).await; + }); + } + }); + Self { port, state, task } + } + + fn exchanges(&self) -> Vec { + self.state.lock().unwrap().exchanges.clone() + } +} + +impl Drop for TokenService { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn read_request(stream: &mut TcpStream) -> std::io::Result<(String, String)> { + let mut request = Vec::new(); + let mut chunk = [0_u8; 8192]; + loop { + let read = stream.read(&mut chunk).await?; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + let text = String::from_utf8_lossy(&request); + if let Some((head, body)) = text.split_once("\r\n\r\n") { + let length = head + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok())? + }) + .unwrap_or(0); + if body.len() >= length { + return Ok((head.to_string(), body.to_string())); + } + } + } + let text = String::from_utf8_lossy(&request).into_owned(); + Ok(text + .split_once("\r\n\r\n") + .map(|(head, body)| (head.to_string(), body.to_string())) + .unwrap_or((text, String::new()))) +} + +fn form(body: &str) -> HashMap { + url::form_urlencoded::parse(body.as_bytes()) + .into_owned() + .collect() +} + +async fn respond(stream: &mut TcpStream, status: &str, body: &str) -> std::io::Result<()> { + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await +} + +async fn answer_token_request( + mut stream: TcpStream, + state: &Mutex, + stored: &str, +) -> std::io::Result<()> { + let (_, body) = read_request(&mut stream).await?; + let fields = form(&body); + let field = |name: &str| fields.get(name).cloned().unwrap_or_default(); + let assertion_subject = claims(&field("client_assertion"))["sub"] + .as_str() + .unwrap_or_default() + .to_string(); + let well_formed = field("grant_type") == "urn:ietf:params:oauth:grant-type:token-exchange" + && field("client_assertion_type") == CLIENT_ASSERTION_TYPE + && field("subject_token_type") == TOKEN_TYPE_ACCESS_TOKEN + && assertion_subject.starts_with("spiffe://"); + let issued = { + let mut state = state.lock().unwrap(); + let subject_token = field("subject_token"); + let audience = field("audience"); + if !well_formed { + None + } else if subject_token == stored { + // The gateway exchanges the user's token for the sandbox identity + // named in `audience`. + let token = format!("intermediate-{}", state.intermediates.len() + 1); + state.intermediates.insert(token.clone(), audience.clone()); + state.exchanges.push(Exchange { + assertion_subject, + audience, + intermediate: true, + }); + Some(token) + } else if state.intermediates.get(&subject_token) == Some(&assertion_subject) { + // Only the identity an intermediate token names may redeem it. + let sandbox = assertion_subject.rsplit('/').next().unwrap_or_default(); + let token = format!("final-{sandbox}-{}", state.exchanges.len() + 1); + state.exchanges.push(Exchange { + assertion_subject, + audience, + intermediate: false, + }); + Some(token) + } else { + None + } + }; + match issued { + Some(token) => { + let body = format!( + r#"{{"access_token":"{token}","issued_token_type":"{TOKEN_TYPE_ACCESS_TOKEN}","token_type":"Bearer","expires_in":300}}"# + ); + respond(&mut stream, "200 OK", &body).await + } + None => { + respond( + &mut stream, + "400 Bad Request", + r#"{"error":"invalid_grant"}"#, + ) + .await + } + } +} + +/// Protected endpoint that reports which sandbox a final token was issued to. +struct Target { + port: u16, + seen_tokens: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Target { + async fn start() -> Self { + let listener = + TcpListener::bind(("0.0.0.0", fixture_port("OPENSHELL_E2E_SPIFFE_TARGET_PORT"))) + .await + .expect("bind protected target"); + let port = listener.local_addr().expect("target address").port(); + let seen_tokens = Arc::new(Mutex::new(HashMap::new())); + let observed = seen_tokens.clone(); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let observed = observed.clone(); + tokio::spawn(async move { + let Ok((head, _)) = read_request(&mut stream).await else { + return; + }; + let authorization = head + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("authorization") + .then(|| value.trim().to_string()) + }) + .unwrap_or_default(); + // "Bearer final--" -> "SAW:" + let seen = authorization + .strip_prefix("Bearer final-") + .and_then(|rest| rest.rsplit_once('-')) + .map_or_else(|| "SAW:NONE".to_string(), |(id, _)| format!("SAW:{id}")); + observed.lock().unwrap().insert(seen.clone(), authorization); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{seen}", + seen.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + }); + } + }); + Self { + port, + seen_tokens, + task, + } + } +} + +impl Drop for Target { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// A selectorless Service in front of the host token service, so token +/// endpoints are the in-cluster `http://..svc` URLs that +/// gateways and supervisors accept without TLS. +async fn expose_token_service(port: u16) -> String { + let host_ip = std::env::var("OPENSHELL_E2E_HOST_GATEWAY_IP") + .ok() + .filter(|ip| !ip.is_empty()) + .expect("OPENSHELL_E2E_HOST_GATEWAY_IP is set by with-kube-gateway.sh"); + let namespace = release_namespace(); + let manifest = format!( + r"apiVersion: v1 +kind: Service +metadata: + name: {STS_SERVICE} + namespace: {namespace} +spec: + ports: + - name: http + port: 80 + targetPort: {port} +--- +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: {STS_SERVICE} + namespace: {namespace} + labels: + kubernetes.io/service-name: {STS_SERVICE} +addressType: IPv4 +ports: + - name: http + port: {port} +endpoints: + - addresses: [{host_ip}] +" + ); + kubectl_with_input(&["apply", "-f", "-"], Some(&manifest)).await; + format!("http://{STS_SERVICE}.{namespace}.svc.cluster.local/token") +} + +async fn remove_token_service() { + let namespace = release_namespace(); + let _ = kubectl_command() + .args([ + "-n", + &namespace, + "delete", + "service,endpointslice", + STS_SERVICE, + "--ignore-not-found", + ]) + .output() + .await; +} + +/// A provider profile whose bearer credential comes from the existing RFC +/// 8693 exchange, bound to `target_port` on the test host. +fn profile(id: &str, token_endpoint: &str, target_port: u16) -> NamedTempFile { + temp_yaml(&format!( + r"id: {id} +display_name: E2E sandbox identity +description: Token exchange with the sandbox's SPIFFE JWT-SVID +category: other +credentials: + - name: subject_token + description: Stored user subject token + required: true + - name: access_token + description: Access token obtained through token exchange + required: false + auth_style: bearer + header_name: Authorization + token_grant: + grant_type: token_exchange + token_endpoint: {token_endpoint} + audience: e2e-service + jwt_svid_audience: {token_endpoint} + client_assertion_type: {CLIENT_ASSERTION_TYPE} + requested_token_type: {TOKEN_TYPE_ACCESS_TOKEN} + cache_ttl_seconds: 300 + subject_token: + source: provider_credential + credential: subject_token + subject_token_type: {TOKEN_TYPE_ACCESS_TOKEN} +endpoints: + - host: {TEST_HOST} + port: {target_port} + path: /** + protocol: rest + access: read-write + enforcement: enforce + allowed_ips: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 +binaries: + - /** +" + )) +} + +/// What the sandbox saw calling the protected target: `SAW:`, +/// `DENIED` for a refused grant, or a failure description. +fn call_script(port: u16) -> String { + format!( + r#" +import urllib.error, urllib.request, uuid + +def canonical_request_id(value): + try: + return str(uuid.UUID(value)) == value + except (TypeError, ValueError, AttributeError): + return False + +try: + with urllib.request.urlopen("http://{TEST_HOST}:{port}/resource", timeout=20) as response: + print(response.read().decode()) +except urllib.error.HTTPError as error: + request_id = error.headers.get("x-openshell-request-id", "") + if error.code == 502 and canonical_request_id(request_id): + print("DENIED") + else: + print(f"HTTP-{{error.code}}") +except OSError: + print("UNREACHABLE") +"# + ) +} + +async fn call(sandbox: &SandboxGuard, port: u16) -> String { + let output = sandbox + .exec(&["python3", "-c", &call_script(port)]) + .await + .unwrap_or_else(|error| panic!("call from {}: {error}", sandbox.name)); + output + .lines() + .map(str::trim) + .find(|line| { + line.starts_with("SAW:") + || *line == "DENIED" + || line.starts_with("HTTP-") + || *line == "UNREACHABLE" + }) + .unwrap_or_else(|| panic!("no call result from {}:\n{output}", sandbox.name)) + .to_string() +} + +/// Call until the sandbox sees `expected`, as SPIRE propagates a new entry to +/// its agent and a restarted sandbox instance becomes ready. +async fn call_until(sandbox: &SandboxGuard, port: u16, expected: &str) { + let mut last = String::new(); + let reached = tokio::time::timeout(Duration::from_mins(2), async { + loop { + last = call(sandbox, port).await; + if last == expected { + return; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + }) + .await; + assert!( + reached.is_ok(), + "{} never saw {expected}; last saw {last}", + sandbox.name + ); +} + +/// The Sandbox resource of an `OpenShell` sandbox. +async fn sandbox_resource(name: &str) -> serde_json::Value { + let output = kubectl(&[ + "get", + "sandboxes.agents.x-k8s.io", + "--all-namespaces", + "-l", + &format!("openshell.ai/sandbox-name={name}"), + "-o", + "json", + ]) + .await; + let list: serde_json::Value = serde_json::from_str(&output).expect("parse Sandbox list"); + let [resource] = list["items"].as_array().expect("Sandbox items").as_slice() else { + panic!("expected one Sandbox resource for {name}: {list}"); + }; + resource.clone() +} + +async fn sandbox_id(name: &str) -> String { + let (ok, output) = cli(&["sandbox", "get", name, "--output", "json"]).await; + assert!(ok, "sandbox get {name} failed:\n{output}"); + let details: serde_json::Value = serde_json::from_str(&output).expect("parse sandbox JSON"); + details["id"].as_str().expect("sandbox id").to_string() +} + +/// The shared-supervisor sandbox instance route of a sandbox's current +/// generation. +async fn instance_route(name: &str) -> String { + let resource = sandbox_resource(name).await; + let annotations = &resource["metadata"]["annotations"]; + assert_eq!(annotations[TOPOLOGY_ANNOTATION], "gateway"); + annotations[INSTANCE_ROUTE_ANNOTATION] + .as_str() + .expect("sandbox instance route") + .to_string() +} + +/// Gateway ID the gateway signs session JWTs with. +async fn gateway_id() -> String { + let output = kubectl(&[ + "-n", + &release_namespace(), + "get", + "configmaps", + "-o", + "jsonpath={.items[*].data.gateway\\.toml}", + ]) + .await; + output + .lines() + .find_map(|line| { + let value = line.trim().strip_prefix("gateway_id = \"")?; + value.strip_suffix('"').map(str::to_string) + }) + .expect("gateway_id in gateway TOML") +} + +/// Stable registration parent shared by every eligible Broker agent. +fn broker_agent_parent_id() -> String { + format!("spiffe://{TRUST_DOMAIN}/openshell/broker-agents") +} + +/// Select one backend and wait for endpoint propagation before acquiring. +async fn select_broker_agent(namespace: &str, pod: &serde_json::Value) { + let name = pod["metadata"]["name"].as_str().unwrap(); + let address = pod["status"]["podIP"].as_str().unwrap(); + kubectl(&[ + "-n", + namespace, + "label", + "pod", + name, + &format!("openshell.ai/broker-target={name}"), + "--overwrite", + ]) + .await; + let selector = serde_json::json!({"spec": {"selector": {"openshell.ai/broker-target": name}}}); + kubectl(&[ + "-n", + namespace, + "patch", + "service", + "spire-broker", + "--type=merge", + "-p", + &selector.to_string(), + ]) + .await; + let ready = tokio::time::timeout(Duration::from_mins(1), async { + loop { + let endpoints = kubectl(&[ + "-n", + namespace, + "get", + "endpoints", + "spire-broker", + "-o", + "json", + ]) + .await; + let endpoints: serde_json::Value = serde_json::from_str(&endpoints).unwrap(); + let addresses: Vec<_> = endpoints["subsets"] + .as_array() + .into_iter() + .flatten() + .flat_map(|subset| subset["addresses"].as_array().into_iter().flatten()) + .filter_map(|endpoint| endpoint["ip"].as_str()) + .collect(); + if addresses == [address] { + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }) + .await; + assert!(ready.is_ok(), "Broker service did not select {name}"); +} + +async fn replace_broker_agent(namespace: &str, previous: &serde_json::Value) -> serde_json::Value { + let name = previous["metadata"]["name"].as_str().unwrap(); + let node = previous["spec"]["nodeName"].as_str().unwrap(); + kubectl(&["-n", namespace, "delete", "pod", name, "--wait=true"]).await; + kubectl(&[ + "-n", + namespace, + "rollout", + "status", + "daemonset/spire-agent", + "--timeout=180s", + ]) + .await; + let replacement = kubectl(&[ + "-n", + namespace, + "get", + "pods", + "-l", + "app.kubernetes.io/name=agent,app.kubernetes.io/instance=spire", + "--field-selector", + &format!("spec.nodeName={node}"), + "-o", + "json", + ]) + .await; + let replacement: serde_json::Value = serde_json::from_str(&replacement).unwrap(); + replacement["items"] + .as_array() + .unwrap() + .iter() + .find(|pod| pod["metadata"]["name"].as_str() != Some(name)) + .expect("replacement SPIRE agent") + .clone() +} + +/// Force a fresh production acquisition through each agent. A random Service +/// connection is insufficient evidence that every backend can authenticate +/// and issue the same registered identity. +async fn assert_every_broker_agent(sandbox: &SandboxGuard, port: u16, sandbox_id: &str) { + let namespace = spire_fixture(); + let output = kubectl(&[ + "-n", + &namespace, + "get", + "pods", + "-l", + "app.kubernetes.io/name=agent,app.kubernetes.io/instance=spire", + "-o", + "json", + ]) + .await; + let pods: serde_json::Value = serde_json::from_str(&output).unwrap(); + let mut agents = pods["items"].as_array().unwrap().clone(); + let minimum = std::env::var("OPENSHELL_E2E_SPIRE_MIN_AGENTS") + .unwrap_or_else(|_| "1".into()) + .parse::() + .unwrap(); + let nodes: BTreeSet<_> = agents + .iter() + .map(|pod| pod["spec"]["nodeName"].as_str().unwrap()) + .collect(); + assert!( + nodes.len() >= minimum, + "expected {minimum} Broker nodes, found {nodes:?}" + ); + let original = kubectl(&[ + "-n", + &namespace, + "get", + "service", + "spire-broker", + "-o", + "json", + ]) + .await; + let original: serde_json::Value = serde_json::from_str(&original).unwrap(); + let initial_agents = agents.len(); + // Replace one agent after the sandbox entries exist. Its replacement must + // recover registrations through the alias without operator re-registration. + for index in 0..=initial_agents { + if index == initial_agents { + let replacement = replace_broker_agent(&namespace, agents.last().unwrap()).await; + agents.push(replacement); + } + let pod = &agents[index]; + let name = pod["metadata"]["name"].as_str().unwrap(); + select_broker_agent(&namespace, pod).await; + let (ok, output) = cli(&["sandbox", "stop", &sandbox.name]).await; + assert!(ok, "{output}"); + let (ok, output) = cli(&["sandbox", "start", &sandbox.name]).await; + assert!(ok, "{output}"); + call_until(sandbox, port, &format!("SAW:{sandbox_id}")).await; + eprintln!( + "Broker acquisition succeeded through {name} on {}", + pod["spec"]["nodeName"] + ); + } + let restore = serde_json::json!([ + {"op": "replace", "path": "/spec/selector", "value": original["spec"]["selector"]} + ]); + kubectl(&[ + "-n", + &namespace, + "patch", + "service", + "spire-broker", + "--type=json", + "-p", + &restore.to_string(), + ]) + .await; +} + +fn spire_fixture() -> String { + std::env::var("OPENSHELL_E2E_SPIRE_NAMESPACE") + .ok() + .filter(|namespace| !namespace.is_empty()) + .expect("OPENSHELL_E2E_SPIRE_NAMESPACE is set by with-kube-gateway.sh") +} + +/// Register `sandbox_id`'s SPIFFE ID for object-reference attestation, scoped to `agent_id`. +async fn register(gateway_id: &str, sandbox_id: &str, agent_id: &str) { + let fixture = spire_fixture(); + // The SPIRE Controller Manager's class is `-`; the + // wrapper installs release `spire` in the fixture namespace. + let class_name = format!("{fixture}-spire"); + let resources = kubectl(&[ + "get", + "sandboxes.agents.x-k8s.io", + "-A", + "-l", + &format!("openshell.ai/sandbox-id={sandbox_id}"), + "-o", + "json", + ]) + .await; + let resources: serde_json::Value = serde_json::from_str(&resources).expect("Sandbox resources"); + let [resource] = resources["items"] + .as_array() + .expect("Sandbox items") + .as_slice() + else { + panic!("expected one Sandbox for {sandbox_id}"); + }; + let uid = resource["metadata"]["uid"].as_str().expect("Sandbox UID"); + let selectors = format!("k8s:resource:sandboxes.agents.x-k8s.io\n - k8s:uid:{uid}"); + let manifest = format!( + r"apiVersion: spire.spiffe.io/v1alpha1 +kind: ClusterStaticEntry +metadata: + name: openshell-sandbox-{sandbox_id} + labels: + {ENTRY_LABEL}: {fixture} +spec: + className: {class_name} + spiffeID: {} + parentID: {agent_id} + selectors: + - {selectors} +", + spiffe_id(gateway_id, sandbox_id) + ); + kubectl_with_input(&["apply", "-f", "-"], Some(&manifest)).await; +} + +async fn unregister_all() { + let _ = kubectl_command() + .args([ + "delete", + "clusterstaticentries.spire.spiffe.io", + "-l", + &format!("{ENTRY_LABEL}={}", spire_fixture()), + "--ignore-not-found", + ]) + .output() + .await; +} + +// Inspect access-bearing fields. Image tags and node names may legitimately +// contain "spiffe" without exposing an identity socket or credential. +fn assert_no_spiffe_access(pod: &serde_json::Value) { + let spec = &pod["spec"]; + let containers = ["containers", "initContainers", "ephemeralContainers"] + .into_iter() + .filter_map(|field| spec[field].as_array()) + .flatten() + .map(|container| { + serde_json::json!({ + "env": container["env"], + "mounts": container["volumeMounts"], + "args": container["args"], + }) + }) + .collect::>(); + let access = serde_json::json!({ "volumes": spec["volumes"], "containers": containers }) + .to_string() + .to_lowercase(); + assert!( + !access.contains("spiffe") && !access.contains("spire"), + "Pod {} carries SPIFFE access", + pod["metadata"]["name"] + ); +} + +/// No sandbox workload Pod receives a SPIFFE socket, SPIRE mount, or SPIFFE +/// environment. +async fn assert_workloads_hold_no_spiffe_material() { + let output = kubectl(&[ + "get", + "pods", + "--all-namespaces", + "-l", + "openshell.ai/boundary-role=workload", + "-o", + "json", + ]) + .await; + let pods: serde_json::Value = serde_json::from_str(&output).expect("parse workload Pods"); + let items = pods["items"].as_array().expect("Pod items"); + assert!(!items.is_empty(), "no sandbox workload Pods found"); + for pod in items { + assert_no_spiffe_access(pod); + } +} + +/// Inside the sandbox: no SPIFFE socket, environment, or SVID. +async fn assert_sandbox_holds_no_spiffe_material(sandbox: &SandboxGuard) { + let script = r#" +import base64, json, os, re +leaks = [name for name in os.environ if "SPIFFE" in name.upper() or "SPIRE" in name.upper()] +leaks += [path for path in ("/spiffe-workload-api", "/run/spire", "/tmp/spire-agent") if os.path.exists(path)] +for value in os.environ.values(): + for token in re.findall(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", value): + try: + payload = token.split(".")[1] + claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + except (ValueError, UnicodeError): + continue + if str(claims.get("sub", "")).startswith("spiffe://openshell.local/"): + leaks.append("svid-in-environment") +print("LEAKS:" + ",".join(leaks) if leaks else "NO-SPIFFE") +"#; + let output = sandbox + .exec(&["python3", "-c", script]) + .await + .unwrap_or_else(|error| panic!("inspect {}: {error}", sandbox.name)); + assert!( + output.lines().any(|line| line.trim() == "NO-SPIFFE"), + "{} holds SPIFFE material:\n{output}", + sandbox.name + ); +} + +/// Operator-side probe of one generation's launch bearer. It never refreshes +/// that bearer, and uses server-authenticated TLS without a user certificate. +/// Tokens stay in memory and are never included in assertion diagnostics. +struct BrokerProbe { + client: OpenShellClient, + bearer: SecretJwt, + credential_key: String, + epoch: u64, +} + +impl BrokerProbe { + async fn capture(sandbox_id: &str) -> Self { + #[derive(serde::Deserialize)] + struct LaunchMaterial { + auth: SupervisorAuthBundle, + } + + let secrets: serde_json::Value = serde_json::from_str( + &kubectl(&[ + "get", "secrets", "--all-namespaces", "-l", + &format!("openshell.ai/sandbox-id={sandbox_id},openshell.ai/component=supervisor-bootstrap"), + "-o", "json", + ]).await, + ).expect("parse launch Secret list"); + let materials = secrets["items"] + .as_array() + .expect("Secret items") + .iter() + .filter_map(|secret| secret["data"]["instance.json"].as_str()) + .collect::>(); + let [material] = materials.as_slice() else { + panic!("expected one shared launch Secret for {sandbox_id}"); + }; + let bytes = base64::engine::general_purpose::STANDARD + .decode(material) + .expect("decode launch material"); + let launch: LaunchMaterial = serde_json::from_slice(&bytes).expect("parse launch material"); + let home = std::env::var_os("XDG_CONFIG_HOME").expect("harness config home"); + let gateway = std::env::var("OPENSHELL_GATEWAY").expect("harness gateway name"); + let directory = std::path::PathBuf::from(home) + .join("openshell/gateways") + .join(gateway); + let metadata: serde_json::Value = serde_json::from_slice( + &std::fs::read(directory.join("metadata.json")).expect("gateway metadata"), + ) + .expect("parse gateway metadata"); + let endpoint = metadata["gateway_endpoint"] + .as_str() + .expect("gateway endpoint"); + assert!( + endpoint.starts_with("https://"), + "the SPIFFE fixture requires TLS" + ); + let ca = std::fs::read(directory.join("mtls/ca.crt")).expect("gateway CA"); + openshell_crypto::tls::ensure_default_provider(); + let channel = Endpoint::from_shared(endpoint.to_string()) + .expect("valid gateway endpoint") + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .tls_config(ClientTlsConfig::new().ca_certificate(Certificate::from_pem(ca))) + .expect("configure gateway TLS") + .connect() + .await + .expect("connect with server-authenticated TLS"); + let mut probe = Self { + client: OpenShellClient::new(channel), + bearer: launch.auth.gateway_token, + credential_key: String::new(), + epoch: launch.auth.auth_epoch.get(), + }; + let request = probe.request(GetSandboxProviderEnvironmentRequest { + sandbox_id: sandbox_id.to_string(), + supports_static_credential_bindings: true, + supports_stable_placeholder_environment_keys: true, + }); + let environment = probe + .client + .get_sandbox_provider_environment(request) + .await + .expect("the launch bearer is currently authorized") + .into_inner(); + let mut keys = environment.dynamic_credentials.into_keys(); + probe.credential_key = keys.next().expect("configured token grant key"); + assert!( + keys.next().is_none(), + "the fixture must have exactly one configured grant" + ); + probe + } + + fn request(&self, message: T) -> tonic::Request { + let mut request = tonic::Request::new(message); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", self.bearer.expose_secret()) + .parse() + .expect("valid bearer header"), + ); + request + } + + async fn access_token(&mut self) -> Result<(String, Duration), tonic::Status> { + let request = self.request(GetSandboxProviderAccessTokenRequest { + credential_key: self.credential_key.clone(), + }); + self.client + .get_sandbox_provider_access_token(request) + .await + .map(|response| { + let response = response.into_inner(); + let lifetime = openshell_core::time::duration_to_std( + response + .expires_after + .as_ref() + .expect("broker token lifetime"), + ) + .expect("valid broker token lifetime"); + (response.access_token, lifetime) + }) + } +} + +/// A warmed broker cache cannot bypass an ownership epoch change. Prove the +/// replacement adopts the same workload and identity, obtains a new provider +/// token, and rejects the previous owner's still-unexpired gateway bearer. +async fn assert_provider_failover( + sandbox: &SandboxGuard, + gateway: &str, + tokens: &TokenService, + target: &Target, +) { + let id = sandbox_id(&sandbox.name).await; + let before = generation(&sandbox.name).await; + let owner = wait_for_owner(&before.route).await; + let process = main_process(sandbox) + .await + .expect("main process before failover"); + let mut probe = BrokerProbe::capture(&id).await; + let (old_token, lifetime) = probe + .access_token() + .await + .expect("warm the broker's credential cache"); + let cache_expires = tokio::time::Instant::now() + lifetime; + let exchange_count = tokens.exchanges().len(); + let killed_at = tokio::time::Instant::now(); + let killed = kill_abruptly(&owner).await; + eprintln!("SPIFFE supervisor {owner} killed: {killed}"); + let replacement = wait_for_owner(&before.route).await; + assert_ne!(replacement, owner, "the lost member cannot own the route"); + assert!( + killed_at.elapsed() >= Duration::from_secs(10), + "takeover must wait for membership expiry" + ); + let logs = gateway_log_lines(&before.route).await; + let term = logs + .iter() + .find(|line| is_takeover(line, &before.route, &owner, "expired")) + .and_then(|line| { + strip_ansi(line) + .split_whitespace() + .find_map(|field| field.strip_prefix("term=")?.parse::().ok()) + }) + .expect("the gateway records the failed owner's replacement epoch"); + assert!( + term > probe.epoch, + "takeover must advance the credential epoch" + ); + counts_past(sandbox, &process).await; + assert_eq!(generation(&sandbox.name).await, before); + call_until(sandbox, target.port, &format!("SAW:{id}")).await; + assert!( + tokio::time::Instant::now() < cache_expires, + "the old cache entry must still be live, so TTL expiry cannot explain the fresh exchange" + ); + assert!( + tokens.exchanges()[exchange_count..] + .iter() + .any(|exchange| !exchange.intermediate + && exchange.assertion_subject == spiffe_id(gateway, &id)), + "the new owner must acquire a provider token for the unchanged sandbox identity" + ); + assert!( + target + .seen_tokens + .lock() + .unwrap() + .get(&format!("SAW:{id}")) + .is_some_and(|token| token != &format!("Bearer {old_token}")), + "the provider request must not reuse the previous epoch's cached token" + ); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("current time") + .as_secs(); + assert!( + claims(probe.bearer.expose_secret())["exp"] + .as_u64() + .is_some_and(|expiry| expiry > now), + "the stale bearer must still be unexpired, so expiry cannot explain its rejection" + ); + let exchanges_before_rejection = tokens.exchanges().len(); + let Err(error) = probe.access_token().await else { + panic!("the broker accepted the old owner bearer"); + }; + assert_eq!(error.code(), tonic::Code::Unauthenticated); + assert_eq!( + tokens.exchanges().len(), + exchanges_before_rejection, + "reject the old owner before contacting the token service" + ); + let pod = shared_supervisor::kubectl_json(&[ + "-n", + &release_namespace(), + "get", + "pod", + &replacement, + "-o", + "json", + ]) + .await; + assert_no_spiffe_access(&pod); + assert_sandbox_holds_no_spiffe_material(sandbox).await; + eprintln!( + "SPIFFE takeover preserved {id}: owner {owner} -> {replacement}, epoch {} -> {term}", + probe.epoch + ); +} + +async fn create(name: &str, provider: &str) -> SandboxGuard { + let main = format!("echo spiffe-ready; {COUNTER_MAIN}"); + let driver_config = std::env::var("OPENSHELL_E2E_SPIFFE_DRIVER_CONFIG_JSON").ok(); + let mut args = vec!["--name", name, "--provider", provider]; + if let Some(config) = driver_config.as_deref() { + args.extend(["--driver-config-json", config]); + } + SandboxGuard::create_keep_with_args(&args, &["sh", "-c", &main], "spiffe-ready") + .await + .unwrap_or_else(|error| panic!("create {name}: {error}")) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn shared_supervisor_acts_as_each_sandboxs_stable_spiffe_identity() { + shared_supervisor::enable_process_control().await; + let members = supervisor_pods().await; + assert!( + members.len() >= 2, + "SPIFFE failover needs two supervisor members" + ); + wait_until_attached(&members).await; + let nonce = format!("{:08x}", rand::random::()); + let stored_token = format!("stored-{nonce}"); + let tokens = TokenService::start(stored_token.clone()).await; + let target = Target::start().await; + let token_endpoint = expose_token_service(tokens.port).await; + let provider = format!("e2e-spiffe-{nonce}"); + let profile = profile(&provider, &token_endpoint, target.port); + let (imported, output) = cli(&[ + "profile", + "import", + "--file", + profile.path().to_str().expect("UTF-8 profile path"), + ]) + .await; + assert!(imported, "profile import failed:\n{output}"); + let credential = format!("subject_token={stored_token}"); + let (created, output) = cli(&[ + "provider", + "create", + "--name", + &provider, + "--type", + &provider, + "--credential", + &credential, + ]) + .await; + assert!(created, "provider create failed:\n{output}"); + + // Sandbox names are limited to 19 characters. + let (alpha_name, beta_name) = (format!("spa-{nonce}"), format!("spb-{nonce}")); + let (alpha, beta) = tokio::join!( + create(&alpha_name, &provider), + create(&beta_name, &provider) + ); + let (mut alpha, mut beta) = (alpha, beta); + let gateway_id = gateway_id().await; + let agent_id = broker_agent_parent_id(); + let alpha_id = sandbox_id(&alpha.name).await; + let beta_id = sandbox_id(&beta.name).await; + + // Without a registration entry the shared supervisor has no identity to + // act as, and never falls back to its own. + assert_eq!(call(&alpha, target.port).await, "DENIED"); + assert!(tokens.exchanges().is_empty()); + + let pods = kubectl(&[ + "-n", + &release_namespace(), + "get", + "pods", + "-l", + "app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell", + "-o", + "json", + ]) + .await; + let pods: serde_json::Value = serde_json::from_str(&pods).expect("gateway pods"); + for pod in pods["items"].as_array().expect("gateway pod items") { + assert!( + pod["spec"]["volumes"] + .as_array() + .expect("gateway volumes") + .iter() + .all(|volume| volume.get("hostPath").is_none()), + "broker gateway must not mount hostPath" + ); + assert!( + pod["spec"]["securityContext"] + .get("supplementalGroups") + .is_none(), + "broker gateway must not require supplemental groups" + ); + let admission_namespace = format!("spiffe-admit-{nonce}"); + let namespace = serde_json::json!({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": admission_namespace, "labels": {"pod-security.kubernetes.io/enforce": "restricted"}}}); + kubectl_with_input(&["apply", "-f", "-"], Some(&namespace.to_string())).await; + let service_account = pod["spec"]["serviceAccountName"] + .as_str() + .unwrap_or("default"); + let account = serde_json::json!({"apiVersion": "v1", "kind": "ServiceAccount", "metadata": {"name": service_account, "namespace": admission_namespace}}); + kubectl_with_input(&["apply", "-f", "-"], Some(&account.to_string())).await; + let probe = serde_json::json!({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "gateway-admission", "namespace": admission_namespace}, "spec": pod["spec"]}); + kubectl_with_input( + &["create", "--dry-run=server", "-f", "-"], + Some(&probe.to_string()), + ) + .await; + kubectl(&["delete", "namespace", &admission_namespace]).await; + } + + register(&gateway_id, &alpha_id, &agent_id).await; + register(&gateway_id, &beta_id, &agent_id).await; + call_until(&alpha, target.port, &format!("SAW:{alpha_id}")).await; + call_until(&beta, target.port, &format!("SAW:{beta_id}")).await; + let finals = |exchanges: &[Exchange]| { + exchanges + .iter() + .filter(|exchange| !exchange.intermediate) + .map(|exchange| exchange.assertion_subject.clone()) + .collect::>() + }; + // A retried exchange may repeat an identity, but only these two appear. + let exchanges = tokens.exchanges(); + let subjects = finals(&exchanges); + assert_eq!( + subjects.iter().cloned().collect::>(), + BTreeSet::from([ + spiffe_id(&gateway_id, &alpha_id), + spiffe_id(&gateway_id, &beta_id) + ]), + "final exchange subjects: {subjects:?}" + ); + // The gateway asked for intermediate tokens for exactly those identities, + // with its own identity rather than a sandbox's. + for exchange in exchanges.iter().filter(|exchange| exchange.intermediate) { + assert!( + [ + spiffe_id(&gateway_id, &alpha_id), + spiffe_id(&gateway_id, &beta_id) + ] + .contains(&exchange.audience), + "unexpected intermediate audience {exchange:?}" + ); + assert!( + !exchange.assertion_subject.starts_with(ID_PREFIX), + "gateway presented a sandbox identity: {exchange:?}" + ); + } + let shared_pods = kubectl(&[ + "-n", + &release_namespace(), + "get", + "pods", + "-l", + "app.kubernetes.io/component=supervisor", + "-o", + "json", + ]) + .await; + let shared: serde_json::Value = serde_json::from_str(&shared_pods).unwrap(); + for pod in shared["items"].as_array().unwrap() { + assert_no_spiffe_access(pod); + } + assert_workloads_hold_no_spiffe_material().await; + assert_sandbox_holds_no_spiffe_material(&alpha).await; + assert_sandbox_holds_no_spiffe_material(&beta).await; + + if std::env::var("OPENSHELL_E2E_KUBE_SPIRE_NODE_LOCAL").as_deref() == Ok("1") { + node_local::exercise(&alpha, target.port, &alpha_id).await; + } else { + assert_every_broker_agent(&alpha, target.port, &alpha_id).await; + } + assert_provider_failover(&alpha, &gateway_id, &tokens, &target).await; + + // The fixture's workload X.509-SVID TTL is 30 seconds. Let the + // credential used for initial acquisition expire before forcing a new + // acquisition; success requires fetching rotated Workload API material. + tokio::time::sleep(Duration::from_secs(35)).await; + + // A restart is a new generation with a new sandbox instance and an empty + // cache, and it acquires under the same SPIFFE ID. + let alpha_route = instance_route(&alpha.name).await; + let (stopped, output) = cli(&["sandbox", "stop", &alpha.name]).await; + assert!(stopped, "sandbox stop failed:\n{output}"); + let (started, output) = cli(&["sandbox", "start", &alpha.name]).await; + assert!(started, "sandbox start failed:\n{output}"); + assert_ne!(instance_route(&alpha.name).await, alpha_route); + let before_restart = finals(&tokens.exchanges()).len(); + call_until(&alpha, target.port, &format!("SAW:{alpha_id}")).await; + let after_restart = finals(&tokens.exchanges()); + assert!( + after_restart.len() > before_restart, + "the restarted generation reused an earlier generation's token" + ); + assert!( + after_restart[before_restart..] + .iter() + .all(|subject| *subject == spiffe_id(&gateway_id, &alpha_id)) + ); + assert_eq!(call(&beta, target.port).await, format!("SAW:{beta_id}")); + + // A sandbox recreated under the same name is a new sandbox with a new + // identity; the old registration does not apply to it. + alpha.cleanup().await; + let mut recreated = create(&alpha_name, &provider).await; + let recreated_id = sandbox_id(&recreated.name).await; + assert_ne!(recreated_id, alpha_id); + let before_recreate = tokens.exchanges().len(); + assert_eq!(call(&recreated, target.port).await, "DENIED"); + assert_eq!(tokens.exchanges().len(), before_recreate); + register(&gateway_id, &recreated_id, &agent_id).await; + call_until(&recreated, target.port, &format!("SAW:{recreated_id}")).await; + let after_recreate = finals(&tokens.exchanges()[before_recreate..]); + assert!( + !after_recreate.is_empty() + && after_recreate + .iter() + .all(|subject| *subject == spiffe_id(&gateway_id, &recreated_id)), + "final exchange subjects after recreate: {after_recreate:?}" + ); + assert_sandbox_holds_no_spiffe_material(&recreated).await; + + // Identifiers only: the exchanges' tokens are never printed. + for exchange in tokens.exchanges() { + eprintln!( + "token exchange: intermediate={} assertion_subject={} audience={}", + exchange.intermediate, exchange.assertion_subject, exchange.audience + ); + } + + recreated.cleanup().await; + beta.cleanup().await; + let _ = cli(&["provider", "delete", &provider]).await; + let _ = cli(&["profile", "delete", &provider]).await; + unregister_all().await; + remove_token_service().await; +} diff --git a/e2e/rust/tests/kubernetes_shared_supervisor_spiffe/node_local.rs b/e2e/rust/tests/kubernetes_shared_supervisor_spiffe/node_local.rs new file mode 100644 index 0000000000..7cd5727c69 --- /dev/null +++ b/e2e/rust/tests/kubernetes_shared_supervisor_spiffe/node_local.rs @@ -0,0 +1,203 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Node-local Broker coverage: no Broker Service, gateway relocation, and +//! fresh acquisition with TCP 8443 restricted to the gateway's current node. + +use super::shared_supervisor::{ + GATEWAY_SELECTOR, release_namespace, supervisor_pods, wait_until_attached, +}; +use super::{call_until, cli, kubectl, kubectl_with_input, replace_broker_agent, spire_fixture}; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serde_json::{Value, json}; +use std::{collections::BTreeSet, time::Duration}; + +async fn gateway_pod() -> Value { + let pods = kubectl(&[ + "-n", + &release_namespace(), + "get", + "pods", + "-l", + GATEWAY_SELECTOR, + "-o", + "json", + ]) + .await; + let pods: Value = serde_json::from_str(&pods).unwrap(); + let active: Vec<_> = pods["items"] + .as_array() + .unwrap() + .iter() + .filter(|pod| { + pod["metadata"]["deletionTimestamp"].is_null() && pod["status"]["phase"] == "Running" + }) + .collect(); + let [pod] = active.as_slice() else { + panic!("expected one active gateway Pod"); + }; + (*pod).clone() +} + +async fn restrict_broker_to_node(ip: &str) { + let ip: std::net::IpAddr = ip.parse().expect("gateway node IP"); + let cidr = format!("{ip}/{}", if ip.is_ipv4() { 32 } else { 128 }); + let policy = json!({ + "apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", + "metadata": {"name": "e2e-node-local-broker", "namespace": release_namespace()}, + "spec": { + "podSelector": {"matchLabels": {"app.kubernetes.io/name": "openshell"}}, + "policyTypes": ["Egress"], + "egress": [ + {"to": [{"ipBlock": {"cidr": cidr}}], "ports": [{"protocol": "TCP", "port": 8443}]}, + {"ports": [{"protocol": "TCP", "port": 1, "endPort": 8442}, {"protocol": "TCP", "port": 8444, "endPort": 65535}]}, + {"ports": [{"protocol": "UDP"}]} + ] + } + }); + kubectl_with_input(&["apply", "-f", "-"], Some(&policy.to_string())).await; +} + +async fn wait_for_gateway() { + tokio::time::timeout(Duration::from_mins(2), async { + while !cli(&["sandbox", "list"]).await.0 { + tokio::time::sleep(Duration::from_secs(1)).await; + } + }) + .await + .expect("gateway port-forward did not reconnect after relocation"); + wait_until_attached(&supervisor_pods().await).await; +} + +async fn fresh_acquisition(sandbox: &SandboxGuard, port: u16, id: &str) { + let (ok, output) = cli(&["sandbox", "stop", &sandbox.name]).await; + assert!(ok, "{output}"); + let (ok, output) = cli(&["sandbox", "start", &sandbox.name]).await; + assert!(ok, "{output}"); + call_until(sandbox, port, &format!("SAW:{id}")).await; +} + +async fn relocate_gateway(deployment: &str, node: &str) { + let namespace = release_namespace(); + if gateway_pod().await["spec"]["nodeName"] != node { + let patch = json!({"spec": {"template": {"spec": {"nodeSelector": {"kubernetes.io/hostname": node}}}}}); + kubectl(&[ + "-n", + &namespace, + "patch", + "deployment", + deployment, + "--type=merge", + "-p", + &patch.to_string(), + ]) + .await; + kubectl(&[ + "-n", + &namespace, + "rollout", + "status", + &format!("deployment/{deployment}"), + "--timeout=180s", + ]) + .await; + wait_for_gateway().await; + } +} + +pub(super) async fn exercise(sandbox: &SandboxGuard, port: u16, id: &str) { + let namespace = release_namespace(); + let fixture = spire_fixture(); + assert!( + kubectl(&[ + "-n", + &fixture, + "get", + "service", + "spire-broker", + "--ignore-not-found", + "-o", + "name" + ]) + .await + .trim() + .is_empty(), + "node-local mode must not create a Broker Service" + ); + let pods = kubectl(&[ + "-n", + &fixture, + "get", + "pods", + "-l", + "app.kubernetes.io/name=agent,app.kubernetes.io/instance=spire", + "-o", + "json", + ]) + .await; + let pods: Value = serde_json::from_str(&pods).unwrap(); + let nodes: BTreeSet = pods["items"] + .as_array() + .unwrap() + .iter() + .map(|pod| pod["spec"]["nodeName"].as_str().unwrap().to_string()) + .collect(); + let minimum: usize = std::env::var("OPENSHELL_E2E_SPIRE_MIN_AGENTS") + .unwrap_or_else(|_| "2".into()) + .parse() + .unwrap(); + assert!( + nodes.len() >= minimum && nodes.len() >= 2, + "node-local relocation needs {minimum} Broker nodes" + ); + let deployments = kubectl(&[ + "-n", + &namespace, + "get", + "deployments", + "-l", + GATEWAY_SELECTOR, + "-o", + "json", + ]) + .await; + let deployments: Value = serde_json::from_str(&deployments).unwrap(); + let gateways: Vec<_> = deployments["items"] + .as_array() + .unwrap() + .iter() + .filter(|deployment| { + deployment["spec"]["selector"]["matchLabels"]["app.kubernetes.io/name"] == "openshell" + }) + .collect(); + let [deployment] = gateways.as_slice() else { + panic!("node-local relocation requires an external-database gateway Deployment"); + }; + let deployment = deployment["metadata"]["name"].as_str().unwrap(); + for node in &nodes { + relocate_gateway(deployment, node).await; + let gateway = gateway_pod().await; + assert_eq!(gateway["spec"]["nodeName"], node.as_str()); + assert!( + gateway["spec"]["containers"][0]["env"] + .as_array() + .unwrap() + .iter() + .any(|env| env["name"] == "OPENSHELL_NODE_IP" + && env["valueFrom"]["fieldRef"]["fieldPath"] == "status.hostIP") + ); + restrict_broker_to_node(gateway["status"]["hostIP"].as_str().unwrap()).await; + fresh_acquisition(sandbox, port, id).await; + eprintln!("Node-local Broker acquisition succeeded with gateway on {node}"); + } + let gateway = gateway_pod().await; + let agent = pods["items"] + .as_array() + .unwrap() + .iter() + .find(|pod| pod["spec"]["nodeName"] == gateway["spec"]["nodeName"]) + .unwrap(); + replace_broker_agent(&fixture, agent).await; + fresh_acquisition(sandbox, port, id).await; + eprintln!("Node-local Broker acquisition succeeded after local agent replacement"); +} diff --git a/e2e/rust/tests/shared_supervisor/mod.rs b/e2e/rust/tests/shared_supervisor/mod.rs index 55ee3c715e..45eed8deef 100644 --- a/e2e/rust/tests/shared_supervisor/mod.rs +++ b/e2e/rust/tests/shared_supervisor/mod.rs @@ -57,6 +57,208 @@ pub const EXEC_PROBE_TIMEOUT: Duration = Duration::from_secs(20); /// Bound on one policy-mediated network probe from a sandbox. pub const NETWORK_PROBE_TIMEOUT: Duration = Duration::from_secs(40); +/// Run `command` in a cluster node's container; k3d and kind name node +/// containers after their nodes. +async fn node_exec(node: &str, command: &[&str]) -> Result { + let output = tokio::process::Command::new("docker") + .arg("exec") + .arg(node) + .args(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("docker exec {node} {command:?}: {error}"))?; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + Err(format!( + "docker exec {node} {command:?}: {}", + String::from_utf8_lossy(&output.stderr).trim() + )) + } +} + +/// SIGKILL the supervisor container of `pod` through its node's container +/// runtime, so it cannot announce that it is leaving, then force-delete the +/// Pod so its member does not come back in place. Kills the task with +/// `ctr`, or failing that the container's process with `kill -KILL`; panics +/// when neither succeeds. Returns how the container was killed. +pub async fn kill_abruptly(pod: &str) -> String { + let namespace = release_namespace(); + if std::env::var("OPENSHELL_E2E_SUPERVISOR_PROCESS_CONTROL").as_deref() == Ok("1") { + kill_with_process_control(&namespace, pod).await; + force_delete_supervisor(&namespace, pod).await; + return "SIGKILL from the test Pod's process-control sidecar".into(); + } + let description = kubectl_json(&["-n", &namespace, "get", "pod", pod, "-o", "json"]).await; + let node = description["spec"]["nodeName"] + .as_str() + .unwrap_or_else(|| panic!("{pod} has no node: {description}")) + .to_string(); + let container = description["status"]["containerStatuses"] + .as_array() + .into_iter() + .flatten() + .find(|status| status["name"] == SUPERVISOR_CONTAINER) + .and_then(|status| status["containerID"].as_str()) + .and_then(|id| id.split_once("://").map(|(_, id)| id.to_string())) + .unwrap_or_else(|| panic!("{pod} has no running {SUPERVISOR_CONTAINER} container")); + let mut failures = Vec::new(); + let mut killed = None; + // k3s ships `ctr` as a multi-call alias; kind ships it on its own. + for ctr in [&["ctr"][..], &["k3s", "ctr"][..]] { + let kill = [ + ctr, + &[ + "-n", "k8s.io", "tasks", "kill", "--signal", "SIGKILL", &container, + ], + ] + .concat(); + match node_exec(&node, &kill).await { + Ok(_) => { + killed = Some(format!("ctr tasks kill --signal SIGKILL on {node}")); + break; + } + Err(error) => failures.push(error), + } + } + if killed.is_none() { + for ctr in [&["ctr"][..], &["k3s", "ctr"][..]] { + let list = [ctr, &["-n", "k8s.io", "tasks", "ls"]].concat(); + let pid = match node_exec(&node, &list).await { + Ok(tasks) => tasks.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next() == Some(container.as_str())) + .then(|| fields.next()) + .flatten() + .filter(|pid| pid.parse::().is_ok_and(|pid| pid > 1)) + .map(str::to_string) + }), + Err(error) => { + failures.push(error); + continue; + } + }; + let Some(pid) = pid else { + failures.push(format!("no running task {container} on {node}")); + continue; + }; + match node_exec(&node, &["kill", "-KILL", &pid]).await { + Ok(_) => { + killed = Some(format!("kill -KILL {pid} on {node}")); + break; + } + Err(error) => failures.push(error), + } + } + } + let killed = killed.unwrap_or_else(|| { + panic!( + "could not hard-kill supervisor {pod}; a graceful stop proves nothing: {failures:#?}" + ) + }); + force_delete_supervisor(&namespace, pod).await; + killed +} + +async fn force_delete_supervisor(namespace: &str, pod: &str) { + kubectl(&[ + "-n", + namespace, + "delete", + "pod", + pod, + "--grace-period=0", + "--force", + "--wait=false", + ]) + .await; +} + +/// An unprivileged test sidecar shares only its Pod's process namespace. +/// PID 1 is the Pod sandbox, so the same UID can send SIGKILL to the supervisor +/// without access to the host PID namespace or container runtime socket. +async fn kill_with_process_control(namespace: &str, pod: &str) { + kubectl(&[ + "-n", + namespace, + "exec", + pod, + "-c", + "e2e-process-control", + "--", + "sh", + "-ec", + r#"for process in /proc/[0-9]*; do + if [ "$(cat "$process/comm" 2>/dev/null)" = openshell-super ]; then + pid=${process#/proc/} + test "$pid" -gt 1 + kill -KILL "$pid" + exit 0 + fi + done + echo 'supervisor process not found' >&2 + exit 1"#, + ]) + .await; +} + +/// Opt-in process control for remote clusters where node-container exec is +/// unavailable. Apply before creating sandboxes; the default image stays intact. +pub async fn enable_process_control() { + if std::env::var("OPENSHELL_E2E_SUPERVISOR_PROCESS_CONTROL").as_deref() != Ok("1") { + return; + } + let namespace = release_namespace(); + let deployments = kubectl_json(&[ + "-n", + &namespace, + "get", + "deployments", + "-l", + SUPERVISOR_COMPONENT, + "-o", + "json", + ]) + .await; + let [deployment] = deployments["items"] + .as_array() + .expect("supervisor deployments") + .as_slice() + else { + panic!("expected one supervisor deployment"); + }; + let name = deployment["metadata"]["name"].as_str().unwrap(); + let patch = serde_json::json!({"spec": {"template": {"spec": { + "shareProcessNamespace": true, + "containers": [{"name": "e2e-process-control", "image": "busybox:1.37.0-musl", "command": ["sleep", "infinity"], + "securityContext": {"runAsNonRoot": true, "allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}, + "resources": {"requests": {"cpu": "10m", "memory": "8Mi"}, "limits": {"cpu": "100m", "memory": "32Mi"}} + }] + }}}}); + kubectl(&[ + "-n", + &namespace, + "patch", + "deployment", + name, + "--type=strategic", + "-p", + &patch.to_string(), + ]) + .await; + kubectl(&[ + "-n", + &namespace, + "rollout", + "status", + &format!("deployment/{name}"), + "--timeout=180s", + ]) + .await; +} + pub fn release_namespace() -> String { std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE").unwrap_or_else(|_| "openshell".to_string()) } diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 0377e49fdc..a4788a3866 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -58,6 +58,14 @@ # installing OpenShell. This is used by HA CI so the gateway can run multiple # replicas without requiring the OpenShell chart to own a database. # +# SPIFFE Broker API fixture: +# Set OPENSHELL_E2E_KUBE_SPIRE_BROKER=1 to install pinned SPIRE hardened +# charts (ci/values-spire-broker-stack.yaml) before OpenShell, with an +# mTLS Broker endpoint and enforced object authorization. The fixture needs a +# cluster without SPIRE CRDs and creates a fresh namespace +# (OPENSHELL_E2E_SPIRE_NAMESPACE, default openshell-e2e-spire); it removes +# both, and the registration entries tests label with that namespace, on exit. +# # Credential-driver fixture: # Set OPENSHELL_E2E_CREDENTIAL_DRIVERS=1 to enable one credential storage # backend. Set OPENSHELL_E2E_CREDENTIAL_DRIVER to `kubernetes-secrets` or @@ -124,6 +132,12 @@ VAULT_DEV_ROOT_TOKEN="${OPENSHELL_E2E_VAULT_DEV_ROOT_TOKEN:-root}" VAULT_CA_CONFIG_MAP="openbao-ca" VAULT_DNS_ALIAS="${VAULT_RELEASE_NAME}-0" VAULT_CA_FILE="${WORKDIR}/openbao-ca.crt" +SPIRE_FIXTURE_DEPLOYED=0 +SPIRE_NAMESPACE="${OPENSHELL_E2E_SPIRE_NAMESPACE:-openshell-e2e-spire}" +SPIRE_CRDS=(clusterspiffeids.spire.spiffe.io clusterstaticentries.spire.spiffe.io clusterfederatedtrustdomains.spire.spiffe.io) +SPIRE_CHART_REPO="https://spiffe.github.io/helm-charts-hardened/" +SPIRE_CHART_VERSION="${OPENSHELL_E2E_SPIRE_CHART_VERSION:-0.30.3}" +SPIRE_CRDS_CHART_VERSION="${OPENSHELL_E2E_SPIRE_CRDS_CHART_VERSION:-0.6.1}" CORPORATE_PROXY_FIXTURE_DEPLOYED=0 CORPORATE_PROXY_FIXTURE_SECRET="openshell-e2e-proxy-auth" CORPORATE_PROXY_FIXTURE_CA_CONFIGMAP="openshell-e2e-proxy-ca" @@ -381,8 +395,7 @@ start_gateway_portforward() { echo "Starting kubectl port-forward svc/${target_service} ${LOCAL_PORT}:${target_port}..." fi - kctl -n "${target_namespace}" port-forward "svc/${target_service}" \ - "${LOCAL_PORT}:${target_port}" >"${PORTFORWARD_LOG}" 2>&1 & + gateway_portforward_loop "svc/${target_service}" "${LOCAL_PORT}:${target_port}" "${PORTFORWARD_LOG}" "${target_namespace}" & PORTFORWARD_PID=$! while [ "${elapsed}" -lt "${pf_timeout}" ]; do @@ -543,6 +556,166 @@ provision_vault_auth() { ttl=1h >/dev/null } +deploy_spire_fixture() { + # The fixture owns only what it creates: refuse a cluster that already has + # the namespace or SPIRE CRDs, so cleanup can never remove another install. + local crd + for crd in "${SPIRE_CRDS[@]}"; do + if kctl get crd "${crd}" >/dev/null 2>&1; then + echo "ERROR: CRD ${crd} already exists; the SPIRE fixture needs a cluster without SPIRE." >&2 + exit 2 + fi + done + if ! kctl create namespace "${SPIRE_NAMESPACE}" >/dev/null; then + echo "ERROR: could not create a fresh SPIRE fixture namespace ${SPIRE_NAMESPACE}; set OPENSHELL_E2E_SPIRE_NAMESPACE to an unused name." >&2 + exit 2 + fi + SPIRE_FIXTURE_DEPLOYED=1 + echo "Installing SPIRE ${SPIRE_CHART_VERSION} fixture in namespace ${SPIRE_NAMESPACE}..." + helmctl install spire-crds spire-crds --repo "${SPIRE_CHART_REPO}" \ + --version "${SPIRE_CRDS_CHART_VERSION}" \ + --namespace "${SPIRE_NAMESPACE}" --wait + local spire_values="${ROOT}/deploy/helm/openshell/ci/values-spire-broker-stack.yaml" + helmctl install spire spire --repo "${SPIRE_CHART_REPO}" \ + --version "${SPIRE_CHART_VERSION}" \ + --namespace "${SPIRE_NAMESPACE}" \ + --values "${spire_values}" \ + --wait --timeout 5m + kctl -n "${SPIRE_NAMESPACE}" rollout status daemonset/spire-agent --timeout=180s + if [ "${OPENSHELL_E2E_KUBE_SPIRE_BROKER:-0}" = "1" ]; then + # UID-only references are resolved by SPIRE, never by a supervisor. The + # agent needs object lookup and SAR access; the broker's grant is namespace scoped. + kctl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kctl apply -f - + if [ "${OPENSHELL_E2E_KUBE_SPIRE_NODE_LOCAL:-0}" != "1" ]; then + kctl apply -f - <"${broker_values}" <"${broker_values}" <>"${broker_values}" </dev/null 2>&1 || true + kctl delete clusterstaticentries.spire.spiffe.io \ + -l "openshell.ai/e2e-spire-fixture=${SPIRE_NAMESPACE}" --ignore-not-found \ + >/dev/null 2>&1 || true + helmctl uninstall spire --namespace "${SPIRE_NAMESPACE}" --wait --timeout 120s \ + >/dev/null 2>&1 || true + helmctl uninstall spire-crds --namespace "${SPIRE_NAMESPACE}" --wait --timeout 60s \ + >/dev/null 2>&1 || true + # The CRD chart keeps its CRDs on uninstall. None existed before the + # fixture, so these are the fixture's own. + kctl delete crd "${SPIRE_CRDS[@]}" --ignore-not-found --wait=false \ + >/dev/null 2>&1 || true + kctl delete namespace "${SPIRE_NAMESPACE}" --wait=true --timeout=120s \ + --ignore-not-found >/dev/null 2>&1 || true + SPIRE_FIXTURE_DEPLOYED=0 +} + cleanup_vault_fixture() { [ -n "${KUBE_CONTEXT}" ] || return 0 [ -n "${VAULT_NAMESPACE}" ] || return 0 @@ -701,6 +874,9 @@ cleanup() { fi fi + # Remove CSI consumers before uninstalling SPIRE and its CSI driver. + cleanup_spire_fixture + if [ "${ENVOY_HELM_INSTALLED}" = "1" ] && [ -n "${KUBE_CONTEXT}" ]; then if command -v helm >/dev/null 2>&1; then helmctl uninstall "${ENVOY_RELEASE_NAME}" --namespace "${ENVOY_NAMESPACE}" \ @@ -831,6 +1007,8 @@ run_scenario() { export OPENSHELL_E2E_KUBE_CONTEXT="${KUBE_CONTEXT}" export OPENSHELL_E2E_KUBE_NAMESPACE="${NAMESPACE}" export OPENSHELL_E2E_KUBE_RELEASE="${RELEASE_NAME}" + export OPENSHELL_E2E_HOST_GATEWAY_IP="${HOST_GATEWAY_IP}" + # Tests that upgrade the release (for example to scale the shared # supervisor pool) reuse its values against this exact chart, through the # kubeconfig this harness uses. @@ -968,6 +1146,22 @@ openshift_register_route_gateway() { "${pki_dir}" } +# Preserve the test endpoint when node-local coverage relocates the gateway. +# The wrapper owns its kubectl child and stops it on cleanup. +gateway_portforward_loop() ( + trap - EXIT + local target="$1" mapping="$2" log="$3" target_namespace="${4:-${NAMESPACE}}" forward_pid="" + trap 'if [ -n "$forward_pid" ]; then kill "$forward_pid" 2>/dev/null || true; wait "$forward_pid" 2>/dev/null || true; fi; exit 0' TERM INT + while true; do + kubectl --context "${KUBE_CONTEXT}" -n "${target_namespace}" port-forward "$target" "$mapping" >>"$log" 2>&1 & + forward_pid=$! + wait "$forward_pid" || true + forward_pid="" + [ "${OPENSHELL_E2E_KUBE_SPIRE_NODE_LOCAL:-0}" = "1" ] || return 0 + sleep 1 + done +) + # Start `kubectl port-forward svc/openshell` for the gRPC endpoint and wait for # it to accept TCP. Sets LOCAL_PORT and PORTFORWARD_PID. Prints the port-forward # log and returns non-zero on failure. Used for the vanilla-Kubernetes transport @@ -975,8 +1169,7 @@ openshift_register_route_gateway() { start_grpc_portforward() { LOCAL_PORT="$(e2e_pick_port)" echo "Starting kubectl port-forward svc/openshell ${LOCAL_PORT}:8080..." - kctl -n "${NAMESPACE}" port-forward "svc/openshell" \ - "${LOCAL_PORT}:8080" >"${PORTFORWARD_LOG}" 2>&1 & + gateway_portforward_loop "svc/openshell" "${LOCAL_PORT}:8080" "${PORTFORWARD_LOG}" & PORTFORWARD_PID=$! local elapsed=0 timeout=30 @@ -1007,8 +1200,7 @@ start_health_portforward() { local workload_ref workload_ref="$(kube_workload_ref "${RELEASE_NAME}")" echo "Starting kubectl port-forward ${workload_ref} ${HEALTH_LOCAL_PORT}:health..." - kctl -n "${NAMESPACE}" port-forward "${workload_ref}" \ - "${HEALTH_LOCAL_PORT}:health" >"${PORTFORWARD_HEALTH_LOG}" 2>&1 & + gateway_portforward_loop "${workload_ref}" "${HEALTH_LOCAL_PORT}:health" "${PORTFORWARD_HEALTH_LOG}" & PORTFORWARD_HEALTH_PID=$! local elapsed=0 timeout=30 @@ -1554,6 +1746,9 @@ if use_envoy_gateway; then helm_values_args+=(--values "${ROOT}/deploy/helm/openshell/ci/values-gateway.yaml") install_envoy_gateway fi +if [ "${OPENSHELL_E2E_KUBE_SPIRE_BROKER:-0}" = "1" ]; then + deploy_spire_fixture +fi if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then # --- Multi-scenario mode: test all database backends --- @@ -1675,6 +1870,8 @@ else export OPENSHELL_E2E_KUBE_CONTEXT="${KUBE_CONTEXT}" export OPENSHELL_E2E_KUBE_NAMESPACE="${NAMESPACE}" export OPENSHELL_E2E_KUBE_RELEASE="${RELEASE_NAME}" + export OPENSHELL_E2E_HOST_GATEWAY_IP="${HOST_GATEWAY_IP}" + # Tests that upgrade the release (for example to scale the shared # supervisor pool) reuse its values against this exact chart, through the # kubeconfig this harness uses. diff --git a/examples/governance-interceptor/Cargo.lock b/examples/governance-interceptor/Cargo.lock index 174d71b8f9..1bb9facc7a 100644 --- a/examples/governance-interceptor/Cargo.lock +++ b/examples/governance-interceptor/Cargo.lock @@ -1996,6 +1996,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2105,6 +2106,17 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "openssl-probe" version = "0.2.1" diff --git a/examples/isolation-backend-v8/Cargo.lock b/examples/isolation-backend-v8/Cargo.lock index 9888e3c403..63edb47fa7 100644 --- a/examples/isolation-backend-v8/Cargo.lock +++ b/examples/isolation-backend-v8/Cargo.lock @@ -2325,6 +2325,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2429,6 +2430,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "openssl-probe" version = "0.2.1" diff --git a/examples/supervisor-middleware-content-guard/Cargo.lock b/examples/supervisor-middleware-content-guard/Cargo.lock index 85d957a0f4..7a3c26f767 100644 --- a/examples/supervisor-middleware-content-guard/Cargo.lock +++ b/examples/supervisor-middleware-content-guard/Cargo.lock @@ -2144,6 +2144,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2219,6 +2220,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "openshell-supervisor-middleware-content-guard" version = "0.0.0" diff --git a/proto/openshell.proto b/proto/openshell.proto index 198990a364..e8fef4acc1 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -557,6 +557,13 @@ service OpenShell { }; } + // Obtain the final provider token for the authenticated sandbox session. + // SPIFFE assertions and intermediate credentials remain in the gateway. + rpc GetSandboxProviderAccessToken(GetSandboxProviderAccessTokenRequest) + returns (GetSandboxProviderAccessTokenResponse) { + option (openshell.options.v1.authorization) = { auth_mode: "sandbox" }; + } + // Fetch recent sandbox logs (one-shot). rpc GetSandboxLogs(GetSandboxLogsRequest) returns (GetSandboxLogsResponse) { option (openshell.options.v1.authorization) = { @@ -2897,6 +2904,21 @@ message ProviderEnvironmentSnapshot { map files = 7; } +// Supervisor-only request. Sandbox/workspace are derived from the verified +// session JWT. No caller-selected identity, endpoint, audience, or scope. +message GetSandboxProviderAccessTokenRequest { + // Opaque endpoint-bound credential key issued in the gateway's provider + // configuration. Echo the configured key without supervisor-local revision + // or installation decorations. Callers must not construct or parse it. + string credential_key = 1; +} + +message GetSandboxProviderAccessTokenResponse { + string access_token = 1 [(openshell.options.v1.secret) = true]; + google.protobuf.Duration expires_after = 2; + string token_type = 3; +} + message ExchangeProviderSubjectTokenRequest { // The sandbox ID. Must match the authenticated sandbox principal. string sandbox_id = 1; diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 7f5c14e26e..cdb721b8ae 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1004,6 +1004,20 @@ kubectl -n openshell get statefulset openshell -o jsonpath='{.spec.template.spec # Should show items filter for ca.crt from openshell-server-tls ``` +With `gatewayConfig.openshell.gateway.spiffe_broker`, the gateway obtains +sandbox JWT-SVIDs through the experimental SPIFFE Broker API over mTLS. +Check the endpoint's exact `server_spiffe_id` or cluster-scoped +`server_spiffe_id_prefix`, gateway X.509-SVID issuance, +the CSI Workload API socket, the SPIRE broker allowlist and reference types, +and enforced `impersonate-via-spire` access to the Sandbox CR. The agent must +be able to resolve its persisted UID and find the corresponding registration. +Supervisors receive final provider access tokens and have no SPIRE material. +With `node_local_port`, check the gateway's `OPENSHELL_NODE_IP` Downward API +field (`status.hostIP`) and the local agent's TCP listener and host port/network +configuration. Broker requests connect directly to that node address; no +Broker Service is used. Registrations must cover every node where a gateway +replica may run. Missing or invalid node addresses fail startup. + #### Secure metrics TLS When `metrics.tls.enabled=true`, the dedicated metrics listener uses its own @@ -1667,6 +1681,8 @@ credential failures. | OpenShift `-supervisor` Deployment never creates pods after setting `supervisor_topology = "gateway"` (SCC `runAsUser`/`fsGroup` not in the namespace range) | `supervisor.podSecurityContext` defaults `runAsUser`, `runAsGroup`, and `fsGroup` to 65534, outside the namespace's assigned UID/GID ranges | `kubectl -n openshell get events --field-selector reason=FailedCreate`; compare with the `openshift.io/sa.scc.*` annotations from `oc describe ns openshell`; set `supervisor.podSecurityContext.runAsUser`, `.runAsGroup`, and `.fsGroup` to `null`, as `deploy/helm/openshell/ci/values-openshift-scc.yaml` does, and keep `runAsNonRoot` and `seccompProfile` | | Shared sandboxes stay `Starting` or report `unavailable` readiness with `supervisor_topology = "gateway"`, gateway logs say no shared supervisor host is connected | The `-supervisor` Pod cannot reach or authenticate to the gateway: wrong `OPENSHELL_TLS_CA`, the `-supervisor-token` Secret is missing or still empty (gateway logs `wrote the supervisor token Secret` once it has delivered a token; `no usable supervisor token` in the supervisor means it has not; right after a fresh install the kubelet can take about 90 seconds to project the written token into the already-ready Pod, so launches can be refused for no live pool member until the supervisor logs `gateway accepted the supervisor host`), or the gateway was deployed without TLS or without the sandbox-JWT signing key | `kubectl -n openshell logs deployment/-supervisor` for `gateway accepted the supervisor host` or a refusal (a refusal or startup failure naming `OPENSHELL_SUPERVISOR_POOL_MEMBER_ID` or `_ADVERTISE_IP` means the Pod lacks its member identity; the chart sets them from the Pod name and IP); gateway logs for `supervisor host attached`, `wrote the supervisor token Secret`, or `failed to write the supervisor token Secret`; `kubectl -n openshell get secret -supervisor-token -o jsonpath='{.metadata.annotations}'` shows the token expiry; the gateway refuses to start the topology without TLS or `gateway_jwt` | | Shared sandbox workloads terminate after a supervisor Pod or node failure (`sandbox-supervisor-recovery-expired`) | One supervisor replica (no member to fail over to), no live pool member took over, or the recovery timeout is shorter than detection plus takeover | `supervisor.replicas` and the `-supervisor` PodDisruptionBudget; gateway `taking over a shared sandbox instance` logs; workload Pod `sandbox-supervisor-*` findings; `supervisor.recoveryTimeoutSeconds`. See "Shared supervisor high availability" in Step 6 | +| Shared-sandbox grants fail with `code=unavailable stage=workload_identity` | The gateway cannot reach the SPIFFE Broker endpoint or obtain credentials from its CSI Workload API socket | Gateway logs for `dynamic token grant failed`; check Broker endpoint connectivity, the gateway's Workload API CSI mount and agent health | +| Shared-sandbox grants fail with `code=workload_identity_rejected stage=workload_identity` | Broker mTLS verification fails, an agent has no Sandbox UID registration, or the gateway lacks `impersonate-via-spire` authorization | Check trust, the exact or prefix peer policy, and Sandbox registrations on the gateway's node-local agent or the configured endpoint's agents. Verify the gateway SPIFFE identity's namespace-scoped authorization. Use a node alias to cover eligible agents | | OpenShift sandbox pod fails to start (`unable to validate against any security context constraint`) | An explicit sandbox UID/GID override or custom security context conflicts with the namespace's SCC range | Inspect pod events and the namespace's `openshift.io/sa.scc.uid-range` and `openshift.io/sa.scc.supplemental-groups` annotations; remove incompatible `sandbox_uid` / `sandbox_gid` overrides so the driver uses the namespace-assigned identity. Current sandbox pods use `restricted-v2` without a privileged SCC grant; see the [OpenShift guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift) and [RFC 0012](https://github.com/NVIDIA/OpenShell/blob/main/rfc/0012-isolation-backend/README.md#motivation). | | OpenShift self-hosted Vault/OpenBao credential store pod never schedules (waits time out with `no matching resources found`) | The store's Helm chart pins `runAsUser`/`fsGroup`/seccomp, which restricted-v2 rejects, so the StatefulSet controller never creates the pod | Deploy the store's chart in its OpenShift mode (`--set global.openshift=true` for the OpenBao/Vault chart) so the namespace SCC assigns a compliant security context — no manual SCC grant needed | | Vault credential driver returns HTTP 403 / `Vault Kubernetes auth denied the configured role` on provider create | Vault's `auth/kubernetes` method or the gateway login role is not provisioned, or the role is not bound to the gateway service account and namespace | In Vault: `bao auth enable kubernetes` and `bao write auth/kubernetes/config kubernetes_host=... kubernetes_ca_cert=@...`; ensure the login role's `bound_service_account_names`/`bound_service_account_namespaces` match the gateway SA and namespace and its policy grants the credential paths | diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 2a5aaf272d..1ac5d78329 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -402,6 +402,7 @@ EOF else echo "Creating k3d cluster '${CLUSTER_NAME}'..." k3d cluster create "${CLUSTER_NAME}" \ + --agents "${HELM_K3S_AGENT_COUNT:-0}" \ --wait \ --kubeconfig-update-default=false \ --kubeconfig-switch-context=false \ diff --git a/tasks/test.toml b/tasks/test.toml index 26c2ddd6a5..b2e599241c 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -369,6 +369,16 @@ env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor_scaling", OPENSH # test. The shared supervisor requires gateway TLS, which the Skaffold values turn off. run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor-ha-scaling.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' +["e2e:kubernetes:shared-supervisor-spiffe"] +description = "Run SPIFFE Broker API provider exchange, credential rotation, and supervisor failover e2e over TLS" +env = { HELM_K3S_AGENT_COUNT = "2", OPENSHELL_E2E_SPIRE_MIN_AGENTS = "3", OPENSHELL_E2E_KUBE_SPIRE_BROKER = "1", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor_spiffe", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor-spiffe" } +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor-spiffe.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' + +["e2e:kubernetes:shared-supervisor-spiffe-node-local"] +description = "Run node-local SPIFFE Broker coverage with gateway relocation across nodes" +env = { HELM_K3S_AGENT_COUNT = "2", OPENSHELL_E2E_SPIRE_MIN_AGENTS = "3", OPENSHELL_E2E_KUBE_SPIRE_NODE_LOCAL = "1", OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET = "e2e-spiffe-postgres", OPENSHELL_E2E_KUBE_SPIRE_BROKER = "1", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor_spiffe", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor-spiffe" } +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor-spiffe-node-local.yaml:deploy/helm/openshell/ci/values-tls-port-forward.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' + ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index cf08ce94fe..76506381c3 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2292,6 +2292,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2393,6 +2394,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "openshell-testkit" version = "0.0.0" diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 459d6889c4..560b02aa72 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2280,6 +2280,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2381,6 +2382,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "openshell-testkit" version = "0.0.0" diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4c65ad4021..b16d3fb67d 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2352,6 +2352,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2453,6 +2454,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "openshell-test-feature-kubernetes-updates" version = "0.0.0" diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index bc18df94cd..28d6e540d2 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2298,6 +2298,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2399,6 +2400,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "openshell-testkit" version = "0.0.0" diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 7e808b028d..bad3c82e89 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2271,6 +2271,7 @@ dependencies = [ "openshell-crypto", "openshell-extension-core", "openshell-policy-schema", + "openshell-spiffe", "prost", "prost-types", "protoc-bin-vendored", @@ -2372,6 +2373,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "openshell-spiffe" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "openshell-testkit" version = "0.0.0"