diff --git a/Cargo.lock b/Cargo.lock index 7c9eb08165..6b743d5ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4635,6 +4635,7 @@ dependencies = [ name = "openshell-driver-kubernetes" version = "0.0.0" dependencies = [ + "base64 0.22.1", "bytes", "clap", "futures", diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 4b7659ffdd..0a582c4e9a 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -33,6 +33,7 @@ tokio-stream = { workspace = true } kube = { workspace = true } kube-runtime = { workspace = true } k8s-openapi = { workspace = true } +base64 = { workspace = true } tower = { workspace = true, features = ["timeout"] } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a18b51e1de..b69b19ca2f 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -161,7 +161,66 @@ impl KubernetesSandboxRuntimeConfig { } } -/// How workspaces map to Kubernetes namespaces. +/// Runtime substrate the Kubernetes driver provisions sandboxes on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum KubernetesRuntimeMode { + /// Provision sandboxes as `agents.x-k8s.io` Sandbox CRs reconciled into + /// pods by the upstream Agent Sandbox controller (default, production + /// path). + #[default] + Pod, + /// Provision sandboxes as `kubevirt.io/v1` `VirtualMachine` objects + /// reconciled by the `KubeVirt` controller (proof of concept). + #[serde(rename = "kubevirt", alias = "kube-virt")] + KubeVirt, +} + +impl std::fmt::Display for KubernetesRuntimeMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Pod => f.write_str("pod"), + Self::KubeVirt => f.write_str("kubevirt"), + } + } +} + +/// How the guest workload runs. +/// +/// `KubeVirt` runtime mode settings (proof of concept). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubeVirtConfig { + /// containerDisk image providing the bootable guest filesystem. The image + /// must contain a Linux guest with the static `openshell-sandbox` + /// supervisor and an init that consumes the driver-rendered bootstrap + /// Secret (cloudInitNoCloud transport). Not an arbitrary sandbox OCI + /// image — see `e2e/kubevirt/` for the purpose-built image build. + pub guest_image: String, + /// Kubernetes `imagePullPolicy` for the guest containerDisk. + /// Empty string omits the field, using the Kubernetes default. + pub guest_image_pull_policy: String, + /// vCPU cores allocated to each guest. + pub cpu_cores: u32, + /// Guest memory request, e.g. `"1Gi"`. + pub memory: String, + /// Attach a per-sandbox PVC at `/sandbox` so the workspace survives + /// stop/start (mirrors the pod path's PVC-backed workspace). + pub workspace_pvc: bool, +} + +impl Default for KubeVirtConfig { + fn default() -> Self { + Self { + guest_image: String::new(), + guest_image_pull_policy: String::new(), + cpu_cores: 1, + memory: "1Gi".to_string(), + workspace_pvc: true, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum WorkspaceMode { @@ -464,6 +523,12 @@ pub struct KubernetesComputeConfig { /// When empty and `sandbox_uid` is set, defaults to the resolved UID. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_gid: Option, + /// Runtime substrate the driver provisions sandboxes on. Defaults to + /// `pod` (Agent Sandbox CRs). `kubevirt` creates `KubeVirt` + /// `VirtualMachine` objects directly (proof of concept). + pub runtime: KubernetesRuntimeMode, + /// `KubeVirt` settings, used only when `runtime = "kubevirt"`. + pub kubevirt: KubeVirtConfig, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -550,6 +615,8 @@ impl Default for KubernetesComputeConfig { provider_spiffe_workload_api_socket_path: String::new(), sandbox_uid: None, sandbox_gid: None, + runtime: KubernetesRuntimeMode::default(), + kubevirt: KubeVirtConfig::default(), } } } @@ -565,7 +632,8 @@ impl KubernetesComputeConfig { self.validate_proxy_uid()?; self.validate_upstream_proxy_config()?; self.validate_allowed_pod_annotations()?; - self.validate_allowed_pod_labels() + self.validate_allowed_pod_labels()?; + self.validate_kubevirt_config() } fn validate_bootstrap_windows(&self) -> Result<(), String> { @@ -674,6 +742,73 @@ impl KubernetesComputeConfig { ) } + /// Validate the `KubeVirt` runtime mode configuration (`PoC` constraints). + /// + /// The `KubeVirt` runtime supports the same supervisor topologies as the + /// pod path: the default is a dedicated supervisor Pod per sandbox + /// (`supervisor_topology` unset), and `supervisor_topology = "gateway"` + /// places each generation on the gateway's shared supervisor. Which + /// objects the driver provisions differs per topology, but the topology + /// rules themselves (`gateway` requires `shared_supervisor`; `workspace` + /// is not implemented) are enforced by [`Self::validate_supervisor_topology`] + /// for both runtimes, so they are not repeated here. + /// + /// `grpc_endpoint` is required for the dedicated topology — its + /// supervisor Pod must reach the gateway, exactly as the pod path's + /// supervisor Pods do — but not for the shared topology, whose guest + /// never learns the gateway endpoint (the boundary resolves the shared + /// supervisor's Service through the API). Any scheme is accepted: a + /// dedicated supervisor carries client TLS material, so an `https://` + /// endpoint (the TLS demo's shape) is as valid as an `http://` one. + pub fn validate_kubevirt_config(&self) -> Result<(), String> { + if self.runtime != KubernetesRuntimeMode::KubeVirt { + return Ok(()); + } + if self.supervisor_topology.is_none() && self.grpc_endpoint.trim().is_empty() { + return Err( + "kubevirt runtime with a dedicated supervisor Pod requires grpc_endpoint: the \ + supervisor Pod must be able to reach the gateway" + .to_string(), + ); + } + if self.kubevirt.guest_image.trim().is_empty() { + return Err( + "kubevirt runtime requires kubevirt.guest_image to be set to a bootable \ + containerDisk image" + .to_string(), + ); + } + if self.kubevirt.cpu_cores == 0 { + return Err("kubevirt.cpu_cores must be at least 1".to_string()); + } + if self.kubevirt.memory.trim().is_empty() { + return Err("kubevirt.memory must be set (e.g. \"1Gi\")".to_string()); + } + if !matches!( + self.workspace_mode, + WorkspaceMode::Shared | WorkspaceMode::Managed + ) { + return Err(format!( + "kubevirt runtime supports the shared and managed workspace \ + modes; configured mode is {}", + self.workspace_mode + )); + } + // Every KubeVirt guest runs its workload as root and its boundary + // is untrusted, so the dedicated supervisor is required: a shared + // supervisor would let a parser bug reached by a hostile root guest + // cross tenants. + if self.supervisor_topology.is_some() { + return Err( + "kubevirt runtime requires the dedicated supervisor topology \ + (supervisor_topology unset); the gateway-shared supervisor is not \ + supported for KubeVirt VMs" + .to_string(), + ); + } + Ok(()) + } + pub fn validate_proxy_uid(&self) -> Result<(), String> { self.sandbox_runtime.validate() } @@ -1168,8 +1303,9 @@ fn validate_provider_spiffe_workload_api_socket_path_value( #[cfg(test)] mod tests { + use super::*; - use std::collections::BTreeMap as HashMap; + use BTreeMap as HashMap; #[test] fn app_endpoint_configuration_has_one_port_source_and_explicit_enablement() { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 7e4643fc9c..c69f76a523 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -17,10 +17,11 @@ use crate::lifecycle::LifecycleGates; use crate::sandbox_runtime::{ BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, ClientTlsMaterial, SHARED_INSTANCE_KEY, SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, - SupervisorClientTls, boundary_service, boundary_service_matches_mode, control_policy, - generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, - sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, shared_control_policy, - shared_instance_secret, supervisor_bootstrap_secret, supervisor_pod, workload_fence, + SupervisorClientTls, boundary_launch_inputs, boundary_service, boundary_service_matches_mode, + control_policy, generate_proxy_ca_material, image_pull_secret_copy, render_boundary_bootstrap, + sandbox_bootstrap_secret, sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, + shared_control_policy, shared_instance_secret, supervisor_bootstrap_secret, supervisor_pod, + workload_fence, }; use crate::shared_supervisor::{ ANNOTATION_SUPERVISOR_SESSION, ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY, @@ -31,8 +32,8 @@ use k8s_openapi::api::authentication::v1::{ TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo, }; use k8s_openapi::api::core::v1::{ - ConfigMap, Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, - Secret, Service, ServiceAccount, Volume, VolumeMount, + ConfigMap, Event as KubeEventObj, Namespace, Node, PersistentVolumeClaim, + PersistentVolumeClaimVolumeSource, Pod, Secret, Service, ServiceAccount, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, @@ -72,8 +73,7 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use openshell_sandbox_backend::boundary_protocol::{ - GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, - generate_sandbox_tls_material, + SandboxTlsClientConfig, SandboxTlsServerConfig, }; use openshell_supervisor_control::{ InstanceSupervision, LaunchError, RecoverableInstance, RecoveryBaseline, @@ -543,6 +543,101 @@ fn is_kube_resource_version_conflict(error: &KubeError) -> bool { matches!(error, KubeError::Api(api) if api.code == 409 && api.reason == "Conflict") } +/// Any 409 from a pinned write: the precondition (UID or +/// resourceVersion) failed, so the write must be retried from a fresh +/// read or aborted — never repeated verbatim. +fn is_kube_conflict(error: &KubeError) -> bool { + matches!(error, KubeError::Api(api) if api.code == 409) +} + +/// Pin a JSON merge patch to the object that was checked (H5): inject +/// the expected `metadata.uid` and the `metadata.resourceVersion` read +/// in the same iteration, so the API server rejects the write when the +/// object was replaced (a new UID under the same name) or concurrently +/// modified. +fn pin_patch_to_checked_object(patch: &mut serde_json::Value, uid: &str, resource_version: &str) { + if !patch["metadata"].is_object() { + patch["metadata"] = serde_json::json!({}); + } + let metadata = patch["metadata"] + .as_object_mut() + .expect("metadata object was just ensured"); + metadata.insert("uid".to_string(), serde_json::json!(uid)); + metadata.insert( + "resourceVersion".to_string(), + serde_json::json!(resource_version), + ); +} + +/// Whether the object's ownerReferences include the expected owner UID. +fn owner_reference_matches( + owner_references: &Option>, + expected_uid: &str, +) -> bool { + owner_references + .as_ref() + .is_some_and(|owners| owners.iter().any(|owner| owner.uid == expected_uid)) +} + +/// Patch a `KubeVirt` `VirtualMachine` pinned to the object the caller +/// checked (H5: lifecycle checks must fence subsequent mutations). +/// +/// Every patch carries the expected `metadata.uid` and the +/// `metadata.resourceVersion` from the same read, so the API server +/// rejects the write when the VM was replaced or concurrently modified. +/// A resourceVersion conflict is a transient race and is retried from a +/// fresh read; a UID change is not: the object the caller verified is +/// gone, so the operation aborts with a Precondition error rather than +/// falling back to a name-only write. +async fn patch_kubevirt_vm_pinned_to_checked_uid( + api: &Api, + kube_name: &str, + expected_vm_uid: &str, + mut patch_for_resource_version: impl FnMut(&str) -> serde_json::Value, +) -> Result { + let operation = async { + loop { + let current = api + .get(kube_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + let resource_version = current + .metadata + .resource_version + .as_deref() + .unwrap_or_default() + .to_string(); + let observed_uid = current.metadata.uid.as_deref().unwrap_or_default(); + if observed_uid != expected_vm_uid { + return Err(KubernetesDriverError::Precondition(format!( + "VirtualMachine {kube_name} was replaced after it was checked: \ + expected UID {expected_vm_uid}, found {observed_uid}" + ))); + } + let mut patch = patch_for_resource_version(&resource_version); + pin_patch_to_checked_object(&mut patch, expected_vm_uid, &resource_version); + match api + .patch(kube_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + { + Ok(updated) => return Ok(updated), + Err(error) if is_kube_conflict(&error) => { + tokio::task::yield_now().await; + } + Err(error) => return Err(KubernetesDriverError::from_kube(error)), + } + } + }; + tokio::time::timeout(KUBE_API_TIMEOUT, operation) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s patching VirtualMachine {kube_name}", + KUBE_API_TIMEOUT.as_secs() + )) + })? +} + async fn patch_dynamic_object_with_resource_version_retry( api: &Api, name: &str, @@ -595,7 +690,17 @@ impl From for openshell_core::ComputeDriverError { /// Timeout for individual Kubernetes API calls (create, delete, get). /// This prevents gRPC handlers from blocking indefinitely when the k8s /// API server is unreachable or slow. -const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +pub(crate) const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); + +/// Upper bound on waiting for a companion resource (bootstrap Secret, +/// workspace PVC) to terminate during create-path orphan cleanup. Bound PVCs +/// carry the `pvc-protection` finalizer and provisioner cleanup, so this is +/// deliberately generous. +const COMPANION_DELETE_TIMEOUT: Duration = Duration::from_secs(90); + +/// Upper bound on waiting for a dead `VirtualMachineInstance` to terminate +/// during the start-path halt bounce. +const VMI_TERMINATION_TIMEOUT: Duration = Duration::from_mins(2); /// How long an operator-mode workspace check waits for the namespace label /// watcher's first full listing before reporting the driver unavailable. const OPERATOR_ALLOWLIST_SYNC_TIMEOUT: Duration = Duration::from_secs(10); @@ -642,25 +747,6 @@ fn decode_launch_authentication( Ok(authentication) } -fn gateway_verification_keys( - keys: &[openshell_core::jwt::SessionVerificationKey], -) -> Result, KubernetesDriverError> { - keys.iter() - .map(|key| { - String::from_utf8(key.public_key_pem.clone()) - .map(|public_key_pem| GatewayVerificationKey { - key_id: key.key_id.clone(), - public_key_pem, - }) - .map_err(|error| { - KubernetesDriverError::Precondition(format!( - "Kubernetes sandbox verification key is not UTF-8 PEM: {error}" - )) - }) - }) - .collect() -} - /// Kubernetes defaults pod termination to 30 seconds when the pod template /// omits `terminationGracePeriodSeconds`. const DEFAULT_POD_TERMINATION_GRACE_PERIOD: Duration = Duration::from_secs(30); @@ -1103,116 +1189,769 @@ impl std::fmt::Debug for KubernetesComputeDriver { } } -impl KubernetesComputeDriver { - /// Environment that lets a supervisor export spans, joining the current - /// trace, OCSF events, and relayed workload traces, identifying the - /// gateway that launched it. - fn supervisor_otlp_environment(&self) -> Vec<(&'static str, String)> { - let mut environment = self.config.supervisor_otlp.environment(); - if self.config.supervisor_otlp.traces.is_some() { - environment.extend(openshell_otel::current_trace_context_environment()); - } - if let Some(name) = &self.config.supervisor_gateway_name { - environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); - } - environment +/// Name of the namespace-wide virt-launcher isolation policy: deny-all +/// ingress and egress for every virt-launcher pod in the sandbox namespace, +/// independent of any per-sandbox object (H1). +const KUBEVIRT_LAUNCHER_ISOLATION_POLICY: &str = "openshell-kubevirt-launcher-isolation"; + +/// Render the namespace-wide virt-launcher isolation policy (H1). No rules +/// with both policy types named: deny all ingress and egress. The +/// per-sandbox fence and control policy add the supervisor allowances on +/// top of this floor. +fn kubevirt_launcher_isolation_policy(namespace: &str) -> NetworkPolicy { + let mut labels = BTreeMap::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert( + "openshell.ai/component".to_string(), + "kubevirt-launcher-isolation".to_string(), + ); + NetworkPolicy { + metadata: ObjectMeta { + name: Some(KUBEVIRT_LAUNCHER_ISOLATION_POLICY.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubevirt.io".to_string(), + "virt-launcher".to_string(), + )])), + ..Default::default() + }), + policy_types: Some(vec!["Ingress".to_string(), "Egress".to_string()]), + ..Default::default() + }), } +} - #[cfg(test)] - pub(crate) fn new_for_test(config: KubernetesComputeConfig) -> Self { - let service = tower::service_fn(|_request: http::Request| async { - Ok::<_, std::convert::Infallible>(http::Response::new(http_body_util::Empty::< - bytes::Bytes, - >::new())) - }); - let client = Client::new(service, "default"); - Self { - client: client.clone(), - watch_client: client, - sandbox_api_version: Arc::new(OnceCell::new()), - lifecycle_gates: Arc::default(), - config, - network_trust_bundle: None, - operator_allowlist: None, - shared_supervisor: None, +/// VM annotation stamped by the fence reconciler when it re-created a +/// drifted or deleted fence (M1), carrying the restore time in Unix +/// milliseconds. +/// +/// The snapshot surfaces it as a `FenceRestored` condition; the reconciler +/// clears it once it has been visible for ten minutes. +pub const FENCE_RESTORED_ANNOTATION: &str = "openshell.ai/kubevirt-fence-restored-at"; +/// How often the running-state fence reconciler checks every sandbox. +const FENCE_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +/// How long a fence-restored marker stays visible on the snapshot. +const FENCE_RESTORED_TTL_MS: i64 = 10 * 60 * 1000; + +/// Failure of a dedicated generation's supervisor Pod, when it is gone, +/// exited, crashed, or cannot pull its image (M2: `restartPolicy: Never` +/// means nothing restarts it, and without a supervisor the boundary can +/// never connect). `None` for shared generations, healthy or pending +/// supervisors, and unreadable pods (never guess a failure). +async fn kubevirt_supervisor_failure( + client: &Client, + namespace: &str, + vm: &DynamicObject, +) -> Option { + if !matches!(supervisor_placement(vm), Ok(SupervisorPlacement::Dedicated)) { + return None; + } + // Only a started VM can have a lost supervisor: during the create's + // Halted window (and while it is still gated) the Pod legitimately does + // not exist yet, and a stopped sandbox's stop deletes the Pod by + // design. + let halted = vm + .data + .get("spec") + .and_then(|spec| spec.get("runStrategy")) + .and_then(serde_json::Value::as_str) + == Some("Halted"); + if halted { + return None; + } + let sandbox_id = sandbox_id_from_object(vm).ok()?; + let names = SandboxRuntimeNames::new(&sandbox_id); + let pods: Api = Api::namespaced(client.clone(), namespace); + match pods.get_opt(&names.supervisor_pod).await { + Ok(Some(pod)) => { + let phase = pod + .status + .as_ref() + .and_then(|status| status.phase.as_deref()); + match phase { + Some("Succeeded") => Some( + "supervisor Pod exited (restartPolicy Never); start a new generation" + .to_string(), + ), + Some("Failed") => Some("supervisor Pod failed (restartPolicy Never)".to_string()), + _ => { + // A Pending pod wedged on an image pull never becomes + // Failed; surface the pull problem instead of + // provisioning forever (L4). + pod.status + .as_ref() + .and_then(|status| status.container_statuses.as_ref()) + .and_then(|statuses| { + statuses.iter().find_map(|status| { + let reason = + status.state.as_ref()?.waiting.as_ref()?.reason.as_deref()?; + matches!(reason, "ImagePullBackOff" | "ErrImagePull") + .then(|| format!("supervisor Pod image pull failed ({reason})")) + }) + }) + } + } } + Ok(None) => Some("supervisor Pod is gone".to_string()), + Err(_) => None, } +} - /// A driver whose new generations get a dedicated supervisor Pod each. - /// A configuration that places them on the gateway's shared supervisor - /// needs [`Self::with_shared_supervision`]; without the gateway's - /// coordination it is refused here, before anything runs. - pub async fn new( - config: KubernetesComputeConfig, - shutdown_rx: tokio::sync::watch::Receiver, - network_trust_bundle: Option, - ) -> Result { - if config.shared_supervisor.is_some() { - return Err(KubernetesDriverError::Precondition( - "the gateway supervisor topology requires the gateway's shared supervision, which only the driver linked into the gateway receives" - .to_string(), - )); +/// Config the running-state reconciler needs to re-render expected fences +/// and re-create the isolation floor without holding the driver. +#[derive(Clone)] +struct KubeVirtReconcileConfig { + gateway_id: String, + boundary_port: u16, + shared_supervisor: Option, +} + +/// The supervisor target a VM's fence should admit, re-resolved from the +/// live cluster (never from the persisted annotation, so a tampered VM +/// annotation cannot inject a fence spec): the dedicated generation's own +/// Service, or the configured shared supervisor's Service. +async fn kubevirt_fence_target( + client: &Client, + config: &KubeVirtReconcileConfig, + namespace: &str, + vm: &DynamicObject, +) -> Result, String> { + let sandbox_id = sandbox_id_from_object(vm)?; + match supervisor_placement(vm) { + Ok(SupervisorPlacement::Dedicated) => { + let names = SandboxRuntimeNames::new(&sandbox_id); + let services = Api::::namespaced(client.clone(), namespace); + let service_name = names.supervisor_service.clone(); + let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get(&service_name)) + .await + .map_err(|_| format!("timed out reading boundary Service {service_name}"))? + .map_err(|error| format!("read boundary Service {service_name}: {error}"))?; + if !boundary_service_matches_mode( + &service, + &sandbox_id, + KubernetesControlConnection::RuntimeInitiated, + ) { + return Err(format!( + "boundary Service {service_name} does not select the runtime-initiated \ + supervisor listener" + )); + } + let spec = service.spec.unwrap_or_default(); + let cluster_ip = spec + .cluster_ip + .filter(|ip| ip != "None") + .ok_or_else(|| format!("boundary Service {service_name} has no ClusterIP"))?; + let address: std::net::SocketAddr = format!("{}:{}", cluster_ip, config.boundary_port) + .parse() + .map_err(|error| format!("invalid boundary Service ClusterIP: {error}"))?; + let pod_selector = spec + .selector + .ok_or_else(|| format!("boundary Service {service_name} has no pod selector"))?; + Ok(Some( + crate::kubevirt_boundary::KubeVirtSupervisorTarget::Dedicated { + authority: format!("{service_name}.{namespace}.svc:{}", config.boundary_port), + address, + supervisor_pod_uid: String::new(), + pod_selector, + port: config.boundary_port, + }, + )) } - Self::build(config, shutdown_rx, None, network_trust_bundle).await + Ok(SupervisorPlacement::Shared { route }) => { + let shared = config.shared_supervisor.as_ref().ok_or( + "the gateway supervisor topology is not configured for a shared generation", + )?; + let services = Api::::namespaced(client.clone(), &shared.service_namespace); + let service = + tokio::time::timeout(KUBE_API_TIMEOUT, services.get(&shared.service_name.clone())) + .await + .map_err(|_| "timed out reading the shared supervisor Service".to_string())? + .map_err(|error| format!("read shared supervisor Service: {error}"))?; + let spec = service.spec.unwrap_or_default(); + let selector = spec + .selector + .filter(|selector| !selector.is_empty()) + .ok_or_else(|| "the shared supervisor Service selects no Pods".to_string())?; + let address: std::net::SocketAddr = format!( + "{}:{}", + spec.cluster_ip + .filter(|ip| ip != "None") + .ok_or_else(|| "the shared supervisor Service has no ClusterIP".to_string())?, + shared.runtime_port + ) + .parse() + .map_err(|error| format!("invalid shared supervisor Service ClusterIP: {error}"))?; + Ok(Some( + crate::kubevirt_boundary::KubeVirtSupervisorTarget::Shared { + authority: format!("{}:{}", shared.service_host(), shared.runtime_port), + address, + route, + namespace: shared.service_namespace.clone(), + pod_selector: selector, + port: shared.runtime_port, + }, + )) + } + Err(error) => Err(error.to_string()), } +} - /// A driver whose new generations become sandbox instances on the shared - /// supervisor the gateway coordinates through `supervision`. The driver - /// adapts that coordination to Kubernetes: it records placements, - /// persists and reads back launch material, provisions egress to the - /// supervisor's Service, and delivers the supervisor token into its - /// Secret. - pub async fn with_shared_supervision( - config: KubernetesComputeConfig, - shutdown_rx: tokio::sync::watch::Receiver, - supervision: SharedSupervision, - ) -> Result { - if config.shared_supervisor.is_none() { - return Err(KubernetesDriverError::Precondition( - "shared supervision was provided but the configuration does not place generations on the shared supervisor" - .to_string(), - )); +/// The running-state fence reconciler (M1): for every driver-owned +/// `VirtualMachine` in the namespace, compare the live egress fence against +/// the expectation re-rendered from the CURRENT supervisor target (never +/// from the persisted annotation, which is not trusted input) and +/// re-create the fence when it was deleted, replaced, or drifted — +/// isolation of a running sandbox never depends on the fence surviving +/// unattended. Also retires fence-restored markers that have been visible +/// long enough. Returns the sandbox ids whose fence was restored. +async fn reconcile_kubevirt_fences( + client: &Client, + namespace: &str, + config: &KubeVirtReconcileConfig, +) -> Vec { + let vm_api = crate::kubevirt::virtual_machine_api(client, namespace); + let selector = openshell_sandbox_label_selector(); + let list = match vm_api.list(&ListParams::default().labels(&selector)).await { + Ok(list) => list, + Err(error) => { + warn!(%error, "KubeVirt fence reconciler could not list VirtualMachines"); + return Vec::new(); } - // The gateway refuses the additional destination CA under the shared - // supervisor topology fail-closed, so no trust bundle reaches this - // constructor. - Self::build(config, shutdown_rx, Some(supervision), None).await + }; + let now = openshell_core::time::now_ms(); + let mut restored = Vec::new(); + for vm in list.items { + if vm.metadata.deletion_timestamp.is_some() { + continue; + } + let Some(kube_name) = vm.metadata.name.clone() else { + continue; + }; + let Ok(sandbox_id) = sandbox_id_from_object(&vm) else { + continue; + }; + // Retire a fence-restored marker that has been visible long enough. + if let Some(marker) = vm + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FENCE_RESTORED_ANNOTATION)) + && marker + .parse::() + .is_ok_and(|stamped| now - stamped > FENCE_RESTORED_TTL_MS) + { + let patch = serde_json::json!({"metadata": {"annotations": { + FENCE_RESTORED_ANNOTATION: null + }}}); + let _ = vm_api + .patch(&kube_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await; + } + // The persisted expectation supplies only the UID the evidence was + // rendered against; the expected spec is re-rendered from the + // current target. + let persisted_uid = vm + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get(crate::kubevirt_boundary::FENCE_EXPECTATION_ANNOTATION) + }) + .and_then(|value| crate::kubevirt_boundary::ExpectedFence::from_annotation(value).ok()) + .map(|expectation| expectation.uid); + let target = match kubevirt_fence_target(client, config, namespace, &vm).await { + Ok(Some(target)) => target, + Ok(None) => continue, + Err(error) => { + warn!( + sandbox_id = %sandbox_id, + %error, + "KubeVirt fence reconciler could not resolve the supervisor target" + ); + continue; + } + }; + let expected_spec = crate::kubevirt_boundary::render_fence( + namespace, + &sandbox_id, + Some(&config.gateway_id), + &target, + )["spec"] + .clone(); + let observed = match crate::kubevirt_boundary::observe_fence(client, namespace, &sandbox_id) + .await + { + Ok(observed) => observed, + Err(error) => { + warn!(sandbox_id = %sandbox_id, %error, "KubeVirt fence reconciler could not read the fence"); + continue; + } + }; + let drifted = match (&observed, &persisted_uid) { + (Some(observed), Some(uid)) => observed.uid != *uid || observed.spec != expected_spec, + // No persisted expectation (pre-annotation VM): compare the + // spec only, then adopt the observed UID. + (Some(observed), None) => observed.spec != expected_spec, + // The fence is gone. + (None, _) => true, + }; + if !drifted { + continue; + } + warn!( + sandbox_id = %sandbox_id, + vm = %kube_name, + "KubeVirt egress fence drifted or was deleted while running; re-fencing" + ); + if let Err(error) = re_fence(client, namespace, &sandbox_id, &vm, &expected_spec).await { + warn!(sandbox_id = %sandbox_id, %error, "KubeVirt fence reconciler could not re-fence"); + continue; + } + restored.push(sandbox_id); } + restored +} - async fn build( - config: KubernetesComputeConfig, - shutdown_rx: tokio::sync::watch::Receiver, - supervision: Option, - network_trust_bundle: Option, - ) -> Result { - config - .resource_admission - .validate() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_workspace_mode() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_provider_spiffe_workload_api_socket_path() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_sandbox_identity_config() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_proxy_uid() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_upstream_proxy_config() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_allowed_pod_annotations() - .map_err(KubernetesDriverError::Precondition)?; - config - .validate_allowed_pod_labels() - .map_err(KubernetesDriverError::Precondition)?; +/// Re-create one sandbox's egress fence from the freshly rendered expected +/// spec: delete the drifted policy (waiting for it to be gone), create a +/// fresh one (owner-referenced to the VM, as at create), and persist the +/// new expectation plus the restore marker on the VM. +async fn re_fence( + client: &Client, + namespace: &str, + sandbox_id: &str, + vm: &DynamicObject, + expected_spec: &serde_json::Value, +) -> Result<(), String> { + let kube_name = vm + .metadata + .name + .clone() + .ok_or("VirtualMachine has no name")?; + let vm_uid = vm.metadata.uid.clone().unwrap_or_default(); + let fence_name = crate::kubevirt_boundary::fence_name(sandbox_id); + let fence_api: Api = Api::namespaced_with( + client.clone(), + namespace, + &crate::kubevirt_boundary::fence_api_resource(), + ); + crate::kubevirt::delete_and_wait( + &fence_api, + &fence_name, + "egress fence", + COMPANION_DELETE_TIMEOUT, + ) + .await + .map_err(|error| error.to_string())?; + let mut labels = BTreeMap::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox_id.to_string()); + if let Some(gateway_id) = vm + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) + { + labels.insert(LABEL_GATEWAY_ID.to_string(), gateway_id.clone()); + } + let mut fence_obj = + DynamicObject::new(&fence_name, &crate::kubevirt_boundary::fence_api_resource()); + fence_obj.data = serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": fence_name, + "namespace": namespace, + "labels": labels, + }, + "spec": expected_spec, + }); + fence_obj.metadata.owner_references = + Some(vec![crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + &kube_name, + &vm_uid, + "kubevirt.io/v1", + false, + )]); + let created_fence = fence_api + .create(&PostParams::default(), &fence_obj) + .await + .map_err(|error| format!("re-create fence: {error}"))?; + let new_expectation = crate::kubevirt_boundary::ExpectedFence { + uid: created_fence.metadata.uid.unwrap_or_default(), + resource_version: created_fence.metadata.resource_version.unwrap_or_default(), + spec: expected_spec.clone(), + }; + let annotation = new_expectation + .to_annotation() + .map_err(|error| format!("encode expectation: {error}"))?; + let restored_at = openshell_core::time::now_ms(); + let patch = serde_json::json!({"metadata": {"annotations": { + crate::kubevirt_boundary::FENCE_EXPECTATION_ANNOTATION: annotation, + FENCE_RESTORED_ANNOTATION: restored_at.to_string(), + }}}); + let vm_api = crate::kubevirt::virtual_machine_api(client, namespace); + vm_api + .patch(&kube_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map_err(|error| format!("persist re-fenced expectation: {error}"))?; + Ok(()) +} + +/// Create or validate the namespace-wide virt-launcher isolation floor +/// (H1) against `client` — used by create, start, and the running-state +/// reconciler, so a deleted floor is restored within one interval. +async fn ensure_kubevirt_launcher_isolation_with( + client: &Client, + namespace: &str, +) -> Result<(), KubernetesDriverError> { + let policies: Api = Api::namespaced(client.clone(), namespace); + let expected = kubevirt_launcher_isolation_policy(namespace); + create_or_validate_sandbox_runtime_fence(&policies, &expected).await +} + +/// Digest-pin the guest image (M6, option A): once the launcher Pod has +/// pulled the guest image, pin the VM spec's containerDisk and kernelBoot +/// images to the resolved `repo@sha256:` so starts after +/// a tag move boot the same image, and the image-identity claim (read from +/// the VM spec) carries the digest. A side-loaded image (k3d/ctr import) +/// reports a bare `sha256:` with no repository, which is +/// not pullable: never patch those — record the unpinnable state so the +/// snapshot surfaces an `ImagePin` condition. The first generation boots the +/// configured tag (the digest does not exist until something pulls it) — +/// the documented first-boot window. +async fn reconcile_kubevirt_image_pins(client: &Client, namespace: &str) { + let vm_api = crate::kubevirt::virtual_machine_api(client, namespace); + let selector = openshell_sandbox_label_selector(); + let Ok(list) = vm_api.list(&ListParams::default().labels(&selector)).await else { + return; + }; + for vm in list.items { + if vm.metadata.deletion_timestamp.is_some() { + continue; + } + let Some(kube_name) = vm.metadata.name.clone() else { + continue; + }; + let Ok(sandbox_id) = sandbox_id_from_object(&vm) else { + continue; + }; + // Already pinned (or the annotation says unpinnable): idempotent. + let boot_image = crate::kubevirt::kubevirt_boot_image(&vm); + let pinned = boot_image.contains("@sha256:"); + let marked_unpinnable = vm + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(crate::kubevirt::IMAGE_PIN_STATE_ANNOTATION)) + .map(String::as_str) + == Some(crate::kubevirt::IMAGE_PIN_NO_REPOSITORY_DIGEST); + if pinned || marked_unpinnable { + continue; + } + // The launcher Pod's init containers carry the guest image. + let pods: Api = Api::namespaced(client.clone(), namespace); + let pod_selector = format!("kubevirt.io=virt-launcher,{LABEL_SANDBOX_ID}={sandbox_id}"); + let Ok(pod_list) = pods + .list(&ListParams::default().labels(&pod_selector)) + .await + else { + continue; + }; + let Some(launcher) = pod_list.items.into_iter().next() else { + continue; + }; + match crate::kubevirt::guest_image_digest(&launcher, &boot_image) { + crate::kubevirt::GuestImageDigest::Patch(pinned_ref) => { + info!( + sandbox_id = %sandbox_id, + vm = %kube_name, + image = %pinned_ref, + "KubeVirt guest image resolved; pinning the VirtualMachine spec to the digest" + ); + if let Err(error) = + pin_kubevirt_guest_image(client, namespace, &vm, &pinned_ref).await + { + warn!(sandbox_id = %sandbox_id, %error, "KubeVirt image pin failed"); + } + } + crate::kubevirt::GuestImageDigest::SideLoaded => { + warn!( + sandbox_id = %sandbox_id, + vm = %kube_name, + "KubeVirt guest image is side-loaded (no repository digest); the VirtualMachine \ + keeps the configured tag and cannot be digest-pinned" + ); + let patch = serde_json::json!({"metadata": {"annotations": { + crate::kubevirt::IMAGE_PIN_STATE_ANNOTATION: + crate::kubevirt::IMAGE_PIN_NO_REPOSITORY_DIGEST + }}}); + let _ = vm_api + .patch(&kube_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await; + } + crate::kubevirt::GuestImageDigest::Absent => { + // The launcher exists but no init container matching the + // VM's guest image has reported an imageID yet. Transient + // while the image pulls; if it persists (the pod spec and + // the VM disagree, or the pull never completes), the VM + // simply keeps the configured tag. Logged at warn ONCE per + // VM: the reconciler runs every 30 seconds and an + // unresolved digest never self-heals, so repeating would + // only add noise. + if kubevirt_absent_pin_warned(&kube_name) { + warn!( + sandbox_id = %sandbox_id, + vm = %kube_name, + "KubeVirt guest image digest not observed; the VirtualMachine keeps \ + the configured tag (the pin is skipped)" + ); + } + } + } + } +} + +/// VMs already warned about an unresolved guest-image digest (once per VM; +/// see the Absent arm of the pin reconciler). +static KUBEVIRT_ABSENT_PIN_WARNED: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +/// Whether this VM's unresolved-digest warning has not been emitted yet, +/// claiming it if so. +fn kubevirt_absent_pin_warned(kube_name: &str) -> bool { + let warned = KUBEVIRT_ABSENT_PIN_WARNED.get_or_init(Default::default); + let mut warned = warned.lock().unwrap(); + warned.insert(kube_name.to_string()) +} + +/// Patch one VM's containerDisk and kernelBoot images to the pinned +/// reference. The rootdisk volume's entry is resolved from the current +/// image, not assumed; merge patch replaces arrays wholesale, so the full +/// volumes array is sent with the one entry mutated. +async fn pin_kubevirt_guest_image( + client: &Client, + namespace: &str, + vm: &DynamicObject, + pinned_ref: &str, +) -> Result<(), String> { + let kube_name = vm + .metadata + .name + .clone() + .ok_or("VirtualMachine has no name")?; + let mut volumes = vm + .data + .pointer("/spec/template/spec/volumes") + .and_then(serde_json::Value::as_array) + .ok_or("VirtualMachine has no volumes")? + .clone(); + let disk_index = volumes + .iter() + .position(|volume| { + volume + .pointer("/containerDisk/image") + .and_then(serde_json::Value::as_str) + .is_some_and(|image| !image.contains("@sha256:")) + }) + .ok_or("VirtualMachine has no unpinned containerDisk volume")?; + volumes[disk_index]["containerDisk"]["image"] = serde_json::json!(pinned_ref); + let patch = serde_json::json!({ + "spec": {"template": {"spec": { + "volumes": volumes, + "domain": {"firmware": {"kernelBoot": {"container": { + "image": pinned_ref + }}}} + }}} + }); + let vm_api = crate::kubevirt::virtual_machine_api(client, namespace); + vm_api + .patch(&kube_name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + .map(|_| ()) + .map_err(|error| format!("pin the guest image: {error}")) +} + +/// Exactly one `VirtualMachine` must match the sandbox's selector; more/// Exactly one `VirtualMachine` must match the sandbox's selector; more/// Exactly one `VirtualMachine` must match the sandbox's selector; more +/// than one is an ambiguous identity the caller must not guess between. +fn select_single_kubevirt_vm( + objects: Vec, +) -> Result { + let [object]: [DynamicObject; 1] = objects.try_into().map_err(|objects: Vec<_>| { + if objects.is_empty() { + KubernetesDriverError::NotFound + } else { + KubernetesDriverError::Precondition( + "multiple VirtualMachines match the sandbox selector".to_string(), + ) + } + })?; + Ok(object) +} + +/// Exactly one `VirtualMachine` must match, and it must carry the persisted +/// runtime identity (namespace + VM UID) — the same guarantees the pod +/// path's start requires of its Sandbox resources. +fn select_expected_kubevirt_vm( + objects: Vec, + encoded_expected_runtime_identity: &str, +) -> Result { + let object = select_single_kubevirt_vm(objects)?; + let expected = parse_kubernetes_runtime_identity(encoded_expected_runtime_identity)?; + let namespace = object.metadata.namespace.as_deref().unwrap_or_default(); + let uid = object.metadata.uid.as_deref().unwrap_or_default(); + if namespace != expected.namespace || uid != expected.resource_uid { + return Err(KubernetesDriverError::Precondition( + "matched VirtualMachine does not match the persisted runtime identity".to_string(), + )); + } + Ok(object) +} + +/// Everything the post-VM provisioning of one `KubeVirt` create needs, +/// captured once so a single rollback in the caller covers every failure +/// point. +struct KubeVirtGeneration<'a> { + sandbox: &'a Sandbox, + namespace: &'a str, + kube_name: &'a str, + vm_uid: &'a str, + placement: &'a SupervisorPlacement, + shared_target: Option, + names: &'a SandboxRuntimeNames, + main_process_spec: &'a str, + log_level: &'a str, + launch_authentication: &'a openshell_core::jwt::SandboxLaunchAuthentication, + /// The guest image the rendered VM actually boots (M6: claimed, not + /// the configured string). + boot_image: String, +} + +impl KubernetesComputeDriver { + /// Environment that lets a supervisor export spans, joining the current + /// trace, OCSF events, and relayed workload traces, identifying the + /// gateway that launched it. + fn supervisor_otlp_environment(&self) -> Vec<(&'static str, String)> { + let mut environment = self.config.supervisor_otlp.environment(); + if self.config.supervisor_otlp.traces.is_some() { + environment.extend(openshell_otel::current_trace_context_environment()); + } + if let Some(name) = &self.config.supervisor_gateway_name { + environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); + } + environment + } + + #[cfg(test)] + pub(crate) fn new_for_test(config: KubernetesComputeConfig) -> Self { + let service = tower::service_fn(|_request: http::Request| async { + Ok::<_, std::convert::Infallible>(http::Response::new(http_body_util::Empty::< + bytes::Bytes, + >::new())) + }); + let client = Client::new(service, "default"); + Self { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + lifecycle_gates: Arc::default(), + config, + network_trust_bundle: None, + operator_allowlist: None, + shared_supervisor: None, + } + } + + /// A driver whose new generations get a dedicated supervisor Pod each. + /// A configuration that places them on the gateway's shared supervisor + /// needs [`Self::with_shared_supervision`]; without the gateway's + /// coordination it is refused here, before anything runs. + pub async fn new( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + network_trust_bundle: Option, + ) -> Result { + if config.shared_supervisor.is_some() { + return Err(KubernetesDriverError::Precondition( + "the gateway supervisor topology requires the gateway's shared supervision, which only the driver linked into the gateway receives" + .to_string(), + )); + } + Self::build(config, shutdown_rx, None, network_trust_bundle).await + } + + /// A driver whose new generations become sandbox instances on the shared + /// supervisor the gateway coordinates through `supervision`. The driver + /// adapts that coordination to Kubernetes: it records placements, + /// persists and reads back launch material, provisions egress to the + /// supervisor's Service, and delivers the supervisor token into its + /// Secret. + pub async fn with_shared_supervision( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: SharedSupervision, + ) -> Result { + if config.shared_supervisor.is_none() { + return Err(KubernetesDriverError::Precondition( + "shared supervision was provided but the configuration does not place generations on the shared supervisor" + .to_string(), + )); + } + // The gateway refuses the additional destination CA under the shared + // supervisor topology fail-closed, so no trust bundle reaches this + // constructor. + Self::build(config, shutdown_rx, Some(supervision), None).await + } + + async fn build( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: Option, + network_trust_bundle: Option, + ) -> Result { + config + .resource_admission + .validate() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_workspace_mode() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_provider_spiffe_workload_api_socket_path() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_sandbox_identity_config() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_proxy_uid() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_upstream_proxy_config() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_annotations() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_labels() + .map_err(KubernetesDriverError::Precondition)?; config .validate_supervisor_topology() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_kubevirt_config() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -1285,7 +2024,9 @@ impl KubernetesComputeDriver { shared_supervisor, }; - if driver.workspace_mode() == WorkspaceMode::Shared { + if driver.workspace_mode() == WorkspaceMode::Shared + && !crate::kubevirt::is_kubevirt_mode(&driver.config) + { driver.backfill_gateway_id_labels().await?; } @@ -1313,8 +2054,10 @@ impl KubernetesComputeDriver { limit_supported: true, }), gpu: Some(GpuResourceCapabilities { - default_selection_supported: true, - count_selection_supported: true, + // KubeVirt VM sandboxes do not support GPU passthrough + // yet (H6); the pod runtime keeps full GPU selection. + default_selection_supported: !crate::kubevirt::is_kubevirt_mode(&self.config), + count_selection_supported: !crate::kubevirt::is_kubevirt_mode(&self.config), }), }), rootfs_tar_staging_dir: String::new(), @@ -2217,6 +2960,17 @@ impl KubernetesComputeDriver { "sandbox external resource identity or attachment inventory changed", )); } + if crate::kubevirt::is_kubevirt_mode(&self.config) { + let namespace = crate::kubevirt::kubevirt_namespace(&self.config) + .map_err(tonic::Status::failed_precondition)?; + crate::kubevirt::kubevirt_available(&self.client, &namespace) + .await + .map_err(|err| { + tonic::Status::failed_precondition(format!( + "kubevirt runtime selected but unusable: {err}" + )) + })?; + } Ok(()) } @@ -2416,6 +3170,9 @@ impl KubernetesComputeDriver { } pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self.get_kubevirt_sandbox(sandbox_id).await; + } info!( sandbox_id = %sandbox_id, workspace_mode = %self.config.workspace_mode, @@ -2471,6 +3228,73 @@ impl KubernetesComputeDriver { } pub async fn list_sandboxes(&self) -> Result, String> { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + let list = crate::kubevirt::list_virtual_machines( + &self.client, + &self.config, + &self.openshell_sandbox_selector(), + ) + .await + .map_err(|err| err.to_string())?; + // Each VM's own namespace (per-workspace in managed mode): + // VMI phases and supervisor-failure probes are fetched per + // namespace so an empty or multi-namespace listing never + // issues a malformed (empty-namespace) request. + let mut vmi_phases_by_namespace: std::collections::HashMap< + String, + std::collections::HashMap, Option)>, + > = std::collections::HashMap::new(); + let mut namespaces: Vec = list + .items + .iter() + .filter_map(|object| object.metadata.namespace.clone()) + .collect(); + namespaces.sort(); + namespaces.dedup(); + for namespace in &namespaces { + vmi_phases_by_namespace.insert( + namespace.clone(), + self.kubevirt_vmi_phases(namespace, &self.openshell_sandbox_selector()) + .await, + ); + } + let empty_phases = std::collections::HashMap::new(); + let mut sandboxes = Vec::with_capacity(list.items.len()); + for obj in list.items { + let Some(namespace) = obj.metadata.namespace.clone() else { + continue; + }; + let vmi_phases = vmi_phases_by_namespace + .get(&namespace) + .unwrap_or(&empty_phases); + let vmi_state = obj + .metadata + .name + .as_deref() + .and_then(|name| vmi_phases.get(name)) + .cloned() + .unwrap_or((None, None)); + let supervisor_failure = + kubevirt_supervisor_failure(&self.client, &namespace, &obj).await; + let fence_restored = obj + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FENCE_RESTORED_ANNOTATION)) + .cloned(); + if let Ok((_, sandbox)) = crate::kubevirt::sandbox_from_vm_object_with_runtime( + &namespace, + obj, + vmi_state.0.as_deref(), + supervisor_failure.as_deref(), + fence_restored.as_deref(), + vmi_state.1, + ) { + sandboxes.push(sandbox); + } + } + return Ok(sandboxes); + } info!( workspace_mode = %self.config.workspace_mode, "Listing sandboxes from Kubernetes" @@ -2580,6 +3404,38 @@ impl KubernetesComputeDriver { KubernetesDriverError::InvalidArgument(status.message().to_string()) })?; + if crate::kubevirt::is_kubevirt_mode(&self.config) { + // GPU passthrough is not supported for VM sandboxes yet (H6): + // reject the request at admission, before anything is created. + if let Some(gpu) = gpu_requirements { + return Err(KubernetesDriverError::InvalidArgument(format!( + "KubeVirt VM sandboxes do not support GPU passthrough yet \ + (requested: {} GPU{}); use the pod runtime for GPU workloads", + gpu.count.unwrap_or(1), + if gpu.count == Some(1) { "" } else { "s" } + ))); + } + // H6: the same resource admission as the pod path — driver-config + // policy, workspace namespace rules, and referenced-resource + // admission all apply before anything is created. + self.admit_requested_resources(sandbox) + .await + .map_err(admission_error)?; + // H6: validate the typed CPU/memory mapping onto the VM domain + // before anything is created (limit below request, memory below + // the safe guest minimum, unparsable quantities). + crate::kubevirt::domain_resources( + sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + &self.config.kubevirt, + ) + .map_err(KubernetesDriverError::InvalidArgument)?; + let runtime_identity = self.create_kubevirt_sandbox(sandbox).await?; + return Ok(runtime_identity); + } + // Validate sandbox name against Kubernetes naming requirements validate_kubernetes_dns1123_label(&sandbox.name, "sandbox name") .map_err(KubernetesDriverError::InvalidArgument)?; @@ -3343,6 +4199,7 @@ impl KubernetesComputeDriver { KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) })?; let dependent_owner = sandbox_runtime_sandbox_owner_reference( + "Sandbox", cr_name, cr_uid, &sandbox_api.resource.api_version, @@ -3814,6 +4671,7 @@ impl KubernetesComputeDriver { sandbox_id, supervisor, sandbox_runtime_sandbox_owner_reference( + "Sandbox", cr_name, cr_uid, &sandbox_api.resource.api_version, @@ -3869,11 +4727,9 @@ impl KubernetesComputeDriver { }) .await?; - let session_id = launch_authentication.supervisor.session_id; - let tls = generate_sandbox_tls_material(session_id) - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let verification_keys = - gateway_verification_keys(&launch_authentication.verification_keys)?; + let launch_inputs = boundary_launch_inputs(launch_authentication)?; + let tls = launch_inputs.tls; + let verification_keys = launch_inputs.verification_keys; let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; let workload_identity = ResolvedWorkloadIdentity::new( agent_uid, @@ -3920,10 +4776,10 @@ impl KubernetesComputeDriver { .supervisor .runtime_generation .to_string(), - session_id, - session_rotation: launch_authentication.supervisor.session_rotation, - auth_epoch: launch_authentication.supervisor.auth_epoch, - gateway_id: launch_authentication.gateway_id.clone(), + session_id: launch_inputs.session_id, + session_rotation: launch_inputs.session_rotation, + auth_epoch: launch_inputs.auth_epoch, + gateway_id: launch_inputs.gateway_id, verification_keys, namespace_uid, sandbox_resource_uid: cr_uid.to_string(), @@ -3949,16 +4805,15 @@ impl KubernetesComputeDriver { .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); let instance_owner = workload_owner.clone(); + let bootstrap = render_boundary_bootstrap(&provisioned.boundary_config, &tls) + .map_err(KubernetesDriverError::Message)?; let sandbox_secret = sandbox_bootstrap_secret( namespace, names, sandbox_id, - provisioned - .boundary_config - .encode() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, - tls.certificate_chain_pem.into_bytes(), - tls.private_key_pem.into_bytes(), + bootstrap.config_json, + bootstrap.certificate_pem, + bootstrap.private_key_pem, workload_owner.clone(), ); let secrets = Api::::namespaced(self.client.clone(), namespace); @@ -4105,189 +4960,2748 @@ impl KubernetesComputeDriver { Ok(()) } - #[tracing::instrument( - name = "kubernetes.stop_sandbox", - skip(self), - fields( - otel.name = "kubernetes.stop_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let _guard = self.lifecycle_gates.gate_for(sandbox_id).lock_owned().await; - let result = Box::pin(self.stop_sandbox_inner(sandbox_id)).await; - span_status.finish(result) - } + /// `KubeVirt` runtime mode: create the bootstrap Secret, workspace PVC + /// and `VirtualMachine` for one sandbox. Returns the VM's UID as the + /// opaque runtime identity (stable across stop/start). + #[allow(clippy::similar_names)] + async fn create_kubevirt_sandbox( + &self, + sandbox: &Sandbox, + ) -> Result { + // Shared mode renders into the configured sandbox namespace; + // managed mode resolves the workspace's own namespace (created by + // the shared pod-path logic above). + let namespace = match self.config.workspace_mode { + WorkspaceMode::Shared => crate::kubevirt::kubevirt_namespace(&self.config) + .map_err(KubernetesDriverError::Precondition)?, + WorkspaceMode::Managed | WorkspaceMode::Operator => match self.config.workspace_mode { + WorkspaceMode::Managed => self.ensure_namespace(&sandbox.workspace).await?, + WorkspaceMode::Operator => sandbox.workspace.clone(), + WorkspaceMode::Shared => unreachable!(), + }, + }; + crate::kubevirt::kubevirt_available(&self.client, &namespace) + .await + .map_err(KubernetesDriverError::Precondition)?; - async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase, placement) = - self.prepare_sandbox_stop(sandbox_id).await?; - if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) { - self.release_generation_supervisor(sandbox_id, &namespace, &placement) - .await?; - patch_dynamic_object_with_resource_version_retry( - &agent_sandbox_api.api, - &kube_name, - |version| { - sandbox_runtime_suspension_patch(&agent_sandbox_api.resource.version, version) - }, - ) - .await?; - } - let pod_api = Api::::namespaced(self.client.clone(), &namespace); + let kube_name = self + .config + .kube_resource_name(&sandbox.workspace, &sandbox.name); + info!( + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + namespace = %namespace, + guest_image = %self.config.kubevirt.guest_image, + "Creating KubeVirt VirtualMachine sandbox" + ); - let deadline = tokio::time::Instant::now() + stop_timeout; - let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; - loop { - let now = tokio::time::Instant::now(); - if now >= deadline { + // Boundary bootstrap: when the sandbox spec carries launch + // authentication, render the boundary-config bootstrap (config + + // TLS + guest payload) instead of the legacy env-var userData. + // The fence NetworkPolicy is created before the VM so the + // boundary evidence can carry its UID. + // W7: missing launch authentication is a Precondition error — the + // boundary path requires it, and silently falling back to the + // legacy env-var payload would put gateway material in the guest. + let launch_authentication: openshell_core::jwt::SandboxLaunchAuthentication = { + let spec = sandbox.spec.as_ref().ok_or_else(|| { + KubernetesDriverError::Precondition("KubeVirt sandbox spec is required".to_string()) + })?; + if spec.launch_authentication.is_empty() { + return Err(KubernetesDriverError::Precondition( + "KubeVirt runtime requires launch authentication; the gateway did not provide it" + .to_string(), + )); + } + crate::kubevirt::decode_launch_authentication(spec.launch_authentication.as_slice()) + .and_then(|opt| { + opt.ok_or_else(|| { + KubernetesDriverError::Precondition( + "launch authentication was non-empty but decoded to None".to_string(), + ) + }) + })? + }; + + let vm_api = crate::kubevirt::virtual_machine_api(&self.client, &namespace); + // The VM is the owning resource: if it already exists this sandbox + // already exists. + match tokio::time::timeout(KUBE_API_TIMEOUT, vm_api.get(&kube_name)).await { + Ok(Ok(_)) => return Err(KubernetesDriverError::AlreadyExists), + Ok(Err(KubeError::Api(err))) if err.code == 404 => {} + Ok(Err(err)) => return Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => { return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() + "timed out after {}s checking for existing VirtualMachine", + KUBE_API_TIMEOUT.as_secs() ))); } - let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); - let object = tokio::time::timeout( - request_timeout, - agent_sandbox_api.api.get(&kube_name), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox stop", - request_timeout.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; - if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(KubernetesDriverError::Message(error)); + } + + // The supervisor topology is decided exactly as the pod path decides + // it: `supervisor_topology = "gateway"` (with its shared_supervisor + // config) places the generation on the gateway's shared supervisor; + // the default is a dedicated supervisor Pod per sandbox. + let placement = self.new_generation_placement(); + let route = match &placement { + SupervisorPlacement::Shared { route } => route.clone(), + SupervisorPlacement::Dedicated => String::new(), + }; + // X2: resolve the shared supervisor through the pod path's single + // resolver (shared_supervisor_target) — authority + // ({service}.{ns}.svc:{port}), ClusterIP, and pod selector — and + // adapt it for the boundary renderers. No second resolver. + let shared_target = match &placement { + SupervisorPlacement::Shared { .. } => { + let (_shared, target) = self.shared_supervisor_target().await?; + Some(crate::kubevirt_boundary::KubeVirtSupervisorTarget::Shared { + authority: target.authority.clone(), + address: target.address, + route: route.clone(), + namespace: target.namespace.clone(), + pod_selector: target.selector.clone(), + port: target.address.port(), + }) } - let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) - .await - .map_err(KubernetesDriverError::Message)?; - let stop_is_complete = kubernetes_sandbox_stop_is_complete( - &agent_sandbox_api.resource.version, - &object, - pod_is_gone, + SupervisorPlacement::Dedicated => None, + }; + + let params = crate::kubevirt::KubeVirtSandboxParams { + sandbox, + config: &self.config, + namespace: &namespace, + kube_name: &kube_name, + generation: &launch_authentication + .supervisor + .runtime_generation + .to_string(), + }; + // W6: create the VirtualMachine first with runStrategy Halted so + // its UID is known before the boundary claims are rendered — the + // cloudInitNoCloud volume references the bootstrap Secret by name + // only, and KubeVirt resolves it when the VMI starts (after the + // runStrategy patch to Once below), not when the VM object is + // created. This lets vm_uid be the VM's real UID, as the claims + // model decided. + let mut vm = crate::kubevirt::render_virtual_machine(¶ms); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + // The in-flight annotation maps the Halted window to a + // `Bootstrapping` condition (Provisioning), not `Suspended` + // (Stopped, which would withdraw the supervisor instance this + // create is about to register). + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + crate::kubevirt::KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION.to_string(), + openshell_core::time::now_ms().to_string(), ); - if stop_is_complete { - self.delete_sandbox_runtime_supervisor( - sandbox_id, - &namespace, - sandbox_runtime_generation(&object).as_slice(), - ) - .await?; - patch_dynamic_object_with_resource_version_retry( - &agent_sandbox_api.api, + // The main process spec is encoded once at create and persisted on + // the VM so the start path can re-register the shared instance + // (the driver's start API does not carry the sandbox spec). + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| { + KubernetesDriverError::InvalidArgument(format!("encode main process spec: {error}")) + })?; + let log_level = openshell_core::driver_utils::sandbox_log_level(sandbox, "info"); + let generation = launch_authentication + .supervisor + .runtime_generation + .to_string(); + let names = SandboxRuntimeNames::for_generation(&sandbox.id, &generation); + if let SupervisorPlacement::Shared { route } = &placement { + // The recorded placement (shared topology + route) lets stop, + // start, and delete dispatch on the sandbox's own topology. + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + GATEWAY_TOPOLOGY.to_string(), + ); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert(ANNOTATION_SUPERVISOR_SESSION.to_string(), route.clone()); + } + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC.to_string(), + main_process_spec.clone(), + ); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL.to_string(), + log_level.clone(), + ); + let vm_uid: String = { + let created_vm = + tokio::time::timeout(KUBE_API_TIMEOUT, vm_api.create(&PostParams::default(), &vm)) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating VirtualMachine (Halted)", + KUBE_API_TIMEOUT.as_secs() + )) + })?; + match created_vm { + Ok(result) => { + info!( + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + "KubeVirt VirtualMachine created (Halted)" + ); + result.metadata.uid.unwrap_or_else(|| kube_name.clone()) + } + Err(KubeError::Api(err)) if err.code == 409 => { + return Err(KubernetesDriverError::AlreadyExists); + } + Err(err) => return Err(KubernetesDriverError::from_kube(err)), + } + }; + + // Roll back everything this create made: the companions (fence, + // bootstrap Secret, PVC, shared-instance Secret), the instance + // registration, and the Halted VM itself (with a UID precondition, + // so a concurrent create's replacement is never deleted). The + // dedicated topology's supervisor Pod, Service, and control + // NetworkPolicy are owned by the VM, so its deletion cascades them. + let rollback_all = |created: Vec, route: String| { + let client = self.client.clone(); + let vm_api = vm_api.clone(); + let namespace = namespace.clone(); + let kube_name = kube_name.clone(); + let vm_uid = vm_uid.clone(); + let shared = self.shared_supervisor.clone(); + async move { + crate::kubevirt::rollback_companions(&client, &namespace, &created).await; + if !route.is_empty() + && let Some(shared) = &shared + { + shared.supervision().withdraw(&route); + } + crate::kubevirt::delete_and_wait_with_uid( + &vm_api, &kube_name, - sandbox_runtime_suspension_completion_patch, + "VirtualMachine", + &vm_uid, + COMPANION_DELETE_TIMEOUT, ) - .await?; - return Ok(()); + .await + .ok(); } - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); + }; + + // One fallible provisioning body: every failure point after the + // Halted VM exists returns through here, so a single rollback + // covers them all (the fence, bootstrap Secret, and PVC are also + // owner-referenced to the VM, so Kubernetes GC cascades them even + // if the gateway dies mid-create). + let mut created: Vec = Vec::new(); + let generation = KubeVirtGeneration { + sandbox, + namespace: &namespace, + kube_name: &kube_name, + vm_uid: &vm_uid, + placement: &placement, + shared_target, + names: &names, + main_process_spec: &main_process_spec, + log_level: &log_level, + launch_authentication: &launch_authentication, + boot_image: crate::kubevirt::kubevirt_boot_image(&vm), + }; + let supervisor_component = match self + .provision_kubevirt_generation(&generation, &mut created) + .await + { + Ok((component, _recorded)) => component, + Err(error) => { + rollback_all(created, route.clone()).await; + return Err(error); } - tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; - poll_interval = next_stop_poll_interval(poll_interval); - } - } + }; - #[tracing::instrument( - name = "kubernetes.start_sandbox", - skip_all, - fields( - otel.name = "kubernetes.start_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn start_sandbox( - &self, - sandbox_id: &str, - generation_id: &str, - launch_authentication: &[u8], - expected_runtime_identity: &str, - ) -> Result { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let _guard = self.lifecycle_gates.gate_for(sandbox_id).lock_owned().await; - let result = Box::pin(self.start_sandbox_runtime_generation( - sandbox_id, - generation_id, - launch_authentication, - expected_runtime_identity, + // The runtime identity in the pod path's format: the VM's namespace + // and UID plus the generation's supervisor identity component (the + // shared route or the dedicated supervisor Pod's UID). + Ok(kubernetes_runtime_identity( + &namespace, + &vm_uid, + &supervisor_component, )) - .await; - span_status.finish(result) } - #[allow(clippy::similar_names)] - async fn start_sandbox_runtime_generation( + /// Provision everything after the Halted `VirtualMachine` exists: the + /// supervisor generation, the fence, the bootstrap Secret(s), the + /// workspace PVC, and the `Once` patch. Returns the generation's + /// supervisor identity component for the runtime identity plus the + /// recorded companions (exact names and UIDs, written to the VM's + /// companions annotation so later deletes need no read and no + /// search). The caller owns the rollback; this body only appends to + /// `created`. + async fn provision_kubevirt_generation( &self, - sandbox_id: &str, - encoded_generation: &str, - encoded_authentication: &[u8], - encoded_expected_runtime_identity: &str, - ) -> Result { - let staged = self.read_staged_bootstrap_material().await?; - let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( - encoded_generation.to_string(), - ) - .map_err(|error| KubernetesDriverError::InvalidArgument(error.to_string()))?; - let launch_authentication = decode_launch_authentication(encoded_authentication)?; - let expected_runtime_identity = - parse_kubernetes_runtime_identity(encoded_expected_runtime_identity)?; - let lookup_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let objects = lookup_api - .api - .list(&ListParams::default().labels(&selector)) - .await - .map_err(KubernetesDriverError::from_kube)? - .items; - let mut object = select_expected_sandbox_runtime(objects, &expected_runtime_identity)?; - self.admit_stored_resources(&object) - .await - .map_err(admission_error)?; - if sandbox_runtime_bootstrap_in_progress(&object) { - let phase = sandbox_runtime_bootstrap_phase(&object); - if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) - && sandbox_runtime_bootstrap_operation(&object) != Some("restart") - { - return Err(KubernetesDriverError::Precondition( - "initial sandbox bootstrap has not completed; wait for reconciliation" - .to_string(), - )); - } - if sandbox_runtime_generation(&object) == Some(generation.as_str()) - && sandbox_runtime_bootstrap_phase(&object) - == Some(SandboxRuntimeBootstrapPhase::Released) - { - let namespace = object - .metadata - .namespace - .as_deref() - .unwrap_or(&self.config.namespace); - if generation_control_availability( - &self.client, - self.shared_supervisor.as_ref(), + generation: &KubeVirtGeneration<'_>, + created: &mut Vec, + ) -> Result<(String, crate::kubevirt::RecordedCompanions), KubernetesDriverError> { + let sandbox = generation.sandbox; + let namespace = generation.namespace; + let kube_name = generation.kube_name; + let vm_uid = generation.vm_uid; + let placement = generation.placement; + let names = generation.names; + let vm_api = crate::kubevirt::virtual_machine_api(&self.client, namespace); + // Every companion below is owner-referenced to the VM (H1): the VM + // exists first, so Kubernetes GC removes the companions after the + // VMI and virt-launcher pod terminate, and the egress fence always + // outlives the workload it fences. + let vm_owner = crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + kube_name, + vm_uid, + "kubevirt.io/v1", + false, + ); + let route = match placement { + SupervisorPlacement::Shared { route } => route.clone(), + SupervisorPlacement::Dedicated => String::new(), + }; + + // H1: the namespace-wide virt-launcher isolation policy. It denies + // all ingress and egress for every virt-launcher pod in the sandbox + // namespace, so isolation never depends on a per-sandbox object's + // lifecycle (a deleted or drifted fence cannot reopen egress; the + // per-sandbox policies only add allowances on top of it). + self.ensure_kubevirt_launcher_isolation(namespace).await?; + + // The dedicated supervisor generation: the pod path's per-sandbox + // Service, the namespace-wide supervisor egress policy, the + // pair-scoped control policy, and the gated supervisor Pod — all + // owned by the VirtualMachine so Kubernetes GC cascades them. + let supervisor_pod_uid = match placement { + SupervisorPlacement::Dedicated => { + let identity = self.resolve_sandbox_identity_in_namespace(namespace).await; + self.create_kubevirt_supervisor_generation( + &sandbox.id, + &sandbox.name, namespace, - sandbox_id, - &object, + names, + generation.main_process_spec, + generation.log_level, + vm_owner.clone(), + identity.0, + identity.1, ) - .await - == SandboxRuntimeControlAvailability::Available - && sandbox_runtime_bootstrap_is_ready_to_complete(&object) - { + .await? + } + SupervisorPlacement::Shared { .. } => String::new(), + }; + + // The supervisor the fence admits and the boundary dials, resolved + // per topology: the dedicated generation's own Service, or the + // shared supervisor's Service with this sandbox's route. + let supervisor = match placement { + SupervisorPlacement::Dedicated => self + .resolve_dedicated_kubevirt_target( + namespace, + names, + &sandbox.id, + &supervisor_pod_uid, + ) + .await + .map_err(KubernetesDriverError::Precondition)?, + SupervisorPlacement::Shared { .. } => { + generation.shared_target.clone().ok_or_else(|| { + KubernetesDriverError::Precondition( + "the shared supervisor target was not resolved".to_string(), + ) + })? + } + }; + + // Create the fence NetworkPolicy. W8: it is a companion with its + // UID, so every failure path below rolls it back. + let bootstrap = { + let fence_json = crate::kubevirt_boundary::render_fence( + namespace, + &sandbox.id, + Some(&self.config.gateway_id), + &supervisor, + ); + let fence_api: Api = Api::namespaced_with( + self.client.clone(), + namespace, + &crate::kubevirt_boundary::fence_api_resource(), + ); + let fence_name = crate::kubevirt_boundary::fence_name(&sandbox.id); + // Delete any orphaned fence and wait for it to be gone. + crate::kubevirt::delete_and_wait( + &fence_api, + &fence_name, + "egress fence", + COMPANION_DELETE_TIMEOUT, + ) + .await?; + let mut fence_obj = + DynamicObject::new(&fence_name, &crate::kubevirt_boundary::fence_api_resource()); + fence_obj.data = fence_json.clone(); + fence_obj.metadata.owner_references = Some(vec![vm_owner.clone()]); + let created_fence_obj = tokio::time::timeout( + KUBE_API_TIMEOUT, + fence_api.create(&PostParams::default(), &fence_obj), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating the egress fence", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + let fence_uid = created_fence_obj.metadata.uid.unwrap_or_default(); + let fence_rv = created_fence_obj + .metadata + .resource_version + .unwrap_or_default(); + let fence_spec = fence_json["spec"].clone(); + created.push(crate::kubevirt::CreatedCompanion { + resource: crate::kubevirt_boundary::fence_api_resource(), + name: fence_name, + uid: fence_uid.clone(), + }); + + // X1: persist the fence expectation (UID + rendered spec) on + // the VM so the start path can verify the fence on every new + // generation. + let fence_expectation = crate::kubevirt_boundary::ExpectedFence { + resource_version: fence_rv.clone(), + spec: fence_spec.clone(), + uid: fence_uid.clone(), + } + .to_annotation() + .map_err(KubernetesDriverError::Precondition)?; + let annotation_patch = serde_json::json!({ + "metadata": {"annotations": { + crate::kubevirt_boundary::FENCE_EXPECTATION_ANNOTATION: fence_expectation + }} + }); + tokio::time::timeout( + KUBE_API_TIMEOUT, + vm_api.patch( + kube_name, + &PatchParams::default(), + &Patch::Merge(&annotation_patch), + ), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out persisting the fence expectation on VirtualMachine {kube_name}" + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + + // The bootstrap Secret's userData carries the boundary + // bootstrap (base64 artifacts + workload identity), rendered + // against the fence evidence and the VM's real UID. + crate::kubevirt_boundary::render_kubevirt_bootstrap( + generation.launch_authentication, + &sandbox.id, + &sandbox.name, + vm_uid, + &generation.boot_image, + &fence_uid, + &fence_rv, + fence_spec, + &supervisor, + ) + .map_err(KubernetesDriverError::Precondition)? + }; + + // Bootstrap Secret: the VM's cloudInitNoCloud volume references it + // by name. The name is scoped to the launch generation + // (-boot-g), so the Secret is immutable per + // generation (H4: the driver never patches or reads a Secret). + // An orphan from an out-of-band VM deletion is deleted and waited + // for, so the recreate cannot collide with a terminating object. + let generation_string = generation + .launch_authentication + .supervisor + .runtime_generation + .to_string(); + let core_secret_resource = ApiResource { + group: String::new(), + version: "v1".to_string(), + api_version: "v1".to_string(), + kind: "Secret".to_string(), + plural: "secrets".to_string(), + }; + let secret_api: Api = Api::namespaced(self.client.clone(), namespace); + // H4: a plain delete — Secrets carry no finalizers, so the delete + // completes when the API returns, and the driver must not GET a + // Secret to poll its disappearance. A same-name object still + // terminating surfaces as a create conflict (retryable). + let secret_cleanup_api: Api = + Api::namespaced_with(self.client.clone(), namespace, &core_secret_resource); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + secret_cleanup_api.delete( + &crate::kubevirt::bootstrap_secret_name(kube_name, &generation_string), + &DeleteParams::default(), + ), + ) + .await + { + Ok(_) | Err(_) => {} + } + let secret = Secret { + metadata: ObjectMeta { + name: Some(crate::kubevirt::bootstrap_secret_name( + kube_name, + &generation_string, + )), + namespace: Some(namespace.to_string()), + labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), + owner_references: Some(vec![vm_owner.clone()]), + ..Default::default() + }, + string_data: Some(BTreeMap::from([( + "userData".to_string(), + bootstrap.guest_payload.clone(), + )])), + type_: Some("Opaque".to_string()), + ..Default::default() + }; + let created_secret = tokio::time::timeout( + KUBE_API_TIMEOUT, + secret_api.create(&PostParams::default(), &secret), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating bootstrap Secret", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(|error| match error { + KubeError::Api(err) if err.code == 409 => KubernetesDriverError::AlreadyExists, + other => KubernetesDriverError::from_kube(other), + })?; + let created_secret_uid = created_secret.metadata.uid.clone().unwrap_or_default(); + created.push(crate::kubevirt::CreatedCompanion { + resource: core_secret_resource, + name: crate::kubevirt::bootstrap_secret_name(kube_name, &generation_string), + uid: created_secret_uid.clone(), + }); + + if self.config.kubevirt.workspace_pvc { + let core_pvc_resource = ApiResource { + group: String::new(), + version: "v1".to_string(), + api_version: "v1".to_string(), + kind: "PersistentVolumeClaim".to_string(), + plural: "persistentvolumeclaims".to_string(), + }; + let pvc_api: Api = + Api::namespaced(self.client.clone(), namespace); + // Bound PVCs terminate slowly (pvc-protection finalizer plus + // provisioner cleanup); wait out the deletion before recreating. + crate::kubevirt::delete_and_wait( + &Api::::namespaced_with( + self.client.clone(), + namespace, + &core_pvc_resource, + ), + &crate::kubevirt::workspace_pvc_name(kube_name), + "workspace PVC", + COMPANION_DELETE_TIMEOUT, + ) + .await?; + let mut pvc = + crate::kubevirt::render_workspace_pvc(&crate::kubevirt::KubeVirtSandboxParams { + sandbox, + config: &self.config, + namespace, + kube_name, + generation: &generation_string, + }); + pvc.metadata.owner_references = Some(vec![vm_owner.clone()]); + let created_pvc = tokio::time::timeout( + KUBE_API_TIMEOUT, + pvc_api.create(&PostParams::default(), &pvc), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating workspace PVC", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(|error| match error { + // A concurrent create owns the claim; roll back + // only what this create made. + KubeError::Api(err) if err.code == 409 => KubernetesDriverError::AlreadyExists, + other => KubernetesDriverError::from_kube(other), + })?; + created.push(crate::kubevirt::CreatedCompanion { + resource: core_pvc_resource, + name: crate::kubevirt::workspace_pvc_name(kube_name), + uid: created_pvc.metadata.uid.unwrap_or_default(), + }); + } + + let mut recorded_supervisor_secret = None; + match placement { + SupervisorPlacement::Dedicated => { + // The dedicated generation's supervisor bootstrap Secret + // (descriptor + auth bundle + proxy CA), owned by the + // supervisor Pod; releasing the Pod's scheduling gate + // starts the supervisor only after the fence and bootstrap + // exist. + let supervisor_secret_uid = self + .install_kubevirt_supervisor_secret( + namespace, + names, + &sandbox.id, + &supervisor_pod_uid, + &bootstrap, + generation.launch_authentication, + ) + .await?; + recorded_supervisor_secret = Some(crate::kubevirt::CompanionRecord { + name: names.supervisor_secret.clone(), + uid: supervisor_secret_uid, + }); + } + SupervisorPlacement::Shared { .. } => { + // W4/X3: register the shared-supervisor instance. The + // supervisor reads the instance Secret and serves this + // sandbox's boundary on the allocated route; without this + // registration the boundary's dial is refused. + self.register_kubevirt_instance( + &route, + &sandbox.id, + &sandbox.name, + generation.main_process_spec.to_string(), + bootstrap.runtime_descriptor, + generation.launch_authentication.supervisor.clone(), + kube_name, + vm_uid, + ) + .await?; + } + } + + // Start the VM: patch runStrategy from Halted to Once, clear the + // in-flight annotation (a later Halted without it is a user stop), + // and record the generation's companions (H4: exact names and UIDs + // on the VM, so teardown and the next generation's turnover never + // read or list the objects). + // H5: the patch is pinned to the VM UID captured before the + // companions were built, so a VM replaced under the same name + // during provisioning is never booted. + let recorded = crate::kubevirt::RecordedCompanions { + boot_secret: crate::kubevirt::CompanionRecord { + name: crate::kubevirt::bootstrap_secret_name(kube_name, &generation_string), + uid: created_secret_uid, + }, + supervisor_secret: recorded_supervisor_secret, + }; + let companions_annotation = recorded.to_annotation(); + patch_kubevirt_vm_pinned_to_checked_uid(&vm_api, kube_name, vm_uid, |_| { + serde_json::json!({ + "metadata": { + "annotations": { + crate::kubevirt::KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION: null, + crate::kubevirt::COMPANIONS_ANNOTATION: companions_annotation, + } + }, + "spec": {"runStrategy": "Once"} + }) + }) + .await?; + + Ok(( + match placement { + SupervisorPlacement::Shared { route } => route.clone(), + SupervisorPlacement::Dedicated => supervisor_pod_uid, + }, + recorded, + )) + } + + /// The supervisor identity component for a VM's runtime identity: the + /// shared route, or the dedicated supervisor Pod's UID ("unknown" + /// still parses — only the namespace and VM UID are verified). + async fn kubevirt_supervisor_component( + &self, + object: &DynamicObject, + namespace: &str, + ) -> String { + match Self::kubevirt_recorded_placement(object) { + Ok(SupervisorPlacement::Shared { route }) => route, + Ok(SupervisorPlacement::Dedicated) => { + let Some(sandbox_id) = sandbox_id_from_object(object).ok() else { + return "unknown".to_string(); + }; + let names = SandboxRuntimeNames::new(&sandbox_id); + let pods: Api = Api::namespaced(self.client.clone(), namespace); + pods.get_opt(&names.supervisor_pod) + .await + .ok() + .flatten() + .and_then(|pod| pod.metadata.uid) + .unwrap_or_else(|| "unknown".to_string()) + } + Err(_) => "unknown".to_string(), + } + } + + /// The recorded supervisor placement of a `KubeVirt` sandbox. New VMs + /// carry the pod path's topology annotation; VMs created before it + /// carry only the shared route annotation, which is treated as shared + /// so a restart under this driver keeps dispatching them correctly. + fn kubevirt_recorded_placement( + object: &DynamicObject, + ) -> Result { + if let Some(route) = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SUPERVISOR_SESSION)) + { + return Ok(SupervisorPlacement::Shared { + route: route.clone(), + }); + } + supervisor_placement(object) + } + + /// Provision one dedicated supervisor generation for a `KubeVirt` + /// sandbox: the pod path's per-sandbox boundary Service, the + /// namespace-wide supervisor egress policy, the pair-scoped control + /// policy, and the gated supervisor Pod — all through the pod path's + /// own renderers, owned by the `VirtualMachine` so Kubernetes GC + /// cascades them. Returns the supervisor Pod's UID (claimed by the + /// boundary as `kubevirt.supervisor_pod_uid`). + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn create_kubevirt_supervisor_generation( + &self, + sandbox_id: &str, + sandbox_name: &str, + namespace: &str, + names: &SandboxRuntimeNames, + main_process_spec: &str, + log_level: &str, + owner: OwnerReference, + agent_uid: u32, + agent_gid: u32, + ) -> Result { + self.ensure_kubevirt_supervisor_networking(sandbox_id, namespace, names, owner.clone()) + .await?; + // The gated supervisor Pod, created before the bootstrap Secret so + // its UID can be claimed; it stays scheduled-gated until the fence, + // bootstrap Secret, and workload config are in place. + let staged = self.read_staged_bootstrap_material().await?; + self.create_sandbox_runtime_supervisor( + namespace, + names, + sandbox_id, + sandbox_name, + owner, + agent_uid, + agent_gid, + main_process_spec, + log_level, + staged.upstream_proxy_ca_bundle.is_some(), + ) + .await + } + + /// Ensure the dedicated generation's network objects: the per-sandbox + /// Service the guest's boundary dials, the namespace-wide supervisor + /// egress policy, and the pair-scoped control policy. Idempotent + /// (create-or-validate), so start re-creates a Service that was + /// deleted under a running sandbox instead of failing every later + /// start with a `NotFound` boundary Service. + async fn ensure_kubevirt_supervisor_networking( + &self, + sandbox_id: &str, + namespace: &str, + names: &SandboxRuntimeNames, + owner: OwnerReference, + ) -> Result<(), KubernetesDriverError> { + // The per-sandbox Service the guest's boundary dials (the pod + // path's runtime-initiated supervisor Service, selecting this + // sandbox's supervisor Pod). + self.ensure_sandbox_runtime_service( + namespace, + names, + sandbox_id, + KubernetesControlConnection::RuntimeInitiated, + owner.clone(), + ) + .await?; + // The namespace-wide supervisor egress policy (the supervisor half + // of the pod path's workload fence): the control policy below + // isolates the supervisor Pod's egress, and this policy re-allows + // its policy-approved DNS and upstream connections. The workload + // half is deliberately not created — the per-sandbox + // virt-launcher fence plays that role for `KubeVirt` sandboxes. + self.ensure_kubevirt_supervisor_egress_policy(namespace, names) + .await?; + // The pair-scoped control ingress (L2): only this pair's supervisor + // Pod accepts boundary-port connections, only from this pair's + // workload-role pods — the virt-launcher never does. The egress + // half is the fence's job here. Its UID is not claimed (the fence + // evidence stays the virt-launcher NetworkPolicy). + let mut expected = serde_json::from_value::( + crate::kubevirt_boundary::render_control_ingress( + namespace, + sandbox_id, + &crate::sandbox_runtime::pair_label_value(sandbox_id), + self.config.sandbox_runtime.boundary_port, + ), + ) + .map_err(|error| { + KubernetesDriverError::Message(format!("render control ingress policy: {error}")) + })?; + expected.metadata.owner_references = Some(vec![owner]); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + create_or_validate_sandbox_runtime_fence(&policies, &expected).await?; + Ok(()) + } + + /// Create or validate the namespace-wide supervisor egress policy (the + /// supervisor half of the pod path's `workload_fence`), labeled exactly + /// as the pod path labels it. + async fn ensure_kubevirt_supervisor_egress_policy( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + ) -> Result<(), KubernetesDriverError> { + let fence = workload_fence(namespace, names, self.config.sandbox_runtime.boundary_port); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let mut policy = fence.supervisor_policy; + let labels = policy.metadata.labels.get_or_insert_default(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert( + "openshell.ai/component".to_string(), + "sandbox-supervisor-egress".to_string(), + ); + create_or_validate_sandbox_runtime_fence(&policies, &policy).await + } + + /// Create or validate the namespace-wide virt-launcher isolation + /// policy (H1): deny-all ingress and egress for every virt-launcher + /// pod in the sandbox namespace, so isolation never depends on a + /// per-sandbox object's lifecycle. The per-sandbox fence and control + /// policy only add allowances on top of it. The policy selects every + /// virt-launcher pod in the namespace — the `PoC`'s sandbox namespaces + /// are OpenShell-managed — including VMs other than this driver's. + async fn ensure_kubevirt_launcher_isolation( + &self, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + ensure_kubevirt_launcher_isolation_with(&self.client, namespace).await + } + + /// Resolve the dedicated supervisor target for a `KubeVirt` sandbox: + /// the per-sandbox Service's authority, `ClusterIP`, and pod selector + /// (read from the live Service, never hard-coded), plus the supervisor + /// Pod's UID for the boundary claim. + async fn resolve_dedicated_kubevirt_target( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + supervisor_pod_uid: &str, + ) -> Result { + let services = Api::::namespaced(self.client.clone(), namespace); + let service_name = &names.supervisor_service; + let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get(service_name)) + .await + .map_err(|_| format!("timed out reading boundary Service {service_name}"))? + .map_err(|error| format!("read boundary Service {service_name}: {error}"))?; + if !boundary_service_matches_mode( + &service, + sandbox_id, + KubernetesControlConnection::RuntimeInitiated, + ) { + return Err(format!( + "boundary Service {service_name} does not select the runtime-initiated supervisor listener" + )); + } + let port = self.config.sandbox_runtime.boundary_port; + let cluster_ip = service + .spec + .as_ref() + .and_then(|spec| spec.cluster_ip.clone()) + .filter(|ip| ip != "None") + .ok_or_else(|| format!("boundary Service {service_name} has no ClusterIP"))?; + let address: std::net::SocketAddr = format!("{cluster_ip}:{port}") + .parse() + .map_err(|error| format!("invalid boundary Service ClusterIP: {error}"))?; + let pod_selector = service + .spec + .and_then(|spec| spec.selector) + .ok_or_else(|| format!("boundary Service {service_name} has no pod selector"))?; + Ok( + crate::kubevirt_boundary::KubeVirtSupervisorTarget::Dedicated { + authority: format!("{service_name}.{namespace}.svc:{port}"), + address, + supervisor_pod_uid: supervisor_pod_uid.to_string(), + pod_selector, + port, + }, + ) + } + + /// Install the dedicated supervisor generation's bootstrap Secret + /// (runtime descriptor, auth bundle, proxy CA, staged client TLS) owned + /// by the supervisor Pod, stage the generation's image-pull Secret + /// copies, and release the supervisor Pod's scheduling gate. + #[allow(clippy::too_many_arguments)] + async fn install_kubevirt_supervisor_secret( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + supervisor_pod_uid: &str, + bootstrap: &crate::kubevirt_boundary::KubeVirtBootstrap, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, + ) -> Result { + let staged = self.read_staged_bootstrap_material().await?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let descriptor = bootstrap + .runtime_descriptor + .backend_descriptor() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, supervisor_pod_uid); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + sandbox_id, + descriptor.payload, + serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { + KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) + })?, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + staged.upstream_proxy_ca_bundle, + staged.client_tls, + supervisor_owner.clone(), + ); + let secrets = Api::::namespaced(self.client.clone(), namespace); + // A retried generation can find this generation's (immutable) + // bootstrap Secret already present; replace it so the material + // always matches this attempt. + let created_supervisor_secret = match secrets + .create(&PostParams::default(), &supervisor_secret) + .await + { + Ok(created) => created, + Err(KubeError::Api(err)) if err.code == 409 => { + // H4: a leftover from an earlier attempt of this same + // generation is deleted by exact name (no read — the + // name is generation-scoped, so a foreign object under + // it is pathological) and the create retried. + let _ = secrets + .delete(&names.supervisor_secret, &DeleteParams::default()) + .await; + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)? + } + Err(err) => return Err(KubernetesDriverError::from_kube(err)), + }; + let supervisor_secret_uid = created_supervisor_secret.metadata.uid.unwrap_or_default(); + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![supervisor_owner], + ) + .await?; + // Start the supervisor only now that its bootstrap exists. + let pods: Api = Api::namespaced(self.client.clone(), namespace); + pods.patch( + &names.supervisor_pod, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + Ok(supervisor_secret_uid) + } + + /// X3: register (or re-register) the shared-supervisor instance for a + /// `KubeVirt` sandbox generation. Shared by create (initial launch) and + /// start (new generation on the same route): the registration goes + /// through the coordinator's `SharedSupervision::launch` (validate → + /// persist via `shared_instance_secret` → place → assign) and the + /// instance Secret is owned by the `VirtualMachine` so Kubernetes GC + /// cascades it. + #[allow(clippy::too_many_arguments)] + async fn register_kubevirt_instance( + &self, + route: &str, + sandbox_id: &str, + sandbox_name: &str, + main_process_spec: String, + runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, + supervisor_auth: openshell_core::jwt::SupervisorAuthBundle, + vm_name: &str, + vm_uid: &str, + ) -> Result<(), KubernetesDriverError> { + let (shared, _target) = self.shared_supervisor_target().await?; + let namespace = crate::kubevirt::kubevirt_namespace(&self.config) + .map_err(KubernetesDriverError::Precondition)?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let registration = shared_instance_registration_from_values( + route, + sandbox_id, + sandbox_name, + main_process_spec, + &self.config.kubevirt.guest_image, + runtime_descriptor, + supervisor_auth, + proxy_ca, + ); + let generation = registration.runtime_descriptor.generation.clone(); + let names = SandboxRuntimeNames::for_generation(sandbox_id, &generation); + let instance_owner = crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + vm_name, + vm_uid, + "kubevirt.io/v1", + true, + ); + let secrets = Api::::namespaced(self.client.clone(), &namespace); + let namespace_for_secret = namespace.clone(); + let sandbox_id_for_secret = sandbox_id.to_string(); + let mut attempts = 0u8; + loop { + attempts += 1; + let secrets = secrets.clone(); + let namespace = namespace_for_secret.clone(); + let names = names.clone(); + let sandbox_id = sandbox_id_for_secret.clone(); + let owner = instance_owner.clone(); + let launch_result = shared + .supervision() + .launch(®istration, move |launch_material| { + let secrets = secrets.clone(); + let namespace = namespace.clone(); + let names = names.clone(); + let sandbox_id = sandbox_id.clone(); + let owner = owner; + async move { + let secret = shared_instance_secret( + &namespace, + &names, + &sandbox_id, + launch_material, + owner, + ); + // A retried launch can find this generation's + // (immutable) instance Secret already created by + // the earlier attempt's persist; replace it so the + // material always matches this attempt. + if let Err(KubeError::Api(err)) = + secrets.create(&PostParams::default(), &secret).await + { + if err.code != 409 { + return Err(KubernetesDriverError::from_kube(KubeError::Api(err))); + } + let _ = secrets + .delete( + &secret.metadata.name.clone().unwrap_or_default(), + &DeleteParams::default(), + ) + .await; + secrets + .create(&PostParams::default(), &secret) + .await + .map(|_| ()) + .map_err(KubernetesDriverError::from_kube)?; + } + Ok(()) + } + }) + .await; + let error = match launch_result { + Ok(()) => return Ok(()), + Err(error) => error, + }; + let retryable = matches!( + &error, + LaunchError::Unplaced(reason) if route_teardown_race(reason) + ); + if retryable && attempts < 12 { + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + return Err(match error { + LaunchError::Persist(error) => error, + LaunchError::Refused(error) => { + KubernetesDriverError::Precondition(error.to_string()) + } + LaunchError::Unplaced(reason) => KubernetesDriverError::Precondition(reason), + LaunchError::Invalid(error) => KubernetesDriverError::Message(error.to_string()), + }); + } + } + + /// `KubeVirt` runtime mode lookup by sandbox-id label. + async fn get_kubevirt_sandbox(&self, sandbox_id: &str) -> Result, String> { + let namespace = self + .kubevirt_sandbox_namespace(sandbox_id) + .await + .map_err(|error| error.to_string())?; + let api = crate::kubevirt::virtual_machine_api(&self.client, &namespace); + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + api.list(&ListParams::default().labels(&selector)), + ) + .await + .map_err(|_| { + format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + ) + })? + .map_err(|err| err.to_string())?; + let Some(vm) = list.items.into_iter().next() else { + return Ok(None); + }; + let vmi_state = self + .kubevirt_vmi_state(&namespace, vm.metadata.name.as_deref().unwrap_or_default()) + .await + .unwrap_or((None, None)); + let supervisor_failure = kubevirt_supervisor_failure(&self.client, &namespace, &vm).await; + let fence_restored = vm + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FENCE_RESTORED_ANNOTATION)) + .cloned(); + Ok(crate::kubevirt::sandbox_from_vm_object_with_runtime( + &namespace, + vm, + vmi_state.0.as_deref(), + supervisor_failure.as_deref(), + fence_restored.as_deref(), + vmi_state.1, + ) + .ok() + .map(|(_, sandbox)| sandbox)) + } + + /// Phase of the paired `VirtualMachineInstance` (same name as the VM), + /// when it exists. `None` covers both a missing and an unreadable VMI. + async fn kubevirt_vmi_phase(&self, namespace: &str, kube_name: &str) -> Option { + self.kubevirt_vmi_state(namespace, kube_name).await?.0 + } + + /// The paired VMI's phase and its not-ready-since time (Unix ms, from + /// the `Ready` condition's `lastTransitionTime` — used by the snapshot's + /// image-pull-backoff terminal rule, L4). + async fn kubevirt_vmi_state( + &self, + namespace: &str, + kube_name: &str, + ) -> Option<(Option, Option)> { + let api = crate::kubevirt::virtual_machine_instance_api(&self.client, namespace); + let vmi = tokio::time::timeout(KUBE_API_TIMEOUT, api.get(kube_name)) + .await + .ok()? + .ok()?; + let phase = vmi + .data + .get("status") + .and_then(|status| status.get("phase")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + Some((phase, crate::kubevirt::vmi_not_ready_since_ms(&vmi))) + } + + /// Map of VM name -> VMI phase for all driver-owned VMIs matching + /// `selector` (the VMI carries the VM template labels). + async fn kubevirt_vmi_phases( + &self, + namespace: &str, + selector: &str, + ) -> std::collections::HashMap, Option)> { + let api = crate::kubevirt::virtual_machine_instance_api(&self.client, namespace); + let Ok(list) = tokio::time::timeout( + KUBE_API_TIMEOUT, + api.list(&ListParams::default().labels(selector)), + ) + .await + else { + return std::collections::HashMap::new(); + }; + let Ok(list) = list else { + return std::collections::HashMap::new(); + }; + list.items + .into_iter() + .filter_map(|vmi| { + let not_ready_since = crate::kubevirt::vmi_not_ready_since_ms(&vmi); + let name = vmi.metadata.name?; + let phase = vmi + .data + .get("status") + .and_then(|status| status.get("phase")) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + Some((name, (phase, not_ready_since))) + }) + .collect() + } + + /// `KubeVirt` runtime mode start: boot the VM. A simple `Once` patch + /// boots a halted or never-started VM, but a completed `Once` VMI makes + /// `Once` -> `Once` a silent no-op, so a not-ready `Once` VM is bounced + /// through `Halted` first. + /// Resolve the namespace a `KubeVirt` sandbox's VM lives in: the + /// configured sandbox namespace in shared mode; in managed mode the + /// per-workspace namespace, discovered from the VM carrying the + /// sandbox-id label (cluster-wide list). + #[allow(clippy::similar_names)] + async fn kubevirt_sandbox_namespace( + &self, + sandbox_id: &str, + ) -> Result { + crate::kubevirt::discover_kubevirt_namespace( + &self.client, + &self.config, + sandbox_id, + &self.sandbox_lookup_selector(sandbox_id), + ) + .await + } + + #[allow(clippy::similar_names)] + async fn start_kubevirt_sandbox( + &self, + sandbox_id: &str, + encoded_authentication: &[u8], + encoded_expected_runtime_identity: &str, + ) -> Result { + let namespace = self.kubevirt_sandbox_namespace(sandbox_id).await?; + let api = crate::kubevirt::virtual_machine_api(&self.client, &namespace); + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + api.list(&ListParams::default().labels(&selector)), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + // Exactly one VM may match the sandbox's label selector, and it + // must carry the persisted runtime identity (namespace + VM UID) — + // the same guarantees the pod path's start requires. + let object = select_expected_kubevirt_vm(list.items, encoded_expected_runtime_identity)?; + let kube_name = object.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("VirtualMachine has no name".to_string()) + })?; + let vm_uid = object.metadata.uid.clone().unwrap_or_default(); + let run_strategy = object + .data + .get("spec") + .and_then(|spec| spec.get("runStrategy")) + .and_then(serde_json::Value::as_str) + .unwrap_or("Once"); + + // M4: decide the start plan before any side effect. A live or + // booting VMI makes start idempotent — no supervisor Pod is + // recreated, no Secret rewritten, no instance re-registered. + let vm_ready = object + .data + .get("status") + .and_then(|status| status.get("ready")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + let printable_status = object + .data + .get("status") + .and_then(|status| status.get("printableStatus")) + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let vmi_phase = self.kubevirt_vmi_phase(&namespace, &kube_name).await; + let plan = crate::kubevirt::kubevirt_start_plan( + run_strategy, + vm_ready, + vmi_phase.as_deref(), + printable_status, + ); + if plan == crate::kubevirt::KubeVirtStartPlan::AlreadyRunning { + info!( + sandbox_id, + vm = %kube_name, + vmi_phase = vmi_phase.as_deref().unwrap_or("absent"), + "KubeVirt start is idempotent: the VirtualMachineInstance is live or booting" + ); + // The gateway persists start's return as the runtime identity; + // return the full identity, not a bare VM UID, or the next + // start cannot parse it (M6 regression). + let component = self + .kubevirt_supervisor_component(&object, &namespace) + .await; + return Ok(kubernetes_runtime_identity(&namespace, &vm_uid, &component)); + } + + // H1: keep the isolation floor present across starts (the + // reconciler also restores it while running). + self.ensure_kubevirt_launcher_isolation(&namespace).await?; + + // Refresh the guest's launch credentials before booting: the + // create-time token frozen in the bootstrap Secret expires under + // the gateway's JWT TTL, leaving a restarted sandbox provisioning + // forever. The gateway mints fresh launch credentials for every + // start. A malformed bundle is an error rather than a silent + // fallback to the stale token. + let authentication = + match crate::kubevirt::decode_launch_authentication(encoded_authentication) { + Ok(Some(authentication)) => authentication, + Ok(None) => { + return Err(KubernetesDriverError::Precondition( + "start carried no launch authentication; the guest cannot re-provision \ + its boundary credentials" + .to_string(), + )); + } + Err(error) => return Err(error), + }; + + // The recorded placement decides the topology exactly as the pod + // path's start does: a shared generation re-registers its route; a + // dedicated generation gets a fresh supervisor Pod (and a new + // `kubevirt.supervisor_pod_uid` claim). + let placement = Self::kubevirt_recorded_placement(&object)?; + + // X4: the sandbox identity comes from the VM's labels + // (sandbox_labels puts them there at create), not annotations. + let sandbox_name = object + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_NAME)) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "VirtualMachine is missing its OpenShell sandbox name label".to_string(), + ) + })?; + + let object_workspace = object + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_WORKSPACE)) + .cloned() + .unwrap_or_else(|| "default".to_string()); + let generation = authentication.supervisor.runtime_generation.to_string(); + let names = SandboxRuntimeNames::for_generation(sandbox_id, &generation); + + // The supervisor the fence is verified against and the boundary + // dials, per topology. + let mut supervisor = match &placement { + SupervisorPlacement::Shared { route } => { + // X2: resolve the shared supervisor through the pod path's + // single resolver. A missing shared supervisor is a + // Precondition error — the boundary's dial would be refused. + let (_shared, target) = self.shared_supervisor_target().await?; + crate::kubevirt_boundary::KubeVirtSupervisorTarget::Shared { + authority: target.authority.clone(), + address: target.address, + route: route.clone(), + namespace: target.namespace.clone(), + pod_selector: target.selector.clone(), + port: target.address.port(), + } + } + SupervisorPlacement::Dedicated => { + // The dedicated generation's Service and selector are + // stable across generations; the supervisor Pod UID is + // bound after the new Pod exists, below. A Service deleted + // out of band (under a running sandbox) is re-created here + // — create-or-validate — so the start does not wedge on a + // NotFound boundary Service. + let owner = crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + &kube_name, + &vm_uid, + "kubevirt.io/v1", + false, + ); + self.ensure_kubevirt_supervisor_networking(sandbox_id, &namespace, &names, owner) + .await?; + self.resolve_dedicated_kubevirt_target(&namespace, &names, sandbox_id, "") + .await + .map_err(KubernetesDriverError::Precondition)? + } + }; + + // X1: verify the fence on every new generation. The expected UID + // was persisted on the VM at create; the expected spec is freshly + // rendered from the current supervisor target, so a fence edited + // to widen egress (same UID, drifted spec) or replaced (new UID) + // fails the start. A resourceVersion-only change is accepted and + // re-captured into the new evidence. + let persisted_expectation = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get(crate::kubevirt_boundary::FENCE_EXPECTATION_ANNOTATION) + }) + .map(|value| crate::kubevirt_boundary::ExpectedFence::from_annotation(value)) + .transpose() + .map_err(KubernetesDriverError::Precondition)? + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "VirtualMachine is missing its persisted fence expectation; the fence \ + cannot be verified for this generation" + .to_string(), + ) + })?; + let expected_fence = crate::kubevirt_boundary::ExpectedFence { + resource_version: persisted_expectation.resource_version.clone(), + spec: crate::kubevirt_boundary::render_fence( + &namespace, + sandbox_id, + Some(&self.config.gateway_id), + &supervisor, + )["spec"] + .clone(), + uid: persisted_expectation.uid.clone(), + }; + let fence = crate::kubevirt_boundary::verify_fence( + &self.client, + &namespace, + sandbox_id, + &expected_fence, + ) + .await + .map_err(KubernetesDriverError::Precondition)?; + + // The main process spec and log level are persisted at create; the + // dedicated generation's supervisor Pod needs both again. + let main_process_spec = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "VirtualMachine is missing its persisted main process spec; the supervisor \ + generation cannot be re-created" + .to_string(), + ) + })?; + let log_level = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL)) + .cloned() + .unwrap_or_else(|| "info".to_string()); + + // A dedicated generation starts with a fresh supervisor Pod (new + // UID, new generation Secret), exactly as the pod path's start + // re-creates the supervisor Pod. + let mut supervisor_pod_uid = String::new(); + if matches!(&placement, SupervisorPlacement::Dedicated) { + self.delete_sandbox_runtime_supervisor_pod(sandbox_id, &namespace) + .await?; + let owner = crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + &kube_name, + &vm_uid, + "kubevirt.io/v1", + false, + ); + let (agent_uid, agent_gid, _) = + self.resolve_sandbox_identity_in_namespace(&namespace).await; + let pod_uid = self + .create_kubevirt_supervisor_generation( + sandbox_id, + &sandbox_name, + &namespace, + &names, + &main_process_spec, + &log_level, + owner, + agent_uid, + agent_gid, + ) + .await?; + supervisor_pod_uid = pod_uid; + supervisor = self + .resolve_dedicated_kubevirt_target( + &namespace, + &names, + sandbox_id, + &supervisor_pod_uid, + ) + .await + .map_err(KubernetesDriverError::Precondition)?; + } + + // Re-render the bootstrap for this generation: the claims use the + // VM's real UID (W6), the recorded route (shared) or the new + // supervisor Pod UID (dedicated), and the evidence carries the + // verified fence's current RV. + // M6: the image identity claims the image the VM actually boots + // (its kernelBoot containerDisk), not the configured string. + let boot_image = crate::kubevirt::kubevirt_boot_image(&object); + let bootstrap = crate::kubevirt_boundary::render_kubevirt_bootstrap( + &authentication, + sandbox_id, + &sandbox_name, + &vm_uid, + &boot_image, + &fence.uid, + &fence.resource_version, + fence.spec, + &supervisor, + ) + .map_err(KubernetesDriverError::Precondition)?; + + // H4: the bootstrap Secret is immutable per generation — a start + // never patches (or reads) a Secret. The new generation's Secret + // is created under its generation-scoped name; the VM's + // cloudInitNoCloud volume is pointed at it by the release patch + // below, and the previous generation's Secret is deleted by + // exact name with a UID precondition taken from the companions + // annotation. A same-name leftover from a retried start of this + // generation is deleted and waited for first, so the create + // cannot collide with a terminating object. + let secret_api: Api = Api::namespaced(self.client.clone(), &namespace); + let current_generation = bootstrap.runtime_descriptor.generation.clone(); + let secret_name = crate::kubevirt::bootstrap_secret_name(&kube_name, ¤t_generation); + let core_secret_resource = ApiResource { + group: String::new(), + version: "v1".to_string(), + api_version: "v1".to_string(), + kind: "Secret".to_string(), + plural: "secrets".to_string(), + }; + // H4: a plain delete (Secrets carry no finalizers; the driver must + // not GET a Secret to poll its disappearance). + let secret_cleanup_api: Api = + Api::namespaced_with(self.client.clone(), &namespace, &core_secret_resource); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + secret_cleanup_api.delete(&secret_name, &DeleteParams::default()), + ) + .await + { + Ok(_) | Err(_) => {} + } + let secret = Secret { + metadata: ObjectMeta { + name: Some(secret_name.clone()), + namespace: Some(namespace.clone()), + labels: Some(kubevirt_restart_labels( + sandbox_id, + &sandbox_name, + &object_workspace, + Some(&self.config.gateway_id), + )), + owner_references: Some(vec![crate::sandbox_runtime::sandbox_owner_reference( + "VirtualMachine", + &kube_name, + &vm_uid, + "kubevirt.io/v1", + false, + )]), + ..Default::default() + }, + string_data: Some(BTreeMap::from([( + "userData".to_string(), + bootstrap.guest_payload.clone(), + )])), + type_: Some("Opaque".to_string()), + ..Default::default() + }; + let created_secret = tokio::time::timeout( + KUBE_API_TIMEOUT, + secret_api.create(&PostParams::default(), &secret), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s creating bootstrap Secret", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + let recorded_boot_secret = crate::kubevirt::CompanionRecord { + name: secret_name, + uid: created_secret.metadata.uid.unwrap_or_default(), + }; + // The previous generation's recorded companions: their names and + // UIDs drive the exact-name deletes after the release patch. + let previous_companions = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations + .get(crate::kubevirt::COMPANIONS_ANNOTATION) + .and_then(|value| crate::kubevirt::RecordedCompanions::from_annotation(value)) + }); + + // X3: install the generation's supervisor material: the shared + // instance re-registers on the same route, the dedicated + // generation gets its supervisor bootstrap Secret and a released + // Pod. + let mut recorded_supervisor_secret = None; + match &placement { + SupervisorPlacement::Shared { route } => { + self.register_kubevirt_instance( + route, + sandbox_id, + &sandbox_name, + main_process_spec, + bootstrap.runtime_descriptor, + authentication.supervisor.clone(), + &kube_name, + &vm_uid, + ) + .await?; + } + SupervisorPlacement::Dedicated => { + let supervisor_secret_uid = self + .install_kubevirt_supervisor_secret( + &namespace, + &names, + sandbox_id, + &supervisor_pod_uid, + &bootstrap, + &authentication, + ) + .await?; + recorded_supervisor_secret = Some(crate::kubevirt::CompanionRecord { + name: names.supervisor_secret.clone(), + uid: supervisor_secret_uid, + }); + } + } + + // H5: every VM patch is pinned to the UID captured at lookup — + // the patch re-reads the VM, refuses a replacement (a new UID + // under the same name), and carries the read resourceVersion, so + // a concurrent modification is rejected and retried from a fresh + // read rather than landing as a name-only write. + let patch_strategy = |strategy: String| { + let api = api.clone(); + let kube_name = kube_name.clone(); + let vm_uid = vm_uid.clone(); + async move { + patch_kubevirt_vm_pinned_to_checked_uid(&api, &kube_name, &vm_uid, |_| { + serde_json::json!({ + "metadata": {"annotations": { + crate::kubevirt::KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION: null + }}, + "spec": {"runStrategy": strategy} + }) + }) + .await + .map(|_| ()) + } + }; + // The release patch boots the VM and carries the generation + // turnover: the config-disk volume points at the new + // generation's Secret, and the companions annotation records the + // new generation's exact names and UIDs (H4). + let release_patch = |api: Api, + kube_name: String, + vm_uid: String, + volumes: Vec, + companions_annotation: String| async move { + patch_kubevirt_vm_pinned_to_checked_uid(&api, &kube_name, &vm_uid, |_| { + serde_json::json!({ + "metadata": {"annotations": { + crate::kubevirt::KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION: null, + crate::kubevirt::COMPANIONS_ANNOTATION: companions_annotation, + }}, + "spec": { + "template": {"spec": {"volumes": serde_json::json!(volumes)}}, + "runStrategy": "Once" + } + }) + }) + .await + .map(|_| ()) + }; + let volumes = + crate::kubevirt::render_volumes(&kube_name, ¤t_generation, &self.config.kubevirt); + let recorded_companions = crate::kubevirt::RecordedCompanions { + boot_secret: recorded_boot_secret, + supervisor_secret: recorded_supervisor_secret, + }; + let companions_annotation = recorded_companions.to_annotation(); + // Delete the previous generation's Secrets by exact name with a + // UID precondition from the annotation (H4: no read, no list). + // Best-effort: a 409 (the object was replaced out of band) is + // logged and left to owner-reference garbage collection — the + // new generation is already live. + let delete_previous_companion_secrets = + |client: Client, + namespace: String, + previous: Option| async move { + let Some(previous) = previous else { + return; + }; + let secrets: Api = Api::namespaced(client.clone(), &namespace); + let _ = secrets + .delete( + &previous.boot_secret.name, + &DeleteParams::default().preconditions(Preconditions { + uid: Some(previous.boot_secret.uid.clone()), + resource_version: None, + }), + ) + .await; + if let Some(supervisor_secret) = previous.supervisor_secret { + let _ = secrets + .delete( + &supervisor_secret.name, + &DeleteParams::default().preconditions(Preconditions { + uid: Some(supervisor_secret.uid.clone()), + resource_version: None, + }), + ) + .await; + } + }; + // H5: re-verify the fence immediately before the patch that boots + // the guest — the check at the top of start must not be stale by + // the time the VM can run workload code. A fence widened or + // replaced during provisioning fails the start instead of + // releasing the VM. + let reverify_fence_before_release = + |client: Client, + namespace: String, + sandbox_id: String, + expected_fence: crate::kubevirt_boundary::ExpectedFence| async move { + crate::kubevirt_boundary::verify_fence( + &client, + &namespace, + &sandbox_id, + &expected_fence, + ) + .await + .map(|_| ()) + .map_err(KubernetesDriverError::Precondition) + }; + + let wait_for_vmi_gone = async { + let vmi_api = crate::kubevirt::virtual_machine_instance_api(&self.client, &namespace); + let deadline = tokio::time::Instant::now() + VMI_TERMINATION_TIMEOUT; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message( + "timed out waiting for the VirtualMachineInstance to terminate before restart" + .to_string(), + )); + } + match tokio::time::timeout( + KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)), + vmi_api.get(&kube_name), + ) + .await + { + Ok(Err(KubeError::Api(err))) if err.code == 404 => return Ok(()), + Ok(Err(err)) => return Err(KubernetesDriverError::from_kube(err)), + Ok(_) | Err(_) => {} + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + }; + + match plan { + crate::kubevirt::KubeVirtStartPlan::PatchOnce => { + // A halted VM always needs the patch, but `stop` only waits + // for the VM's ready flag: a VMI in graceful shutdown can + // still exist here. Wait it out before patching so the + // controller does not reject or ignore the boot. + wait_for_vmi_gone.await?; + reverify_fence_before_release( + self.client.clone(), + namespace.clone(), + sandbox_id.to_string(), + expected_fence.clone(), + ) + .await?; + release_patch( + api.clone(), + kube_name.clone(), + vm_uid.clone(), + volumes.clone(), + companions_annotation.clone(), + ) + .await?; + } + // AlreadyRunning returned before any side effect above. + crate::kubevirt::KubeVirtStartPlan::AlreadyRunning => {} + crate::kubevirt::KubeVirtStartPlan::BounceHaltedThenOnce => { + // A completed `Once` VMI makes an `Once`->`Once` patch a + // no-op: halt first so KubeVirt discards the dead VMI, wait + // for it to go away, then boot a fresh one. + patch_strategy("Halted".to_string()).await?; + wait_for_vmi_gone.await?; + reverify_fence_before_release( + self.client.clone(), + namespace.clone(), + sandbox_id.to_string(), + expected_fence.clone(), + ) + .await?; + release_patch( + api.clone(), + kube_name.clone(), + vm_uid.clone(), + volumes.clone(), + companions_annotation.clone(), + ) + .await?; + } + } + // The new generation is live: drop the previous generation's + // Secrets by exact name with a UID precondition (X6: they hold + // gateway auth bundles). + delete_previous_companion_secrets( + self.client.clone(), + namespace.clone(), + previous_companions, + ) + .await; + + info!( + sandbox_id, + vm = %kube_name, + "Patched KubeVirt VirtualMachine run strategy to start" + ); + let component = match &placement { + SupervisorPlacement::Shared { route } => route.clone(), + SupervisorPlacement::Dedicated => supervisor_pod_uid, + }; + Ok(kubernetes_runtime_identity(&namespace, &vm_uid, &component)) + } + + /// `KubeVirt` runtime mode stop: halt the VM and wait for the guest to go + /// down (status.ready == false). The VM, bootstrap Secret and workspace + /// PVC are retained. + async fn stop_kubevirt_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let namespace = self.kubevirt_sandbox_namespace(sandbox_id).await?; + let api = crate::kubevirt::virtual_machine_api(&self.client, &namespace); + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + api.list(&ListParams::default().labels(&selector)), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + let object = select_single_kubevirt_vm(list.items)?; + let kube_name = object.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("VirtualMachine has no name".to_string()) + })?; + // Halt the VirtualMachine BEFORE releasing the supervisor (S1): the + // stop used to delete the dedicated supervisor Pod first, and in + // that window the watch mapped the exiting Pod (restartPolicy + // Never -> Failed) plus the not-yet-halted VM to SupervisorFailed, + // aborting the stop RPC. With the halt patch first, the supervisor + // exiting — as its boundary goes away or as the explicit delete + // lands — is observed against a Halted (or Stopping) VM and is + // correctly treated as the expected end of a stop. + // + // Also clear any lingering create-in-flight annotation: a create + // interrupted before its Once patch leaves one, and the + // provisioning-timeout reclaim reaches this stop. + // H5: the halt is pinned to the UID from the lookup — a VM + // replaced under the same name is never halted by this stop. + let checked_vm_uid = object.metadata.uid.clone().unwrap_or_default(); + patch_kubevirt_vm_pinned_to_checked_uid(&api, &kube_name, &checked_vm_uid, |_| { + serde_json::json!({ + "metadata": {"annotations": { + crate::kubevirt::KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION: null + }}, + "spec": {"runStrategy": "Halted"} + }) + }) + .await?; + + info!( + sandbox_id, + vm = %kube_name, + run_strategy = "Halted", + "Patched KubeVirt VirtualMachine run strategy" + ); + + // The stopped generation releases its supervisor, mirroring the pod + // path's stop. A shared generation ends its sandbox instance: the + // supervisor stops serving this sandbox's route, and the pool's + // record is withdrawn now (not by the later async sweep) so an + // immediate start can re-register the route for the new + // generation. A dedicated generation's supervisor Pod is deleted + // (cascading its bootstrap Secret), exactly as the pod path's stop + // deletes both Pods. + match Self::kubevirt_recorded_placement(&object)? { + SupervisorPlacement::Shared { route } => { + if let Some(shared) = &self.shared_supervisor { + shared.supervision().withdraw(&route); + } + } + SupervisorPlacement::Dedicated => { + self.delete_sandbox_runtime_supervisor_pod(sandbox_id, &namespace) + .await?; + } + } + + let deadline = tokio::time::Instant::now() + VMI_TERMINATION_TIMEOUT; + let mut poll_interval = Duration::from_millis(500); + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message( + "timed out after 120s waiting for KubeVirt sandbox to stop".to_string(), + )); + } + let object = tokio::time::timeout( + KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)), + api.get(&kube_name), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message("timed out checking VirtualMachine stop".to_string()) + })? + .map_err(KubernetesDriverError::from_kube)?; + let ready = object + .data + .get("status") + .and_then(|status| status.get("ready")) + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if !ready { + return Ok(()); + } + tokio::time::sleep(poll_interval).await; + poll_interval = (poll_interval * 2).min(Duration::from_secs(5)); + } + } + + /// `KubeVirt` runtime mode delete: remove the VM (which cascades the VMI + /// and virt-launcher pod), then the bootstrap Secret, workspace PVC and + /// proxy-pod companions. + async fn delete_kubevirt_sandbox(&self, sandbox_id: &str) -> Result { + let namespace = self + .kubevirt_sandbox_namespace(sandbox_id) + .await + .map_err(|error| error.to_string())?; + let api = crate::kubevirt::virtual_machine_api(&self.client, &namespace); + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + api.list(&ListParams::default().labels(&selector)), + ) + .await + .map_err(|_| { + format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + ) + })? + .map_err(|err| err.to_string())?; + // Exactly one VM may match the sandbox's selector; more than one is + // an ambiguous identity this delete must not guess between. + let object = match select_single_kubevirt_vm(list.items) { + Ok(object) => object, + Err(KubernetesDriverError::NotFound) => { + debug!(sandbox_id = %sandbox_id, "VirtualMachine not found (already deleted)"); + // H4: with the VM gone, every owner-referenced companion is + // removed by Kubernetes garbage collection (the bootstrap + // Secret included — it cannot leak, the cascade removes + // it). The driver no longer searches by label: the only + // derivable name without the VM is the egress fence (it + // comes from the sandbox ID), which is deleted by exact + // name as a best effort. + let fence_api: Api = Api::namespaced_with( + self.client.clone(), + &namespace, + &crate::kubevirt_boundary::fence_api_resource(), + ); + let _ = fence_api + .delete( + &crate::kubevirt_boundary::fence_name(sandbox_id), + &DeleteParams::default(), + ) + .await; + return Ok(false); + } + Err(error) => return Err(error.to_string()), + }; + let kube_name = object + .metadata + .name + .clone() + .ok_or("VirtualMachine has no name")?; + // W4: withdraw the shared-supervisor instance before the VM goes, + // so the supervisor stops serving this sandbox's boundary route + // (the route was persisted on the VM at create). A dedicated + // generation's supervisor Pod, Service, control policy, and + // bootstrap Secrets are owned by the VM and cascade with it. + if let Ok(SupervisorPlacement::Shared { route }) = + Self::kubevirt_recorded_placement(&object) + { + if let Some(shared) = &self.shared_supervisor { + shared.supervision().withdraw(&route); + } else { + warn!( + route, + "shared supervisor is not configured; its sandbox instance cannot be withdrawn" + ); + } + } + let checked_vm_uid = object.metadata.uid.clone().unwrap_or_default(); + let preconditions = Preconditions { + uid: Some(checked_vm_uid.clone()), + resource_version: None, + }; + + info!(sandbox_id = %sandbox_id, vm = %kube_name, "Deleting KubeVirt VirtualMachine sandbox"); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.delete( + &kube_name, + &DeleteParams::default().preconditions(preconditions), + ), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(KubeError::Api(err))) if err.code == 404 => { + return Ok(false); + } + Ok(Err(err)) => return Err(err.to_string()), + Err(_elapsed) => { + return Err(format!( + "timed out after {}s deleting VirtualMachine", + KUBE_API_TIMEOUT.as_secs() + )); + } + } + + // Explicit cleanup of the companion resources (labelled and deleted + // by name; GC usually wins through the VM owner references). No + // wait for the VM to be gone first (user decision): the + // namespace-wide launcher isolation floor keeps the pod fenced + // after the per-sandbox fence goes, and the owner references + // cascade the companions once the VM is actually removed. + // + // H5: each companion delete is pinned to the object that was read + // — the object's ownerReference UID must be the checked VM, and + // the delete carries the read UID as a precondition, so a + // replaced or foreign-owned object under a derived name is + // skipped, never deleted. + // + // H4: the bootstrap Secret is never read — its exact name and UID + // come from the companions annotation recorded on the VM, so the + // delete needs no get and no list. A missing annotation (a + // pre-annotation VM) relies on the owner-reference cascade. + let secret_api: Api = Api::namespaced(self.client.clone(), &namespace); + if let Some(recorded) = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(crate::kubevirt::COMPANIONS_ANNOTATION)) + .and_then(|value| crate::kubevirt::RecordedCompanions::from_annotation(value)) + { + let _ = secret_api + .delete( + &recorded.boot_secret.name, + &DeleteParams::default().preconditions(Preconditions { + uid: Some(recorded.boot_secret.uid.clone()), + resource_version: None, + }), + ) + .await; + if let Some(supervisor_secret) = recorded.supervisor_secret { + let _ = secret_api + .delete( + &supervisor_secret.name, + &DeleteParams::default().preconditions(Preconditions { + uid: Some(supervisor_secret.uid.clone()), + resource_version: None, + }), + ) + .await; + } + } else { + debug!( + sandbox_id = %sandbox_id, + "no recorded companions on the VM; relying on owner-reference garbage collection for the Secrets" + ); + } + if self.config.kubevirt.workspace_pvc { + let pvc_api: Api = + Api::namespaced(self.client.clone(), &namespace); + if let Ok(Some(pvc)) = pvc_api + .get_opt(&crate::kubevirt::workspace_pvc_name(&kube_name)) + .await + { + if owner_reference_matches(&pvc.metadata.owner_references, &checked_vm_uid) { + let _ = pvc_api + .delete( + &crate::kubevirt::workspace_pvc_name(&kube_name), + &DeleteParams::default().preconditions(Preconditions { + uid: pvc.metadata.uid, + resource_version: None, + }), + ) + .await; + } else { + warn!( + sandbox_id = %sandbox_id, + "workspace PVC is not owned by this sandbox's VirtualMachine; skipping its deletion" + ); + } + } + } + // W8: delete the egress fence in the normal path too — its name + // derives from the sandbox ID, so the name-based companion cleanup + // above misses it. H5: the fence is read first and its UID is a + // delete precondition (a replaced fence under the same name is + // not deleted). + let fence_api: Api = Api::namespaced_with( + self.client.clone(), + &namespace, + &crate::kubevirt_boundary::fence_api_resource(), + ); + if let Ok(Some(fence)) = fence_api + .get_opt(&crate::kubevirt_boundary::fence_name(sandbox_id)) + .await + { + if owner_reference_matches(&fence.metadata.owner_references, &checked_vm_uid) { + let _ = fence_api + .delete( + &crate::kubevirt_boundary::fence_name(sandbox_id), + &DeleteParams::default().preconditions(Preconditions { + uid: fence.metadata.uid, + resource_version: None, + }), + ) + .await; + } else { + warn!( + sandbox_id = %sandbox_id, + "egress fence is not owned by this sandbox's VirtualMachine; skipping its deletion" + ); + } + } + Ok(true) + } + + /// `KubeVirt` runtime mode watch: label-selected watchers for + /// `VirtualMachine` and `VirtualMachineInstance` objects. The `KubeVirt` + /// controller updates `VM.status` as the VMI progresses; the VMI stream + /// additionally supplies the VMI phase, which distinguishes a failed or + /// completed guest from one that has not booted yet (both show + /// `printableStatus: Stopped`). The gateway still composes public + /// readiness from the supervisor session. + #[allow(clippy::unused_async)] + #[allow( + clippy::unused_async, + reason = "the stream setup itself is synchronous; the watcher tasks await inside their spawned closures" + )] + async fn watch_sandboxes_kubevirt(&self) -> Result { + // Shared mode watches the sandbox namespace; managed mode watches + // cluster-wide (the VMs live in per-workspace namespaces). + let (vm_api, instance_api) = match self.config.workspace_mode { + WorkspaceMode::Shared => { + let namespace = crate::kubevirt::kubevirt_namespace(&self.config)?; + ( + crate::kubevirt::virtual_machine_api(&self.watch_client.clone(), &namespace), + crate::kubevirt::virtual_machine_instance_api( + &self.watch_client.clone(), + &namespace, + ), + ) + } + WorkspaceMode::Managed | WorkspaceMode::Operator => ( + Api::all_with( + self.watch_client.clone(), + &crate::kubevirt::virtual_machine_api_resource(), + ), + Api::all_with( + self.watch_client.clone(), + &crate::kubevirt::virtual_machine_instance_api_resource(), + ), + ), + }; + // Shared mode has one namespace; managed mode resolves each + // object's own namespace at the emit sites below. + let namespace = match self.config.workspace_mode { + WorkspaceMode::Shared => crate::kubevirt::kubevirt_namespace(&self.config)?, + WorkspaceMode::Managed | WorkspaceMode::Operator => String::new(), + }; + let selector = openshell_sandbox_label_selector(); + let watcher_config = watcher::Config::default().labels(&selector); + let mut vm_stream = recovering_watcher_stream( + watcher::watcher(vm_api, watcher_config.clone()), + "kubevirt-virtualmachine", + ) + .boxed(); + let mut instance_stream = recovering_watcher_stream( + watcher::watcher(instance_api, watcher_config), + "kubevirt-virtualmachineinstance", + ) + .boxed(); + let (tx, rx) = mpsc::channel(256); + let watch_client = self.watch_client.clone(); + + let reconcile_client = watch_client.clone(); + let reconcile_namespace = namespace.clone(); + // Built before the spawn: the cluster-wide (managed) or + // namespaced (shared) VM list the reconcile tick groups by + // namespace. + let reconcile_list_api = + crate::kubevirt::kubevirt_list_api(&reconcile_client, &self.config).ok(); + let reconcile_config = KubeVirtReconcileConfig { + gateway_id: self.config.gateway_id.clone(), + boundary_port: self.config.sandbox_runtime.boundary_port, + shared_supervisor: self + .shared_supervisor + .as_ref() + .map(|shared| shared.config().clone()), + }; + let mut reconcile_tick = tokio::time::interval(FENCE_RECONCILE_INTERVAL); + // R3: a root workload whose generation ended terminally keeps + // executing unsupervised (fenced, but burning CPU and writing the + // workspace PVC) while the user sees Error — the watch halts the + // VM (runStrategy Halted, a trusted control-plane action) when a + // root-mode sandbox reaches SupervisorFailed (which covers the + // reset-detection path). + tokio::spawn(async move { + async fn emit( + vm: DynamicObject, + namespace: &str, + vmi_phases: &std::collections::HashMap, Option)>, + client: &Client, + tx: &mpsc::Sender>, + ) -> bool { + let Some(kube_name) = vm.metadata.name.clone() else { + return true; + }; + let vmi_state = vmi_phases.get(&kube_name).cloned().unwrap_or((None, None)); + // Enrich the snapshot with the dedicated supervisor Pod's + // health (M2) and the fence-restored marker (M1). + let supervisor_failure = kubevirt_supervisor_failure(client, namespace, &vm).await; + let fence_restored = vm + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(FENCE_RESTORED_ANNOTATION)) + .cloned(); + let Ok((_, sandbox)) = crate::kubevirt::sandbox_from_vm_object_with_runtime( + namespace, + vm.clone(), + vmi_state.0.as_deref(), + supervisor_failure.as_deref(), + fence_restored.as_deref(), + vmi_state.1, + ) else { + return true; + }; + // R3: halt the VM when a root-mode generation ended while + // the guest keeps running (SupervisorFailed with a live + // VMI — the P1/P2 reset-detection outcome). Already-halted + // VMs and terminal VM failures (the guest is dead) are + // skipped. + if supervisor_failure.is_some() { + let halted = vm + .data + .get("spec") + .and_then(|spec| spec.get("runStrategy")) + .and_then(serde_json::Value::as_str) + == Some("Halted"); + let vmi_running = matches!(vmi_state.0.as_deref(), Some("Running")); + if !halted && vmi_running { + // H5: the auto-halt is pinned to the watched VM's + // UID — a replaced VM is never halted by this + // watch loop. + let checked_vm_uid = vm.metadata.uid.clone().unwrap_or_default(); + let vm_api = crate::kubevirt::virtual_machine_api(client, namespace); + let halt = patch_kubevirt_vm_pinned_to_checked_uid( + &vm_api, + &kube_name, + &checked_vm_uid, + |_| { + serde_json::json!({ + "spec": {"runStrategy": "Halted"} + }) + }, + ) + .await; + match halt { + Ok(_) => { + warn!( + sandbox = %kube_name, + "halted root-workload VM after terminal generation end (R3)" + ); + } + Err(error) => { + warn!( + sandbox = %kube_name, + %error, + "could not halt root-workload VM after terminal generation end (R3)" + ); + } + } + } + } + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }; + tx.send(Ok(event)).await.is_ok() + } + + // VM name -> current VMI phase, maintained by the instance + // stream. + let mut vmi_phases: std::collections::HashMap, Option)> = + std::collections::HashMap::new(); + + loop { + tokio::select! { + event = vm_stream.next() => match event { + Some(Event::Apply(obj) | Event::InitApply(obj)) => { + let object_namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.clone()); + if !emit(obj, &object_namespace, &vmi_phases, &watch_client, &tx).await { + break; + } + } + Some(Event::Delete(obj)) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id }, + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Init | Event::InitDone) => {} + None => { + let _ = tx + .send(Err(KubernetesDriverError::Message( + "VirtualMachine watcher stream ended unexpectedly".to_string(), + ))) + .await; + break; + } + }, + vmi_event = instance_stream.next() => { + let is_delete = matches!(vmi_event, Some(Event::Delete(_))); + match vmi_event { + Some(Event::Apply(obj) | Event::InitApply(obj) | Event::Delete(obj)) => { + if let Some(name) = obj.metadata.name.clone() { + if is_delete { + vmi_phases.remove(&name); + } else { + let state = ( + obj.data + .get("status") + .and_then(|status| status.get("phase")) + .and_then(serde_json::Value::as_str) + .map(str::to_string), + crate::kubevirt::vmi_not_ready_since_ms(&obj), + ); + vmi_phases.insert(name, state); + } + } + // Re-emit the paired VM snapshot with the new + // phase so terminal VMI states surface promptly. + // PoC note: this re-GETs the VM inside the + // select loop for every VMI event — a plain GET + // per phase change, not a watch join. Fine at + // PoC scale; a shared snapshot cache is the + // follow-up if VMI churn ever matters. + if let Some(name) = obj.metadata.name.clone() { + let vm_api = crate::kubevirt::virtual_machine_api( + &watch_client, + &obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.clone()), + ); + if let Ok(vm) = + tokio::time::timeout(KUBE_API_TIMEOUT, vm_api.get(&name)).await + && let Ok(vm) = vm + && let vm_namespace = vm + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.clone()) + && !emit(vm, &vm_namespace, &vmi_phases, &watch_client, &tx) + .await + { + break; + } + } + } + Some(Event::Init | Event::InitDone) => {} + None => { + let _ = tx + .send(Err(KubernetesDriverError::Message( + "VirtualMachineInstance watcher stream ended unexpectedly" + .to_string(), + ))) + .await; + break; + } + } + } + // M1: verify every running sandbox's fence on an + // interval and re-fence drift, so a fence deleted or + // widened under a Ready sandbox is contained by the + // namespace-wide default-deny and restored within one + // interval. + _ = reconcile_tick.tick() => { + // H1: restore a deleted isolation floor within one + // interval, then reconcile the per-sandbox fences. + // Managed mode reconciles every namespace that + // holds a sandbox VM (the cluster-wide list + // groups them); shared mode reconciles the one + // sandbox namespace. + let reconcile_namespaces: Vec = match &reconcile_list_api { + Some(api) => { + match api.list(&ListParams::default().labels(&selector)).await { + Ok(list) => { + let mut namespaces: Vec = list + .items + .iter() + .filter_map(|object| { + object.metadata.namespace.clone() + }) + .collect(); + namespaces.sort(); + namespaces.dedup(); + namespaces + } + Err(_) => vec![reconcile_namespace.clone()], + } + } + None => vec![reconcile_namespace.clone()], + }; + for reconcile_namespace in reconcile_namespaces { + if let Err(error) = ensure_kubevirt_launcher_isolation_with( + &reconcile_client, + &reconcile_namespace, + ) + .await + { + warn!( + %error, + "KubeVirt reconciler could not ensure the launcher isolation floor" + ); + } + let _ = reconcile_kubevirt_fences( + &reconcile_client, + &reconcile_namespace, + &reconcile_config, + ) + .await; + // M6 (option A): digest-pin the guest image once + // the launcher has pulled it. + reconcile_kubevirt_image_pins(&reconcile_client, &reconcile_namespace) + .await; + } + // M2: re-emit every sandbox's snapshot so a + // dedicated supervisor Pod deleted under a quiet + // Ready sandbox surfaces as Error within one + // interval (nothing else fires a VM event for a + // Pod deletion). The list is cluster-wide in + // managed mode and each VM's own namespace is + // used for the emit. + let vm_api = match &reconcile_list_api { + Some(api) => api.clone(), + None => crate::kubevirt::virtual_machine_api( + &reconcile_client, + &reconcile_namespace, + ), + }; + let selector = openshell_sandbox_label_selector(); + if let Ok(list) = vm_api + .list(&ListParams::default().labels(&selector)) + .await + { + for vm in list.items { + let vm_namespace = vm + .metadata + .namespace + .clone() + .unwrap_or_else(|| reconcile_namespace.clone()); + if !emit( + vm, + &vm_namespace, + &vmi_phases, + &watch_client, + &tx, + + ) + .await + { + break; + } + } + } + } + () = tx.closed() => break, + } + } + }); + + Ok(Box::pin(ReceiverStream::new(rx))) + } + + #[tracing::instrument( + name = "kubernetes.stop_sandbox", + skip(self), + fields( + otel.name = "kubernetes.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let _guard = self.lifecycle_gates.gate_for(sandbox_id).lock_owned().await; + let result = Box::pin(self.stop_sandbox_inner(sandbox_id)).await; + span_status.finish(result) + } + + async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self.stop_kubevirt_sandbox(sandbox_id).await; + } + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase, placement) = + self.prepare_sandbox_stop(sandbox_id).await?; + if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) { + self.release_generation_supervisor(sandbox_id, &namespace, &placement) + .await?; + patch_dynamic_object_with_resource_version_retry( + &agent_sandbox_api.api, + &kube_name, + |version| { + sandbox_runtime_suspension_patch(&agent_sandbox_api.resource.version, version) + }, + ) + .await?; + } + let pod_api = Api::::namespaced(self.client.clone(), &namespace); + + let deadline = tokio::time::Instant::now() + stop_timeout; + let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); + let object = tokio::time::timeout( + request_timeout, + agent_sandbox_api.api.get(&kube_name), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox stop", + request_timeout.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if let Some(error) = kubernetes_sandbox_stop_failure(&object) { + return Err(KubernetesDriverError::Message(error)); + } + let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)?; + let stop_is_complete = kubernetes_sandbox_stop_is_complete( + &agent_sandbox_api.resource.version, + &object, + pod_is_gone, + ); + if stop_is_complete { + self.delete_sandbox_runtime_supervisor( + sandbox_id, + &namespace, + sandbox_runtime_generation(&object).as_slice(), + ) + .await?; + patch_dynamic_object_with_resource_version_retry( + &agent_sandbox_api.api, + &kube_name, + sandbox_runtime_suspension_completion_patch, + ) + .await?; + return Ok(()); + } + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; + poll_interval = next_stop_poll_interval(poll_interval); + } + } + + #[tracing::instrument( + name = "kubernetes.start_sandbox", + skip_all, + fields( + otel.name = "kubernetes.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn start_sandbox( + &self, + sandbox_id: &str, + generation_id: &str, + launch_authentication: &[u8], + expected_runtime_identity: &str, + ) -> Result { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let _guard = self.lifecycle_gates.gate_for(sandbox_id).lock_owned().await; + let result = Box::pin(self.start_sandbox_runtime_generation( + sandbox_id, + generation_id, + launch_authentication, + expected_runtime_identity, + )) + .await; + span_status.finish(result) + } + + #[allow(clippy::similar_names)] + async fn start_sandbox_runtime_generation( + &self, + sandbox_id: &str, + encoded_generation: &str, + encoded_authentication: &[u8], + encoded_expected_runtime_identity: &str, + ) -> Result { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self + .start_kubevirt_sandbox( + sandbox_id, + encoded_authentication, + encoded_expected_runtime_identity, + ) + .await; + } + let staged = self.read_staged_bootstrap_material().await?; + let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( + encoded_generation.to_string(), + ) + .map_err(|error| KubernetesDriverError::InvalidArgument(error.to_string()))?; + let launch_authentication = decode_launch_authentication(encoded_authentication)?; + let expected_runtime_identity = + parse_kubernetes_runtime_identity(encoded_expected_runtime_identity)?; + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let objects = lookup_api + .api + .list(&ListParams::default().labels(&selector)) + .await + .map_err(KubernetesDriverError::from_kube)? + .items; + let mut object = select_expected_sandbox_runtime(objects, &expected_runtime_identity)?; + self.admit_stored_resources(&object) + .await + .map_err(admission_error)?; + if sandbox_runtime_bootstrap_in_progress(&object) { + let phase = sandbox_runtime_bootstrap_phase(&object); + if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) + && sandbox_runtime_bootstrap_operation(&object) != Some("restart") + { + return Err(KubernetesDriverError::Precondition( + "initial sandbox bootstrap has not completed; wait for reconciliation" + .to_string(), + )); + } + if sandbox_runtime_generation(&object) == Some(generation.as_str()) + && sandbox_runtime_bootstrap_phase(&object) + == Some(SandboxRuntimeBootstrapPhase::Released) + { + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + if generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + sandbox_id, + &object, + ) + .await + == SandboxRuntimeControlAvailability::Available + && sandbox_runtime_bootstrap_is_ready_to_complete(&object) + { self.complete_sandbox_runtime_bootstrap(&lookup_api, &object) .await; let cr_uid = object.metadata.uid.as_deref().ok_or_else(|| { @@ -4383,6 +7797,7 @@ impl KubernetesComputeDriver { // generation that follows one creates both. if placement == SupervisorPlacement::Dedicated { let owner = sandbox_runtime_sandbox_owner_reference( + "Sandbox", cr_name, cr_uid, &sandbox_api.resource.api_version, @@ -4461,6 +7876,7 @@ impl KubernetesComputeDriver { sandbox_id, &sandbox_name, sandbox_runtime_sandbox_owner_reference( + "Sandbox", cr_name, cr_uid, &sandbox_api.resource.api_version, @@ -4796,6 +8212,9 @@ impl KubernetesComputeDriver { ) )] pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self.delete_kubevirt_sandbox(sandbox_id).await; + } let span_status = openshell_otel::ErrorStatusGuard::current(); let _guard = self.lifecycle_gates.gate_for(sandbox_id).lock_owned().await; let result = self.delete_sandbox_inner(sandbox_id).await; @@ -4950,6 +8369,12 @@ impl KubernetesComputeDriver { } pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self + .get_kubevirt_sandbox(sandbox_id) + .await + .map(|sandbox| sandbox.is_some()); + } let agent_sandbox_api = self .supported_sandbox_api_for_lookup(self.client.clone()) .await?; @@ -5703,6 +9128,9 @@ impl KubernetesComputeDriver { // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + if crate::kubevirt::is_kubevirt_mode(&self.config) { + return self.watch_sandboxes_kubevirt().await; + } self.reconcile_sandbox_runtime_resources().await; if self.config.is_multi_namespace() { self.watch_sandboxes_cluster_wide().await @@ -5898,22 +9326,16 @@ fn shared_instance_registration( launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, ) -> Result { - Ok(SandboxInstanceRegistration { - route: route.to_string(), - sandbox_id: sandbox_id.to_string(), - sandbox_name: annotation_or_label(object, LABEL_SANDBOX_NAME).ok_or_else(|| { + Ok(shared_instance_registration_from_values( + route, + sandbox_id, + &annotation_or_label(object, LABEL_SANDBOX_NAME).ok_or_else(|| { KubernetesDriverError::Precondition( "Sandbox resource is missing its OpenShell sandbox name".to_string(), ) })?, - runtime_descriptor, - auth: launch_authentication.supervisor.clone(), - main_process_spec: required_sandbox_annotation( - object, - ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC, - )?, - workdir: Some(WORKSPACE_MOUNT_PATH.to_string()), - container_image: object + required_sandbox_annotation(object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?, + object .data .pointer("/spec/podTemplate/spec/containers") .and_then(serde_json::Value::as_array) @@ -5924,14 +9346,66 @@ fn shared_instance_registration( }) .and_then(|container| container.get("image")) .and_then(serde_json::Value::as_str) - .unwrap_or_default() - .to_string(), + .unwrap_or_default(), + runtime_descriptor, + launch_authentication.supervisor.clone(), + proxy_ca, + )) +} + +/// The pool's durable route record is removed asynchronously after a +/// withdrawal (the gateway reconciles every ~2s), so an immediate +/// stop -> start can race it: the launch is refused with "the sandbox +/// instance route is recorded for another generation". The `KubeVirt` +/// registration retries that specific refusal for a bounded window. +/// The message is pinned by a test in `openshell-server`'s supervisor-pool +/// suite; if the wording changes, that test fails and this constant needs +/// updating. +const ROUTE_RECORD_RACE_MESSAGE: &str = "route is recorded for another generation"; + +/// The same teardown race one phase later: the route's record still names +/// the previous generation while the withdrawal that stop issued is being +/// reconciled away (the supervisor has usually already removed the +/// instance). Pinned by `openshell-server`'s supervisor-pool suite the same +/// way. +const ROUTE_OWNER_RACE_MESSAGE: &str = "route is owned by another generation"; + +/// Whether a shared-instance launch refusal is the withdraw-teardown race: +/// the record of the route this sandbox owns is mid-reconciliation after a +/// stop, and a bounded retry re-launches once the pool has cleared it. +fn route_teardown_race(reason: &str) -> bool { + reason.contains(ROUTE_RECORD_RACE_MESSAGE) || reason.contains(ROUTE_OWNER_RACE_MESSAGE) +} + +/// The value-taking core of `shared_instance_registration`, shared by the +/// pod path (which extracts values from the Sandbox CR) and the `KubeVirt` +/// path (which has them from the gateway's `Sandbox` object directly). +#[allow(clippy::too_many_arguments)] +fn shared_instance_registration_from_values( + route: &str, + sandbox_id: &str, + sandbox_name: &str, + main_process_spec: String, + container_image: &str, + runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, + auth: openshell_core::jwt::SupervisorAuthBundle, + proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, +) -> SandboxInstanceRegistration { + SandboxInstanceRegistration { + route: route.to_string(), + sandbox_id: sandbox_id.to_string(), + sandbox_name: sandbox_name.to_string(), + runtime_descriptor, + auth, + main_process_spec, + workdir: Some(WORKSPACE_MOUNT_PATH.to_string()), + container_image: container_image.to_string(), proxy_ca: openshell_supervisor_control::ProxyCaMaterial { certificate_pem: proxy_ca.certificate_pem, private_key_pem: openshell_supervisor_control::SecretPem::new(proxy_ca.private_key_pem), }, trace_context: RegistrationTraceContext::current(), - }) + } } /// What bootstrap completion waits for before the runtime is declared ready. @@ -6338,7 +9812,33 @@ fn openshell_sandbox_selector_for(gateway_id: &str) -> String { selector } -fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { +/// The sandbox labels for a `KubeVirt` start, where only the id, name, +/// and workspace are known (the VM's labels carry them; the start +/// re-renders the same set for the new generation's companions). +pub(crate) fn kubevirt_restart_labels( + sandbox_id: &str, + sandbox_name: &str, + workspace: &str, + gateway_id: Option<&str>, +) -> BTreeMap { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox_id.to_string()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox_name.to_string()); + labels.insert(LABEL_SANDBOX_WORKSPACE.to_string(), workspace.to_string()); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + if let Some(gw_id) = gateway_id { + labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + } + labels +} + +pub(crate) fn sandbox_labels( + sandbox: &Sandbox, + gateway_id: Option<&str>, +) -> BTreeMap { let mut labels = BTreeMap::new(); labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); @@ -6448,7 +9948,7 @@ fn replace_sandbox_bootstrap_init_container( Ok(()) } -fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { +pub(crate) fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { let mut annotations = BTreeMap::new(); annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); annotations.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); @@ -8197,980 +11697,2163 @@ fn apply_pod_driver_config( merge_string_map(node_selector, &config.node_selector); } - if !config.priority_class_name.is_empty() { - spec.entry("priorityClassName".to_string()) - .or_insert_with(|| serde_json::json!(config.priority_class_name)); - } + if !config.priority_class_name.is_empty() { + spec.entry("priorityClassName".to_string()) + .or_insert_with(|| serde_json::json!(config.priority_class_name)); + } + + if !config.tolerations.is_empty() { + let tolerations = spec + .entry("tolerations".to_string()) + .or_insert_with(|| serde_json::json!([])); + if let Some(existing) = tolerations.as_array_mut() { + existing.extend(config.tolerations.iter().cloned()); + } else { + *tolerations = serde_json::Value::Array(config.tolerations.clone()); + } + } +} + +fn apply_agent_driver_resources( + container: &mut serde_json::Map, + resources: &KubernetesContainerResourceConfig, +) { + if resources.requests.is_empty() && resources.limits.is_empty() { + return; + } + + let target = container + .entry("resources".to_string()) + .or_insert_with(|| serde_json::json!({})); + apply_resource_quantity_map(target, "requests", &resources.requests); + apply_resource_quantity_map(target, "limits", &resources.limits); +} + +fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { + if !target.is_object() { + *target = serde_json::json!({}); + } + let target = target + .as_object_mut() + .expect("target was converted to object"); + for (key, value) in values { + target + .entry(key.clone()) + .or_insert_with(|| serde_json::json!(value)); + } +} + +fn apply_resource_quantity_map( + target: &mut serde_json::Value, + section: &str, + values: &BTreeMap, +) { + if values.is_empty() { + return; + } + if !target.is_object() { + *target = serde_json::json!({}); + } + let target = target + .as_object_mut() + .expect("target was converted to object"); + let section_value = target + .entry(section.to_string()) + .or_insert_with(|| serde_json::json!({})); + merge_string_map(section_value, values); +} + +fn image_pull_secret_refs(secrets: &[String]) -> Vec { + secrets + .iter() + .map(|secret| secret.trim()) + .filter(|secret| !secret.is_empty()) + .map(|secret| serde_json::json!({ "name": secret })) + .collect() +} + +fn container_resources( + template: &SandboxTemplate, + gpu_requirements: Option<&GpuResourceRequirements>, +) -> Option { + // Start from the raw resources passthrough in platform_config (preserves + // custom resource types like GPU limits that users set via the public API + // Struct), then overlay the typed DriverResourceRequirements on top. + let mut resources = + platform_config_struct(template, "resources_raw").unwrap_or_else(|| serde_json::json!({})); + + // Overlay typed CPU/memory from DriverResourceRequirements. + if let Some(ref req) = template.resources { + let obj = resources.as_object_mut().unwrap(); + let mut apply = |section: &str, key: &str, value: &str| { + if !value.is_empty() { + let sec = obj.entry(section).or_insert_with(|| serde_json::json!({})); + sec[key] = serde_json::json!(value); + } + }; + apply("limits", "cpu", &req.cpu_limit); + apply("limits", "memory", &req.memory_limit); + + let cpu_request = if req.cpu_request.is_empty() { + &req.cpu_limit + } else { + &req.cpu_request + }; + let memory_request = if req.memory_request.is_empty() { + &req.memory_limit + } else { + &req.memory_request + }; + apply("requests", "cpu", cpu_request); + apply("requests", "memory", memory_request); + } + + if let Some(gpu) = gpu_requirements { + let quantity = gpu.count.unwrap_or(1).to_string(); + apply_gpu_limit(&mut resources, &quantity); + } + if resources.as_object().is_some_and(serde_json::Map::is_empty) { + None + } else { + Some(resources) + } +} + +fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { + let Some(resources_obj) = resources.as_object_mut() else { + *resources = serde_json::json!({}); + return apply_gpu_limit(resources, quantity); + }; + + let limits = resources_obj + .entry("limits") + .or_insert_with(|| serde_json::json!({})); + let Some(limits_obj) = limits.as_object_mut() else { + *limits = serde_json::json!({}); + return apply_gpu_limit(resources, quantity); + }; + + limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); +} + +fn build_sandbox_env( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, +) -> Vec { + let mut env = Vec::new(); + for (name, value) in template_environment.iter().chain(spec_environment) { + if !name.starts_with("OPENSHELL_") || name == openshell_core::sandbox_env::LOG_LEVEL { + upsert_env(&mut env, name, value); + } + } + upsert_env( + &mut env, + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + env +} + +fn upsert_env(env: &mut Vec, name: &str, value: &str) { + if let Some(existing) = env + .iter_mut() + .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name)) + { + *existing = serde_json::json!({"name": name, "value": value}); + return; + } + + env.push(serde_json::json!({"name": name, "value": value})); +} + +fn apply_resolved_identity_env(env: &mut Vec, uid: u32, gid: u32) { + remove_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER); + remove_env(env, openshell_core::sandbox_env::SANDBOX_UID); + remove_env(env, openshell_core::sandbox_env::SANDBOX_GID); + upsert_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER, ""); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_UID, + &uid.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_GID, + &gid.to_string(), + ); +} + +fn remove_env(env: &mut Vec, name: &str) { + env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); +} + +/// Extract a string value from the template's `platform_config` Struct. +fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { + let config = template.platform_config.as_ref()?; + let value = config.fields.get(key)?; + match value.kind.as_ref() { + Some(prost_types::value::Kind::StringValue(s)) if !s.is_empty() => Some(s.clone()), + _ => None, + } +} + +/// Reject template pod annotations and labels whose keys are not in the +/// operator's `allowed_pod_annotations` / `allowed_pod_labels` lists. +fn validate_pod_metadata( + template: &SandboxTemplate, + config: &KubernetesComputeConfig, +) -> Result<(), String> { + if let Some(serde_json::Value::Object(annotations)) = + platform_config_struct(template, "annotations") + { + reject_disallowed_pod_metadata_keys( + "annotations", + "allowed_pod_annotations", + annotations.keys().map(String::as_str), + |key| config.pod_annotation_allowed(key), + )?; + } + reject_disallowed_pod_metadata_keys( + "labels", + "allowed_pod_labels", + template.labels.keys().map(String::as_str), + |key| config.pod_label_allowed(key), + ) +} + +fn reject_disallowed_pod_metadata_keys<'a>( + kind: &str, + field: &str, + keys: impl Iterator, + allowed: impl Fn(&str) -> bool, +) -> Result<(), String> { + let mut rejected: Vec<&str> = keys.filter(|key| !allowed(key)).collect(); + if rejected.is_empty() { + return Ok(()); + } + rejected.sort_unstable(); + Err(format!( + "pod {kind} not permitted by the gateway's {field}: {}", + rejected.join(", ") + )) +} + +/// Extract a nested Struct value from the template's `platform_config`, +/// converting it to `serde_json::Value`. +fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { + let config = template.platform_config.as_ref()?; + let value = config.fields.get(key)?; + let json = value_to_json(value); + // Return None for null/empty objects so callers can distinguish + // "field absent" from "field present but empty". + match &json { + serde_json::Value::Null => None, + serde_json::Value::Object(m) if m.is_empty() => None, + _ => Some(json), + } +} + +fn child_environment_from_sandbox_object( + object: &DynamicObject, +) -> std::collections::HashMap { + object + .data + .pointer("/spec/podTemplate/spec/containers/0/env") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| { + let name = entry.get("name")?.as_str()?; + let value = entry.get("value")?.as_str()?; + (!name.starts_with("OPENSHELL_")).then(|| (name.to_string(), value.to_string())) + }) + .collect() +} - if !config.tolerations.is_empty() { - let tolerations = spec - .entry("tolerations".to_string()) - .or_insert_with(|| serde_json::json!([])); - if let Some(existing) = tolerations.as_array_mut() { - existing.extend(config.tolerations.iter().cloned()); - } else { - *tolerations = serde_json::Value::Array(config.tolerations.clone()); - } - } +fn required_sandbox_annotation( + object: &DynamicObject, + name: &str, +) -> Result { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(name)) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "Sandbox resource is missing required annotation {name}" + )) + }) } -fn apply_agent_driver_resources( - container: &mut serde_json::Map, - resources: &KubernetesContainerResourceConfig, -) { - if resources.requests.is_empty() && resources.limits.is_empty() { - return; +fn condition_observes_current_generation( + object: &DynamicObject, + condition: &serde_json::Value, +) -> bool { + match ( + object.metadata.generation, + condition + .get("observedGeneration") + .and_then(serde_json::Value::as_i64), + ) { + (Some(generation), Some(observed_generation)) => generation == observed_generation, + // Older Agent Sandbox API versions can omit observedGeneration. + _ => true, } - - let target = container - .entry("resources".to_string()) - .or_insert_with(|| serde_json::json!({})); - apply_resource_quantity_map(target, "requests", &resources.requests); - apply_resource_quantity_map(target, "limits", &resources.limits); } -fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { - if !target.is_object() { - *target = serde_json::json!({}); - } - let target = target - .as_object_mut() - .expect("target was converted to object"); - for (key, value) in values { - target - .entry(key.clone()) - .or_insert_with(|| serde_json::json!(value)); - } +fn status_from_object(obj: &DynamicObject) -> Option { + let status = obj.data.get("status")?; + let status_obj = status.as_object()?; + + let conditions = status_obj + .get("conditions") + .and_then(|val| val.as_array()) + .map(|items| { + items + .iter() + .filter(|condition| condition_observes_current_generation(obj, condition)) + .filter_map(condition_from_value) + .collect::>() + }) + .unwrap_or_default(); + + Some(SandboxStatus { + name: status_obj + .get("sandboxName") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + instance_id: status_obj + .get("agentPod") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + agent_fd: status_obj + .get("agentFd") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + sandbox_fd: status_obj + .get("sandboxFd") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + conditions, + deleting: obj.metadata.deletion_timestamp.is_some(), + ..Default::default() + }) } -fn apply_resource_quantity_map( - target: &mut serde_json::Value, - section: &str, - values: &BTreeMap, -) { - if values.is_empty() { - return; +async fn create_or_validate_sandbox_runtime_fence( + policies: &Api, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + match tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(name)).await { + Ok(Ok(Some(existing))) => { + return validate_sandbox_runtime_fence(&existing, expected); + } + Ok(Ok(None)) => {} + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message( + "timed out reading sandbox-runtime workload fence".to_string(), + )); + } } - if !target.is_object() { - *target = serde_json::json!({}); + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.create(&PostParams::default(), expected), + ) + .await + { + Ok(Ok(_)) => Ok(()), + Ok(Err(KubeError::Api(error))) if error.code == 409 => { + let existing = tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out validating existing sandbox-runtime workload fence".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_sandbox_runtime_fence(&existing, expected) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message( + "timed out creating sandbox-runtime workload fence".to_string(), + )), } - let target = target - .as_object_mut() - .expect("target was converted to object"); - let section_value = target - .entry(section.to_string()) - .or_insert_with(|| serde_json::json!({})); - merge_string_map(section_value, values); } -fn image_pull_secret_refs(secrets: &[String]) -> Vec { - secrets - .iter() - .map(|secret| secret.trim()) - .filter(|secret| !secret.is_empty()) - .map(|secret| serde_json::json!({ "name": secret })) - .collect() +fn validate_sandbox_runtime_fence( + existing: &NetworkPolicy, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + if sandbox_runtime_fence_matches(existing, expected) { + Ok(()) + } else { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + Err(KubernetesDriverError::Precondition(format!( + "sandbox-runtime NetworkPolicy {name} exists but does not match the intended enforcement" + ))) + } } -fn container_resources( - template: &SandboxTemplate, - gpu_requirements: Option<&GpuResourceRequirements>, -) -> Option { - // Start from the raw resources passthrough in platform_config (preserves - // custom resource types like GPU limits that users set via the public API - // Struct), then overlay the typed DriverResourceRequirements on top. - let mut resources = - platform_config_struct(template, "resources_raw").unwrap_or_else(|| serde_json::json!({})); - - // Overlay typed CPU/memory from DriverResourceRequirements. - if let Some(ref req) = template.resources { - let obj = resources.as_object_mut().unwrap(); - let mut apply = |section: &str, key: &str, value: &str| { - if !value.is_empty() { - let sec = obj.entry(section).or_insert_with(|| serde_json::json!({})); - sec[key] = serde_json::json!(value); +fn sandbox_runtime_fence_matches(existing: &NetworkPolicy, expected: &NetworkPolicy) -> bool { + fn normalized_spec(mut spec: Option) -> Option { + if let Some(spec) = spec.as_mut() { + // The Kubernetes API server omits explicitly empty rule arrays when it + // persists a NetworkPolicy. For a policy type named in `policyTypes`, + // an omitted rule array and an empty rule array both deny all traffic. + if spec.egress.as_ref().is_some_and(Vec::is_empty) { + spec.egress = None; } - }; - apply("limits", "cpu", &req.cpu_limit); - apply("limits", "memory", &req.memory_limit); - - let cpu_request = if req.cpu_request.is_empty() { - &req.cpu_limit - } else { - &req.cpu_request - }; - let memory_request = if req.memory_request.is_empty() { - &req.memory_limit - } else { - &req.memory_request - }; - apply("requests", "cpu", cpu_request); - apply("requests", "memory", memory_request); + if spec.ingress.as_ref().is_some_and(Vec::is_empty) { + spec.ingress = None; + } + } + spec } - if let Some(gpu) = gpu_requirements { - let quantity = gpu.count.unwrap_or(1).to_string(); - apply_gpu_limit(&mut resources, &quantity); - } - if resources.as_object().is_some_and(serde_json::Map::is_empty) { - None - } else { - Some(resources) + fn contains_required_metadata( + actual: &Option>, + required: &Option>, + ) -> bool { + required.as_ref().is_none_or(|required| { + actual.as_ref().is_some_and(|actual| { + required + .iter() + .all(|(key, value)| actual.get(key) == Some(value)) + }) + }) } + + normalized_spec(existing.spec.clone()) == normalized_spec(expected.spec.clone()) + && contains_required_metadata(&existing.metadata.labels, &expected.metadata.labels) + && contains_required_metadata( + &existing.metadata.annotations, + &expected.metadata.annotations, + ) } -fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { - let Some(resources_obj) = resources.as_object_mut() else { - *resources = serde_json::json!({}); - return apply_gpu_limit(resources, quantity); - }; +fn sandbox_runtime_namespace_fence_generation_matches( + policy: &NetworkPolicy, + sandbox: &DynamicObject, +) -> bool { + sandbox_runtime_policy_generation_matches( + policy, + sandbox, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION, + ) +} - let limits = resources_obj - .entry("limits") - .or_insert_with(|| serde_json::json!({})); - let Some(limits_obj) = limits.as_object_mut() else { - *limits = serde_json::json!({}); - return apply_gpu_limit(resources, quantity); +/// Whether `policy` is the exact object and generation recorded on the +/// sandbox under the given annotation keys. +fn sandbox_runtime_policy_generation_matches( + policy: &NetworkPolicy, + sandbox: &DynamicObject, + uid_annotation: &str, + generation_annotation: &str, +) -> bool { + let Some(annotations) = sandbox.metadata.annotations.as_ref() else { + return false; }; - - limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); + policy.metadata.uid.as_deref() == annotations.get(uid_annotation).map(String::as_str) + && annotations + .get(generation_annotation) + .and_then(|generation| generation.parse::().ok()) + == policy.metadata.generation } -fn build_sandbox_env( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, -) -> Vec { - let mut env = Vec::new(); - for (name, value) in template_environment.iter().chain(spec_environment) { - if !name.starts_with("OPENSHELL_") || name == openshell_core::sandbox_env::LOG_LEVEL { - upsert_env(&mut env, name, value); - } - } - upsert_env( - &mut env, - openshell_core::sandbox_env::TELEMETRY_ENABLED, - openshell_core::telemetry::enabled_env_value(), - ); - env +fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { + obj.data + .get("status") + .and_then(|status| status.get("conditions")) + .and_then(serde_json::Value::as_array) + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition_observes_current_generation(obj, condition) + && condition.get("type").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_CONDITION) + && condition + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("true")) + }) + }) } -fn upsert_env(env: &mut Vec, name: &str, value: &str) { - if let Some(existing) = env - .iter_mut() - .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name)) - { - *existing = serde_json::json!({"name": name, "value": value}); - return; +fn kubernetes_sandbox_stop_is_complete( + api_version: &str, + obj: &DynamicObject, + pod_is_gone: bool, +) -> bool { + if api_version == SANDBOX_VERSION_V1ALPHA1 { + // v1alpha1 omits a usable stopped condition. + pod_is_gone + } else { + kubernetes_sandbox_has_stopped_condition(obj) && pod_is_gone } - - env.push(serde_json::json!({"name": name, "value": value})); } -fn apply_resolved_identity_env(env: &mut Vec, uid: u32, gid: u32) { - remove_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER); - remove_env(env, openshell_core::sandbox_env::SANDBOX_UID); - remove_env(env, openshell_core::sandbox_env::SANDBOX_GID); - upsert_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER, ""); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_UID, - &uid.to_string(), - ); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_GID, - &gid.to_string(), - ); -} +fn kubernetes_sandbox_stop_failure(obj: &DynamicObject) -> Option { + obj.data + .get("status")? + .get("conditions")? + .as_array()? + .iter() + .find_map(|condition| { + let is_terminal = condition_observes_current_generation(obj, condition) + && condition.get("type").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_CONDITION) + && condition + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("false")) + && condition.get("reason").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON); + if !is_terminal { + return None; + } -fn remove_env(env: &mut Vec, name: &str) { - env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); + let message = condition + .get("message") + .and_then(serde_json::Value::as_str) + .filter(|message| !message.is_empty()) + .unwrap_or("backing pod is not owned by this sandbox"); + Some(format!("Kubernetes sandbox stop rejected: {message}")) + }) } -/// Extract a string value from the template's `platform_config` Struct. -fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { - let config = template.platform_config.as_ref()?; - let value = config.fields.get(key)?; - match value.kind.as_ref() { - Some(prost_types::value::Kind::StringValue(s)) if !s.is_empty() => Some(s.clone()), - _ => None, +async fn kubernetes_sandbox_pod_is_gone( + pod_api: &Api, + pod_name: &str, + deadline: tokio::time::Instant, +) -> Result { + let request_timeout = + KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(tokio::time::Instant::now())); + if request_timeout.is_zero() { + return Ok(false); } -} -/// Reject template pod annotations and labels whose keys are not in the -/// operator's `allowed_pod_annotations` / `allowed_pod_labels` lists. -fn validate_pod_metadata( - template: &SandboxTemplate, - config: &KubernetesComputeConfig, -) -> Result<(), String> { - if let Some(serde_json::Value::Object(annotations)) = - platform_config_struct(template, "annotations") - { - reject_disallowed_pod_metadata_keys( - "annotations", - "allowed_pod_annotations", - annotations.keys().map(String::as_str), - |key| config.pod_annotation_allowed(key), - )?; + match tokio::time::timeout(request_timeout, pod_api.get(pod_name)).await { + Ok(Ok(_)) => Ok(false), + Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(true), + Ok(Err(err)) => Err(err.to_string()), + Err(_) => Err(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox pod termination", + request_timeout.as_secs() + )), } - reject_disallowed_pod_metadata_keys( - "labels", - "allowed_pod_labels", - template.labels.keys().map(String::as_str), - |key| config.pod_label_allowed(key), - ) } -fn reject_disallowed_pod_metadata_keys<'a>( - kind: &str, - field: &str, - keys: impl Iterator, - allowed: impl Fn(&str) -> bool, -) -> Result<(), String> { - let mut rejected: Vec<&str> = keys.filter(|key| !allowed(key)).collect(); - if rejected.is_empty() { - return Ok(()); - } - rejected.sort_unstable(); - Err(format!( - "pod {kind} not permitted by the gateway's {field}: {}", - rejected.join(", ") - )) -} +fn kubernetes_sandbox_stop_timeout(obj: &DynamicObject) -> Duration { + let termination_grace_period = obj + .data + .get("spec") + .and_then(|spec| spec.get("podTemplate")) + .and_then(|template| template.get("spec")) + .and_then(|spec| spec.get("terminationGracePeriodSeconds")) + .and_then(serde_json::Value::as_u64) + .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); -/// Extract a nested Struct value from the template's `platform_config`, -/// converting it to `serde_json::Value`. -fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { - let config = template.platform_config.as_ref()?; - let value = config.fields.get(key)?; - let json = value_to_json(value); - // Return None for null/empty objects so callers can distinguish - // "field absent" from "field present but empty". - match &json { - serde_json::Value::Null => None, - serde_json::Value::Object(m) if m.is_empty() => None, - _ => Some(json), - } + // The controller must observe the desired state, wait for the pod grace + // period and kubelet teardown, then reconcile the deleted pod into the + // Sandbox status. Keep one API timeout of headroom around that grace. + termination_grace_period.saturating_add(KUBE_API_TIMEOUT) } -fn child_environment_from_sandbox_object( - object: &DynamicObject, -) -> std::collections::HashMap { - object - .data - .pointer("/spec/podTemplate/spec/containers/0/env") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .filter_map(|entry| { - let name = entry.get("name")?.as_str()?; - let value = entry.get("value")?.as_str()?; - (!name.starts_with("OPENSHELL_")).then(|| (name.to_string(), value.to_string())) - }) - .collect() +fn kubernetes_pod_termination_timeout(pod: &Pod) -> Duration { + let default_grace = u64::try_from(SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS) + .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); + let termination_grace = pod + .spec + .as_ref() + .and_then(|spec| spec.termination_grace_period_seconds) + .and_then(|seconds| u64::try_from(seconds).ok()) + .map_or(default_grace, Duration::from_secs); + termination_grace.saturating_add(KUBE_API_TIMEOUT) } -fn required_sandbox_annotation( - object: &DynamicObject, - name: &str, -) -> Result { - object - .metadata - .annotations - .as_ref() - .and_then(|annotations| annotations.get(name)) - .cloned() - .ok_or_else(|| { - KubernetesDriverError::Precondition(format!( - "Sandbox resource is missing required annotation {name}" - )) - }) +fn next_stop_poll_interval(current: Duration) -> Duration { + current.saturating_mul(2).min(STOP_MAX_POLL_INTERVAL) } -fn condition_observes_current_generation( - object: &DynamicObject, - condition: &serde_json::Value, -) -> bool { - match ( - object.metadata.generation, - condition - .get("observedGeneration") - .and_then(serde_json::Value::as_i64), - ) { - (Some(generation), Some(observed_generation)) => generation == observed_generation, - // Older Agent Sandbox API versions can omit observedGeneration. - _ => true, +fn sandbox_operating_state_patch( + api_version: &str, + resource_version: &str, + running: bool, +) -> serde_json::Value { + if api_version == SANDBOX_VERSION_V1BETA1 { + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"operatingMode": "Running"} + }) + } else { + sandbox_runtime_suspension_patch(api_version, resource_version) + } + } else { + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"replicas": 1} + }) + } else { + sandbox_runtime_suspension_patch(api_version, resource_version) + } } } -fn status_from_object(obj: &DynamicObject) -> Option { - let status = obj.data.get("status")?; - let status_obj = status.as_object()?; +fn sandbox_runtime_stop_begin_patch(resource_version: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "stop", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Releasing.as_str(), + ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", + }, + }, + }) +} - let conditions = status_obj - .get("conditions") - .and_then(|val| val.as_array()) - .map(|items| { - items - .iter() - .filter(|condition| condition_observes_current_generation(obj, condition)) - .filter_map(condition_from_value) - .collect::>() - }) - .unwrap_or_default(); +fn sandbox_runtime_suspension_patch( + api_version: &str, + resource_version: &str, +) -> serde_json::Value { + let desired_state = if api_version == SANDBOX_VERSION_V1BETA1 { + serde_json::json!({"operatingMode": "Suspended"}) + } else { + serde_json::json!({"replicas": 0}) + }; + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "stop", + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Suspending.as_str(), + ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", + }, + }, + "spec": desired_state, + }) +} - Some(SandboxStatus { - name: status_obj - .get("sandboxName") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - instance_id: status_obj - .get("agentPod") +fn sandbox_runtime_suspension_completion_patch(resource_version: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_GENERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_TOPOLOGY: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_SESSION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: serde_json::Value::Null, + ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", + }, + } + }) +} + +fn condition_from_value(value: &serde_json::Value) -> Option { + let obj = value.as_object()?; + Some(SandboxCondition { + r#type: obj.get("type")?.as_str()?.to_string(), + status: obj.get("status")?.as_str()?.to_string(), + reason: obj + .get("reason") .and_then(|val| val.as_str()) .unwrap_or_default() .to_string(), - agent_fd: status_obj - .get("agentFd") + message: obj + .get("message") .and_then(|val| val.as_str()) .unwrap_or_default() .to_string(), - sandbox_fd: status_obj - .get("sandboxFd") + transition_time: obj + .get("lastTransitionTime") .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - conditions, - deleting: obj.metadata.deletion_timestamp.is_some(), - ..Default::default() + .and_then(|value| value.parse().ok()), }) } -async fn create_or_validate_sandbox_runtime_fence( - policies: &Api, - expected: &NetworkPolicy, -) -> Result<(), KubernetesDriverError> { - let name = expected.metadata.name.as_deref().unwrap_or_default(); - match tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(name)).await { - Ok(Ok(Some(existing))) => { - return validate_sandbox_runtime_fence(&existing, expected); - } - Ok(Ok(None)) => {} - Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), - Err(_) => { - return Err(KubernetesDriverError::Message( - "timed out reading sandbox-runtime workload fence".to_string(), - )); - } - } +fn spawn_namespace_label_watcher( + client: Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + let ns_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + let jitter_seed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_secs() ^ u64::from(duration.subsec_nanos()) + }); + + tokio::spawn(async move { + let mut retry_attempt = 0; + let mut relisted_names = std::collections::BTreeSet::new(); + loop { + let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + + loop { + let event = tokio::select! { + result = stream.try_next() => result, + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + match event { + Ok(Some(Event::Apply(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); + } + } + Ok(Some(Event::Delete(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); + } + } + Ok(Some(Event::Init)) => { + retry_attempt = 0; + relisted_names.clear(); + } + Ok(Some(Event::InitApply(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name { + relisted_names.insert(name); + } + } + Ok(Some(Event::InitDone)) => { + retry_attempt = 0; + let count = relisted_names.len(); + allowlist.replace(std::mem::take(&mut relisted_names)); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(err) => { + warn!(error = %err, "operator namespace watcher stream error"); + break; + } + } + } - match tokio::time::timeout( - KUBE_API_TIMEOUT, - policies.create(&PostParams::default(), expected), - ) - .await - { - Ok(Ok(_)) => Ok(()), - Ok(Err(KubeError::Api(error))) if error.code == 409 => { - let existing = tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(name)) - .await - .map_err(|_| { - KubernetesDriverError::Message( - "timed out validating existing sandbox-runtime workload fence".to_string(), - ) - })? - .map_err(KubernetesDriverError::from_kube)?; - validate_sandbox_runtime_fence(&existing, expected) + let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); + warn!(?retry_delay, "operator namespace watcher reconnecting"); + tokio::select! { + () = tokio::time::sleep(retry_delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + retry_attempt = retry_attempt.saturating_add(1); } - Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), - Err(_) => Err(KubernetesDriverError::Message( - "timed out creating sandbox-runtime workload fence".to_string(), - )), - } + }); + + info!( + label_selector = %label_selector, + "operator namespace label watcher spawned" + ); } -fn validate_sandbox_runtime_fence( - existing: &NetworkPolicy, - expected: &NetworkPolicy, -) -> Result<(), KubernetesDriverError> { - if sandbox_runtime_fence_matches(existing, expected) { - Ok(()) - } else { - let name = expected.metadata.name.as_deref().unwrap_or_default(); - Err(KubernetesDriverError::Precondition(format!( - "sandbox-runtime NetworkPolicy {name} exists but does not match the intended enforcement" - ))) - } +fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { + let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); + let max_jitter_secs = base_secs / 4; + let mixed_seed = + jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let jitter_secs = mixed_seed % (max_jitter_secs + 1); + Duration::from_secs(base_secs + jitter_secs) } -fn sandbox_runtime_fence_matches(existing: &NetworkPolicy, expected: &NetworkPolicy) -> bool { - fn normalized_spec(mut spec: Option) -> Option { - if let Some(spec) = spec.as_mut() { - // The Kubernetes API server omits explicitly empty rule arrays when it - // persists a NetworkPolicy. For a policy type named in `policyTypes`, - // an omitted rule array and an empty rule array both deny all traffic. - if spec.egress.as_ref().is_some_and(Vec::is_empty) { - spec.egress = None; - } - if spec.ingress.as_ref().is_some_and(Vec::is_empty) { - spec.ingress = None; - } +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher( + path: PathBuf, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + path = %path.display(), + total = count, + "operator namespace allowlist loaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to load initial operator namespace file, allowlist empty" + ); + // The file is the only source, so an unreadable file is an empty + // allowlist rather than one still loading. + allowlist.mark_synced(); } - spec } - fn contains_required_metadata( - actual: &Option>, - required: &Option>, - ) -> bool { - required.as_ref().is_none_or(|required| { - actual.as_ref().is_some_and(|actual| { - required - .iter() - .all(|(key, value)| actual.get(key) == Some(value)) - }) - }) - } + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let debounce = Duration::from_secs(1); - normalized_spec(existing.spec.clone()) == normalized_spec(expected.spec.clone()) - && contains_required_metadata(&existing.metadata.labels, &expected.metadata.labels) - && contains_required_metadata( - &existing.metadata.annotations, - &expected.metadata.annotations, - ) -} + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); -fn sandbox_runtime_namespace_fence_generation_matches( - policy: &NetworkPolicy, - sandbox: &DynamicObject, -) -> bool { - sandbox_runtime_policy_generation_matches( - policy, - sandbox, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION, - ) -} + let mut watcher = + match notify::recommended_watcher(move |res: Result| { + if let Ok(event) = res + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }) { + Ok(w) => w, + Err(e) => { + warn!( + error = %e, + "failed to start operator namespace file watcher, hot-reload disabled" + ); + return; + } + }; -/// Whether `policy` is the exact object and generation recorded on the -/// sandbox under the given annotation keys. -fn sandbox_runtime_policy_generation_matches( - policy: &NetworkPolicy, - sandbox: &DynamicObject, - uid_annotation: &str, - generation_annotation: &str, -) -> bool { - let Some(annotations) = sandbox.metadata.annotations.as_ref() else { - return false; - }; - policy.metadata.uid.as_deref() == annotations.get(uid_annotation).map(String::as_str) - && annotations - .get(generation_annotation) - .and_then(|generation| generation.parse::().ok()) - == policy.metadata.generation -} + if let Err(e) = notify::Watcher::watch( + &mut watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!( + error = %e, + dir = %watch_dir.display(), + "failed to watch operator namespace file directory, hot-reload disabled" + ); + return; + } -fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { - obj.data - .get("status") - .and_then(|status| status.get("conditions")) - .and_then(serde_json::Value::as_array) - .is_some_and(|conditions| { - conditions.iter().any(|condition| { - condition_observes_current_generation(obj, condition) - && condition.get("type").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_CONDITION) - && condition - .get("status") - .and_then(serde_json::Value::as_str) - .is_some_and(|status| status.eq_ignore_ascii_case("true")) - }) - }) -} + info!( + path = %path.display(), + "operator namespace file watcher started" + ); -fn kubernetes_sandbox_stop_is_complete( - api_version: &str, - obj: &DynamicObject, - pod_is_gone: bool, -) -> bool { - if api_version == SANDBOX_VERSION_V1ALPHA1 { - // v1alpha1 omits a usable stopped condition. - pod_is_gone - } else { - kubernetes_sandbox_has_stopped_condition(obj) && pod_is_gone - } -} + loop { + let got_event = tokio::select! { + event = rx.recv() => event.is_some(), + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + if !got_event { + warn!("operator namespace file watcher disconnected"); + break; + } -fn kubernetes_sandbox_stop_failure(obj: &DynamicObject) -> Option { - obj.data - .get("status")? - .get("conditions")? - .as_array()? - .iter() - .find_map(|condition| { - let is_terminal = condition_observes_current_generation(obj, condition) - && condition.get("type").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_CONDITION) - && condition - .get("status") - .and_then(serde_json::Value::as_str) - .is_some_and(|status| status.eq_ignore_ascii_case("false")) - && condition.get("reason").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON); - if !is_terminal { - return None; + loop { + tokio::select! { + () = tokio::time::sleep(debounce) => { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist reloaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to reload operator namespace file, keeping existing allowlist" + ); + } + } + break; + } + r = rx.recv() => { + if r.is_some() { + continue; + } + warn!("operator namespace file watcher disconnected"); + return; + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } } - - let message = condition - .get("message") - .and_then(serde_json::Value::as_str) - .filter(|message| !message.is_empty()) - .unwrap_or("backing pod is not owned by this sandbox"); - Some(format!("Kubernetes sandbox stop rejected: {message}")) - }) + } + }); } -async fn kubernetes_sandbox_pod_is_gone( - pod_api: &Api, - pod_name: &str, - deadline: tokio::time::Instant, -) -> Result { - let request_timeout = - KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(tokio::time::Instant::now())); - if request_timeout.is_zero() { - return Ok(false); - } +#[cfg(test)] +mod tests { + use super::*; - match tokio::time::timeout(request_timeout, pod_api.get(pod_name)).await { - Ok(Ok(_)) => Ok(false), - Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(true), - Ok(Err(err)) => Err(err.to_string()), - Err(_) => Err(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox pod termination", - request_timeout.as_secs() - )), + /// Serve Pod GETs over one reusable HTTP/1 connection per client, + /// delaying a request for Pod `slow-` by that many milliseconds. + async fn serve_delayed_pods() -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(hyper::server::conn::http1::Builder::new().serve_connection( + hyper_util::rt::TokioIo::new(stream), + hyper::service::service_fn(|request: http::Request<_>| async move { + let delay = request + .uri() + .path() + .rsplit_once("/slow-") + .and_then(|(_, millis)| millis.parse().ok()) + .map_or(Duration::ZERO, Duration::from_millis); + tokio::time::sleep(delay).await; + Ok::<_, std::convert::Infallible>(http::Response::new( + http_body_util::Full::new(bytes::Bytes::from_static( + br#"{"apiVersion":"v1","kind":"Pod","metadata":{"name":"pod"}}"#, + )), + )) + }), + )); + } + }); + addr } -} - -fn kubernetes_sandbox_stop_timeout(obj: &DynamicObject) -> Duration { - let termination_grace_period = obj - .data - .get("spec") - .and_then(|spec| spec.get("podTemplate")) - .and_then(|template| template.get("spec")) - .and_then(|spec| spec.get("terminationGracePeriodSeconds")) - .and_then(serde_json::Value::as_u64) - .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); - - // The controller must observe the desired state, wait for the pod grace - // period and kubelet teardown, then reconcile the deleted pod into the - // Sandbox status. Keep one API timeout of headroom around that grace. - termination_grace_period.saturating_add(KUBE_API_TIMEOUT) -} - -fn kubernetes_pod_termination_timeout(pod: &Pod) -> Duration { - let default_grace = u64::try_from(SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS) - .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); - let termination_grace = pod - .spec - .as_ref() - .and_then(|spec| spec.termination_grace_period_seconds) - .and_then(|seconds| u64::try_from(seconds).ok()) - .map_or(default_grace, Duration::from_secs); - termination_grace.saturating_add(KUBE_API_TIMEOUT) -} -fn next_stop_poll_interval(current: Duration) -> Duration { - current.saturating_mul(2).min(STOP_MAX_POLL_INTERVAL) -} + // ------------------------------------------------------------------- + // Stateful mock Kubernetes API (KubeVirt behavioural tests) + // ------------------------------------------------------------------- + + /// A stateful in-memory Kubernetes API: enough of GET / LIST / POST / + /// `PATCH` / `DELETE` for the `KubeVirt` create, start, and delete paths, + /// with request logging, `Service` `ClusterIP` allocation, owner-reference + /// garbage collection, and single-shot failure injection. + #[derive(Clone, Default)] + struct MockKube { + inner: Arc>, + } + + /// An armed `swap_on_get` hook: swap `replacement` in when the + /// `target`-th GET of `path` arrives after arming. + struct MockSwapOnGet { + path: String, + target: u32, + seen: u32, + replacement: serde_json::Value, + } + + #[derive(Default)] + struct MockKubeInner { + objects: std::collections::HashMap, + requests: Vec, + fail_next: Option<(String, String)>, + counter: u64, + cluster_ip: u32, + /// Armed swaps: on the Nth exact-path GET *after arming*, swap + /// the stored object for the replacement before responding + /// (simulates an out-of-band replacement between reads). + swap_on_get: Vec, + /// Paths whose stored resourceVersion is bumped after each GET + /// (simulates a concurrent writer racing the read-modify-write). + bump_rv_after_get: Vec, + /// When set, DELETE does not cascade owner-referenced dependents + /// (simulates the real API's asynchronous garbage collector, so + /// tests can exercise the explicit companion deletes). + disable_gc: bool, + } + + impl MockKube { + fn seed(&self, path: &str, object: serde_json::Value) { + self.inner + .lock() + .unwrap() + .objects + .insert(path.to_string(), object); + } -fn sandbox_operating_state_patch( - api_version: &str, - resource_version: &str, - running: bool, -) -> serde_json::Value { - if api_version == SANDBOX_VERSION_V1BETA1 { - if running { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"operatingMode": "Running"} - }) - } else { - sandbox_runtime_suspension_patch(api_version, resource_version) + fn requests(&self) -> Vec { + self.inner.lock().unwrap().requests.clone() } - } else { - if running { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"replicas": 1} - }) - } else { - sandbox_runtime_suspension_patch(api_version, resource_version) + + /// Fail the next request whose method matches and whose path + /// contains `path_contains` with a 500, once. + fn fail_next(&self, method: &str, path_contains: &str) { + self.inner.lock().unwrap().fail_next = + Some((method.to_string(), path_contains.to_string())); } - } -} -fn sandbox_runtime_stop_begin_patch(resource_version: &str) -> serde_json::Value { - serde_json::json!({ - "metadata": { - "resourceVersion": resource_version, - "annotations": { - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "stop", - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Releasing.as_str(), - ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", - }, - }, - }) -} + fn exists(&self, path: &str) -> bool { + self.inner.lock().unwrap().objects.contains_key(path) + } -fn sandbox_runtime_suspension_patch( - api_version: &str, - resource_version: &str, -) -> serde_json::Value { - let desired_state = if api_version == SANDBOX_VERSION_V1BETA1 { - serde_json::json!({"operatingMode": "Suspended"}) - } else { - serde_json::json!({"replicas": 0}) - }; - serde_json::json!({ - "metadata": { - "resourceVersion": resource_version, - "annotations": { - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: "stop", - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Suspending.as_str(), - ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", - }, - }, - "spec": desired_state, - }) -} + /// On the `occurrence`-th GET of `path` after this call (1-based), + /// replace the stored object with `replacement` before responding. + fn swap_on_get(&self, path: &str, occurrence: u32, replacement: serde_json::Value) { + self.inner.lock().unwrap().swap_on_get.push(MockSwapOnGet { + path: path.to_string(), + target: occurrence, + seen: 0, + replacement, + }); + } -fn sandbox_runtime_suspension_completion_patch(resource_version: &str) -> serde_json::Value { - serde_json::json!({ - "metadata": { - "resourceVersion": resource_version, - "annotations": { - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_GENERATION: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: serde_json::Value::Null, - ANNOTATION_SUPERVISOR_TOPOLOGY: serde_json::Value::Null, - ANNOTATION_SUPERVISOR_SESSION: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: serde_json::Value::Null, - ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", - }, + /// Bump the stored resourceVersion of `path` after every GET, so + /// a value just read is stale by the time a patch lands with it. + fn bump_rv_after_get(&self, path: &str) { + self.inner + .lock() + .unwrap() + .bump_rv_after_get + .push(path.to_string()); } - }) -} -fn condition_from_value(value: &serde_json::Value) -> Option { - let obj = value.as_object()?; - Some(SandboxCondition { - r#type: obj.get("type")?.as_str()?.to_string(), - status: obj.get("status")?.as_str()?.to_string(), - reason: obj - .get("reason") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - message: obj - .get("message") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - transition_time: obj - .get("lastTransitionTime") - .and_then(|val| val.as_str()) - .and_then(|value| value.parse().ok()), - }) -} + /// Disable the mock's synchronous owner-reference garbage + /// collection (the real collector is asynchronous). + fn disable_gc(&self) { + self.inner.lock().unwrap().disable_gc = true; + } -fn spawn_namespace_label_watcher( - client: Client, - label_selector: String, - allowlist: OperatorNamespaceAllowlist, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - let ns_api: Api = Api::all(client); - let watcher_config = watcher::Config::default().labels(&label_selector); - let jitter_seed = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |duration| { - duration.as_secs() ^ u64::from(duration.subsec_nanos()) - }); + /// The currently stored object at `path`, if any. + fn object(&self, path: &str) -> Option { + self.inner.lock().unwrap().objects.get(path).cloned() + } + } - tokio::spawn(async move { - let mut retry_attempt = 0; - let mut relisted_names = std::collections::BTreeSet::new(); - loop { - let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + fn mock_kube_kind(path: &str) -> (&'static str, &'static str) { + // (apiVersion, kind) derived from the request path; DynamicObject + // responses require TypeMeta. + let plural = path.rsplit('/').next().unwrap_or_default(); + let api_version = if path.starts_with("/apis/") { + let mut segments = path.trim_start_matches("/apis/").split('/'); + let group = segments.next().unwrap_or_default(); + let version = segments.next().unwrap_or_default(); + Box::leak(format!("{group}/{version}").into_boxed_str()) + } else { + "v1" + }; + let kind = match plural { + "virtualmachines" => "VirtualMachine", + "virtualmachineinstances" => "VirtualMachineInstance", + "networkpolicies" => "NetworkPolicy", + "secrets" => "Secret", + "pods" => "Pod", + "services" => "Service", + "persistentvolumeclaims" => "PersistentVolumeClaim", + "namespaces" => "Namespace", + "configmaps" => "ConfigMap", + _ => "Unknown", + }; + (api_version, kind) + } - loop { - let event = tokio::select! { - result = stream.try_next() => result, - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - continue; - } - }; - match event { - Ok(Some(Event::Apply(ns))) => { - retry_attempt = 0; - if let Some(name) = ns.metadata.name.as_deref() - && allowlist.insert(name.to_string()) - { - info!(namespace = name, "operator namespace added to allowlist"); - } + fn merge_json(base: &mut serde_json::Value, patch: &serde_json::Value) { + if let (serde_json::Value::Object(base), serde_json::Value::Object(patch)) = (base, patch) { + for (key, value) in patch { + if value.is_null() { + // JSON merge patch semantics: a null value removes the + // key (the real API server drops null-valued + // annotations). + base.remove(key); + continue; + } + match (base.get_mut(key), value) { + (Some(serde_json::Value::Object(_)), serde_json::Value::Object(_)) => { + merge_json(base.get_mut(key).expect("checked"), value); } - Ok(Some(Event::Delete(ns))) => { - retry_attempt = 0; - if let Some(name) = ns.metadata.name.as_deref() - && allowlist.remove(name) - { - info!( - namespace = name, - "operator namespace removed from allowlist" - ); - } + _ => { + base.insert(key.clone(), value.clone()); } - Ok(Some(Event::Init)) => { - retry_attempt = 0; - relisted_names.clear(); + } + } + } + } + + fn mock_percent_decode(value: &str) -> String { + let mut decoded = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(character) = chars.next() { + match character { + '%' => { + let hex: String = chars.by_ref().take(2).collect(); + if let Ok(byte) = u8::from_str_radix(&hex, 16) { + decoded.push(byte as char); } - Ok(Some(Event::InitApply(ns))) => { - retry_attempt = 0; - if let Some(name) = ns.metadata.name { - relisted_names.insert(name); + } + '+' => decoded.push(' '), + other => decoded.push(other), + } + } + decoded + } + + fn mock_kube_response_with_ok( + body: &serde_json::Value, + ) -> http::Response> { + mock_kube_response(200, body) + } + + fn mock_kube_response( + status: u16, + body: &serde_json::Value, + ) -> http::Response> { + http::Response::builder() + .status(status) + .header("content-type", "application/json") + .body(http_body_util::Full::new(bytes::Bytes::from( + body.to_string(), + ))) + .expect("static response builds") + } + + fn mock_kube_status(code: u16, reason: &str) -> serde_json::Value { + serde_json::json!({ + "kind": "Status", "apiVersion": "v1", + "status": "Failure", "reason": reason, "code": code, + }) + } + + async fn mock_kube_handle( + state: MockKube, + request: http::Request, + ) -> http::Response> { + use http_body_util::BodyExt; + let (parts, body) = request.into_parts(); + let payload = body.collect().await.expect("mock body reads").to_bytes(); + let method = parts.method.as_str().to_string(); + let path = parts.uri.path().to_string(); + let query = parts.uri.query().unwrap_or_default().to_string(); + let mut inner = state.inner.lock().unwrap(); + inner.requests.push(format!("{method} {path}")); + let body_text = String::from_utf8_lossy(&payload).to_string(); + if let Some((fail_method, contains)) = inner.fail_next.clone() + && method == fail_method + && (path.contains(&contains) || body_text.contains(&contains)) + { + inner.fail_next = None; + return mock_kube_response(500, &mock_kube_status(500, "MockFailure")); + } + let counter = inner.counter; + inner.counter += 1; + match method.as_str() { + "GET" if query.is_empty() => { + // H5 hooks: a configured swap replaces the stored object + // first (an out-of-band replacement), and a configured + // bump invalidates the resourceVersion this response + // carries (a concurrent writer). + let mut swap_index = None; + for (index, hook) in inner.swap_on_get.iter_mut().enumerate() { + if hook.path == path { + hook.seen += 1; + if hook.seen == hook.target { + swap_index = Some(index); } } - Ok(Some(Event::InitDone)) => { - retry_attempt = 0; - let count = relisted_names.len(); - allowlist.replace(std::mem::take(&mut relisted_names)); - info!( - total = count, - "operator namespace allowlist replaced from full relist" - ); + } + if let Some(index) = swap_index { + let hook = inner.swap_on_get.remove(index); + inner.objects.insert(path.clone(), hook.replacement); + } + let found = inner.objects.get(&path).cloned(); + if found.is_some() + && let Some(index) = inner + .bump_rv_after_get + .iter() + .position(|bump| bump == &path) + { + // One shot: invalidate the resourceVersion this GET is + // about to serve, then stop (a racing writer that + // touched the object once, not forever). + inner.bump_rv_after_get.remove(index); + if let Some(stored) = inner.objects.get_mut(&path) + && let Some(rv) = stored["metadata"]["resourceVersion"].as_str() + { + let bumped = format!("bumped-{rv}"); + stored["metadata"]["resourceVersion"] = serde_json::json!(bumped); } - Ok(None) => { - warn!("operator namespace watcher stream ended unexpectedly"); + } + found.map_or_else( + || mock_kube_response(404, &mock_kube_status(404, "NotFound")), + |object| mock_kube_response_with_ok(&object), + ) + } + // Listing requests always carry a query (limit or selector). + "GET" => { + let selector = query + .split('&') + .find_map(|pair| pair.strip_prefix("labelSelector=")) + .map(mock_percent_decode); + let required: Vec<(String, String)> = selector + .as_deref() + .map(|selector| { + selector + .split(',') + .filter_map(|pair| pair.split_once('=')) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }) + .unwrap_or_default(); + let prefix = format!("{path}/"); + let items: Vec = inner + .objects + .iter() + .filter(|(stored, _)| stored.starts_with(&prefix)) + .map(|(_, object)| object.clone()) + .filter(|object| { + required.iter().all(|(key, value)| { + object["metadata"]["labels"] + .get(key) + .and_then(serde_json::Value::as_str) + == Some(value.as_str()) + }) + }) + .collect(); + mock_kube_response( + 200, + &serde_json::json!({ + "apiVersion": "v1", "kind": "List", "items": items, + }), + ) + } + "POST" => { + let mut object: serde_json::Value = + serde_json::from_slice(&payload).unwrap_or_else(|_| serde_json::json!({})); + let (api_version, kind) = mock_kube_kind(&path); + object["apiVersion"] = serde_json::json!(api_version); + object["kind"] = serde_json::json!(kind); + let metadata = object + .as_object_mut() + .expect("posted objects are JSON objects") + .entry("metadata") + .or_insert_with(|| serde_json::json!({})); + metadata["uid"] = serde_json::json!(format!("uid-{counter}")); + metadata["resourceVersion"] = serde_json::json!(format!("rv-{counter}")); + metadata["generation"] = serde_json::json!(counter); + metadata["creationTimestamp"] = serde_json::json!("2026-01-01T00:00:00Z"); + // A create stores the object at the collection path plus + // its metadata name, as the real API does. + let name = metadata + .get("name") + .and_then(serde_json::Value::as_str) + .unwrap_or("unnamed") + .to_string(); + metadata["name"] = serde_json::json!(name); + // Simulate ClusterIP allocation for Services. + if kind == "Service" { + let ip = 10_433_000 + inner.cluster_ip; + inner.cluster_ip += 1; + let spec = object + .as_object_mut() + .expect("service is an object") + .entry("spec") + .or_insert_with(|| serde_json::json!({})); + spec["clusterIP"] = + serde_json::json!(format!("10.43.{}.{}", (ip / 256) % 256, ip % 256)); + } + let stored_path = format!("{path}/{name}"); + inner.objects.insert(stored_path, object.clone()); + mock_kube_response(201, &object) + } + "PATCH" => { + let Some(object) = inner.objects.get_mut(&path) else { + return mock_kube_response(404, &mock_kube_status(404, "NotFound")); + }; + let merge_patch: serde_json::Value = + serde_json::from_slice(&payload).unwrap_or_else(|_| serde_json::json!({})); + // H5: the real API server rejects a patch whose pinned + // metadata.uid or metadata.resourceVersion does not match + // the stored object (a replaced or concurrently modified + // object); the mock enforces the same contract. + if let Some(patch_uid) = merge_patch["metadata"]["uid"].as_str() + && object["metadata"]["uid"] + .as_str() + .is_some_and(|stored| stored != patch_uid) + { + return mock_kube_response(409, &mock_kube_status(409, "Conflict")); + } + if let Some(patch_rv) = merge_patch["metadata"]["resourceVersion"].as_str() + && object["metadata"]["resourceVersion"] + .as_str() + .is_some_and(|stored| stored != patch_rv) + { + return mock_kube_response(409, &mock_kube_status(409, "Conflict")); + } + merge_json(object, &merge_patch); + mock_kube_response(200, object) + } + "DELETE" => { + let Some(object) = inner.objects.remove(&path) else { + return mock_kube_response(404, &mock_kube_status(404, "NotFound")); + }; + // H5: DeleteOptions.preconditions.uid from the request body + // must match the stored object's UID, as the real API + // server enforces (a replaced object under the same name + // is not deleted). + if let Ok(options) = serde_json::from_slice::(&payload) + && let Some(expected) = options["preconditions"]["uid"].as_str() + && object["metadata"]["uid"] + .as_str() + .is_some_and(|stored| stored != expected) + { + inner.objects.insert(path, object); + return mock_kube_response(409, &mock_kube_status(409, "Conflict")); + } + let mut deleted_uids = vec![object["metadata"]["uid"].clone()]; + // Simple transitive owner-reference garbage collection + // (skipped when the test simulates the real asynchronous + // collector): + // dependents of a deleted object go with it (as the real + // API does), so tests can observe the owner-reference + // cleanup (H1) — including second-level dependents like a + // supervisor Secret owned by the cascaded supervisor Pod. + loop { + if inner.disable_gc { break; } - Err(err) => { - warn!(error = %err, "operator namespace watcher stream error"); + let dependents: Vec<(String, serde_json::Value)> = inner + .objects + .iter() + .filter(|(_, candidate)| { + candidate["metadata"]["ownerReferences"] + .as_array() + .is_some_and(|owners| { + owners + .iter() + .any(|owner| deleted_uids.contains(&owner["uid"])) + }) + }) + .map(|(stored, candidate)| { + (stored.clone(), candidate["metadata"]["uid"].clone()) + }) + .collect(); + if dependents.is_empty() { break; } - } - } - - let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); - warn!(?retry_delay, "operator namespace watcher reconnecting"); - tokio::select! { - () = tokio::time::sleep(retry_delay) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; + for (dependent, uid) in dependents { + inner.objects.remove(&dependent); + deleted_uids.push(uid); } } + mock_kube_response( + 200, + &serde_json::json!({"kind": "Status", "status": "Success"}), + ) } - retry_attempt = retry_attempt.saturating_add(1); + _ => mock_kube_response(405, &mock_kube_status(405, "MethodNotAllowed")), } - }); + } - info!( - label_selector = %label_selector, - "operator namespace label watcher spawned" - ); -} + fn mock_kube_client(state: MockKube) -> Client { + let service = tower::service_fn(move |request: http::Request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(mock_kube_handle(state, request).await) } + }); + Client::new(service, "default") + } -fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { - let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); - let max_jitter_secs = base_secs / 4; - let mixed_seed = - jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); - let jitter_secs = mixed_seed % (max_jitter_secs + 1); - Duration::from_secs(base_secs + jitter_secs) -} + fn kubevirt_mock_config() -> KubernetesComputeConfig { + KubernetesComputeConfig { + runtime: crate::KubernetesRuntimeMode::KubeVirt, + namespace: "openshell-e2e-ns".to_string(), + gateway_id: "e2e-gateway".to_string(), + grpc_endpoint: "http://127.0.0.1:18085".to_string(), + kubevirt: crate::KubeVirtConfig { + guest_image: "openshell/sandbox-vm:test".to_string(), + ..Default::default() + }, + ..Default::default() + } + } -fn load_namespace_file(path: &Path) -> Result, String> { - let contents = std::fs::read_to_string(path) - .map_err(|e| format!("failed to read {}: {e}", path.display()))?; - let names: Vec = serde_json::from_str(&contents) - .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; - Ok(names.into_iter().collect()) -} + fn kubevirt_mock_driver(client: Client) -> KubernetesComputeDriver { + KubernetesComputeDriver { + client: client.clone(), + watch_client: client, + sandbox_api_version: Arc::new(OnceCell::new()), + lifecycle_gates: Arc::default(), + config: kubevirt_mock_config(), + network_trust_bundle: None, + operator_allowlist: None, + shared_supervisor: None, + } + } -fn spawn_namespace_file_watcher( - path: PathBuf, - allowlist: OperatorNamespaceAllowlist, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - match load_namespace_file(&path) { - Ok(names) => { - let count = names.len(); - allowlist.replace(names); - info!( - path = %path.display(), - total = count, - "operator namespace allowlist loaded from file" - ); + fn kubevirt_test_launch_authentication() -> openshell_core::jwt::SandboxLaunchAuthentication { + use openshell_core::jwt::{ + CredentialEpoch, SecretJwt, SessionRotation, SessionVerificationKey, + SupervisorAuthBundle, + }; + openshell_core::jwt::SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: SessionRotation::new(1).unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, + traces_otlp_headers: None, + ocsf_otlp_headers: None, + agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, + }, + gateway_id: "e2e-gateway".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "key-1".to_string(), + public_key_pem: b"public-key".to_vec(), + }], } - Err(err) => { - warn!( - error = %err, - "failed to load initial operator namespace file, allowlist empty" - ); - // The file is the only source, so an unreadable file is an empty - // allowlist rather than one still loading. - allowlist.mark_synced(); + } + + fn kubevirt_test_sandbox() -> Sandbox { + Sandbox { + id: "sbx-id".to_string(), + name: "demo".to_string(), + namespace: "openshell-e2e-ns".to_string(), + workspace: "default".to_string(), + status: None, + spec: Some(openshell_core::proto::compute::v1::DriverSandboxSpec { + launch_authentication: serde_json::to_vec(&kubevirt_test_launch_authentication()) + .unwrap(), + ..Default::default() + }), } } - let watch_dir = path - .parent() - .unwrap_or_else(|| Path::new(".")) - .to_path_buf(); - let debounce = Duration::from_secs(1); + /// H6: a GPU request in `KubeVirt` mode is rejected at admission, before + /// anything is created. + #[tokio::test] + async fn kubevirt_gpu_requests_are_rejected_before_create() { + use openshell_core::proto::compute::v1::{ + DriverSandboxSpec, GpuResourceRequirements, ResourceRequirements, + }; + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let mut sandbox = kubevirt_test_sandbox(); + sandbox.spec = Some(DriverSandboxSpec { + resource_requirements: Some(ResourceRequirements { + gpu: Some(GpuResourceRequirements { count: Some(2) }), + }), + ..Default::default() + }); + let error = driver + .create_sandbox(&sandbox) + .await + .expect_err("a GPU request must be refused in KubeVirt mode"); + assert!( + error.to_string().contains("GPU"), + "the error must name the GPU refusal: {error}" + ); + assert!( + !state.exists(MOCK_VM_PATH), + "nothing may be created for a refused GPU request" + ); + } - tokio::spawn(async move { - let (tx, mut rx) = mpsc::unbounded_channel(); + /// H6: resource admission runs in `KubeVirt` mode too — a sandbox whose + /// template carries caller driver config while the gateway disallows + /// it is refused before anything is created. + #[tokio::test] + async fn kubevirt_create_runs_resource_admission() { + use openshell_core::proto::compute::v1::DriverSandboxTemplate; + use prost_types::{Struct, Value, value::Kind}; + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + assert!( + !driver.config.allow_driver_config, + "the mock gateway disallows caller driver config" + ); + let mut sandbox = kubevirt_test_sandbox(); + sandbox.spec = Some(SandboxSpec { + template: Some(DriverSandboxTemplate { + driver_config: Some(Struct { + fields: std::iter::once(( + "key".to_string(), + Value { + kind: Some(Kind::StringValue("value".to_string())), + }, + )) + .collect(), + }), + ..Default::default() + }), + ..Default::default() + }); + let error = driver + .create_sandbox(&sandbox) + .await + .expect_err("driver config must be refused by admission"); + assert!( + error.to_string().contains("driver config"), + "the error must come from resource admission: {error}" + ); + assert!( + !state.exists(MOCK_VM_PATH), + "nothing may be created when admission refuses" + ); + } - let mut watcher = - match notify::recommended_watcher(move |res: Result| { - if let Ok(event) = res - && matches!( - event.kind, - notify::EventKind::Modify(_) | notify::EventKind::Create(_) - ) - { - let _ = tx.send(()); - } - }) { - Ok(w) => w, - Err(e) => { - warn!( - error = %e, - "failed to start operator namespace file watcher, hot-reload disabled" - ); - return; + /// H6: capabilities report GPU selection as unsupported in `KubeVirt` + /// mode and unchanged in pod mode. + #[tokio::test] + async fn kubevirt_capabilities_report_gpu_support_per_mode() { + let state = MockKube::default(); + let _ = state; // the capabilities call needs no cluster + let driver = kubevirt_mock_driver(mock_kube_client(state)); + let capabilities = driver.capabilities().expect("capabilities build"); + let gpu = capabilities + .resource_capabilities + .as_ref() + .and_then(|resources| resources.gpu.as_ref()) + .expect("gpu capabilities are reported"); + assert!( + !gpu.default_selection_supported && !gpu.count_selection_supported, + "KubeVirt mode must not advertise GPU selection: {gpu:?}" + ); + let cpu = capabilities + .resource_capabilities + .as_ref() + .and_then(|resources| resources.cpu.as_ref()) + .expect("cpu capabilities are reported"); + assert!(cpu.limit_supported, "CPU limits stay supported"); + let memory = capabilities + .resource_capabilities + .as_ref() + .and_then(|resources| resources.memory.as_ref()) + .expect("memory capabilities are reported"); + assert!(memory.limit_supported, "memory limits stay supported"); + + let mut pod_config = kubevirt_mock_config(); + pod_config.runtime = crate::config::KubernetesRuntimeMode::Pod; + let mut pod_driver = kubevirt_mock_driver(mock_kube_client(MockKube::default())); + pod_driver.config = pod_config; + let pod_capabilities = pod_driver.capabilities().expect("capabilities build"); + let pod_gpu = pod_capabilities + .resource_capabilities + .as_ref() + .and_then(|resources| resources.gpu.as_ref()) + .expect("gpu capabilities are reported"); + assert!( + pod_gpu.default_selection_supported && pod_gpu.count_selection_supported, + "pod mode keeps GPU selection: {pod_gpu:?}" + ); + } + + const MOCK_VM_PATH: &str = + "/apis/kubevirt.io/v1/namespaces/openshell-e2e-ns/virtualmachines/default--demo"; + + fn seed_kubevirt_namespace(state: &MockKube) { + state.seed( + "/api/v1/namespaces/openshell-e2e-ns", + serde_json::json!({"apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": "openshell-e2e-ns", "uid": "ns-uid-1"}}), + ); + } + + /// Every per-sandbox object the dedicated create provisions, for the + /// rollback assertions. + fn kubevirt_per_sandbox_paths() -> Vec<&'static str> { + vec![ + MOCK_VM_PATH, + "/apis/kubevirt.io/v1/namespaces/openshell-e2e-ns/virtualmachines/default--demo", + "/api/v1/namespaces/openshell-e2e-ns/services/os-supervisor-sbx-id", + "/apis/networking.k8s.io/v1/namespaces/openshell-e2e-ns/networkpolicies/os-control-sbx-id", + "/apis/networking.k8s.io/v1/namespaces/openshell-e2e-ns/networkpolicies/openshell-kubevirt-fence-sbx-id", + "/api/v1/namespaces/openshell-e2e-ns/pods/os-supervisor-sbx-id", + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation1", + "/api/v1/namespaces/openshell-e2e-ns/secrets/os-supervisor-sbx-id-generation1", + "/api/v1/namespaces/openshell-e2e-ns/persistentvolumeclaims/default--demo-workspace", + ] + } + + /// M3: a create that fails at any point after the Halted `VirtualMachine` + /// exists rolls back everything it made — companions by UID-guarded + /// delete, owner-referenced objects through the VM's cascading delete — + /// leaving no per-sandbox object behind. + #[tokio::test] + async fn kubevirt_create_rolls_back_at_each_failure_point() { + for failure in [ + // The dedicated supervisor Pod creation (the only pods POST). + ("POST", "/pods"), + // The egress fence creation (its name is unique to that body). + ("POST", "openshell-kubevirt-fence-sbx-id"), + // The guest bootstrap Secret creation (its userData key is + // unique to that body). + ("POST", "userData"), + // The workspace PVC creation (its access mode is unique to + // that body). + ("POST", "ReadWriteOnce"), + // The supervisor bootstrap Secret creation (its generation + // name is unique to that body). + ("POST", "os-supervisor-sbx-id-generation1"), + ] { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + state.fail_next(failure.0, failure.1); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let error = match driver.create_sandbox(&kubevirt_test_sandbox()).await { + Err(error) => error, + Ok(identity) => { + panic!("create unexpectedly succeeded at {failure:?}: {identity}") } }; - - if let Err(e) = notify::Watcher::watch( - &mut watcher, - &watch_dir, - notify::RecursiveMode::NonRecursive, - ) { - warn!( - error = %e, - dir = %watch_dir.display(), - "failed to watch operator namespace file directory, hot-reload disabled" + assert!(!error.to_string().is_empty()); + // The rollback removed every per-sandbox object: companions by + // UID-guarded delete, owner-referenced objects through the VM. + let leftovers: Vec<&str> = kubevirt_per_sandbox_paths() + .into_iter() + .filter(|path| state.exists(path)) + .collect(); + assert!( + leftovers.is_empty(), + "failure at {failure:?} left objects behind: {leftovers:?}; requests: {:?}", + state.requests() ); - return; } + } - info!( - path = %path.display(), - "operator namespace file watcher started" + /// H5: a VM replaced under the same name between the start's lookup + /// and its first patch is refused — the patch is pinned to the + /// checked UID, so the replacement is never booted by this start. + #[tokio::test] + async fn kubevirt_start_refuses_a_replaced_vm() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + // Simulate the stopped state start operates on: halt the VM the + // create left running. + let mut halted = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + halted["spec"]["runStrategy"] = serde_json::json!("Halted"); + state.seed(MOCK_VM_PATH, halted); + // Replace the VM (same name, new UID) after the start's lookup: + // the pinned patch's first read sees the replacement. + let mut replaced = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + replaced["metadata"]["uid"] = serde_json::json!("vm-uid-replaced"); + state.swap_on_get(MOCK_VM_PATH, 1, replaced); + let requests_before_start = state.requests().len(); + let authentication = serde_json::to_vec(&kubevirt_test_launch_authentication()).unwrap(); + let error = driver + .start_kubevirt_sandbox("sbx-id", &authentication, &created) + .await + .expect_err("a replaced VM must refuse the start"); + assert!( + error.to_string().contains("was replaced"), + "the error must name the replacement: {error}" ); + assert!( + error.to_string().contains("vm-uid-replaced"), + "the error must name the observed UID: {error}" + ); + // The replacement was never patched: no write reached the VM + // during the start. + let start_requests = &state.requests()[requests_before_start..]; + assert!( + !start_requests + .iter() + .any(|request| request.starts_with("PATCH") && request.contains("virtualmachines/")), + "no VM patch may land after a replacement is detected: {start_requests:?}" + ); + } - loop { - let got_event = tokio::select! { - event = rx.recv() => event.is_some(), - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - continue; - } - }; - if !got_event { - warn!("operator namespace file watcher disconnected"); - break; - } + /// H4: start never reads or patches a bootstrap Secret. A new + /// generation creates its own Secret under its generation-scoped + /// name, the VM's config-disk volume is pointed at it by the release + /// patch, and the previous generation's Secret is deleted by exact + /// name with a UID precondition taken from the companions + /// annotation. + #[tokio::test] + async fn kubevirt_start_creates_a_new_generation_secret_and_deletes_the_old() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + // The generation-1 Secret the create recorded. + let old_secret_path = + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation1"; + assert!( + state.exists(old_secret_path), + "the create provisions the generation-1 bootstrap Secret; stored: {:?}", + state.requests() + ); + // Simulate the stopped state start operates on: halt the VM the + // create left running. + let mut halted = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + halted["spec"]["runStrategy"] = serde_json::json!("Halted"); + state.seed(MOCK_VM_PATH, halted); + // Start a NEW generation: the authentication carries generation-2. + let mut authentication = kubevirt_test_launch_authentication(); + authentication.supervisor.runtime_generation = + openshell_core::sandbox_generation::SandboxGenerationId::parse("generation-2").unwrap(); + let encoded = serde_json::to_vec(&authentication).unwrap(); + driver + .start_kubevirt_sandbox("sbx-id", &encoded, &created) + .await + .expect("the generation-2 start succeeds"); + let requests = state.requests(); + // No Secret was ever read or patched — create and delete only. + assert!( + !requests.iter().any(|request| { + request.contains("/secrets/") + && (request.starts_with("GET") || request.starts_with("PATCH")) + }), + "the driver must never read or patch a Secret: {requests:?}" + ); + // The new generation's Secret was created and the VM points at it. + let new_secret_path = + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation2"; + assert!( + state.exists(new_secret_path), + "the generation-2 Secret exists" + ); + let vm = state.object(MOCK_VM_PATH).expect("the VM is stored"); + assert_eq!( + vm["spec"]["template"]["spec"]["volumes"][1]["cloudInitNoCloud"]["secretRef"]["name"], + serde_json::json!("default--demo-boot-generation2"), + "the config-disk volume points at the new generation's Secret" + ); + assert!( + vm["metadata"]["annotations"]["openshell.ai/kubevirt-companions"] + .to_string() + .contains("default--demo-boot-generation2"), + "the companions annotation records the new generation: {}", + vm["metadata"]["annotations"]["openshell.ai/kubevirt-companions"] + ); + // The previous generation's Secret was deleted. + assert!( + !state.exists(old_secret_path), + "the generation-1 Secret was deleted by the generation-2 start" + ); + } - loop { - tokio::select! { - () = tokio::time::sleep(debounce) => { - match load_namespace_file(&path) { - Ok(names) => { - let count = names.len(); - allowlist.replace(names); - info!( - total = count, - "operator namespace allowlist reloaded from file" - ); - } - Err(err) => { - warn!( - error = %err, - "failed to reload operator namespace file, keeping existing allowlist" - ); - } - } - break; - } - r = rx.recv() => { - if r.is_some() { - continue; - } - warn!("operator namespace file watcher disconnected"); - return; - } - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - } - } - } - } - }); -} + /// H4: a stale companions annotation (the recorded Secret was + /// replaced out of band) does not break the start — the + /// preconditioned delete of the stale record fails closed and the + /// replacement is left to owner-reference garbage collection, while + /// the new generation is live. + #[tokio::test] + async fn kubevirt_start_tolerates_a_stale_companions_annotation() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + // Replace the recorded boot Secret out of band (same name, new + // UID): the annotation's UID is now stale. + let old_secret_path = + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation1"; + let mut replaced = state.object(old_secret_path).expect("the secret exists"); + replaced["metadata"]["uid"] = serde_json::json!("replaced-out-of-band"); + state.seed(old_secret_path, replaced); + let mut halted = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + halted["spec"]["runStrategy"] = serde_json::json!("Halted"); + state.seed(MOCK_VM_PATH, halted); + let mut authentication = kubevirt_test_launch_authentication(); + authentication.supervisor.runtime_generation = + openshell_core::sandbox_generation::SandboxGenerationId::parse("generation-2").unwrap(); + let encoded = serde_json::to_vec(&authentication).unwrap(); + driver + .start_kubevirt_sandbox("sbx-id", &encoded, &created) + .await + .expect("a stale annotation must not break the start"); + // The stale record's preconditioned delete failed closed: the + // out-of-band replacement is still there (GC owns it), and the + // new generation is live. + assert!( + state.exists(old_secret_path), + "the replaced object is left to garbage collection" + ); + assert!( + state.exists( + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation2" + ), + "the new generation's Secret is live" + ); + } -#[cfg(test)] -mod tests { - use super::*; + /// H5: the fence is re-verified immediately before the patch that + /// boots the VM — a fence widened after the start's first check fails + /// the start instead of releasing the guest. + #[tokio::test] + async fn kubevirt_start_reverifies_the_fence_before_release() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + // Simulate the stopped state start operates on: halt the VM the + // create left running. + let mut halted = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + halted["spec"]["runStrategy"] = serde_json::json!("Halted"); + state.seed(MOCK_VM_PATH, halted); + // Widen the fence (same UID, an extra egress allow) between the + // start's first verification and its immediate pre-release + // re-check: the second GET of the fence serves the widened spec. + let fence_path = "/apis/networking.k8s.io/v1/namespaces/openshell-e2e-ns/networkpolicies/openshell-kubevirt-fence-sbx-id"; + let mut widened = state.object(fence_path).expect("the fence exists"); + widened["spec"]["egress"] = serde_json::json!([{}]); + state.swap_on_get(fence_path, 2, widened); + let authentication = serde_json::to_vec(&kubevirt_test_launch_authentication()).unwrap(); + let error = driver + .start_kubevirt_sandbox("sbx-id", &authentication, &created) + .await + .expect_err("a fence widened before release must refuse the start"); + assert!( + error.to_string().contains("fence"), + "the error must name the fence: {error}" + ); + // The VM was never booted: the run strategy is not Once. + let vm = state.object(MOCK_VM_PATH).expect("the VM is stored"); + assert_ne!( + vm["spec"]["runStrategy"], + serde_json::json!("Once"), + "the release patch must not land after fence drift" + ); + } - /// Serve Pod GETs over one reusable HTTP/1 connection per client, - /// delaying a request for Pod `slow-` by that many milliseconds. - async fn serve_delayed_pods() -> std::net::SocketAddr { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - tokio::spawn(async move { - loop { - let (stream, _) = listener.accept().await.unwrap(); - tokio::spawn(hyper::server::conn::http1::Builder::new().serve_connection( - hyper_util::rt::TokioIo::new(stream), - hyper::service::service_fn(|request: http::Request<_>| async move { - let delay = request - .uri() - .path() - .rsplit_once("/slow-") - .and_then(|(_, millis)| millis.parse().ok()) - .map_or(Duration::ZERO, Duration::from_millis); - tokio::time::sleep(delay).await; - Ok::<_, std::convert::Infallible>(http::Response::new( - http_body_util::Full::new(bytes::Bytes::from_static( - br#"{"apiVersion":"v1","kind":"Pod","metadata":{"name":"pod"}}"#, - )), - )) - }), - )); - } + /// H5: a resourceVersion conflict on a pinned VM patch retries from a + /// fresh read and completes — the conflict is a transient race, never + /// a name-only fallback or an abort. + #[tokio::test] + async fn kubevirt_patches_retry_cleanly_after_a_conflict() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + // Simulate the stopped state start operates on: halt the VM the + // create left running. + let mut halted = state + .object(MOCK_VM_PATH) + .expect("the created VM is stored"); + halted["spec"]["runStrategy"] = serde_json::json!("Halted"); + state.seed(MOCK_VM_PATH, halted); + // A concurrent writer bumps the VM's resourceVersion right after + // the pinned patch's first read: the first patch conflicts (409) + // and the retry succeeds with the fresh version. + state.bump_rv_after_get(MOCK_VM_PATH); + let authentication = serde_json::to_vec(&kubevirt_test_launch_authentication()).unwrap(); + let identity = driver + .start_kubevirt_sandbox("sbx-id", &authentication, &created) + .await + .expect("a transient conflict retries and the start completes"); + assert!( + identity.starts_with("kubernetes://openshell-e2e-ns/"), + "the retried start returns a Kubernetes identity, got {identity}" + ); + let vm_patches = state + .requests() + .iter() + .filter(|request| request.starts_with("PATCH") && request.contains("virtualmachines/")) + .count(); + assert!( + vm_patches >= 2, + "the conflicted patch must have been retried (VM patches: {vm_patches})" + ); + } + + /// M4: a start on a live or booting VMI is idempotent — the plan is + /// computed first, so no supervisor Pod is recreated, no Secret + /// rewritten, and no instance registered. + #[tokio::test] + async fn kubevirt_start_plan_runs_before_side_effects() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let expectation = serde_json::json!({ + "uid": "fence-uid-1", + "spec": {"podSelector": {"matchLabels": { + "kubevirt.io": "virt-launcher", + "openshell.ai/sandbox-id": "sbx-id", + }}}, }); - addr + let annotations = serde_json::json!({ + "openshell.ai/sandbox-runtime-supervisor-session": "", + "openshell.ai/sandbox-runtime-main-process-spec": "e30=", + "openshell.ai/sandbox-runtime-log-level": "info", + "openshell.ai/kubevirt-fence-expectation": + serde_json::to_string(&expectation).unwrap(), + }); + state.seed( + MOCK_VM_PATH, + serde_json::json!({ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "metadata": { + "name": "default--demo", "uid": "vm-uid-1", + "namespace": "openshell-e2e-ns", + "labels": { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-id": "sbx-id", + "openshell.ai/gateway-id": "e2e-gateway", + }, + "annotations": annotations, + }, + "spec": {"runStrategy": "Once"}, + "status": {"ready": true, "printableStatus": "Running"}, + }), + ); + state.seed( + "/apis/kubevirt.io/v1/namespaces/openshell-e2e-ns/virtualmachineinstances/default--demo", + serde_json::json!({ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachineInstance", + "metadata": {"name": "default--demo", "uid": "vmi-uid-1"}, + "status": {"phase": "Running"}, + }), + ); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let identity = driver + .start_kubevirt_sandbox( + "sbx-id", + &serde_json::to_vec(&kubevirt_test_launch_authentication()).unwrap(), + "kubernetes://openshell-e2e-ns/vm-uid-1/anything", + ) + .await + .expect("idempotent start succeeds"); + assert!( + identity.starts_with("kubernetes://openshell-e2e-ns/vm-uid-1/"), + "the idempotent start returns the full runtime identity, got {identity}" + ); + let writes: Vec = state + .requests() + .into_iter() + .filter(|request| { + request.starts_with("POST") + || request.starts_with("PATCH") + || request.starts_with("DELETE") + }) + .collect(); + assert!( + writes.is_empty(), + "an AlreadyRunning start must not write, got: {writes:?}" + ); + } + + /// M6: the runtime identity round-trips through create, start, and + /// start again — every return parses as a Kubernetes identity, so the + /// gateway's persisted identity never breaks the next start. + #[tokio::test] + async fn kubevirt_identity_round_trips_through_create_and_starts() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + let created = driver + .create_sandbox(&kubevirt_test_sandbox()) + .await + .expect("create succeeds against the mock"); + assert!( + created.starts_with("kubernetes://openshell-e2e-ns/"), + "create returns a Kubernetes identity, got {created}" + ); + let authentication = serde_json::to_vec(&kubevirt_test_launch_authentication()).unwrap(); + let first_start = driver + .start_kubevirt_sandbox("sbx-id", &authentication, &created) + .await + .expect("the first start accepts the created identity"); + assert!( + first_start.starts_with("kubernetes://openshell-e2e-ns/"), + "the first start returns a Kubernetes identity, got {first_start}" + ); + let second_start = driver + .start_kubevirt_sandbox("sbx-id", &authentication, &first_start) + .await + .expect("the second start accepts the first start's identity"); + assert_eq!(first_start, second_start); + } + + /// H1 / delete ordering: delete removes the `VirtualMachine` (with its UID + /// precondition) and waits for it to be gone before touching the + /// companions, so the fence outlives the virt-launcher pod it fences. + #[tokio::test] + async fn kubevirt_delete_waits_for_the_vm_before_companions() { + let state = MockKube::default(); + seed_kubevirt_namespace(&state); + // Keep the companions alive past the VM delete (the real garbage + // collector is asynchronous) so the explicit pinned deletes run. + state.disable_gc(); + let labels = serde_json::json!({ + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-id": "sbx-id", + "openshell.ai/gateway-id": "e2e-gateway", + }); + let vm_uid = serde_json::json!("vm-uid-1"); + // H4: the VM carries the recorded companions (exact names and + // UIDs), so the Secret deletes need no read and no list. + let companions = serde_json::json!({ + "boot_secret": { + "name": "default--demo-boot-generation1", + "uid": "boot-secret-uid-1", + }, + "supervisor_secret": { + "name": "os-supervisor-sbx-id-generation1", + "uid": "supervisor-secret-uid-1", + }, + }); + state.seed( + MOCK_VM_PATH, + serde_json::json!({ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "metadata": {"name": "default--demo", "uid": vm_uid.clone(), + "labels": labels.clone(), + "annotations": { + "openshell.ai/kubevirt-companions": + serde_json::to_string(&companions).unwrap(), + }}, + "spec": {"runStrategy": "Once"}, + }), + ); + for path in [ + "/apis/networking.k8s.io/v1/namespaces/openshell-e2e-ns/networkpolicies/openshell-kubevirt-fence-sbx-id", + "/api/v1/namespaces/openshell-e2e-ns/persistentvolumeclaims/default--demo-workspace", + ] { + state.seed( + path, + serde_json::json!({ + "metadata": {"labels": labels.clone(), + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": "default--demo", "uid": vm_uid.clone(), + "controller": false, "blockOwnerDeletion": false, + }]}, + }), + ); + } + // The recorded Secrets exist with the recorded UIDs (the delete + // path never reads them — it uses the annotation). + for (path, uid) in [ + ( + "/api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation1", + "boot-secret-uid-1", + ), + ( + "/api/v1/namespaces/openshell-e2e-ns/secrets/os-supervisor-sbx-id-generation1", + "supervisor-secret-uid-1", + ), + ] { + state.seed( + path, + serde_json::json!({ + "metadata": {"labels": labels.clone(), "uid": uid, + "ownerReferences": [{ + "apiVersion": "kubevirt.io/v1", "kind": "VirtualMachine", + "name": "default--demo", "uid": vm_uid.clone(), + "controller": false, "blockOwnerDeletion": false, + }]}, + }), + ); + } + let driver = kubevirt_mock_driver(mock_kube_client(state.clone())); + assert!(driver.delete_kubevirt_sandbox("sbx-id").await.unwrap()); + let requests = state.requests(); + let vm_delete = requests + .iter() + .position(|request| *request == format!("DELETE {MOCK_VM_PATH}")) + .expect("the VM delete was issued"); + // The companion deletes come after the VM delete. There is no + // wait for the VM to be gone first (user decision): the isolation + // floor plus owner references carry the window. + for companion in [ + "DELETE /apis/networking.k8s.io/v1/namespaces/openshell-e2e-ns/networkpolicies/openshell-kubevirt-fence-sbx-id", + "DELETE /api/v1/namespaces/openshell-e2e-ns/secrets/default--demo-boot-generation1", + ] { + let position = requests + .iter() + .position(|request| request == companion) + .unwrap_or_else(|| panic!("companion delete missing: {companion}")); + assert!( + position > vm_delete, + "{companion} must be deleted after the VM ({requests:?})" + ); + } + // The owner-referenced companions are gone (mock GC, mirroring the + // real cascade). + assert!( + kubevirt_per_sandbox_paths() + .into_iter() + .all(|path| !state.exists(path)) + ); } #[tokio::test] @@ -9193,6 +13876,105 @@ mod tests { ); } + #[test] + fn route_teardown_race_matches_both_withdrawal_phases() { + // The recording phase: the route's record still names the previous + // generation while the withdrawal is being recorded. + assert!(route_teardown_race( + "the sandbox instance route is recorded for another generation" + )); + // The admission phase: the record is being reconciled away after + // the supervisor already removed the instance. + assert!(route_teardown_race( + "the sandbox instance route is owned by another generation" + )); + // Anything else is a real refusal (another sandbox's route, pool + // capacity) and must fail closed without retries. + assert!(!route_teardown_race( + "no live member has room for the instance" + )); + assert!(!route_teardown_race( + "the sandbox instance is being recorded by another gateway task" + )); + } + + #[tokio::test] + async fn kubevirt_config_rules_run_at_construction() { + // Z1 guard: the kubevirt runtime rules must run through the real + // construction entry points, so an invalid runtime=kubevirt config + // cannot build a driver. Validation happens before any cluster + // connection, so no kubeconfig is needed to assert the refusal. + let config = KubernetesComputeConfig { + runtime: crate::KubernetesRuntimeMode::KubeVirt, + namespace: "openshell-e2e-ns".to_string(), + gateway_id: "e2e-gateway".to_string(), + grpc_endpoint: "https://172.18.0.1:18085".to_string(), + kubevirt: crate::KubeVirtConfig { + guest_image: "openshell/sandbox-vm:test".to_string(), + ..Default::default() + }, + ..Default::default() + }; + let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + // The default (dedicated) topology refuses an empty grpc_endpoint: + // its supervisor Pod could not reach the gateway. + let mut dedicated = config.clone(); + dedicated.grpc_endpoint = String::new(); + let error = KubernetesComputeDriver::new(dedicated, shutdown_rx.clone(), None) + .await + .expect_err("dedicated without grpc_endpoint is refused"); + assert!( + error.to_string().contains("grpc_endpoint"), + "message should name the key: {error}" + ); + + // The gateway topology without shared_supervisor is refused by the + // shared topology rules. + let mut shared = config; + shared.supervisor_topology = Some(crate::SupervisorTopology::Gateway); + let error = KubernetesComputeDriver::new(shared, shutdown_rx.clone(), None) + .await + .expect_err("gateway topology without shared supervisor is refused"); + assert!( + error.to_string().contains("requires shared_supervisor"), + "message should name the requirement: {error}" + ); + + // With the gateway topology and shared supervision wired, an empty + // guest image is refused by the same path (the wiring cannot + // silently disappear again). + let mut kubevirt = KubernetesComputeConfig { + runtime: crate::KubernetesRuntimeMode::KubeVirt, + namespace: "openshell-e2e-ns".to_string(), + gateway_id: "e2e-gateway".to_string(), + supervisor_topology: Some(crate::SupervisorTopology::Gateway), + kubevirt: crate::KubeVirtConfig { + guest_image: "openshell/sandbox-vm:test".to_string(), + ..Default::default() + }, + ..Default::default() + }; + kubevirt.shared_supervisor = Some(crate::SharedSupervisorConfig { + service_name: "openshell-supervisor".to_string(), + service_namespace: "openshell".to_string(), + runtime_port: 5500, + token_secret_name: "openshell-supervisor-token".to_string(), + }); + kubevirt.kubevirt.guest_image = String::new(); + let error = KubernetesComputeDriver::with_shared_supervision( + kubevirt, + shutdown_rx, + SharedSupervision::new(), + ) + .await + .expect_err("empty guest_image is refused"); + assert!( + error.to_string().contains("guest_image"), + "message should name the key: {error}" + ); + } + #[tokio::test] async fn supervisor_otlp_environment_carries_only_configured_signals() { use openshell_core::sandbox_env::{ @@ -9714,6 +14496,7 @@ mod tests { let mut sandbox_owned = expected.clone(); sandbox_owned.metadata.owner_references = Some(vec![sandbox_runtime_sandbox_owner_reference( + "Sandbox", "sandbox", "sandbox-uid", "agents.x-k8s.io/v1beta1", diff --git a/crates/openshell-driver-kubernetes/src/kubevirt.rs b/crates/openshell-driver-kubernetes/src/kubevirt.rs new file mode 100644 index 0000000000..f0779cd630 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/kubevirt.rs @@ -0,0 +1,2741 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `KubeVirt` runtime mode (proof of concept). +//! +//! When `[openshell.drivers.kubernetes] runtime = "kubevirt"` is configured, +//! the driver provisions sandboxes as `kubevirt.io/v1` `VirtualMachine` +//! objects instead of `agents.x-k8s.io` Sandbox CRs. The `KubeVirt` controller +//! (VM -> VMI -> virt-launcher pod) plays the role the Agent Sandbox +//! controller plays in the pod runtime. +//! +//! Resource model per sandbox: +//! - one `VirtualMachine` carrying the standard `OpenShell` ownership labels +//! (the durable lifecycle handle; `runStrategy: Once` mirrors the pod +//! path's `restartPolicy: Never` — supervisor exit is terminal), +//! - one bootstrap `Secret` referenced through a `cloudInitNoCloud` volume +//! (transport only — the guest init parses the ISO itself, no cloud-init +//! software required). It carries the boundary bootstrap artifacts only: +//! the boundary config (claims, supervisor address, public verification +//! keys), the session TLS key/cert, and the workload uid/gid. No gateway +//! token, no endpoint, no JWT — nothing the guest could use against the +//! gateway, +//! - one egress-fence `NetworkPolicy` plus the namespace-wide launcher +//! isolation floor (deny-all ingress and egress for every +//! virt-launcher pod; the per-sandbox fence adds only the supervisor +//! allowance), both owner-referenced to the `VirtualMachine`, +//! - one workspace `PersistentVolumeClaim` mounted by the guest init at +//! `/sandbox` (mirrors the pod path's PVC-backed workspace), and a +//! dedicated supervisor Pod with its per-sandbox Service when the +//! dedicated topology is configured. +//! +//! What the `PoC` deliberately skips: GPU, managed/operator workspace modes, +//! mTLS to the gateway, live-migration/VMI-replacement semantics, and +//! arbitrary OCI sandbox images as VM disks (the guest image is a +//! purpose-built containerDisk). + +use std::collections::BTreeMap; + +use std::time::Duration; + +use k8s_openapi::api::core::v1::{ + PersistentVolumeClaim, PersistentVolumeClaimSpec, VolumeResourceRequirements, +}; +use k8s_openapi::apimachinery::pkg::api::resource::Quantity; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::api::{Api, ApiResource, DeleteParams, ListParams, ObjectList, Preconditions}; +use kube::core::DynamicObject; +use kube::{Client, Error as KubeError}; +use openshell_core::driver_utils::{ + LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, + LABEL_SANDBOX_WORKSPACE, +}; +use openshell_core::proto::compute::v1::{ + DriverCondition, DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, +}; + +use crate::KubernetesDriverError; +use crate::config::{KubernetesComputeConfig, KubernetesRuntimeMode, WorkspaceMode}; +use crate::driver::{sandbox_annotations, sandbox_labels}; + +pub const KUBEVIRT_GROUP: &str = "kubevirt.io"; +pub const KUBEVIRT_VERSION: &str = "v1"; +const KUBEVIRT_VM_PLURAL: &str = "virtualmachines"; +const KUBEVIRT_RUN_STRATEGY: &str = "Once"; + +/// Kernel and initramfs locations inside the purpose-built guest image +/// (see `e2e/kubevirt/` for the image build). +const KUBEVIRT_GUEST_KERNEL_PATH: &str = "/boot/vmlinuz-virt"; +const KUBEVIRT_GUEST_INITRD_PATH: &str = "/boot/initramfs-virt"; +const KUBEVIRT_KERNEL_ARGS: &str = + "console=ttyS0 root=/dev/vda rw rootfstype=ext4 modules=ext4 init=/opt/openshell/init"; + +/// Condition reason set on `Ready=False` while the VM is booting. Matches the +/// gateway's transient-reason vocabulary so the sandbox stays `Provisioning` +/// rather than `Error`. +const REASON_VM_NOT_READY: &str = "Starting"; + +/// Non-transient condition reason: the gateway's phase mapper maps this to +/// the public `Error` phase rather than leaving the sandbox provisioning. +const REASON_VM_FAILED: &str = "VMFailed"; + +/// Non-transient condition reason for a dedicated generation whose +/// supervisor Pod is gone (`restartPolicy: Never`): without a supervisor +/// the boundary can never connect, so the sandbox maps to the public +/// `Error` phase instead of provisioning forever. +pub const REASON_SUPERVISOR_FAILED: &str = "SupervisorFailed"; + +/// Non-transient condition reason for a create whose Halted window +/// outlived the deadline (M7): the gateway died mid-create and nothing is +/// driving the VM to `Once`. +pub const REASON_CREATE_TIMED_OUT: &str = "CreateTimedOut"; + +/// How long the launcher's image-pull backoff may persist before the +/// snapshot maps it to a terminal failure (L4): kubelet retries with +/// exponential backoff (capped at five minutes), so a pull that has not +/// succeeded after two minutes — four failed attempts — is effectively +/// permanent, and the CLI's provision timeout would otherwise expire with +/// only the configuration gate's message. Below the threshold the state +/// stays transient: a slow registry can legitimately back off once. +pub const IMAGE_PULL_BACKOFF_TERMINAL_AFTER_MS: i64 = 2 * 60 * 1000; + +/// Parse the Kubernetes RFC 3339 timestamp format to Unix milliseconds, +/// through `k8s-openapi`'s `Time` (already a dependency — the same +/// deserializer the API types use). +fn rfc3339_to_unix_ms(value: &str) -> Option { + serde_json::from_value::( + serde_json::Value::String(value.to_string()), + ) + .ok() + .map(|time| time.0.as_millisecond()) +} + +/// VM annotation recording why the guest image could not be digest-pinned +/// (the side-loaded case: the launcher's init container reports a bare +/// `sha256:` config digest with no repository, which is not pullable). +/// Surfaced as an `ImagePin` condition by the snapshot. +pub const IMAGE_PIN_STATE_ANNOTATION: &str = "openshell.ai/kubevirt-image-pin-state"; +/// The annotation's value for the unpinnable side-loaded case. +pub const IMAGE_PIN_NO_REPOSITORY_DIGEST: &str = "no-repository-digest"; + +/// The digest decision for one observed launcher init-container imageID +/// (M6 digest pinning, option A): +/// - `Patch(String)`: a registry-backed `repo@sha256:` — +/// the VM spec can be pinned to it verbatim; +/// - `SideLoaded`: a bare `sha256:` with no repository +/// (the k3d/ctr side-load path) — never patchable, the tag stays; +/// - `Absent`: nothing observed yet. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GuestImageDigest { + Patch(String), + SideLoaded, + Absent, +} + +/// Read the guest image digest from the launcher Pod's init containers: +/// `KubeVirt` runs the containerDisk as init containers (`volumerootdisk`, +/// `volumekernel-boot-volume`), so `containerStatuses` holds the +/// virt-launcher image instead. The launcher also runs other init +/// containers (`guest-console-log`) on the VIRT-LAUNCHER image, whose +/// digest must never be pinned — so the status is matched by the image +/// reference the VM spec currently names, not by position or name. +pub fn guest_image_digest( + launcher_pod: &k8s_openapi::api::core::v1::Pod, + guest_image: &str, +) -> GuestImageDigest { + let Some(status) = launcher_pod.status.as_ref() else { + return GuestImageDigest::Absent; + }; + let Some(init_statuses) = status.init_container_statuses.as_ref() else { + return GuestImageDigest::Absent; + }; + // Match by the POD SPEC's init container image — exactly what KubeVirt + // rendered from the VM spec — not the status `.image`: the CRI may + // report the status image under a DIFFERENT name the node knows the + // same digest as (e.g. a side-loaded copy of a registry-pulled tag), + // and matching on the status reference silently skips the pin. + let spec_images: std::collections::HashMap<&str, &str> = launcher_pod + .spec + .as_ref() + .and_then(|spec| spec.init_containers.as_ref()) + .map(|containers| { + containers + .iter() + .map(|container| { + ( + container.name.as_str(), + container.image.as_deref().unwrap_or(""), + ) + }) + .collect() + }) + .unwrap_or_default(); + for container in init_statuses.iter().filter(|container| { + spec_images + .get(container.name.as_str()) + .is_some_and(|spec_image| image_ref_matches(spec_image, guest_image)) + }) { + let image_id = container.image_id.as_str(); + if image_id.is_empty() { + continue; + } + if image_id.contains('@') { + // A registry-backed reference: repo@sha256:. + return GuestImageDigest::Patch(image_id.to_string()); + } + // A bare sha256:: the side-loaded path, not + // pullable as a reference. + return GuestImageDigest::SideLoaded; + } + GuestImageDigest::Absent +} + +/// Whether a container status `image` reference names the same image as the +/// VM spec's guest image. Containerd normalizes references (adds +/// `docker.io/` and `library/`), so the comparison is on the last path +/// components (repository and tag). +fn image_ref_matches(status_image: &str, guest_image: &str) -> bool { + fn tail(reference: &str) -> &str { + // Strip a registry host (contains a port, a dot, or is + // "localhost"; a repository path's first segment never does), + // then drop a leading "library/" for docker hub official images. + let no_host = match reference.split_once('/') { + Some((host, rest)) + if host.contains(':') || host.contains('.') || host == "localhost" => + { + rest + } + _ => reference, + }; + no_host.strip_prefix("library/").unwrap_or(no_host) + } + tail(status_image) == tail(guest_image) +} + +/// The VMI's not-ready-since time in Unix milliseconds, from its `Ready`/// The VMI's not-ready-since time in Unix milliseconds, from its `Ready` +/// condition's `lastTransitionTime`. `None` when the VMI carries no such +/// condition — the caller then keeps the pull-backoff state transient +/// (timing fails open, isolation never does). +pub fn vmi_not_ready_since_ms(vmi: &DynamicObject) -> Option { + let condition = vmi + .data + .get("status")? + .get("conditions")? + .as_array()? + .iter() + .find(|condition| { + condition.get("type").and_then(serde_json::Value::as_str) == Some("Ready") + })?; + let stamped = condition.get("lastTransitionTime")?.as_str()?; + rfc3339_to_unix_ms(stamped) +} + +/// Classify a terminal VM failure from the printable status and the paired +/// VMI's phase. Returns a human-readable summary when the VM has failed. +/// +/// Terminal printable statuses (per `KubeVirt` v1.9's +/// `VirtualMachinePrintableStatus`): `CrashLoopBackOff`, +/// `ErrorUnschedulable`, `ErrorPvcNotFound`, and `DataVolumeError` mean the +/// VM cannot run and will not recover on its own. +/// +/// Deliberately **not** terminal: +/// - `Unknown` (the controller could not obtain the VM state) can appear +/// transiently during apiserver/communication hiccups; mapping it to +/// `Error` would flip a booting sandbox to `Error` on a blip. +/// - `ErrImagePull` / `ImagePullBackOff`: kubelet retries the pull, so they +/// stay transient — but the condition message carries the printable +/// status so the cause is visible while the sandbox provisions. +/// - A `Failed` or `Succeeded` VMI under a not-ready VM **is** terminal: the +/// guest exited (kernel panic, supervisor death, or a completed `Once` +/// run). Without the VMI signal those states are indistinguishable from +/// a not-yet-booted VM, whose printable status is also `Stopped`. +pub fn terminal_vm_failure( + printable_status: &str, + vmi_phase: Option<&str>, + vm_ready: bool, +) -> Option { + match printable_status { + "CrashLoopBackOff" | "ErrorUnschedulable" | "ErrorPvcNotFound" | "DataVolumeError" => { + return Some(format!("printableStatus {printable_status}")); + } + _ => {} + } + if !vm_ready { + match vmi_phase { + Some("Failed") => return Some("VirtualMachineInstance phase Failed".to_string()), + Some("Succeeded") => { + return Some("VirtualMachineInstance phase Succeeded (guest exited)".to_string()); + } + _ => {} + } + } + None +} + +/// Namespaced API for `kubevirt.io/v1` `virtualmachines`. +pub fn virtual_machine_api(client: &Client, namespace: &str) -> Api { + Api::namespaced_with(client.clone(), namespace, &virtual_machine_api_resource()) +} + +pub fn virtual_machine_api_resource() -> ApiResource { + ApiResource { + group: KUBEVIRT_GROUP.to_string(), + version: KUBEVIRT_VERSION.to_string(), + api_version: format!("{KUBEVIRT_GROUP}/{KUBEVIRT_VERSION}"), + kind: "VirtualMachine".to_string(), + plural: KUBEVIRT_VM_PLURAL.to_string(), + } +} + +/// Check that the cluster serves `kubevirt.io/v1` virtualmachines. +pub async fn kubevirt_available(client: &Client, namespace: &str) -> Result<(), String> { + let api = virtual_machine_api(client, namespace); + match api.list(&ListParams::default().limit(1)).await { + Ok(_) => Ok(()), + Err(KubeError::Api(err)) if err.code == 404 => Err(format!( + "kubevirt.io/{KUBEVIRT_VERSION} virtualmachines is not served by this cluster; \ + install KubeVirt to use runtime = \"kubevirt\"" + )), + Err(err) => Err(err.to_string()), + } +} + +/// Everything the renderer needs for one sandbox. +pub struct KubeVirtSandboxParams<'a> { + pub sandbox: &'a Sandbox, + pub config: &'a KubernetesComputeConfig, + pub namespace: &'a str, + /// `KubeVirt` resource name (`{workspace}--{name}` in shared mode). + pub kube_name: &'a str, + /// The launch generation scoping the bootstrap Secret's name. + pub generation: &'a str, +} + +/// The `KubeVirt` domain resources mapped from a sandbox's typed +/// resource requirements. +/// +/// Mapping rules (validated by [`domain_resources`], enforced at +/// admission before anything is created): +/// +/// - **vCPU cores**: the effective CPU limit (or request when no limit is +/// set, or the global `kubevirt.cpu_cores` when neither is) rounded +/// **up** to a whole number of vCPUs — a VM's vCPU count is integral. +/// A fractional limit keeps its exact value on the virt-launcher +/// container (`limits.cpu`), so the guest sees the rounded vCPU count +/// while the launcher is throttled to the fraction. +/// - **Guest memory**: the effective memory limit (or request when no +/// limit is set, or the global `kubevirt.memory` when neither is) — +/// `domain.memory.guest`, what the guest OS sees. +/// - **virt-launcher requests/limits** (`domain.resources`): the sandbox's +/// CPU and memory requests and limits verbatim, with the pod path's +/// mirroring (an omitted request defaults to the limit, and the guest +/// memory stands in for an omitted memory request). These make the +/// launcher container enforceable and leave headroom for `KubeVirt`'s +/// overhead. +/// - **Validation**: a limit below its request, a memory below the safe +/// guest minimum ([`MIN_GUEST_MEMORY_BYTES`]), or an unparsable +/// quantity is an `InvalidArgument` at admission. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct KubeVirtDomainResources { + /// Whole vCPUs the guest sees. + pub cores: u32, + /// `domain.memory.guest` quantity string. + pub guest_memory: String, + /// `domain.resources.requests.cpu`, when the sandbox asked for CPU. + pub cpu_request: Option, + /// `domain.resources.limits.cpu`, when the sandbox set a CPU limit. + pub cpu_limit: Option, + /// `domain.resources.requests.memory`, when the sandbox asked for + /// memory. + pub memory_request: Option, + /// `domain.resources.limits.memory`, when the sandbox set a memory + /// limit. + pub memory_limit: Option, +} + +/// The smallest guest memory the purpose-built image boots with: the +/// kernel, the Alpine userland, and the in-guest boundary need headroom +/// below this before the workload starts. +pub const MIN_GUEST_MEMORY_BYTES: u64 = 256 * 1024 * 1024; + +/// Parse a Kubernetes CPU quantity into millicores (`"2"` → 2000, +/// `"500m"` → 500, `"1.5"` → 1500). Zero and negative values are +/// invalid. +fn parse_cpu_millis(quantity: &str) -> Result { + let trimmed = quantity.trim(); + if let Some(millis_str) = trimmed.strip_suffix('m') { + let millis: u64 = millis_str.parse().map_err(|_| { + format!("invalid CPU quantity {quantity:?}: expected m or ") + })?; + if millis == 0 { + return Err(format!("invalid CPU quantity {quantity:?}: zero")); + } + return Ok(millis); + } + let cores: f64 = trimmed + .parse() + .map_err(|_| format!("invalid CPU quantity {quantity:?}: expected m or "))?; + if cores <= 0.0 || cores.is_nan() || cores.is_infinite() { + return Err(format!( + "invalid CPU quantity {quantity:?}: must be positive" + )); + } + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let millis = (cores * 1000.0) as u64; + if millis == 0 { + return Err(format!("invalid CPU quantity {quantity:?}: rounds to zero")); + } + Ok(millis) +} + +/// Parse a Kubernetes memory quantity into bytes (binary `Ki`..`Ei` and +/// decimal `k`..`E` suffixes, or plain bytes). +fn parse_memory_bytes(quantity: &str) -> Result { + let trimmed = quantity.trim(); + let suffixes: &[(&str, u64)] = &[ + ("Ei", 1_u64 << 60), + ("Pi", 1 << 50), + ("Ti", 1 << 40), + ("Gi", 1 << 30), + ("Mi", 1 << 20), + ("Ki", 1 << 10), + ("E", 1_000_000_000_000_000_000), + ("P", 1_000_000_000_000_000), + ("T", 1_000_000_000_000), + ("G", 1_000_000_000), + ("M", 1_000_000), + ("k", 1_000), + ("K", 1_000), + ]; + for (suffix, multiplier) in suffixes { + if let Some(number) = trimmed.strip_suffix(suffix) { + let value: u64 = number.parse().map_err(|_| { + format!("invalid memory quantity {quantity:?}: expected {suffix}") + })?; + return value + .checked_mul(*multiplier) + .ok_or_else(|| format!("invalid memory quantity {quantity:?}: overflow")); + } + } + trimmed + .parse() + .map_err(|_| format!("invalid memory quantity {quantity:?}: expected [suffix]")) +} + +/// Map a sandbox's typed resource requirements onto the `KubeVirt` +/// domain resources. See [`KubeVirtDomainResources`] for the rules; +/// errors are admission-time `InvalidArgument`s. +pub fn domain_resources( + template: Option<&openshell_core::proto::compute::v1::DriverSandboxTemplate>, + kubevirt: &crate::config::KubeVirtConfig, +) -> Result { + let default_requirements = + openshell_core::proto::compute::v1::DriverResourceRequirements::default(); + let requirements = template + .and_then(|template| template.resources.as_ref()) + .unwrap_or(&default_requirements); + let cpu_request = + (!requirements.cpu_request.is_empty()).then(|| requirements.cpu_request.clone()); + let cpu_limit = (!requirements.cpu_limit.is_empty()).then(|| requirements.cpu_limit.clone()); + let memory_request = + (!requirements.memory_request.is_empty()).then(|| requirements.memory_request.clone()); + let memory_limit = + (!requirements.memory_limit.is_empty()).then(|| requirements.memory_limit.clone()); + + // A limit below its request is a contradiction in any mapping. + if let (Some(request), Some(limit)) = (&cpu_request, &cpu_limit) + && parse_cpu_millis(request)? > parse_cpu_millis(limit)? + { + return Err(format!( + "cpu limit {limit:?} is below the cpu request {request:?}" + )); + } + if let (Some(request), Some(limit)) = (&memory_request, &memory_limit) + && parse_memory_bytes(request)? > parse_memory_bytes(limit)? + { + return Err(format!( + "memory limit {limit:?} is below the memory request {request:?}" + )); + } + + // vCPU count: the effective CPU, rounded up to whole vCPUs. + let effective_cpu = cpu_limit + .clone() + .or_else(|| cpu_request.clone()) + .unwrap_or_else(|| kubevirt.cpu_cores.to_string()); + let cores = parse_cpu_millis(&effective_cpu)? + .div_ceil(1000) + .try_into() + .map_err(|_| format!("cpu quantity {effective_cpu:?} maps to too many vCPUs"))?; + + // Guest memory: the effective memory, floored at the safe minimum. + let guest_memory = memory_limit + .clone() + .or_else(|| memory_request.clone()) + .unwrap_or_else(|| kubevirt.memory.clone()); + let guest_bytes = parse_memory_bytes(&guest_memory)?; + if guest_bytes < MIN_GUEST_MEMORY_BYTES { + return Err(format!( + "memory {guest_memory:?} is below the safe guest minimum of 256Mi \ + (the guest kernel, userland, and boundary need it before the workload starts)" + )); + } + + // The launcher's requests mirror the pod path's rule: an omitted + // request defaults to the limit (and the guest memory stands in for + // an omitted memory request). + let cpu_request = cpu_request.or_else(|| cpu_limit.clone()); + let memory_request = memory_request.or_else(|| memory_limit.clone()); + + Ok(KubeVirtDomainResources { + cores, + guest_memory, + cpu_request, + cpu_limit, + memory_request, + memory_limit, + }) +} + +/// Render the workspace PVC. The guest init formats it on first boot (sentinel +/// guarded, same pattern as the pod path's init-container seeding) and mounts +/// it at `/sandbox`. +pub fn render_workspace_pvc(params: &KubeVirtSandboxParams<'_>) -> PersistentVolumeClaim { + let storage_class = (!params.config.workspace_storage_class.is_empty()) + .then(|| params.config.workspace_storage_class.clone()); + PersistentVolumeClaim { + metadata: ObjectMeta { + name: Some(workspace_pvc_name(params.kube_name)), + namespace: Some(params.namespace.to_string()), + labels: Some(sandbox_labels( + params.sandbox, + Some(¶ms.config.gateway_id), + )), + annotations: Some(sandbox_annotations(params.sandbox)), + ..Default::default() + }, + spec: Some(PersistentVolumeClaimSpec { + access_modes: Some(vec!["ReadWriteOnce".to_string()]), + resources: Some(VolumeResourceRequirements { + requests: Some(BTreeMap::from([( + "storage".to_string(), + Quantity(params.config.workspace_default_storage_size.clone()), + )])), + ..Default::default() + }), + storage_class_name: storage_class, + ..Default::default() + }), + status: None, + } +} + +/// Render the `VirtualMachine`. +/// The VM's `volumes` array: the root containerDisk, the bootstrap +/// Secret's cloudInitNoCloud volume (named for the launch generation), +/// and the workspace virtio-fs share. Start re-renders this array to +/// point the config disk at a new generation's Secret. +pub fn render_volumes( + kube_name: &str, + generation: &str, + kubevirt: &crate::config::KubeVirtConfig, +) -> Vec { + let mut container_disk = serde_json::json!({ + "image": kubevirt.guest_image, + }); + if !kubevirt.guest_image_pull_policy.is_empty() { + container_disk["imagePullPolicy"] = serde_json::json!(kubevirt.guest_image_pull_policy); + } + let mut volumes = vec![ + serde_json::json!({"name": "rootdisk", "containerDisk": container_disk}), + serde_json::json!({ + "name": "cloudinit", + "cloudInitNoCloud": { + "secretRef": {"name": bootstrap_secret_name(kube_name, generation)} + } + }), + ]; + if kubevirt.workspace_pvc { + volumes.push(serde_json::json!({ + "name": "workspace", + "persistentVolumeClaim": {"claimName": workspace_pvc_name(kube_name)} + })); + } + volumes +} + +pub fn render_virtual_machine(params: &KubeVirtSandboxParams<'_>) -> DynamicObject { + let sandbox = params.sandbox; + let kubevirt = ¶ms.config.kubevirt; + // The kernel args boot the purpose-built initramfs init, which mounts + // the containerDisk root read-only and stacks a writable tmpfs overlay + // (the guest root is writable at runtime and fresh from the image on + // every boot — the guest is untrusted once a workload has run). + let kernel_args = KUBEVIRT_KERNEL_ARGS; + let mut labels = sandbox_labels(sandbox, Some(¶ms.config.gateway_id)); + let annotations = sandbox_annotations(sandbox); + + let mut container_disk = serde_json::json!({ + "image": kubevirt.guest_image, + }); + if !kubevirt.guest_image_pull_policy.is_empty() { + container_disk["imagePullPolicy"] = serde_json::json!(kubevirt.guest_image_pull_policy); + } + + let disks = vec![ + serde_json::json!({"name": "rootdisk", "disk": {"bus": "virtio"}}), + serde_json::json!({"name": "cloudinit", "disk": {"bus": "virtio"}}), + ]; + let volumes = render_volumes(params.kube_name, params.generation, ¶ms.config.kubevirt); + // The workspace PVC is a filesystem volume (the cluster's default + // storage class provisions filesystem PVCs), which KubeVirt shares + // with the guest via virtio-fs rather than as a block disk. + let mut filesystems = Vec::new(); + if kubevirt.workspace_pvc { + filesystems.push(serde_json::json!({ + "name": "workspace", + "virtiofs": {} + })); + } + + // VMI inherits template labels; keeping the OpenShell ownership labels + // there makes VMI-scoped tooling (describe, events) correlate as well. + labels.insert( + KUBEVIRT_VM_LABEL.to_string(), + KUBEVIRT_VM_LABEL_VALUE.to_string(), + ); + // The boundary-role workload label: KubeVirt propagates template labels + // to the virt-launcher pod, and the shared supervisor's ingress + // NetworkPolicy (deploy/helm/openshell/templates/ + // supervisor-networkpolicy.yaml) admits connections only from pods + // carrying this label. + labels.insert( + crate::isolation::BOUNDARY_ROLE_LABEL.to_string(), + "workload".to_string(), + ); + // The boundary-pair label binds the launcher pod to this sandbox's pair, + // exactly as the pod path's workload pods carry it: the dedicated + // topology's control NetworkPolicy and egress fence select the pair's + // two roles with it (isolation.rs pair_role_labels). + labels.insert( + crate::isolation::BOUNDARY_PAIR_LABEL.to_string(), + crate::sandbox_runtime::pair_label_value(&sandbox.id), + ); + + // Direct kernel boot: the guest image is purpose-built (see + // `e2e/kubevirt/`), so no bootloader is installed in the rootfs. The + // kernel and initramfs live next to the rootfs disk inside the same + // containerDisk image. + let mut kernel_boot_container = serde_json::json!({ + "image": kubevirt.guest_image, + "kernelPath": KUBEVIRT_GUEST_KERNEL_PATH, + "initrdPath": KUBEVIRT_GUEST_INITRD_PATH, + }); + if !kubevirt.guest_image_pull_policy.is_empty() { + kernel_boot_container["imagePullPolicy"] = + serde_json::json!(kubevirt.guest_image_pull_policy); + } + + // The sandbox's typed resource requirements map onto the domain (see + // `domain_resources`); admission validated them before the VM was + // rendered, so the mapping here cannot fail. + let resources = domain_resources( + params + .sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()), + kubevirt, + ) + .expect("admission validated the domain resources before rendering"); + let mut domain_resources_json = serde_json::Map::new(); + if resources.cpu_request.is_some() || resources.memory_request.is_some() { + let mut requests = serde_json::Map::new(); + if let Some(cpu) = &resources.cpu_request { + requests.insert("cpu".to_string(), serde_json::json!(cpu)); + } + if let Some(memory) = &resources.memory_request { + requests.insert("memory".to_string(), serde_json::json!(memory)); + } + domain_resources_json.insert("requests".to_string(), serde_json::Value::Object(requests)); + } + if resources.cpu_limit.is_some() || resources.memory_limit.is_some() { + let mut limits = serde_json::Map::new(); + if let Some(cpu) = &resources.cpu_limit { + limits.insert("cpu".to_string(), serde_json::json!(cpu)); + } + if let Some(memory) = &resources.memory_limit { + limits.insert("memory".to_string(), serde_json::json!(memory)); + } + domain_resources_json.insert("limits".to_string(), serde_json::Value::Object(limits)); + } + + let data = serde_json::json!({ + "spec": { + "runStrategy": KUBEVIRT_RUN_STRATEGY, + "template": { + "metadata": {"labels": labels}, + "spec": { + "domain": { + "firmware": { + "kernelBoot": { + "container": kernel_boot_container, + "kernelArgs": kernel_args + } + }, + "cpu": {"cores": resources.cores}, + "memory": {"guest": resources.guest_memory}, + "resources": domain_resources_json, + "devices": { + "disks": disks, + "filesystems": filesystems, + "interfaces": [{"name": "default", "masquerade": {}}], + // Minimal device surface: a root guest drives every + // emulated device, so nothing optional is attached. + // No graphics, no memory balloon, no VSOCK — the + // serial console stays for the boot log. Still + // present per the live domain XML: the guest-agent + // virtio-serial channel (KubeVirt adds it + // unconditionally; a hook sidecar is the removal + // path, pending a design call), the q35 implicit + // PS/2 input devices, the iTCO watchdog (a root + // guest can arm it for an in-place reset — the + // reset detection covers it), and empty SATA/SCSI + // controllers. The RFC documents the full + // inventory. + "autoattachGraphicsDevice": false, + "autoattachMemBalloon": false, + "autoattachInputDevice": false, + "autoattachVSOCK": false, + "autoattachSerialConsole": true + } + }, + "volumes": volumes, + "networks": [{"name": "default", "pod": {}}] + } + } + } + }); + + let mut obj = DynamicObject::new(params.kube_name, &virtual_machine_api_resource()); + obj.metadata = ObjectMeta { + name: Some(params.kube_name.to_string()), + namespace: Some(params.namespace.to_string()), + labels: Some(sandbox_labels(sandbox, Some(¶ms.config.gateway_id))), + annotations: Some(annotations), + ..Default::default() + }; + obj.data = data; + obj +} + +/// Label set on the VM template so the VMI (and virt-launcher pod) carry the +/// `OpenShell` ownership markers. +const KUBEVIRT_VM_LABEL: &str = "kubevirt.io/domain"; +const KUBEVIRT_VM_LABEL_VALUE: &str = "openshell"; + +/// Set on the `VirtualMachine` while the driver's create still holds it +/// `Halted` (the W6 window between capturing the VM's UID and patching +/// `runStrategy` to `Once`). A `Halted` VM with this annotation maps to a +/// `Bootstrapping=True` condition — the gateway composes `Provisioning`, +/// the same escape the pod path uses for a starting generation — instead +/// of `Suspended=True`, which would compose `Stopped` and withdraw the +/// shared-supervisor instance before the guest's boundary can dial. A +/// `Halted` VM WITHOUT the annotation is a user stop (`Suspended=True`). +pub const KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION: &str = "openshell.ai/kubevirt-create-in-flight"; + +/// How long the create's Halted window may last before the driver maps the +/// VM to a terminal create timeout (M7). The window is seconds in a healthy +/// create; a gateway that died mid-create leaves the VM Halted with the +/// in-flight annotation, and a gateway restarted from a clean database has +/// no sandbox record for its own provisioning-deadline reclaim — the +/// driver-side deadline makes the wedge visible (public Error) either way. +pub const KUBEVIRT_CREATE_DEADLINE_MS: i64 = 10 * 60 * 1000; + +/// The create's in-flight annotation value: the Unix-millisecond timestamp +/// of the Halted-window start (older revisions wrote a bare "true", which +/// parses to no deadline). +/// The guest image the VM actually boots: the containerDisk reference in +/// the VM spec's direct kernel boot (M6: the claim tracks what runs, not +/// the configured string — after a config change, start must not claim an +/// image the VM is not booting). +pub fn kubevirt_boot_image(vm: &DynamicObject) -> String { + vm.data + .pointer("/spec/template/spec/domain/firmware/kernelBoot/container/image") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string() +} + +fn kubevirt_create_in_flight_since(obj: &DynamicObject) -> Option { + obj.metadata + .annotations + .as_ref()? + .get(KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION)? + .parse::() + .ok() +} + +/// The bootstrap Secret for one generation: immutable per generation +/// (`-boot-`), so a start creates the new generation's +/// Secret and deletes the previous one by exact name with a UID +/// precondition instead of rewriting (patching) a live Secret. The +/// generation is the launch authentication's runtime generation, the +/// same value that scopes the supervisor Pod's bootstrap Secret. +pub fn bootstrap_secret_name(kube_name: &str, generation: &str) -> String { + // The same generation normalization `SandboxRuntimeNames` applies + // (alphanumeric, lowercase, bounded), so the name stays DNS-safe and + // matches the supervisor Secret's generation suffix. + let normalized = generation + .chars() + .filter(char::is_ascii_alphanumeric) + .take(12) + .collect::() + .to_ascii_lowercase(); + format!("{kube_name}-boot-{normalized}") +} + +/// A companion recorded on the VM: its exact name and the UID from its +/// create response, so later deletes need no read and no search. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CompanionRecord { + pub name: String, + pub uid: String, +} + +/// The VM annotation carrying the current generation's recorded +/// companions (name → UID), so teardown and generation turnover never +/// list or read the objects themselves. +pub const COMPANIONS_ANNOTATION: &str = "openshell.ai/kubevirt-companions"; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RecordedCompanions { + pub boot_secret: CompanionRecord, + pub supervisor_secret: Option, +} + +impl RecordedCompanions { + /// Parse the annotation value. `None` when the annotation is absent + /// or malformed (a pre-annotation VM): callers treat that as "no + /// recorded companion" and rely on owner-reference garbage + /// collection rather than guessing. + pub fn from_annotation(value: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(value).ok()?; + let boot_secret = CompanionRecord { + name: parsed["boot_secret"]["name"].as_str()?.to_string(), + uid: parsed["boot_secret"]["uid"] + .as_str() + .unwrap_or_default() + .to_string(), + }; + let supervisor_secret = + parsed["supervisor_secret"]["name"] + .as_str() + .map(|name| CompanionRecord { + name: name.to_string(), + uid: parsed["supervisor_secret"]["uid"] + .as_str() + .unwrap_or_default() + .to_string(), + }); + Some(Self { + boot_secret, + supervisor_secret, + }) + } + + /// Render the annotation value. + pub fn to_annotation(&self) -> String { + serde_json::json!({ + "boot_secret": {"name": self.boot_secret.name, "uid": self.boot_secret.uid}, + "supervisor_secret": self.supervisor_secret.as_ref().map(|record| { + serde_json::json!({"name": record.name, "uid": record.uid}) + }), + }) + .to_string() + } +} + +pub fn workspace_pvc_name(kube_name: &str) -> String { + format!("{kube_name}-workspace") +} + +/// Map a `VirtualMachine` object to a driver sandbox snapshot without VMI +/// phase context. Production paths join the VMI phase (see +/// [`sandbox_from_vm_object_with_vmi`]); this convenience wrapper exists for +/// callers with only a VM object at hand. +#[cfg(test)] +fn sandbox_from_vm_object( + namespace: &str, + obj: DynamicObject, +) -> Result<(String, Sandbox), String> { + sandbox_from_vm_object_with_vmi(namespace, obj, None) +} + +/// Map a `VirtualMachine` object, joined with its paired VMI's phase, to a +/// driver sandbox snapshot. +/// +/// Backend observations only: the gateway still composes public `Ready` from +/// the supervisor relay session, exactly as with Sandbox CRs. `vmi_phase` is +/// the phase of the paired `VirtualMachineInstance` when known (the VMI +/// carries the same labels); it distinguishes a failed or completed guest +/// from one that has not booted yet. +#[cfg(test)] +fn sandbox_from_vm_object_with_vmi( + namespace: &str, + obj: DynamicObject, + vmi_phase: Option<&str>, +) -> Result<(String, Sandbox), String> { + sandbox_from_vm_object_with_runtime(namespace, obj, vmi_phase, None, None, None) +} + +/// The full snapshot join: the VM object, its VMI's phase, the dedicated +/// supervisor Pod's failure (when its generation's supervisor is gone or +/// crashed), and the fence drift marker (when the watch reconciler restored +/// a drifted or deleted fence). +pub fn sandbox_from_vm_object_with_runtime( + namespace: &str, + obj: DynamicObject, + vmi_phase: Option<&str>, + supervisor_failure: Option<&str>, + fence_restored_at: Option<&str>, + vmi_not_ready_since_ms: Option, +) -> Result<(String, Sandbox), String> { + let kube_name = obj.metadata.name.clone().unwrap_or_default(); + + if !is_openshell_managed(&obj) { + return Err(format!("object {kube_name} not managed by openshell")); + } + let Some(id) = annotation_or_label(&obj, LABEL_SANDBOX_ID) else { + return Err(format!("object {kube_name} missing sandbox id")); + }; + let Some(name) = annotation_or_label(&obj, LABEL_SANDBOX_NAME) else { + return Err(format!("object {kube_name} missing sandbox name")); + }; + let Some(workspace) = annotation_or_label(&obj, LABEL_SANDBOX_WORKSPACE) else { + return Err(format!("object {kube_name} missing sandbox workspace")); + }; + + let namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.to_string()); + let status = vm_status_from_object( + &obj, + &name, + vmi_phase, + supervisor_failure, + fence_restored_at, + vmi_not_ready_since_ms, + ); + + Ok(( + kube_name, + Sandbox { + id, + name, + namespace, + spec: None, + status, + workspace, + }, + )) +} + +/// Translate VM status into driver conditions: +/// - `status.ready == true` becomes a backend `Ready=True` observation. +/// Without a connected supervisor session the gateway still reports +/// `Provisioning` ("`SupervisorNotConnected`") — the central readiness +/// invariant. +/// - `runStrategy: Halted` becomes `Suspended=True` (public `Stopped`). +/// - a terminal VMI phase (`Failed`/`Succeeded` while the VM is not ready) or +/// a terminal printable status (e.g. `CrashLoopBackOff`) +/// becomes `Ready=False` with the non-transient `VMFailed` reason so the +/// gateway surfaces the sandbox as `Error` instead of provisioning forever. +/// - a deletion timestamp becomes `deleting`. +fn vm_status_from_object( + obj: &DynamicObject, + sandbox_name: &str, + vmi_phase: Option<&str>, + supervisor_failure: Option<&str>, + fence_restored_at: Option<&str>, + vmi_not_ready_since_ms: Option, +) -> Option { + let status = obj.data.get("status")?; + let status_obj = status.as_object()?; + + let ready = status_obj + .get("ready") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + // Only an explicit `runStrategy: Halted` maps to `Suspended`. A fresh VM + // (or one whose `Once` VMI exited) also reports `printableStatus: + // Stopped`; treating that as Suspended would flip a provisioning sandbox + // to public `Stopped`. + let halted = obj + .data + .get("spec") + .and_then(|spec| spec.get("runStrategy")) + .and_then(serde_json::Value::as_str) + == Some("Halted"); + // The create path holds the VM `Halted` while it captures the UID and + // builds the fence, bootstrap Secret, and shared-instance registration + // before patching `runStrategy` to `Once`. During that window the + // annotation above is present: report `Bootstrapping` (the gateway + // composes `Provisioning`) rather than `Suspended` (which composes + // `Stopped` and withdraws the supervisor instance the create just + // registered). + let create_in_flight = obj + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION)) + .is_some_and(|value| !value.is_empty()); + let stopped = halted && !create_in_flight; + + let printable = status_obj + .get("printableStatus") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + + let terminal = terminal_vm_failure(printable, vmi_phase, ready).or_else(|| { + // L4: an image-pull backoff that has persisted past the threshold + // is terminal — the pull is effectively permanent, and the gateway + // would otherwise surface only the configuration gate's message + // until the CLI's provision timeout expires. + (printable == "ImagePullBackOff" + && vmi_not_ready_since_ms.is_some_and(|since| { + openshell_core::time::now_ms() - since > IMAGE_PULL_BACKOFF_TERMINAL_AFTER_MS + })) + .then(|| "guest image pull stuck in backoff (the image cannot be pulled)".to_string()) + }); + + // A create whose Halted window outlived the deadline is a terminal + // failure (M7): the gateway died mid-create, and a gateway restarted + // from a clean database has no record to reclaim with. + let create_timed_out = halted + && create_in_flight + && kubevirt_create_in_flight_since(obj).is_some_and(|since| { + openshell_core::time::now_ms() - since > KUBEVIRT_CREATE_DEADLINE_MS + }); + + // The image-pin state annotation (M6 digest pinning, side-loaded case) + // surfaces as an ImagePin condition so operators see that the guest + // image cannot be digest-pinned. + let image_not_pinned = matches!( + obj.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(IMAGE_PIN_STATE_ANNOTATION)) + .map(String::as_str), + Some(IMAGE_PIN_NO_REPOSITORY_DIGEST) + ); + + // A dedicated generation whose supervisor Pod is gone or crashed is a + // terminal failure (`restartPolicy: Never`): the boundary can never + // connect, so the sandbox surfaces as Error rather than provisioning + // forever. It wins over the not-ready fallback but not over an + // already-terminal VM failure. Suppressed while the VM is halted OR + // halting (`printableStatus: Stopping` — the VMI is terminating): the + // supervisor exiting then is the expected end of a stop, not a failure + // (S1 — mapping it aborted stops whose supervisor Pod exited while the + // stop was still in flight). + let supervisor_failed = + supervisor_failure.filter(|_| terminal.is_none() && !halted && printable != "Stopping"); + + let mut conditions = Vec::new(); + conditions.push(DriverCondition { + r#type: "Ready".to_string(), + status: if ready && terminal.is_none() && supervisor_failed.is_none() { + "True" + } else { + "False" + } + .to_string(), + reason: if terminal.is_some() { + REASON_VM_FAILED.to_string() + } else if supervisor_failed.is_some() { + REASON_SUPERVISOR_FAILED.to_string() + } else if ready { + "VMReady".to_string() + } else if create_timed_out { + REASON_CREATE_TIMED_OUT.to_string() + } else { + REASON_VM_NOT_READY.to_string() + }, + message: terminal + .map(|terminal| format!("VirtualMachine failed: {terminal}")) + .or_else(|| { + supervisor_failed + .map(|failure| format!("dedicated supervisor Pod failed: {failure}")) + }) + .or_else(|| ready.then(|| "VirtualMachineInstance is running".to_string())) + .or_else(|| { + create_timed_out.then(|| { + "the create's Halted window outlived the deadline (the gateway \ + likely died mid-create); stop or delete the sandbox" + .to_string() + }) + }) + .unwrap_or_else(|| { + // Surface the VMI's phase alongside the printable status + // so launcher-side problems are visible (L4). + vmi_phase.map_or_else( + || format!("VirtualMachine status: {printable}"), + |phase| format!("VirtualMachine status: {printable} (VMI {phase})"), + ) + }), + transition_time: None, + }); + // The watch reconciler re-fenced a drifted or deleted fence and stamped + // this marker; surface the event so operators see that isolation + // briefly depended on the namespace-wide default-deny floor. + if image_not_pinned { + conditions.push(DriverCondition { + r#type: "ImagePin".to_string(), + status: "False".to_string(), + reason: "NoRepositoryDigest".to_string(), + message: "image not pinned: no repository digest (side-loaded guest \ + image); the configured tag is used directly" + .to_string(), + transition_time: None, + }); + } + if let Some(restored_at) = fence_restored_at { + conditions.push(DriverCondition { + r#type: "FenceRestored".to_string(), + status: "True".to_string(), + reason: "ReFencedAfterDrift".to_string(), + message: format!( + "the egress fence drifted or was deleted and was re-created by the driver \ + (restored at {} Unix seconds)", + restored_at + .parse::() + .map(|millis| millis / 1000) + .map_or_else(|_| restored_at.to_string(), |seconds| seconds.to_string()) + ), + transition_time: None, + }); + } + if halted && create_in_flight { + conditions.push(DriverCondition { + r#type: "Bootstrapping".to_string(), + status: "True".to_string(), + reason: "SandboxRuntimeGenerationStarting".to_string(), + message: "driver create is provisioning the VirtualMachine's companions".to_string(), + transition_time: None, + }); + } + if stopped { + conditions.push(DriverCondition { + r#type: "Suspended".to_string(), + status: "True".to_string(), + reason: "VMHalted".to_string(), + message: "VirtualMachine runStrategy is Halted".to_string(), + transition_time: None, + }); + } + + Some(SandboxStatus { + name: sandbox_name.to_string(), + // The VMI is created with the same name as the VM. + instance_id: obj.metadata.name.clone().unwrap_or_default(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions, + deleting: obj.metadata.deletion_timestamp.is_some(), + ..Default::default() + }) +} + +fn is_openshell_managed(obj: &DynamicObject) -> bool { + obj.metadata + .annotations + .as_ref() + .and_then(|a| a.get(LABEL_MANAGED_BY)) + .or_else(|| { + obj.metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_MANAGED_BY)) + }) + .map(String::as_str) + == Some(LABEL_MANAGED_BY_VALUE) +} + +fn annotation_or_label(obj: &DynamicObject, key: &str) -> Option { + obj.metadata + .annotations + .as_ref() + .and_then(|a| a.get(key)) + .or_else(|| obj.metadata.labels.as_ref().and_then(|l| l.get(key))) + .cloned() +} + +/// Single-quote a value for the shell-sourceable user-data file. +#[cfg(test)] +fn shell_quote(value: &str) -> String { + if value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "._-/:+=@".contains(c)) + { + return value.to_string(); + } + format!("'{}'", value.replace('\'', "'\\''")) +} + +/// True when the driver is configured for the `KubeVirt` runtime. +pub fn is_kubevirt_mode(config: &KubernetesComputeConfig) -> bool { + config.runtime == KubernetesRuntimeMode::KubeVirt +} + +/// Shared-mode namespace the VM resources render into. Managed mode +/// resolves per-workspace namespaces through the config's +/// `namespace_for_workspace` (create) and cluster-wide label lookups +/// (every other entry point). +pub fn kubevirt_namespace(config: &KubernetesComputeConfig) -> Result { + if config.workspace_mode != WorkspaceMode::Shared { + return Err(format!( + "kubevirt namespace resolution requires the workspace; the \ + configured mode is {}", + config.workspace_mode + )); + } + Ok(config.namespace.clone()) +} + +/// The API scope the `KubeVirt` driver lists and watches VMs over: the +/// sandbox namespace in shared mode, the whole cluster in managed mode +/// (sandbox VMs live in per-workspace namespaces the sandbox id alone +/// does not name). +pub fn kubevirt_list_api( + client: &Client, + config: &KubernetesComputeConfig, +) -> Result, String> { + match config.workspace_mode { + WorkspaceMode::Shared => { + let namespace = kubevirt_namespace(config)?; + Ok(virtual_machine_api(client, &namespace)) + } + WorkspaceMode::Managed | WorkspaceMode::Operator => Ok(Api::all_with( + client.clone(), + &virtual_machine_api_resource(), + )), + } +} + +/// Timeout for the managed-mode namespace discovery lookup. +const NAMESPACE_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(30); + +/// Discover the namespace of the single VM carrying the sandbox-id label. +/// Shared mode reads the configured namespace; managed mode looks +/// cluster-wide (the sandbox id alone does not name the per-workspace +/// namespace). More than one match is an ambiguous identity. +pub async fn discover_kubevirt_namespace( + client: &Client, + config: &KubernetesComputeConfig, + sandbox_id: &str, + selector: &str, +) -> Result { + if config.workspace_mode == WorkspaceMode::Shared { + return kubevirt_namespace(config).map_err(KubernetesDriverError::Precondition); + } + let api = kubevirt_list_api(client, config).map_err(KubernetesDriverError::Precondition)?; + let list = tokio::time::timeout( + NAMESPACE_DISCOVERY_TIMEOUT, + api.list(&ListParams::default().labels(selector)), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API", + NAMESPACE_DISCOVERY_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + let items = list.items; + if items.is_empty() { + return Err(KubernetesDriverError::NotFound); + } + if items.len() > 1 { + return Err(KubernetesDriverError::Precondition(format!( + "sandbox id {sandbox_id} matches VirtualMachines in multiple namespaces" + ))); + } + items[0] + .metadata + .namespace + .clone() + .ok_or_else(|| KubernetesDriverError::Message("VirtualMachine has no namespace".into())) +} + +/// List driver-owned VMs by label selector. +pub async fn list_virtual_machines( + client: &Client, + config: &KubernetesComputeConfig, + selector: &str, +) -> Result, KubernetesDriverError> { + // Shared mode lists the sandbox namespace; managed mode lists + // cluster-wide (the VMs live in per-workspace namespaces). + let api = match config.workspace_mode { + WorkspaceMode::Shared => { + let namespace = kubevirt_namespace(config).map_err(KubernetesDriverError::Message)?; + virtual_machine_api(client, &namespace) + } + WorkspaceMode::Managed | WorkspaceMode::Operator => Api::all_with( + client.clone(), + &ApiResource { + group: KUBEVIRT_GROUP.to_string(), + version: KUBEVIRT_VERSION.to_string(), + api_version: format!("{KUBEVIRT_GROUP}/{KUBEVIRT_VERSION}"), + kind: "VirtualMachine".to_string(), + plural: "virtualmachines".to_string(), + }, + ), + }; + api.list(&ListParams::default().labels(selector)) + .await + .map_err(KubernetesDriverError::from_kube) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[allow( + unused_imports, + reason = "used by the tests below; the inline test above also imports super" + )] + use super::*; + use crate::KubernetesComputeConfig; + use openshell_core::proto::compute::v1::DriverSandbox as Sandbox; + + fn sandbox() -> Sandbox { + Sandbox { + id: "sbx-id".to_string(), + name: "demo".to_string(), + namespace: "openshell-e2e-ns".to_string(), + spec: None, + status: None, + workspace: "default".to_string(), + } + } + + fn config() -> KubernetesComputeConfig { + KubernetesComputeConfig { + runtime: KubernetesRuntimeMode::KubeVirt, + namespace: "openshell-e2e-ns".to_string(), + gateway_id: "e2e-gateway".to_string(), + grpc_endpoint: "http://172.18.0.1:18085/".to_string(), + kubevirt: crate::KubeVirtConfig { + guest_image: "openshell/sandbox-vm:test".to_string(), + guest_image_pull_policy: "IfNotPresent".to_string(), + cpu_cores: 2, + memory: "2Gi".to_string(), + workspace_pvc: true, + }, + shared_supervisor: Some(crate::SharedSupervisorConfig { + service_name: "openshell-supervisor".to_string(), + service_namespace: "openshell".to_string(), + runtime_port: 5500, + token_secret_name: "openshell-supervisor-token".to_string(), + }), + ..KubernetesComputeConfig::default() + } + } + + fn params<'a>( + sandbox: &'a Sandbox, + config: &'a KubernetesComputeConfig, + ) -> KubeVirtSandboxParams<'a> { + KubeVirtSandboxParams { + sandbox, + config, + namespace: "openshell-e2e-ns", + kube_name: "default--demo", + generation: "1", + } + } + + #[test] + fn virtual_machine_renders_kubevirt_contract() { + let cfg = config(); + let vm = render_virtual_machine(¶ms(&sandbox(), &cfg)); + + assert_eq!(vm.metadata.name.as_deref(), Some("default--demo")); + assert_eq!(vm.metadata.namespace.as_deref(), Some("openshell-e2e-ns")); + let labels = vm.metadata.labels.expect("labels"); + assert_eq!( + labels.get(LABEL_MANAGED_BY).map(String::as_str), + Some("openshell") + ); + + assert_eq!(vm.data["spec"]["runStrategy"], "Once"); + + let domain = &vm.data["spec"]["template"]["spec"]["domain"]; + assert_eq!(domain["cpu"]["cores"], 2); + assert_eq!(domain["memory"]["guest"], "2Gi"); + // Direct kernel boot from the purpose-built image. + assert_eq!( + domain["firmware"]["kernelBoot"]["container"]["image"], + "openshell/sandbox-vm:test" + ); + assert_eq!( + domain["firmware"]["kernelBoot"]["container"]["kernelPath"], + "/boot/vmlinuz-virt" + ); + let kernel_args = domain["firmware"]["kernelBoot"]["kernelArgs"] + .as_str() + .expect("kernel args"); + assert!(kernel_args.contains("root=/dev/vda")); + assert!(kernel_args.contains("init=/opt/openshell/init")); + + let volumes = vm.data["spec"]["template"]["spec"]["volumes"] + .as_array() + .unwrap(); + let names: Vec<&str> = volumes + .iter() + .map(|v| v["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, vec!["rootdisk", "cloudinit", "workspace"]); + assert_eq!( + volumes[1]["cloudInitNoCloud"]["secretRef"]["name"], + "default--demo-boot-1" + ); + assert_eq!( + volumes[2]["persistentVolumeClaim"]["claimName"], + "default--demo-workspace" + ); + + // The template labels carry the boundary pair and workload role: + // KubeVirt propagates them to the VMI and virt-launcher pod, where + // the pod path's control NetworkPolicy and the egress fence select + // the pair's two roles (isolation.rs pair_role_labels). + let template_labels = &vm.data["spec"]["template"]["metadata"]["labels"]; + assert_eq!( + template_labels["openshell.ai/boundary-pair"], "sbx-id", + "the launcher pod must carry this sandbox's pair label" + ); + assert_eq!( + template_labels["openshell.ai/boundary-role"], "workload", + "the launcher pod must carry the workload role label" + ); + + let filesystems = vm.data["spec"]["template"]["spec"]["domain"]["devices"]["filesystems"] + .as_array() + .unwrap(); + assert_eq!(filesystems.len(), 1); + assert_eq!(filesystems[0]["name"], "workspace"); + } + + #[test] + fn virtual_machine_minimizes_the_device_surface() { + // The reviewer's condition 6: a root guest drives every emulated + // device, so nothing optional is attached — no graphics, no memory + // balloon, no input. The serial console stays for the boot log. + let cfg = config(); + let vm = render_virtual_machine(¶ms(&sandbox(), &cfg)); + let devices = &vm.data["spec"]["template"]["spec"]["domain"]["devices"]; + assert_eq!(devices["autoattachGraphicsDevice"], false); + assert_eq!(devices["autoattachMemBalloon"], false); + assert_eq!(devices["autoattachInputDevice"], false); + assert_eq!(devices["autoattachSerialConsole"], true); + } + + #[test] + fn virtual_machine_omits_workspace_without_pvc() { + let mut cfg = config(); + cfg.kubevirt.workspace_pvc = false; + let vm = render_virtual_machine(¶ms(&sandbox(), &cfg)); + let volumes = vm.data["spec"]["template"]["spec"]["volumes"] + .as_array() + .unwrap(); + assert_eq!(volumes.len(), 2); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["devices"]["filesystems"] + .as_array() + .map(Vec::len), + Some(0) + ); + } + + #[test] + fn vm_status_maps_conditions() { + let cfg = config(); + let vm = render_virtual_machine(¶ms(&sandbox(), &cfg)); + let mut with_status = vm.clone(); + with_status.data["status"] = serde_json::json!({"ready": true}); + let (_, sbx) = + sandbox_from_vm_object("openshell-e2e-ns", with_status).expect("managed vm converts"); + let status = sbx.status.expect("status"); + let ready = status + .conditions + .iter() + .find(|c| c.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.status, "True"); + + // A halted VM reports Suspended. + let mut halted = vm; + halted.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + halted.data["status"] = serde_json::json!({"ready": false}); + let (_, sbx) = sandbox_from_vm_object("openshell-e2e-ns", halted).expect("managed vm"); + let status = sbx.status.expect("status"); + assert!( + status + .conditions + .iter() + .any(|c| c.r#type == "Suspended" && c.status == "True") + ); + + // A freshly created VM (runStrategy Once, not ready) must NOT report + // Suspended even though printableStatus is Stopped. + let mut fresh = render_virtual_machine(¶ms(&sandbox(), &config())); + fresh.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + let (_, sbx) = sandbox_from_vm_object("openshell-e2e-ns", fresh).expect("managed vm"); + let status = sbx.status.expect("status"); + assert!(!status.conditions.iter().any(|c| c.r#type == "Suspended")); + } + + #[test] + fn unmanaged_vm_is_rejected() { + let cfg = config(); + let mut vm = render_virtual_machine(¶ms(&sandbox(), &cfg)); + vm.metadata + .labels + .as_mut() + .unwrap() + .remove(LABEL_MANAGED_BY); + assert!(sandbox_from_vm_object("openshell-e2e-ns", vm).is_err()); + } + + #[test] + fn kubevirt_namespace_rejects_non_shared_modes() { + let mut cfg = config(); + cfg.workspace_mode = WorkspaceMode::Managed; + assert!(kubevirt_namespace(&cfg).is_err()); + assert!(kubevirt_namespace(&config()).is_ok()); + } + + #[test] + fn shell_quote_keeps_values_lossless() { + assert_eq!(shell_quote("simple-value.1"), "simple-value.1"); + assert_eq!(shell_quote("has space"), "'has space'"); + assert_eq!(shell_quote("it's"), "'it'\\''s'"); + } +} + +/// How `start` should drive a `runStrategy: Once` VM back to running. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KubeVirtStartPlan { + /// The VM is halted: a single `runStrategy: Once` patch boots a fresh + /// VMI. + PatchOnce, + /// The VMI is live or booting (Pending/Scheduling/Scheduled/Running): + /// start is idempotent. The gateway admits retries from the `Starting` + /// phase, and tearing down a healthy boot to "really start" would turn + /// every retry into a restart. The start returns the current runtime + /// identity. + AlreadyRunning, + /// The VMI is terminal (`Succeeded`/`Failed`) or gone while the strategy + /// is `Once` and the VM reports `Stopped`: a completed VMI makes an + /// `Once` -> `Once` patch a silent no-op, so boot requires bouncing + /// through `Halted` first so `KubeVirt` discards the dead VMI. + BounceHaltedThenOnce, +} + +/// Decide how `start` must drive the VM, from the run strategy, the VM's +/// ready flag and printable status, and the paired VMI's phase. Extracted +/// for unit tests. Never tears down a VMI it does not positively know is +/// terminal: unknown phases and missing-context combinations are treated as +/// a boot in progress. +#[must_use] +pub fn kubevirt_start_plan( + run_strategy: &str, + vm_ready: bool, + vmi_phase: Option<&str>, + printable_status: &str, +) -> KubeVirtStartPlan { + // A halted VM always needs the `Once` patch, whatever VMI phase is + // still reported: `stop` only waits for the VM's ready flag to clear, + // so a VMI in graceful shutdown can still show Running (or any other + // phase) here. Returning AlreadyRunning for those would silently leave + // the VM halted. The driver waits out a lingering VMI before patching. + if run_strategy == "Halted" { + return KubeVirtStartPlan::PatchOnce; + } + match vmi_phase { + // Live or booting VMI: idempotent no-op. + Some("Pending" | "Scheduling" | "Scheduled" | "Running" | "WaitingForSync") => { + KubeVirtStartPlan::AlreadyRunning + } + // Terminal VMI: the Once->Once patch would be ignored. + Some("Succeeded" | "Failed") => KubeVirtStartPlan::BounceHaltedThenOnce, + // Unknown phase (KubeVirt could not obtain state): conservatively + // treat as live. + Some(_) => KubeVirtStartPlan::AlreadyRunning, + None => { + if run_strategy == "Once" && !vm_ready && printable_status == "Stopped" { + // The VMI object is gone and the VM reports Stopped under + // Once: a dead VMI the controller will not replace on a + // no-op patch. + KubeVirtStartPlan::BounceHaltedThenOnce + } else { + // No VMI object yet under Once with a printable status that + // implies the controller is creating one: boot in progress. + KubeVirtStartPlan::AlreadyRunning + } + } + } +} + +/// Namespaced API for `kubevirt.io/v1` `virtualmachineinstances`. +pub fn virtual_machine_instance_api_resource() -> ApiResource { + ApiResource { + group: KUBEVIRT_GROUP.to_string(), + version: KUBEVIRT_VERSION.to_string(), + api_version: format!("{KUBEVIRT_GROUP}/{KUBEVIRT_VERSION}"), + kind: "VirtualMachineInstance".to_string(), + plural: "virtualmachineinstances".to_string(), + } +} + +pub fn virtual_machine_instance_api(client: &Client, namespace: &str) -> Api { + Api::namespaced_with( + client.clone(), + namespace, + &ApiResource { + group: KUBEVIRT_GROUP.to_string(), + version: KUBEVIRT_VERSION.to_string(), + api_version: format!("{KUBEVIRT_GROUP}/{KUBEVIRT_VERSION}"), + kind: "VirtualMachineInstance".to_string(), + plural: "virtualmachineinstances".to_string(), + }, + ) +} + +/// Delete an object and wait until it is actually gone from the API. +/// +/// Companion resources (the bootstrap Secret, the workspace PVC with its +/// `pvc-protection` finalizer and provisioner cleanup, proxy companions) +/// take time to terminate; an immediate recreate collides with the dying +/// object and surfaces as a confusing `AlreadyExists`. Fails with a clear +/// error when the object is still present after `timeout`. +pub async fn delete_and_wait( + api: &Api, + name: &str, + what: &str, + timeout: Duration, +) -> Result<(), KubernetesDriverError> { + let _ = api.delete(name, &DeleteParams::default()).await; + wait_until_gone(api, name, what, timeout).await +} + +/// Delete an object with a UID precondition and wait until it is gone, so a +/// concurrent replacement of the same name is never deleted. +pub async fn delete_and_wait_with_uid( + api: &Api, + name: &str, + what: &str, + uid: &str, + timeout: Duration, +) -> Result<(), KubernetesDriverError> { + let preconditions = DeleteParams::default().preconditions(Preconditions { + uid: Some(uid.to_string()), + resource_version: None, + }); + let _ = api.delete(name, &preconditions).await; + wait_until_gone(api, name, what, timeout).await +} + +async fn wait_until_gone( + api: &Api, + name: &str, + what: &str, + timeout: Duration, +) -> Result<(), KubernetesDriverError> { + let deadline = tokio::time::Instant::now() + timeout; + loop { + match api.get(name).await { + Err(KubeError::Api(err)) if err.code == 404 => return Ok(()), + Err(err) => { + return Err(KubernetesDriverError::Message(format!( + "failed to confirm deletion of {what} {name}: {err}" + ))); + } + Ok(_) => {} + } + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for {what} {name} to terminate", + timeout.as_secs() + ))); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// A companion resource this create path provisioned, identified by its API +/// resource and UID so rollback only ever deletes objects this create made +/// (a concurrent create may have replaced an orphan with its own). +#[derive(Debug, Clone)] +pub struct CreatedCompanion { + pub resource: ApiResource, + pub name: String, + pub uid: String, +} + +/// Best-effort rollback of exactly the companions this create made. Deletes +/// carry UID preconditions, so a companion already replaced by a concurrent +/// create is left alone. +pub async fn rollback_companions(client: &Client, namespace: &str, created: &[CreatedCompanion]) { + for companion in created { + let api = + Api::::namespaced_with(client.clone(), namespace, &companion.resource); + let preconditions = DeleteParams::default().preconditions(Preconditions { + uid: Some(companion.uid.clone()), + resource_version: None, + }); + let _ = api.delete(&companion.name, &preconditions).await; + } +} + +#[cfg(test)] +mod kubevirt_review_tests { + use super::*; + use crate::KubernetesComputeConfig; + use openshell_core::proto::compute::v1::DriverSandbox as Sandbox; + + fn review_sandbox() -> Sandbox { + Sandbox { + id: "sbx-id".to_string(), + name: "demo".to_string(), + namespace: "openshell-e2e-ns".to_string(), + spec: None, + status: None, + workspace: "default".to_string(), + } + } + + fn review_config() -> KubernetesComputeConfig { + KubernetesComputeConfig { + runtime: KubernetesRuntimeMode::KubeVirt, + namespace: "openshell-e2e-ns".to_string(), + gateway_id: "e2e-gateway".to_string(), + grpc_endpoint: "http://172.18.0.1:18085/".to_string(), + kubevirt: crate::KubeVirtConfig { + guest_image: "openshell/sandbox-vm:test".to_string(), + guest_image_pull_policy: "IfNotPresent".to_string(), + cpu_cores: 2, + memory: "2Gi".to_string(), + workspace_pvc: true, + }, + shared_supervisor: Some(crate::SharedSupervisorConfig { + service_name: "openshell-supervisor".to_string(), + service_namespace: "openshell".to_string(), + runtime_port: 5500, + token_secret_name: "openshell-supervisor-token".to_string(), + }), + ..KubernetesComputeConfig::default() + } + } + + fn review_params<'a>( + sandbox: &'a Sandbox, + config: &'a KubernetesComputeConfig, + ) -> KubeVirtSandboxParams<'a> { + KubeVirtSandboxParams { + sandbox, + config, + namespace: "openshell-e2e-ns", + kube_name: "default--demo", + generation: "1", + } + } + + fn template_with_resources( + cpu_request: &str, + cpu_limit: &str, + memory_request: &str, + memory_limit: &str, + ) -> openshell_core::proto::compute::v1::DriverSandboxTemplate { + use openshell_core::proto::compute::v1::{ + DriverResourceRequirements, DriverSandboxTemplate, + }; + DriverSandboxTemplate { + resources: Some(DriverResourceRequirements { + cpu_request: cpu_request.to_string(), + cpu_limit: cpu_limit.to_string(), + memory_request: memory_request.to_string(), + memory_limit: memory_limit.to_string(), + }), + ..Default::default() + } + } + + #[test] + fn domain_resources_default_when_the_sandbox_asks_for_nothing() { + let resources = domain_resources(None, &review_config().kubevirt).expect("defaults map"); + assert_eq!(resources.cores, 2, "the global cpu_cores default"); + assert_eq!(resources.guest_memory, "2Gi", "the global memory default"); + assert_eq!(resources.cpu_request, None); + assert_eq!(resources.cpu_limit, None); + assert_eq!(resources.memory_request, None); + assert_eq!(resources.memory_limit, None); + } + + #[test] + fn domain_resources_requests_only() { + let template = template_with_resources("1500m", "", "1Gi", ""); + let resources = + domain_resources(Some(&template), &review_config().kubevirt).expect("requests map"); + // Fractional request rounds up to whole vCPUs. + assert_eq!(resources.cores, 2); + assert_eq!( + resources.guest_memory, "1Gi", + "guest memory from the request" + ); + assert_eq!(resources.cpu_request.as_deref(), Some("1500m")); + assert_eq!(resources.cpu_limit, None); + assert_eq!(resources.memory_request.as_deref(), Some("1Gi")); + assert_eq!(resources.memory_limit, None); + } + + #[test] + fn domain_resources_limits_only() { + let template = template_with_resources("", "4", "", "4Gi"); + let resources = + domain_resources(Some(&template), &review_config().kubevirt).expect("limits map"); + assert_eq!(resources.cores, 4); + assert_eq!(resources.guest_memory, "4Gi", "guest memory from the limit"); + // An omitted request mirrors the limit, exactly as the pod path. + assert_eq!(resources.cpu_request.as_deref(), Some("4")); + assert_eq!(resources.cpu_limit.as_deref(), Some("4")); + assert_eq!(resources.memory_request.as_deref(), Some("4Gi")); + assert_eq!(resources.memory_limit.as_deref(), Some("4Gi")); + } + + #[test] + fn domain_resources_both_requests_and_limits() { + let template = template_with_resources("1", "2", "1Gi", "2Gi"); + let resources = + domain_resources(Some(&template), &review_config().kubevirt).expect("both map"); + assert_eq!(resources.cores, 2, "the limit sizes the vCPU count"); + assert_eq!(resources.guest_memory, "2Gi"); + assert_eq!(resources.cpu_request.as_deref(), Some("1")); + assert_eq!(resources.cpu_limit.as_deref(), Some("2")); + assert_eq!(resources.memory_request.as_deref(), Some("1Gi")); + assert_eq!(resources.memory_limit.as_deref(), Some("2Gi")); + } + + #[test] + fn domain_resources_fractional_cpu_limit_rounds_up() { + let template = template_with_resources("", "1500m", "", "1Gi"); + let resources = + domain_resources(Some(&template), &review_config().kubevirt).expect("fractional maps"); + // The guest sees 2 vCPUs; the launcher is limited to 1500m. + assert_eq!(resources.cores, 2); + assert_eq!(resources.cpu_limit.as_deref(), Some("1500m")); + } + + #[test] + fn domain_resources_rejects_a_limit_below_the_request() { + let template = template_with_resources("2", "1", "1Gi", "2Gi"); + let error = domain_resources(Some(&template), &review_config().kubevirt) + .expect_err("cpu limit below request"); + assert!(error.contains("below"), "{error}"); + + let template = template_with_resources("1", "2", "2Gi", "1Gi"); + let error = domain_resources(Some(&template), &review_config().kubevirt) + .expect_err("memory limit below request"); + assert!(error.contains("below"), "{error}"); + } + + #[test] + fn domain_resources_rejects_memory_below_the_guest_minimum() { + let template = template_with_resources("", "", "", "128Mi"); + let error = domain_resources(Some(&template), &review_config().kubevirt) + .expect_err("memory below the guest minimum"); + assert!(error.contains("256Mi"), "{error}"); + } + + #[test] + fn domain_resources_rejects_unparsable_quantities() { + let template = template_with_resources("fast", "", "", ""); + assert!(domain_resources(Some(&template), &review_config().kubevirt).is_err()); + let template = template_with_resources("", "", "lots", ""); + assert!(domain_resources(Some(&template), &review_config().kubevirt).is_err()); + } + + #[test] + fn rendered_vm_carries_the_mapped_domain_resources() { + use openshell_core::proto::compute::v1::DriverSandboxSpec; + let mut sandbox = review_sandbox(); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(template_with_resources("1", "1500m", "1Gi", "2Gi")), + ..Default::default() + }); + let cfg = review_config(); + let vm = render_virtual_machine(&review_params(&sandbox, &cfg)); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["cpu"]["cores"], + serde_json::json!(2) + ); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["memory"]["guest"], + serde_json::json!("2Gi") + ); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["cpu"], + serde_json::json!("1") + ); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["cpu"], + serde_json::json!("1500m") + ); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["resources"]["requests"]["memory"], + serde_json::json!("1Gi") + ); + assert_eq!( + vm.data["spec"]["template"]["spec"]["domain"]["resources"]["limits"]["memory"], + serde_json::json!("2Gi") + ); + } + + #[test] + fn start_plan_is_idempotent_during_boot() { + use super::KubeVirtStartPlan; + // A start admitted from Starting during the boot window must not + // restart a healthy, booting VM (the gateway admits retries from + // Starting, and the VM is Once + not ready throughout the window). + for phase in ["Pending", "Scheduling", "Scheduled", "Running"] { + assert_eq!( + kubevirt_start_plan("Once", false, Some(phase), "Starting"), + KubeVirtStartPlan::AlreadyRunning, + "a booting VMI ({phase}) must not be bounced" + ); + } + // So is a ready VM and an unrecognized VMI phase. + assert_eq!( + kubevirt_start_plan("Once", true, Some("Running"), "Running"), + KubeVirtStartPlan::AlreadyRunning + ); + assert_eq!( + kubevirt_start_plan("Once", false, Some("Unknown"), "Unknown"), + KubeVirtStartPlan::AlreadyRunning + ); + // No VMI object yet while the controller creates one: boot starting. + assert_eq!( + kubevirt_start_plan("Once", false, None, "Provisioning"), + KubeVirtStartPlan::AlreadyRunning + ); + } + + #[test] + fn start_plan_bounces_dead_once_vm() { + use super::KubeVirtStartPlan; + // A terminal VMI under Once makes the Once->Once patch a no-op. + for phase in ["Succeeded", "Failed"] { + assert_eq!( + kubevirt_start_plan("Once", false, Some(phase), "Stopped"), + KubeVirtStartPlan::BounceHaltedThenOnce, + "a terminal VMI ({phase}) must be bounced" + ); + } + // A gone VMI with a Stopped printable under Once is also dead. + assert_eq!( + kubevirt_start_plan("Once", false, None, "Stopped"), + KubeVirtStartPlan::BounceHaltedThenOnce + ); + // A halted VM boots with one patch; a ready VM is left alone. + assert_eq!( + kubevirt_start_plan("Halted", false, None, "Stopped"), + KubeVirtStartPlan::PatchOnce + ); + assert_eq!( + kubevirt_start_plan("Once", true, None, "Running"), + KubeVirtStartPlan::AlreadyRunning + ); + } + + #[test] + fn start_plan_halved_vm_always_patches_despite_lingering_vmi() { + use super::KubeVirtStartPlan; + // `stop` only waits for the VM's ready flag; a VMI in graceful + // shutdown can still report any phase. A start issued right after + // stop must patch Once (after waiting out the VMI), never report + // AlreadyRunning and silently leave the VM halted. + for phase in ["Running", "Succeeded", "Failed", "Pending", "Unknown"] { + assert_eq!( + kubevirt_start_plan("Halted", false, Some(phase), "Stopping"), + KubeVirtStartPlan::PatchOnce, + "a halted VM with a lingering VMI ({phase}) must patch" + ); + } + assert_eq!( + kubevirt_start_plan("Halted", false, Some("Succeeded"), "Stopped"), + KubeVirtStartPlan::PatchOnce + ); + } + + #[test] + fn terminal_failure_maps_vmi_and_printable_states() { + // A failed or completed VMI under a not-ready VM is terminal. + assert!(terminal_vm_failure("Stopped", Some("Failed"), false).is_some()); + assert!(terminal_vm_failure("Stopped", Some("Succeeded"), false).is_some()); + // KubeVirt's terminal printable statuses stand on their own + // (VirtualMachinePrintableStatus, KubeVirt v1.9). + for status in [ + "CrashLoopBackOff", + "ErrorUnschedulable", + "ErrorPvcNotFound", + "DataVolumeError", + ] { + assert!( + terminal_vm_failure(status, None, false).is_some(), + "{status} must be terminal" + ); + } + // Image-pull problems are transient (kubelet retries) but visible: + // the caller puts the printableStatus in the condition message. + for status in ["ErrImagePull", "ImagePullBackOff"] { + assert!( + terminal_vm_failure(status, None, false).is_none(), + "{status} must stay transient" + ); + } + // Unknown (controller could not obtain state) can appear transiently; + // mapping it to Error would flip a booting sandbox on a blip. + assert!(terminal_vm_failure("Unknown", None, false).is_none()); + // A running VM, a not-yet-booted VM, and a missing VMI are not + // terminal: those stay provisioning. + assert!(terminal_vm_failure("Running", Some("Running"), true).is_none()); + assert!(terminal_vm_failure("Stopped", None, false).is_none()); + assert!(terminal_vm_failure("Starting", Some("Pending"), false).is_none()); + assert!(terminal_vm_failure("WaitingForVolumeBinding", None, false).is_none()); + // A Succeeded VMI under a ready VM is not a failure signal. + assert!(terminal_vm_failure("Running", Some("Succeeded"), true).is_none()); + } + + #[test] + fn halted_vm_in_create_window_maps_to_bootstrapping_not_suspended() { + // The create path holds the VM Halted while it builds the fence, + // bootstrap Secret, and shared-instance registration. With the + // in-flight annotation present, the snapshot must carry + // `Bootstrapping=True` and no `Suspended=True` — the gateway + // composes Provisioning (keeping the supervisor instance) rather + // than Stopped (which withdraws it before the guest dials). + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION.to_string(), + openshell_core::time::now_ms().to_string(), + ); + let (_, sbx) = + sandbox_from_vm_object_with_vmi("openshell-e2e-ns", vm, None).expect("converts"); + let conditions = sbx.status.unwrap().conditions; + assert!( + conditions + .iter() + .any(|condition| condition.r#type == "Bootstrapping" && condition.status == "True"), + "in-flight Halted VM must report Bootstrapping: {conditions:?}" + ); + assert!( + !conditions + .iter() + .any(|condition| condition.r#type == "Suspended" && condition.status == "True"), + "in-flight Halted VM must not report Suspended: {conditions:?}" + ); + + // Without the annotation the same Halted VM is a user stop: + // Suspended=True, no Bootstrapping. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + let (_, sbx) = + sandbox_from_vm_object_with_vmi("openshell-e2e-ns", vm, None).expect("converts"); + let conditions = sbx.status.unwrap().conditions; + assert!( + conditions + .iter() + .any(|condition| condition.r#type == "Suspended" && condition.status == "True"), + "user-stopped Halted VM must report Suspended: {conditions:?}" + ); + assert!( + !conditions + .iter() + .any(|condition| condition.r#type == "Bootstrapping"), + "user-stopped Halted VM must not report Bootstrapping: {conditions:?}" + ); + } + + #[test] + fn supervisor_failure_while_vm_halting_is_not_an_error() { + // S1: while the VM is halting (`printableStatus: Stopping` — the + // VMI is terminating), the dedicated supervisor Pod exiting + // (restartPolicy Never -> Failed) is the EXPECTED end of a stop, + // not a failure. Mapping it to SupervisorFailed aborted in-flight + // stops ("sandbox lifecycle changed while stop completed"). + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopping"}); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + Some("Running"), + Some("supervisor Pod failed (restartPolicy Never)"), + None, + None, + ) + .expect("converts"); + let ready = sbx.status.clone().unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_ne!( + ready.reason, REASON_SUPERVISOR_FAILED, + "a supervisor exiting while the VM halts must not be SupervisorFailed" + ); + // And the sandbox is NOT terminal: no Error-mapping reason, so the + // gateway keeps composing Stopping/Stopped from the desired state. + assert!( + !sbx.status + .unwrap() + .conditions + .iter() + .any(|condition| condition.reason == REASON_SUPERVISOR_FAILED), + "no condition may carry SupervisorFailed while the VM is halting" + ); + + // Control: the same failed supervisor Pod on a Running (not + // halting) VM still maps to the terminal Error — the suppression + // must not hide genuine crashes. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": true, "printableStatus": "Running"}); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + Some("Running"), + Some("supervisor Pod failed (restartPolicy Never)"), + None, + None, + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, REASON_SUPERVISOR_FAILED); + } + + #[test] + fn supervisor_failure_maps_to_terminal_error() { + // M2: a dedicated generation whose supervisor Pod is gone or + // crashed maps the sandbox to the public Error phase (a + // non-transient Ready=False reason) instead of provisioning + // forever. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Running"}); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + Some("Running"), + Some("supervisor Pod is gone"), + None, + None, + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, REASON_SUPERVISOR_FAILED); + assert!(ready.message.contains("supervisor Pod is gone")); + // The real M2 case: the VM still reports ready when its supervisor + // Pod is deleted under it — the failure must still surface as a + // terminal Error, not hide behind VMReady. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": true, "printableStatus": "Running"}); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + Some("Running"), + Some("supervisor Pod is gone"), + None, + None, + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, REASON_SUPERVISOR_FAILED); + // Without the failure the same VM stays transient. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Running"}); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + Some("Running"), + None, + None, + None, + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.reason, REASON_VM_NOT_READY); + } + + #[test] + fn create_window_watch_event_stays_bootstrapping() { + // The watch re-emits snapshots during the create's Halted window, + // before the dedicated supervisor Pod exists; even if a failure + // were reported for that window, the snapshot must stay + // Bootstrapping (Provisioning), never SupervisorFailed (Error). + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION.to_string(), + openshell_core::time::now_ms().to_string(), + ); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + None, + Some("supervisor Pod is gone"), + None, + None, + ) + .expect("converts"); + let conditions = sbx.status.unwrap().conditions; + let ready = conditions.iter().find(|c| c.r#type == "Ready").unwrap(); + assert_eq!(ready.reason, REASON_VM_NOT_READY); + assert!( + conditions + .iter() + .any(|condition| condition.r#type == "Bootstrapping" && condition.status == "True") + ); + } + + #[test] + fn stale_create_window_maps_to_terminal_timeout() { + // M7: a create whose Halted window outlived the deadline (the + // gateway died mid-create; a gateway restarted from a clean + // database has no record to reclaim with) maps the sandbox to the + // public Error phase instead of provisioning forever. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION.to_string(), + (openshell_core::time::now_ms() - KUBEVIRT_CREATE_DEADLINE_MS - 1).to_string(), + ); + let (_, sbx) = + sandbox_from_vm_object_with_runtime("openshell-e2e-ns", vm, None, None, None, None) + .expect("converts"); + let conditions = sbx.status.unwrap().conditions; + let ready = conditions.iter().find(|c| c.r#type == "Ready").unwrap(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, REASON_CREATE_TIMED_OUT); + assert!(ready.message.contains("outlived the deadline")); + // A fresh window stays Bootstrapping (Provisioning), not Error. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Halted"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + KUBEVIRT_CREATE_IN_FLIGHT_ANNOTATION.to_string(), + openshell_core::time::now_ms().to_string(), + ); + let (_, sbx) = + sandbox_from_vm_object_with_runtime("openshell-e2e-ns", vm, None, None, None, None) + .expect("converts"); + let conditions = sbx.status.unwrap().conditions; + assert!( + conditions + .iter() + .any(|condition| condition.r#type == "Bootstrapping" && condition.status == "True") + ); + let ready = conditions.iter().find(|c| c.r#type == "Ready").unwrap(); + assert_eq!(ready.reason, REASON_VM_NOT_READY); + } + + #[test] + fn guest_image_digest_classifies_launcher_init_containers() { + use k8s_openapi::api::core::v1::{Container, ContainerStatus, Pod, PodSpec, PodStatus}; + const GUEST: &str = "openshell/sandbox-vm:test"; + // statuses: (name, status image, imageID); specs: the pod spec's + // init container image references (what KubeVirt rendered from the + // VM spec — the selection key). + let pod = |statuses: &[(&str, &str, &str)], specs: &[(&str, &str)]| { + let status = PodStatus { + init_container_statuses: Some( + statuses + .iter() + .map(|(name, image, image_id)| ContainerStatus { + name: name.to_string(), + image: image.to_string(), + image_id: image_id.to_string(), + ..ContainerStatus::default() + }) + .collect(), + ), + ..PodStatus::default() + }; + let spec = PodSpec { + init_containers: Some( + specs + .iter() + .map(|(name, image)| Container { + name: name.to_string(), + image: Some(image.to_string()), + ..Container::default() + }) + .collect(), + ), + ..PodSpec::default() + }; + Pod { + status: Some(status), + spec: Some(spec), + ..Pod::default() + } + }; + // The launcher runs other init containers (guest-console-log) on + // the VIRT-LAUNCHER image whose digest must never be pinned: the + // status is matched by the POD SPEC image (what KubeVirt rendered + // from the VM spec), keyed by container name. + let spec = [ + ("guest-console-log", "quay.io/kubevirt/virt-launcher:v1.9.1"), + ("volumerootdisk", GUEST), + ("volumekernel-boot-volume", GUEST), + ]; + let launcher_statuses = [ + ( + "guest-console-log", + "quay.io/kubevirt/virt-launcher:v1.9.1", + "quay.io/kubevirt/virt-launcher@sha256:53e6d121", + ), + ( + "volumerootdisk", + "docker.io/openshell/sandbox-vm:test", + "docker.io/openshell/sandbox-vm@sha256:5a58c279", + ), + ( + "volumekernel-boot-volume", + "docker.io/openshell/sandbox-vm:test", + "docker.io/openshell/sandbox-vm@sha256:5a58c279", + ), + ]; + assert_eq!( + guest_image_digest(&pod(&launcher_statuses, &spec), GUEST), + GuestImageDigest::Patch("docker.io/openshell/sandbox-vm@sha256:5a58c279".to_string()) + ); + // The node knows the same digest under ANOTHER name (a side-loaded + // copy of the registry tag): the status .image disagrees with the + // VM spec, but the POD SPEC image — the selection key — matches, + // so the pin still resolves. + let renamed = [ + ( + "guest-console-log", + "quay.io/kubevirt/virt-launcher:v1.9.1", + "quay.io/kubevirt/virt-launcher@sha256:53e6d121", + ), + ( + "volumerootdisk", + "docker.io/kvreview/openshell-sandbox-vm:dev", + "k3d-kvreview-registry:5050/openshell-sandbox-vm@sha256:fe7f1fae", + ), + ( + "volumekernel-boot-volume", + "docker.io/kvreview/openshell-sandbox-vm:dev", + "k3d-kvreview-registry:5050/openshell-sandbox-vm@sha256:fe7f1fae", + ), + ]; + assert_eq!( + guest_image_digest(&pod(&renamed, &spec), GUEST), + GuestImageDigest::Patch( + "k3d-kvreview-registry:5050/openshell-sandbox-vm@sha256:fe7f1fae".to_string() + ) + ); + // Side-loaded (k3d/ctr import): a bare config digest, no + // repository — never patchable. + let side_loaded = [ + ( + "guest-console-log", + "quay.io/kubevirt/virt-launcher:v1.9.1", + "quay.io/kubevirt/virt-launcher@sha256:53e6d121", + ), + ( + "volumerootdisk", + "docker.io/openshell/sandbox-vm:test", + "sha256:5a58c279", + ), + ]; + assert_eq!( + guest_image_digest(&pod(&side_loaded, &spec), GUEST), + GuestImageDigest::SideLoaded + ); + // Nothing observed yet (no statuses, or none matching the guest + // image). + assert_eq!( + guest_image_digest(&Pod::default(), GUEST), + GuestImageDigest::Absent + ); + let empty = [("volumerootdisk", "docker.io/openshell/sandbox-vm:test", "")]; + assert_eq!( + guest_image_digest(&pod(&empty, &spec), GUEST), + GuestImageDigest::Absent + ); + // Reference normalization: containerd's docker.io/library form + // matches a bare short reference. + assert!(image_ref_matches( + "docker.io/library/busybox:1.36", + "busybox:1.36" + )); + assert!(!image_ref_matches( + "quay.io/kubevirt/virt-launcher:v1.9.1", + GUEST + )); + } + + #[test] + fn side_loaded_image_surfaces_image_pin_condition() { + // M6 (option A): a side-loaded guest image cannot be digest-pinned; + // the reconciler's annotation surfaces as an ImagePin condition so + // operators see the unpinned state. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = serde_json::json!({"ready": true}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + IMAGE_PIN_STATE_ANNOTATION.to_string(), + IMAGE_PIN_NO_REPOSITORY_DIGEST.to_string(), + ); + let (_, sbx) = + sandbox_from_vm_object_with_runtime("openshell-e2e-ns", vm, None, None, None, None) + .expect("converts"); + let conditions = sbx.status.unwrap().conditions; + let pin = conditions + .iter() + .find(|condition| condition.r#type == "ImagePin") + .expect("ImagePin condition present"); + assert_eq!(pin.status, "False"); + assert_eq!(pin.reason, "NoRepositoryDigest"); + assert!(pin.message.contains("no repository digest")); + // Without the annotation there is no condition. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = serde_json::json!({"ready": true}); + let (_, sbx) = + sandbox_from_vm_object_with_runtime("openshell-e2e-ns", vm, None, None, None, None) + .expect("converts"); + assert!( + !sbx.status + .unwrap() + .conditions + .iter() + .any(|condition| condition.r#type == "ImagePin") + ); + } + + #[test] + fn fence_restored_marker_surfaces_as_condition() { + // M1: the reconciler's restore marker surfaces as a FenceRestored + // condition so operators see that isolation briefly depended on + // the namespace-wide default-deny floor. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = serde_json::json!({"ready": true}); + vm.metadata + .annotations + .get_or_insert_with(Default::default) + .insert( + "openshell.ai/kubevirt-fence-restored-at".to_string(), + "1760000000000".to_string(), + ); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + vm, + None, + None, + Some("1760000000000"), + None, + ) + .expect("converts"); + let conditions = sbx.status.unwrap().conditions; + let restored = conditions + .iter() + .find(|condition| condition.r#type == "FenceRestored") + .expect("FenceRestored condition present"); + assert_eq!(restored.status, "True"); + assert_eq!(restored.reason, "ReFencedAfterDrift"); + // Without the marker there is no condition. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = serde_json::json!({"ready": true}); + let (_, sbx) = + sandbox_from_vm_object_with_runtime("openshell-e2e-ns", vm, None, None, None, None) + .expect("converts"); + assert!( + !sbx.status + .unwrap() + .conditions + .iter() + .any(|condition| condition.r#type == "FenceRestored") + ); + } + + #[test] + fn rfc3339_parser_reads_kubernetes_timestamps() { + // The parser backs the image-pull-backoff terminal rule (L4). + assert_eq!( + rfc3339_to_unix_ms("2026-11-02T21:34:56Z"), + Some(1_793_655_296_000) + ); + assert_eq!(rfc3339_to_unix_ms("1970-01-01T00:00:00Z"), Some(0)); + assert_eq!( + rfc3339_to_unix_ms("2026-11-02T21:34:56.123Z"), + Some(1_793_655_296_123) + ); + assert_eq!(rfc3339_to_unix_ms("not-a-timestamp"), None); + } + + #[test] + fn image_pull_backoff_becomes_terminal_after_the_threshold() { + // L4: below the threshold the backoff stays transient (a slow + // registry can legitimately back off); past it the pull is + // effectively permanent and the sandbox maps to Error with a + // message users see, instead of hiding behind the gateway's + // configuration gate until the CLI's provision timeout. + let cfg = review_config(); + let stale = || { + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = + serde_json::json!({"ready": false, "printableStatus": "ImagePullBackOff"}); + vm + }; + let fresh_since = openshell_core::time::now_ms(); + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + stale(), + None, + None, + None, + Some(fresh_since), + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.reason, REASON_VM_NOT_READY); + assert!(ready.message.contains("ImagePullBackOff")); + + let stale_since = openshell_core::time::now_ms() - IMAGE_PULL_BACKOFF_TERMINAL_AFTER_MS - 1; + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + stale(), + None, + None, + None, + Some(stale_since), + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, REASON_VM_FAILED); + assert!(ready.message.contains("image pull")); + + // No timestamp: the rule cannot fire (timing fails open). + let (_, sbx) = sandbox_from_vm_object_with_runtime( + "openshell-e2e-ns", + stale(), + None, + None, + None, + None, + ) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.reason, REASON_VM_NOT_READY); + } + + #[test] + fn transient_image_pull_status_is_visible_in_condition_message() { + // ErrImagePull stays transient but the condition message must carry + // the printableStatus so the cause reaches the user. + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["status"] = + serde_json::json!({"ready": false, "printableStatus": "ImagePullBackOff"}); + let (_, sbx) = + sandbox_from_vm_object_with_vmi("openshell-e2e-ns", vm, None).expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.reason, "Starting"); + assert!( + ready.message.contains("ImagePullBackOff"), + "message must name the pull problem: {}", + ready.message + ); + } + + #[test] + fn vm_status_marks_terminal_vmi_as_error() { + let cfg = review_config(); + let mut vm = render_virtual_machine(&review_params(&review_sandbox(), &cfg)); + vm.data["spec"]["runStrategy"] = serde_json::json!("Once"); + vm.data["status"] = serde_json::json!({"ready": false, "printableStatus": "Stopped"}); + + // Without the VMI phase the not-ready VM stays transient (Starting). + let (_, sbx) = sandbox_from_vm_object_with_vmi("openshell-e2e-ns", vm.clone(), None) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.reason, "Starting"); + + // With a Failed VMI the same VM reports the non-transient VMFailed + // reason, which the gateway maps to the public Error phase. + let (_, sbx) = sandbox_from_vm_object_with_vmi("openshell-e2e-ns", vm, Some("Failed")) + .expect("converts"); + let ready = sbx.status.unwrap().conditions[0].clone(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "VMFailed"); + assert!(ready.message.contains("Failed")); + } + + #[test] + fn grpc_endpoint_is_required_only_for_the_dedicated_topology() { + // The dedicated topology's supervisor Pod must reach the gateway, + // exactly as the pod path's supervisor Pods do, so grpc_endpoint is + // required (and any scheme is fine: the supervisor carries client + // TLS material, so the https:// shape the TLS demo uses is valid). + let mut cfg = review_config(); + cfg.shared_supervisor = None; + cfg.grpc_endpoint = "https://openshell-gateway.ns.svc:8080".to_string(); + assert!(cfg.validate_kubevirt_config().is_ok()); + cfg.grpc_endpoint = String::new(); + let err = cfg + .validate_kubevirt_config() + .expect_err("dedicated without grpc_endpoint is refused"); + assert!( + err.contains("grpc_endpoint"), + "message should name the key: {err}" + ); + } + + #[test] + fn kubevirt_runtime_requires_dedicated_supervisor() { + // Every KubeVirt guest is the untrusted root class, so the runtime + // requires the dedicated supervisor: a shared supervisor would let + // a parser bug reached by a hostile root guest cross tenants. + let mut cfg = review_config(); + cfg.shared_supervisor = None; + assert!(cfg.validate_kubevirt_config().is_ok()); + + cfg.supervisor_topology = Some(crate::SupervisorTopology::Gateway); + let err = cfg + .validate_kubevirt_config() + .expect_err("kubevirt + gateway topology is refused"); + assert!( + err.contains("requires the dedicated supervisor topology"), + "message should name the requirement: {err}" + ); + } + + #[test] + fn validate_configuration_surfaces_kubevirt_rules() { + // The startup validation entry point must include the kubevirt + // rules and the shared topology rules for the kubevirt runtime. + // The default (dedicated) topology validates cleanly. + let mut cfg = review_config(); + cfg.shared_supervisor = None; + assert!(cfg.validate_configuration().is_ok()); + + // A dedicated generation without grpc_endpoint cannot run its + // supervisor Pod. + let mut cfg = review_config(); + cfg.shared_supervisor = None; + cfg.grpc_endpoint = String::new(); + let err = cfg + .validate_configuration() + .expect_err("dedicated without grpc_endpoint is refused"); + assert!( + err.contains("grpc_endpoint"), + "message should name the key: {err}" + ); + + // The gateway topology requires shared_supervisor (enforced by the + // shared validate_supervisor_topology, not kubevirt-specific rules). + let mut cfg = review_config(); + cfg.supervisor_topology = Some(crate::SupervisorTopology::Gateway); + cfg.shared_supervisor = None; + let err = cfg + .validate_configuration() + .expect_err("gateway topology without shared supervisor is refused"); + assert!( + err.contains("requires shared_supervisor"), + "message should name the requirement: {err}" + ); + + // The workspace topology is rejected, as on the pod path. + let mut cfg = review_config(); + cfg.supervisor_topology = Some(crate::SupervisorTopology::Workspace); + cfg.shared_supervisor = None; + assert!(cfg.validate_configuration().is_err()); + + let mut cfg = review_config(); + cfg.supervisor_topology = Some(crate::SupervisorTopology::Gateway); + cfg.kubevirt.guest_image = String::new(); + let err = cfg + .validate_configuration() + .expect_err("empty guest_image is refused"); + assert!( + err.contains("guest_image"), + "message should name the key: {err}" + ); + } +} + +/// Decode the gateway's launch authentication for the start-path token +/// refresh. Empty input skips the refresh (with a warning: the guest keeps +/// the create-time token, which expires under the gateway JWT TTL); a +/// malformed bundle is an error rather than a silent stale-token fallback. +pub fn decode_launch_authentication( + encoded: &[u8], +) -> Result, KubernetesDriverError> { + if encoded.is_empty() { + tracing::warn!( + "start carried no launch authentication; the guest keeps the create-time \ + sandbox token, which expires under the gateway JWT TTL" + ); + return Ok(None); + } + serde_json::from_slice::(encoded) + .map(Some) + .map_err(|error| { + KubernetesDriverError::Precondition(format!( + "invalid launch authentication bundle: {error}" + )) + }) +} diff --git a/crates/openshell-driver-kubernetes/src/kubevirt_boundary.rs b/crates/openshell-driver-kubernetes/src/kubevirt_boundary.rs new file mode 100644 index 0000000000..ccf3f4a6f5 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/kubevirt_boundary.rs @@ -0,0 +1,1685 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `KubeVirt` boundary provisioning for the shared authenticated boundary +//! protocol, modeled on the VM driver's `VmBoundarySpec` +//! (`crates/openshell-driver-vm/src/isolation/mod.rs`). +//! +//! Claims model (user decision, option A): the boundary claims the +//! `VirtualMachine`'s UID (stable across stop/start) and the generation, +//! plus the image identity. The outer-fence evidence carries the +//! `NetworkPolicy` UID that fences the virt-launcher pod's egress, +//! projected against the generation. The virt-launcher pod's own UID +//! cannot be claimed: `KubeVirt` creates that pod asynchronously, so its +//! UID does not exist when the boundary config is rendered. +//! +//! Image identity: the VM driver claims the image's registry digest; +//! the `KubeVirt` `PoC` claims the configured `guest_image` reference (a tag), +//! which is weaker. The production path is a digest-pinned guest image. +//! +//! Who verifies the fence: +//! +//! - the **driver** attests the fence by re-reading the `NetworkPolicy` +//! each generation (`verify_fence`): it fails the start when the fence +//! was deleted, its UID changed (replaced), or its spec drifted; +//! - the **boundary** checks the evidence's generation binding +//! (`contract.rs:254 validate`); +//! - **descriptor/config evidence and claims consistency** is enforced at +//! `isolation_protocol.rs:100` (the driver descriptor must match the +//! provisioned boundary config exactly) and on confirmation +//! (`runtime.rs:757`). +//! +//! Neither the boundary nor the supervisor can independently query the +//! cluster; the driver is the attestation authority for the fence. + +use std::collections::{BTreeMap, HashMap}; +use std::net::SocketAddr; + +use openshell_isolation_interface::contract::{ + BackendError, ProvisioningEvidence, ResolvedWorkloadIdentity, +}; +use openshell_sandbox_backend::boundary_protocol::{ + BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor, + SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport, +}; +use serde::Serialize; + +/// Image identity note. +/// +/// The VM driver claims the image's registry digest +/// (`bootstrap_image_cache_identity` in +/// `crates/openshell-driver-vm/src/driver.rs`). The `KubeVirt` `PoC` claims +/// the configured `guest_image` reference (a tag), which is weaker: a tag +/// does not identify an image. The production path is a digest-pinned +/// Resource-claim keys. +/// +/// The generation marker is recognized by the shared `is_vm_guest` +/// predicate in `boundary_protocol` (user decision, option A: both +/// `vm.generation` and `kubevirt.generation` mark a VM guest). +pub const CLAIM_VM_UID: &str = "kubevirt.vm_uid"; +pub const CLAIM_GENERATION: &str = + openshell_sandbox_backend::boundary_protocol::KUBEVIRT_GENERATION_CLAIM; +pub const CLAIM_IMAGE_IDENTITY: &str = "kubevirt.image_identity"; +/// The dedicated supervisor Pod's UID, claimed only in the dedicated +/// topology (mirrors the pod path's `kubernetes.supervisor_pod_uid`). +pub const CLAIM_SUPERVISOR_POD_UID: &str = "kubevirt.supervisor_pod_uid"; + +/// Where the guest init stages the boundary bootstrap artifacts. These are +/// the paths written into the `BoundaryConfig`'s TLS config and read by the +/// boundary at startup. +pub const GUEST_BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; +pub const GUEST_BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt"; +pub const GUEST_BOUNDARY_PRIVATE_KEY_PATH: &str = "/.openshell/state/bootstrap/tls.key"; + +/// Native evidence that the virt-launcher `NetworkPolicy` fence exists and +/// belongs to this generation. +#[derive(Serialize)] +struct KubeVirtOuterFenceEvidence<'a> { + generation: &'a str, + network_policy_uid: &'a str, + network_policy_resource_version: &'a str, +} + +impl KubeVirtOuterFenceEvidence<'_> { + fn project(&self) -> Result { + if self.generation.is_empty() + || self.network_policy_uid.is_empty() + || self.network_policy_resource_version.is_empty() + { + return Err(BackendError::Descriptor( + "KubeVirt provisioning evidence is incomplete: the NetworkPolicy fence must \ + exist with a UID and resourceVersion before the boundary config is rendered" + .to_string(), + )); + } + let encoded = serde_json::to_vec(self).map_err(|error| { + BackendError::Descriptor(format!("encode KubeVirt outer fence evidence: {error}")) + })?; + let projection = + ProvisioningEvidence::from_enforcement_evidence(self.generation, &encoded)?; + projection.validate(self.generation)?; + Ok(projection) + } +} + +/// Where a `KubeVirt` boundary dials its supervisor, and what the fence +/// admits egress to. +/// +/// Mirrors the pod path's two supervisor topologies: a dedicated supervisor +/// Pod per sandbox (the default), or a sandbox instance on the gateway's +/// shared supervisor. +#[derive(Clone, Debug)] +pub enum KubeVirtSupervisorTarget { + /// This sandbox's dedicated supervisor Pod, fronted by its per-sandbox + /// Service (`os-supervisor-`) in the sandbox namespace. + Dedicated { + /// The per-sandbox Service's DNS authority + /// (`os-supervisor-..svc:`). + authority: String, + /// The Service's `ClusterIP` endpoint. + address: SocketAddr, + /// The supervisor Pod's UID, claimed as `kubevirt.supervisor_pod_uid` + /// and re-bound on every start (a new generation gets a new Pod). + supervisor_pod_uid: String, + /// Labels selecting the supervisor Pod (the boundary pair's + /// supervisor role); the fence's egress target. + pod_selector: BTreeMap, + /// The boundary port the supervisor listens on. + port: u16, + }, + /// A sandbox instance on the gateway's shared supervisor, fronted by + /// its Service. + Shared { + /// The shared supervisor Service's DNS authority + /// (`{service}.{ns}.svc:{port}`). + authority: String, + /// The Service's `ClusterIP` endpoint. + address: SocketAddr, + /// Route handle for this sandbox instance. + route: String, + /// Namespace of the shared supervisor Service. + namespace: String, + /// The Service's `spec.selector` (pod labels). + pod_selector: BTreeMap, + /// The runtime port. + port: u16, + }, +} + +impl KubeVirtSupervisorTarget { + fn authority(&self) -> &str { + match self { + Self::Dedicated { authority, .. } | Self::Shared { authority, .. } => authority, + } + } + + fn address(&self) -> SocketAddr { + match self { + Self::Dedicated { address, .. } | Self::Shared { address, .. } => *address, + } + } + + fn port(&self) -> u16 { + match self { + Self::Dedicated { port, .. } | Self::Shared { port, .. } => *port, + } + } + + /// The egress `to` peers the fence admits: this sandbox's dedicated + /// supervisor Pod (same namespace, pair-scoped selector) or the shared + /// supervisor's Pods (cross-namespace, Service selector). + fn fence_egress_to(&self) -> Vec { + let selector = |labels: &BTreeMap| { + labels + .iter() + .map(|(k, v)| (k.clone(), serde_json::json!(v))) + .collect::>() + }; + match self { + Self::Dedicated { pod_selector, .. } => vec![serde_json::json!({ + "podSelector": {"matchLabels": selector(pod_selector)} + })], + Self::Shared { + namespace, + pod_selector, + .. + } => vec![serde_json::json!({ + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": namespace + } + }, + "podSelector": {"matchLabels": selector(pod_selector)} + })], + } + } +} + +/// Driver-owned inputs that bind one `KubeVirt` VM generation to one +/// supervisor boundary. +pub struct KubeVirtBoundarySpec { + pub boundary_id: String, + pub generation: String, + pub session_id: openshell_core::SandboxSessionId, + pub session_rotation: openshell_core::jwt::SessionRotation, + pub auth_epoch: openshell_core::jwt::CredentialEpoch, + pub gateway_id: String, + pub verification_keys: Vec, + /// The `VirtualMachine` object's UID, stable across stop/start. + pub vm_uid: String, + /// The guest image reference, claimed as the image identity. + pub image_identity: String, + /// UID of the `NetworkPolicy` that fences the virt-launcher pod's + /// egress. Re-read and verified on every generation. + pub network_policy_uid: String, + /// resourceVersion of the fence `NetworkPolicy` at render time. + pub network_policy_resource_version: String, + /// Where the boundary dials its supervisor: the per-sandbox Service of + /// a dedicated supervisor Pod, or the gateway-shared supervisor's + /// Service with this sandbox's route handle. + pub supervisor: KubeVirtSupervisorTarget, + /// TLS client config the supervisor uses to authenticate the boundary. + pub supervisor_tls: SandboxTlsClientConfig, + /// The workload identity (uid/gid) the boundary runs as. + pub workload_identity: ResolvedWorkloadIdentity, + /// Environment the boundary passes to workload children. + pub child_env: HashMap, +} + +/// The protected guest config and matching runtime descriptor for one +/// `KubeVirt` VM. +pub struct KubeVirtBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub runtime_descriptor: SandboxRuntimeDescriptor, +} + +impl KubeVirtBoundarySpec { + /// Produce both sides of the common protocol from one set of immutable + /// driver inputs so their identity claims cannot drift. + pub fn provision(self) -> Result { + let mut resource_claims = BTreeMap::from([ + (CLAIM_VM_UID.to_string(), self.vm_uid.clone()), + (CLAIM_GENERATION.to_string(), self.generation.clone()), + ( + CLAIM_IMAGE_IDENTITY.to_string(), + self.image_identity.clone(), + ), + ]); + // The dedicated topology claims the supervisor Pod's UID (mirroring + // the pod path's `kubernetes.supervisor_pod_uid`); the shared + // topology instead names its instance by the route handle carried in + // the transport. No control-policy UID is claimed in either + // topology (user decision). + let listener_route = match &self.supervisor { + KubeVirtSupervisorTarget::Dedicated { + supervisor_pod_uid, .. + } => { + if supervisor_pod_uid.trim().is_empty() { + return Err(BackendError::Descriptor( + "KubeVirt boundary requires the dedicated supervisor Pod's UID; an \ + empty claim would bind the boundary to no supervisor" + .to_string(), + )); + } + resource_claims.insert( + CLAIM_SUPERVISOR_POD_UID.to_string(), + supervisor_pod_uid.clone(), + ); + None + } + KubeVirtSupervisorTarget::Shared { route, .. } => { + if route.trim().is_empty() { + return Err(BackendError::Descriptor( + "KubeVirt boundary requires a shared-supervisor route handle; an \ + empty route would route ambiguously" + .to_string(), + )); + } + Some(route.clone()) + } + }; + let transport = match &self.supervisor { + // A dedicated supervisor serves its one boundary on a plain + // runtime-initiated TCP listener (the pod path's dedicated + // transport); no route preface names an instance. + KubeVirtSupervisorTarget::Dedicated { + authority, port, .. + } => SandboxTransport::RuntimeInitiatedTcp { + authority: authority.clone(), + listen_address: SocketAddr::from(([0, 0, 0, 0], *port)), + }, + // The shared supervisor routes the connection to this sandbox's + // instance by the preface route handle. + KubeVirtSupervisorTarget::Shared { + authority, route, .. + } => SandboxTransport::RuntimeInitiatedRouted { + authority: authority.clone(), + route: route.clone(), + }, + }; + let provisioning_evidence = KubeVirtOuterFenceEvidence { + generation: &self.generation, + network_policy_uid: &self.network_policy_uid, + network_policy_resource_version: &self.network_policy_resource_version, + } + .project()?; + let sandbox_tls = SandboxTlsServerConfig { + certificate_chain_path: GUEST_BOUNDARY_CERTIFICATE_PATH.into(), + private_key_path: GUEST_BOUNDARY_PRIVATE_KEY_PATH.into(), + }; + Ok(KubeVirtBoundaryProvisioning { + boundary_config: BoundaryConfig { + boundary_id: self.boundary_id.clone(), + generation: self.generation.clone(), + session_id: self.session_id, + session_rotation: self.session_rotation, + auth_epoch: self.auth_epoch, + gateway_id: self.gateway_id, + verification_keys: self.verification_keys, + listener: BoundaryListener::RuntimeInitiatedTlsTcp { + authority: self.supervisor.authority().to_string(), + addresses: vec![self.supervisor.address()], + tls: sandbox_tls, + route: listener_route, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: self.workload_identity.clone(), + provisioning_evidence: provisioning_evidence.clone(), + child_env: self.child_env, + }, + runtime_descriptor: SandboxRuntimeDescriptor { + boundary_id: self.boundary_id, + generation: self.generation, + session_id: self.session_id, + workload_identity: self.workload_identity, + transport, + tls: self.supervisor_tls, + host_gateway_ip: None, + resource_claims, + provisioning_evidence, + }, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The real descriptor/config consistency check from sandbox-backend + /// (`isolation_protocol.rs` calls the same function), not a local copy. + fn descriptor_matches_config( + descriptor: &SandboxRuntimeDescriptor, + config: &BoundaryConfig, + ) -> Result<(), ()> { + if openshell_sandbox_backend::boundary_protocol::descriptor_matches_config( + descriptor, config, + ) { + Ok(()) + } else { + Err(()) + } + } + + fn shared_target() -> KubeVirtSupervisorTarget { + KubeVirtSupervisorTarget::Shared { + authority: "supervisor.openshell.svc:5500".to_string(), + address: "10.43.0.10:5500".parse().unwrap(), + route: "route-1".to_string(), + namespace: "openshell".to_string(), + pod_selector: BTreeMap::from([ + ( + "app.kubernetes.io/name".to_string(), + "openshell-supervisor".to_string(), + ), + ( + "app.kubernetes.io/component".to_string(), + "supervisor".to_string(), + ), + ]), + port: 5500, + } + } + + fn dedicated_target() -> KubeVirtSupervisorTarget { + KubeVirtSupervisorTarget::Dedicated { + authority: "os-supervisor-sbx-1.openshell-e2e-ns.svc:5500".to_string(), + address: "10.43.5.10:5500".parse().unwrap(), + supervisor_pod_uid: "supervisor-pod-uid-1".to_string(), + pod_selector: BTreeMap::from([ + ( + "openshell.ai/boundary-pair".to_string(), + "sbx-1".to_string(), + ), + ( + "openshell.ai/boundary-role".to_string(), + "supervisor".to_string(), + ), + ]), + port: 5500, + } + } + + fn spec_with(supervisor: KubeVirtSupervisorTarget) -> KubeVirtBoundarySpec { + KubeVirtBoundarySpec { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: openshell_core::SandboxSessionId::new(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + auth_epoch: openshell_core::jwt::CredentialEpoch::new(1).unwrap(), + gateway_id: "gateway-1".to_string(), + verification_keys: vec![GatewayVerificationKey { + key_id: "key-1".to_string(), + public_key_pem: "public-key".to_string(), + }], + vm_uid: "vm-uid-1".to_string(), + image_identity: "openshell/sandbox-vm:test".to_string(), + network_policy_uid: "np-uid-1".to_string(), + network_policy_resource_version: "1945".to_string(), + supervisor, + supervisor_tls: SandboxTlsClientConfig { + server_name: "boundary.sandbox.openshell".to_string(), + trust_anchor_pem: "test-ca".to_string(), + }, + workload_identity: ResolvedWorkloadIdentity::new_privileged( + "kubevirt-config".to_string(), + format!("sandbox:{}", "vm-uid-1"), + ) + .unwrap(), + child_env: HashMap::new(), + } + } + + fn spec() -> KubeVirtBoundarySpec { + spec_with(shared_target()) + } + + fn dedicated_spec() -> KubeVirtBoundarySpec { + spec_with(dedicated_target()) + } + + #[test] + fn provision_produces_matching_claims_and_evidence() { + let provisioned = spec().provision().expect("provision"); + assert_eq!( + provisioned.boundary_config.resource_claims[CLAIM_VM_UID], + "vm-uid-1" + ); + assert_eq!( + provisioned.runtime_descriptor.resource_claims[CLAIM_VM_UID], + "vm-uid-1" + ); + // Evidence is bound to the generation and validates. + provisioned + .boundary_config + .provisioning_evidence + .validate("generation-1") + .expect("evidence validates"); + } + + #[test] + fn evidence_rejects_mismatched_generation() { + let provisioned = spec().provision().expect("provision"); + assert!( + provisioned + .boundary_config + .provisioning_evidence + .validate("generation-2") + .is_err() + ); + } + + #[test] + fn evidence_rejects_missing_fence() { + let mut s = spec(); + s.network_policy_uid = String::new(); + assert!(s.provision().is_err(), "empty fence UID must fail closed"); + } + + #[test] + fn evidence_rejects_missing_resource_version() { + let mut s = spec(); + s.network_policy_resource_version = String::new(); + assert!( + s.provision().is_err(), + "empty fence resourceVersion must fail closed" + ); + } + + #[test] + fn evidence_rejects_replaced_fence() { + // Render the boundary config from fence A and the descriptor from + // fence B: the descriptor/config consistency check + // (isolation_protocol.rs:100) must reject the mismatched pair. + let original = spec().provision().expect("provision"); + let mut replaced = spec(); + replaced.network_policy_uid = "np-uid-2".to_string(); + let replacement = replaced.provision().expect("provision"); + assert!( + descriptor_matches_config(&original.runtime_descriptor, &replacement.boundary_config) + .is_err(), + "a descriptor rendered against fence B must not match a config from fence A" + ); + } + + #[test] + fn evidence_rejects_stale_bootstrap_from_previous_generation() { + let generation_1 = spec().provision().expect("provision"); + let mut s = spec(); + s.generation = "generation-2".to_string(); + let generation_2 = s.provision().expect("provision"); + assert!( + generation_1 + .boundary_config + .provisioning_evidence + .validate("generation-2") + .is_err() + ); + assert!( + descriptor_matches_config( + &generation_2.runtime_descriptor, + &generation_1.boundary_config + ) + .is_err(), + "a current descriptor must not match a stale previous-generation config" + ); + } + + #[test] + fn vm_uid_mismatch_rejected_by_descriptor_config_consistency() { + let original = spec().provision().expect("provision"); + let mut replaced = spec(); + replaced.vm_uid = "vm-uid-2".to_string(); + replaced.workload_identity = ResolvedWorkloadIdentity::new_privileged( + "kubevirt-config".to_string(), + "sandbox:vm-uid-2".to_string(), + ) + .unwrap(); + let other = replaced.provision().expect("provision"); + assert!( + descriptor_matches_config(&other.runtime_descriptor, &original.boundary_config) + .is_err(), + "a descriptor rendered for vm-uid-2 must not match a config from vm-uid-1" + ); + } + + #[test] + fn vm_uid_survives_stop_start_while_generation_advances() { + // The VM UID is stable across stop/start; only the generation + // advances. The claims carry the same vm_uid in both renders. + let first = spec().provision().expect("provision"); + let mut s = spec(); + s.generation = "generation-2".to_string(); + let second = s.provision().expect("provision"); + assert_eq!( + first.boundary_config.resource_claims[CLAIM_VM_UID], + second.boundary_config.resource_claims[CLAIM_VM_UID] + ); + assert_ne!( + first.boundary_config.generation, + second.boundary_config.generation + ); + // Both generations' evidence validate against their own generation. + first + .boundary_config + .provisioning_evidence + .validate("generation-1") + .expect("first generation validates"); + second + .boundary_config + .provisioning_evidence + .validate("generation-2") + .expect("second generation validates"); + } + + #[test] + fn rendered_config_is_recognized_as_vm_guest() { + let provisioned = spec().provision().expect("provision"); + assert!( + openshell_sandbox_backend::boundary_protocol::is_vm_guest( + &provisioned.boundary_config.resource_claims + ), + "the rendered KubeVirt boundary config must satisfy is_vm_guest" + ); + } + + #[test] + fn dedicated_provision_claims_the_supervisor_pod_uid() { + let provisioned = dedicated_spec().provision().expect("provision"); + // Both sides carry the claim, so descriptor/config consistency + // (isolation_protocol.rs) binds the boundary to this supervisor Pod. + assert_eq!( + provisioned.boundary_config.resource_claims[CLAIM_SUPERVISOR_POD_UID], + "supervisor-pod-uid-1" + ); + assert_eq!( + provisioned.runtime_descriptor.resource_claims[CLAIM_SUPERVISOR_POD_UID], + "supervisor-pod-uid-1" + ); + assert!( + descriptor_matches_config( + &provisioned.runtime_descriptor, + &provisioned.boundary_config + ) + .is_ok() + ); + } + + #[test] + fn bootstrap_renders_with_uid_zero_identity() { + // B1: the untrusted-guest class must render a bootstrap end to end. + // The standard ResolvedWorkloadIdentity::new refuses UID/GID zero, + // so the privileged constructor is used. + let auth = payload_tests::test_launch_authentication(); + let target = shared_target(); + let bootstrap = render_kubevirt_bootstrap( + &auth, + "sbx-1", + "sandbox-1", + "vm-uid-1", + "openshell/sandbox-vm:test", + "np-uid-1", + "1945", + serde_json::json!({}), + &target, + ) + .expect("untrusted-guest bootstrap renders"); + // The descriptor carries the kubevirt guest claim and the uid-0 + // identity. + assert!( + bootstrap + .runtime_descriptor + .resource_claims + .contains_key(CLAIM_GENERATION), + "the kubevirt generation claim marks the guest class" + ); + let identity = &bootstrap.runtime_descriptor.workload_identity; + assert_eq!(identity.uid, 0, "the guest workload runs as uid 0"); + assert_eq!(identity.gid, 0, "the guest workload runs as gid 0"); + } + + #[test] + fn shared_provision_carries_no_supervisor_pod_uid_claim() { + let provisioned = spec().provision().expect("provision"); + assert!( + !provisioned + .boundary_config + .resource_claims + .contains_key(CLAIM_SUPERVISOR_POD_UID) + ); + assert!( + !provisioned + .runtime_descriptor + .resource_claims + .contains_key(CLAIM_SUPERVISOR_POD_UID) + ); + } + + #[test] + fn dedicated_listener_omits_the_route_and_uses_plain_tcp_transport() { + let provisioned = dedicated_spec().provision().expect("provision"); + match &provisioned.boundary_config.listener { + BoundaryListener::RuntimeInitiatedTlsTcp { route, .. } => { + assert!( + route.is_none(), + "a dedicated supervisor is not routed: {route:?}" + ); + } + other => panic!("expected RuntimeInitiatedTlsTcp listener, got {other:?}"), + } + // The dedicated descriptor transport mirrors the pod path's + // dedicated KubernetesControlPath (isolation.rs): a plain + // runtime-initiated TCP listener with no route preface. + match &provisioned.runtime_descriptor.transport { + SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } => { + assert_eq!(listen_address.port(), 5500); + } + other => panic!("expected RuntimeInitiatedTcp transport, got {other:?}"), + } + } + + #[test] + fn shared_transport_stays_routed() { + let provisioned = spec().provision().expect("provision"); + match &provisioned.runtime_descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } => { + assert_eq!(route, "route-1"); + } + other => panic!("expected RuntimeInitiatedRouted transport, got {other:?}"), + } + } + + fn with_dedicated_pod_uid(uid: &str) -> KubeVirtSupervisorTarget { + let mut target = dedicated_target(); + if let KubeVirtSupervisorTarget::Dedicated { + supervisor_pod_uid, .. + } = &mut target + { + *supervisor_pod_uid = uid.to_string(); + } + target + } + + #[test] + fn dedicated_rejects_an_empty_supervisor_pod_uid() { + let Err(error) = spec_with(with_dedicated_pod_uid("")).provision() else { + panic!("an empty pod UID must fail closed") + }; + assert!( + error.to_string().contains("supervisor Pod's UID"), + "message should name the claim: {error}" + ); + } + + #[test] + fn dedicated_descriptor_does_not_match_a_config_for_another_supervisor_pod() { + let original = dedicated_spec().provision().expect("provision"); + let replaced = spec_with(with_dedicated_pod_uid("supervisor-pod-uid-2")) + .provision() + .expect("provision"); + assert!( + descriptor_matches_config(&replaced.runtime_descriptor, &original.boundary_config) + .is_err(), + "a descriptor rendered for another supervisor Pod must not match" + ); + } + + #[test] + fn dedicated_and_shared_configs_never_match() { + let dedicated = dedicated_spec().provision().expect("provision"); + let shared = spec().provision().expect("provision"); + assert!( + descriptor_matches_config(&shared.runtime_descriptor, &dedicated.boundary_config) + .is_err(), + "a shared descriptor must not match a dedicated config" + ); + assert!( + descriptor_matches_config(&dedicated.runtime_descriptor, &shared.boundary_config) + .is_err(), + "a dedicated descriptor must not match a shared config" + ); + } + + #[test] + fn guest_tls_paths_match_staging_constants() { + let provisioned = spec().provision().expect("provision"); + match &provisioned.boundary_config.listener { + BoundaryListener::RuntimeInitiatedTlsTcp { tls, .. } => { + assert_eq!( + tls.certificate_chain_path, + std::path::Path::new(GUEST_BOUNDARY_CERTIFICATE_PATH) + ); + assert_eq!( + tls.private_key_path, + std::path::Path::new(GUEST_BOUNDARY_PRIVATE_KEY_PATH) + ); + } + other => panic!("expected RuntimeInitiatedTlsTcp listener, got {other:?}"), + } + } +} + +// --------------------------------------------------------------------------- +// Virt-launcher egress fence +// --------------------------------------------------------------------------- + +use kube::Client; +use kube::api::Api; +use kube::core::DynamicObject; + +/// The fence's observed state: UID, resourceVersion, and the full spec +/// (for drift comparison). +#[derive(Debug, Clone)] +pub struct ObservedFence { + pub uid: String, + pub resource_version: String, + pub spec: serde_json::Value, +} + +/// The fence identity and shape the boundary evidence was rendered against. +#[derive(Debug, Clone)] +pub struct ExpectedFence { + pub uid: String, + pub resource_version: String, + pub spec: serde_json::Value, +} + +/// VM annotation persisting the fence expectation (UID and rendered spec) +/// the create-time evidence was rendered against. +/// +/// The start path verifies the fence on every new generation with it: +/// deletion, UID change, and spec drift are Precondition errors; a +/// resourceVersion-only change is accepted and re-captured. +pub const FENCE_EXPECTATION_ANNOTATION: &str = "openshell.ai/kubevirt-fence-expectation"; + +impl ExpectedFence { + /// Encode the expectation (UID + spec, not the volatile RV) as a VM + /// annotation value. + pub fn to_annotation(&self) -> Result { + serde_json::to_string(&serde_json::json!({ + "uid": self.uid, + "spec": self.spec, + })) + .map_err(|error| format!("encode fence expectation: {error}")) + } + + /// Decode a VM annotation value back into an expectation. The + /// resourceVersion is not persisted (it changes on every label-only + /// edit); the caller supplies the freshly rendered spec separately. + pub fn from_annotation(value: &str) -> Result { + let decoded: serde_json::Value = serde_json::from_str(value) + .map_err(|error| format!("decode fence expectation: {error}"))?; + Ok(Self { + uid: decoded["uid"] + .as_str() + .ok_or("fence expectation has no uid")? + .to_string(), + resource_version: String::new(), + spec: decoded["spec"].clone(), + }) + } +} + +impl ExpectedFence { + /// Build from a freshly rendered fence JSON and the observed UID/RV. + #[must_use] + pub fn from_rendered( + rendered: serde_json::Value, + uid: String, + resource_version: String, + ) -> Self { + Self { + uid, + resource_version, + spec: rendered["spec"].clone(), + } + } +} + +/// Render the virt-launcher fence `NetworkPolicy`. +/// +/// The sandbox's only permitted egress is its supervisor on the boundary +/// port — the dedicated topology's own supervisor Pod (pair-scoped +/// selector, same namespace), or the shared supervisor's Pods (the +/// Service's own pod selector, read at render time — never hard-coded, +/// because the Helm chart's supervisor labels vary by release). No DNS, no +/// gateway: the boundary's in-guest DNS relay forwards through the +/// supervisor's network mediation, and the supervisor connection uses +/// driver-resolved addresses. +/// +/// Ingress is deny-all (`policyTypes: [Ingress, Egress]` with no ingress +/// rules): the boundary dials out (`RuntimeInitiatedTlsTcp`), so the guest +/// needs no inbound connections. Serial console and virt-handler access go +/// through unix sockets on the node, not pod networking. +/// +/// The policy carries the standard `OpenShell` ownership labels so the +/// driver's label-based cleanup and the demo's zero-leftovers check find +/// it. +#[must_use] +pub fn render_fence( + namespace: &str, + sandbox_id: &str, + gateway_id: Option<&str>, + target: &KubeVirtSupervisorTarget, +) -> serde_json::Value { + let mut labels = BTreeMap::new(); + labels.insert( + openshell_core::driver_utils::LABEL_MANAGED_BY.to_string(), + openshell_core::driver_utils::LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert( + openshell_core::driver_utils::LABEL_SANDBOX_ID.to_string(), + sandbox_id.to_string(), + ); + if let Some(gw_id) = gateway_id { + labels.insert( + openshell_core::driver_utils::LABEL_GATEWAY_ID.to_string(), + gw_id.to_string(), + ); + } + + serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": fence_name(sandbox_id), + "namespace": namespace, + "labels": labels, + }, + "spec": { + "podSelector": { + "matchLabels": { + "kubevirt.io": "virt-launcher", + openshell_core::driver_utils::LABEL_SANDBOX_ID: sandbox_id, + } + }, + "policyTypes": ["Ingress", "Egress"], + "egress": [{ + "to": target.fence_egress_to(), + "ports": [{ + "port": target.port(), + "protocol": "TCP" + }] + }] + } + }) +} + +/// Render the `KubeVirt` control-ingress `NetworkPolicy` (L2). +/// +/// The ONLY pods that may accept connections on the boundary port are this +/// pair's supervisor Pods, and the only admitted sources are this pair's +/// workload-role pods — the virt-launcher pod itself never accepts +/// boundary-port traffic. The pod path's pair-wide control policy applies +/// its ingress rule to the whole pair (workload pods there listen too); +/// the `KubeVirt` workload is a VM whose boundary dials out, so the ingress +/// half alone is needed and the fence already carries the egress half. +/// Residual, inherent to `NetworkPolicy`: a pod that forges the pair labels +/// can still REACH the supervisor's port, but cannot authenticate (the +/// supervisor verifies the boundary's pinned TLS and descriptor claims); +/// binding source identity needs namespace governance (pod creation +/// restricted to the driver's `ServiceAccount`). +#[must_use] +pub fn render_control_ingress( + namespace: &str, + sandbox_id: &str, + pair: &str, + port: u16, +) -> serde_json::Value { + use std::collections::BTreeMap; + let pair_role = |role: &str| { + BTreeMap::from([ + ("openshell.ai/boundary-pair".to_string(), pair.to_string()), + ("openshell.ai/boundary-role".to_string(), role.to_string()), + ]) + }; + let selector = |labels: &BTreeMap| { + labels + .iter() + .map(|(k, v)| (k.clone(), serde_json::json!(v))) + .collect::>() + }; + let mut labels = BTreeMap::new(); + labels.insert( + openshell_core::driver_utils::LABEL_MANAGED_BY.to_string(), + openshell_core::driver_utils::LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert( + openshell_core::driver_utils::LABEL_SANDBOX_ID.to_string(), + sandbox_id.to_string(), + ); + serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": control_ingress_name(sandbox_id), + "namespace": namespace, + "labels": labels, + }, + "spec": { + "podSelector": {"matchLabels": selector(&pair_role("supervisor"))}, + "policyTypes": ["Ingress"], + "ingress": [{ + "from": [{"podSelector": {"matchLabels": selector(&pair_role("workload"))}}], + "ports": [{"port": port, "protocol": "TCP"}] + }] + } + }) +} + +/// Name of the control-ingress `NetworkPolicy` for one sandbox (the same +/// name the pod path's control policy uses, so a namespace never carries +/// both shapes). +#[must_use] +pub fn control_ingress_name(sandbox_id: &str) -> String { + format!("os-control-{}", sandbox_id.to_ascii_lowercase()) +} + +/// Name of the fence `NetworkPolicy` for one sandbox. +#[must_use] +pub fn fence_name(sandbox_id: &str) -> String { + format!("openshell-kubevirt-fence-{sandbox_id}") +} + +/// Read the fence's current state. `None` when the policy does not exist. +pub async fn observe_fence( + client: &Client, + namespace: &str, + sandbox_id: &str, +) -> Result, String> { + let api: Api = Api::namespaced_with( + client.clone(), + namespace, + &kube::api::ApiResource { + group: "networking.k8s.io".to_string(), + version: "v1".to_string(), + api_version: "networking.k8s.io/v1".to_string(), + kind: "NetworkPolicy".to_string(), + plural: "networkpolicies".to_string(), + }, + ); + let name = fence_name(sandbox_id); + match api.get(&name).await { + Ok(obj) => Ok(Some(ObservedFence { + uid: obj.metadata.uid.unwrap_or_default(), + resource_version: obj.metadata.resource_version.unwrap_or_default(), + spec: obj + .data + .get("spec") + .cloned() + .unwrap_or_else(|| serde_json::json!({})), + })), + Err(kube::Error::Api(err)) if err.code == 404 => Ok(None), + Err(err) => Err(err.to_string()), + } +} + +/// Pure fence verification (unit-testable without a cluster): +/// - deleted (None) → Err; +/// - UID changed (replaced policy) → Err; +/// - spec drift (podSelector, policyTypes, egress rules) → Err; +/// - resourceVersion-only change (a label or annotation from a controller) +/// → Ok with the new RV, so the driver re-captures it into the new +/// generation's evidence. +pub fn check_fence( + observed: Option<&ObservedFence>, + expected: &ExpectedFence, + sandbox_id: &str, +) -> Result { + let Some(observed) = observed else { + return Err(format!( + "the virt-launcher egress fence {} was deleted; the boundary \ + evidence was rendered against a fence that no longer exists", + fence_name(sandbox_id) + )); + }; + if observed.uid != expected.uid { + return Err(format!( + "the virt-launcher egress fence was replaced (expected UID {}, got {}); \ + the boundary evidence was rendered against the old fence and cannot \ + be reused", + expected.uid, observed.uid + )); + } + if observed.spec != expected.spec { + return Err( + "the virt-launcher egress fence spec drifted from the rendered shape \ + (podSelector, policyTypes, or egress rules changed); the fence no \ + longer matches what the boundary evidence was rendered against" + .to_string(), + ); + } + Ok(observed.clone()) +} + +/// Verify the fence on every new generation. +/// +/// The async wrapper around [`check_fence`]. Returns the current +/// `ObservedFence` so the caller can re-capture the resourceVersion into +/// the new generation's evidence. +pub async fn verify_fence( + client: &Client, + namespace: &str, + sandbox_id: &str, + expected: &ExpectedFence, +) -> Result { + let observed = observe_fence(client, namespace, sandbox_id).await?; + check_fence(observed.as_ref(), expected, sandbox_id) +} + +#[cfg(test)] +mod fence_tests { + use super::*; + + pub(super) fn supervisor() -> KubeVirtSupervisorTarget { + KubeVirtSupervisorTarget::Shared { + authority: "openshell-supervisor.openshell.svc:5500".to_string(), + address: "10.43.0.10:5500".parse().unwrap(), + route: "route-1".to_string(), + namespace: "openshell".to_string(), + pod_selector: BTreeMap::from([ + ( + "app.kubernetes.io/name".to_string(), + "openshell-supervisor".to_string(), + ), + ( + "app.kubernetes.io/component".to_string(), + "supervisor".to_string(), + ), + ]), + port: 5500, + } + } + + fn rendered_fence() -> serde_json::Value { + render_fence("openshell-e2e-ns", "sbx-1", Some("gw"), &supervisor()) + } + + fn dedicated_fence() -> serde_json::Value { + render_fence( + "openshell-e2e-ns", + "sbx-1", + Some("gw"), + &KubeVirtSupervisorTarget::Dedicated { + authority: "os-supervisor-sbx-1.openshell-e2e-ns.svc:5500".to_string(), + address: "10.43.5.10:5500".parse().unwrap(), + supervisor_pod_uid: "supervisor-pod-uid-1".to_string(), + pod_selector: BTreeMap::from([ + ( + "openshell.ai/boundary-pair".to_string(), + "sbx-1".to_string(), + ), + ( + "openshell.ai/boundary-role".to_string(), + "supervisor".to_string(), + ), + ]), + port: 5500, + }, + ) + } + + fn observed(uid: &str, rv: &str, spec: serde_json::Value) -> ObservedFence { + ObservedFence { + uid: uid.to_string(), + resource_version: rv.to_string(), + spec, + } + } + + #[test] + fn fence_renders_deny_all_ingress_and_supervisor_only_egress() { + let fence = rendered_fence(); + let policy_types = fence["spec"]["policyTypes"].as_array().unwrap(); + assert!(policy_types.contains(&serde_json::json!("Ingress"))); + assert!(policy_types.contains(&serde_json::json!("Egress"))); + // No ingress rules: deny-all ingress. + assert!(fence["spec"]["ingress"].is_null()); + // The podSelector matches the virt-launcher pod. + assert_eq!( + fence["spec"]["podSelector"]["matchLabels"]["kubevirt.io"], + "virt-launcher" + ); + assert_eq!( + fence["spec"]["podSelector"]["matchLabels"]["openshell.ai/sandbox-id"], + "sbx-1" + ); + // Egress allows only the supervisor's port, matched by the Service + // selector (not hard-coded labels). + assert_eq!(fence["spec"]["egress"][0]["ports"][0]["port"], 5500); + assert_eq!( + fence["spec"]["egress"][0]["to"][0]["podSelector"]["matchLabels"]["app.kubernetes.io/name"], + "openshell-supervisor" + ); + assert_eq!( + fence["spec"]["egress"][0]["to"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshell" + ); + } + + #[test] + fn fence_carries_openshell_labels_for_cleanup() { + let fence = rendered_fence(); + assert_eq!( + fence["metadata"]["labels"]["openshell.ai/managed-by"], + "openshell" + ); + assert_eq!( + fence["metadata"]["labels"]["openshell.ai/sandbox-id"], + "sbx-1" + ); + assert_eq!(fence["metadata"]["labels"]["openshell.ai/gateway-id"], "gw"); + } + + #[test] + fn fence_name_is_derived_from_sandbox_id() { + assert_eq!(fence_name("sbx-1"), "openshell-kubevirt-fence-sbx-1"); + } + + #[test] + fn dedicated_fence_selects_only_this_sandboxes_supervisor_pod() { + let fence = dedicated_fence(); + let policy_types = fence["spec"]["policyTypes"].as_array().unwrap(); + assert!(policy_types.contains(&serde_json::json!("Ingress"))); + assert!(policy_types.contains(&serde_json::json!("Egress"))); + // No ingress rules: deny-all ingress, same as the shared fence. + assert!(fence["spec"]["ingress"].is_null()); + // The egress peer is a same-namespace podSelector (no + // namespaceSelector): only this sandbox's dedicated supervisor Pod. + let to = &fence["spec"]["egress"][0]["to"][0]; + assert!( + to.get("namespaceSelector").is_none(), + "a dedicated supervisor lives in the sandbox namespace: {to}" + ); + assert_eq!( + to["podSelector"]["matchLabels"]["openshell.ai/boundary-pair"], + "sbx-1" + ); + assert_eq!( + to["podSelector"]["matchLabels"]["openshell.ai/boundary-role"], + "supervisor" + ); + assert_eq!(fence["spec"]["egress"][0]["ports"][0]["port"], 5500); + } + + #[test] + fn control_ingress_scopes_the_boundary_port_to_the_supervisor_pod() { + // L2: only this pair's supervisor Pod accepts boundary-port + // connections, and the policy carries no egress rules — the + // virt-launcher never accepts boundary-port inbound, and its + // egress allowance is the fence's job. + let policy = render_control_ingress("openshell-e2e-ns", "sbx-1", "sbx-1", 5500); + assert_eq!( + policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/boundary-role"], + "supervisor", + "the policy must select only the supervisor role" + ); + assert_eq!( + policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/boundary-pair"], + "sbx-1" + ); + let policy_types = policy["spec"]["policyTypes"].as_array().unwrap(); + assert_eq!(policy_types, &vec![serde_json::json!("Ingress")]); + assert!(policy["spec"]["egress"].is_null()); + let ingress = &policy["spec"]["ingress"][0]; + assert_eq!( + ingress["from"][0]["podSelector"]["matchLabels"]["openshell.ai/boundary-role"], + "workload", + "only this pair's workload role may connect" + ); + assert_eq!(ingress["ports"][0]["port"], 5500); + assert_eq!( + policy["metadata"]["labels"]["openshell.ai/sandbox-id"], + "sbx-1" + ); + assert_eq!( + control_ingress_name("sbx-1"), + "os-control-sbx-1", + "the same name the pod path's control policy uses, so a namespace \ + never carries both shapes" + ); + } + + #[test] + fn dedicated_and_shared_fence_specs_differ() { + // The two topologies render different egress targets, so a fence + // created for one topology fails the other's start-time + // verification (spec drift). + let dedicated = dedicated_fence(); + let shared = rendered_fence(); + assert_ne!(dedicated["spec"], shared["spec"]); + } + + #[test] + fn check_fence_rejects_deleted() { + let expected = ExpectedFence::from_rendered(rendered_fence(), "uid-1".into(), "1".into()); + let err = check_fence(None, &expected, "sbx-1").unwrap_err(); + assert!(err.contains("was deleted")); + } + + #[test] + fn check_fence_rejects_replaced_uid() { + let expected = ExpectedFence::from_rendered(rendered_fence(), "uid-1".into(), "1".into()); + let replaced = observed("uid-2", "1", rendered_fence()["spec"].clone()); + let err = check_fence(Some(&replaced), &expected, "sbx-1").unwrap_err(); + assert!(err.contains("was replaced")); + } + + #[test] + fn check_fence_rejects_spec_drift() { + let expected = ExpectedFence::from_rendered(rendered_fence(), "uid-1".into(), "1".into()); + let mut drifted = rendered_fence(); + drifted["spec"]["egress"][0]["ports"][0]["port"] = serde_json::json!(9999); + let observed_drift = observed("uid-1", "1", drifted["spec"].clone()); + let err = check_fence(Some(&observed_drift), &expected, "sbx-1").unwrap_err(); + assert!(err.contains("spec drifted")); + } + + #[test] + fn check_fence_accepts_rv_only_change() { + let expected = ExpectedFence::from_rendered(rendered_fence(), "uid-1".into(), "1".into()); + let rv_changed = observed("uid-1", "42", rendered_fence()["spec"].clone()); + let result = + check_fence(Some(&rv_changed), &expected, "sbx-1").expect("RV-only change must pass"); + assert_eq!(result.resource_version, "42"); + } + + #[test] + fn check_fence_accepts_unchanged() { + let expected = ExpectedFence::from_rendered(rendered_fence(), "uid-1".into(), "1".into()); + let unchanged = observed("uid-1", "1", rendered_fence()["spec"].clone()); + check_fence(Some(&unchanged), &expected, "sbx-1").expect("unchanged must pass"); + } +} + +// --------------------------------------------------------------------------- +// Cloud-init delivery +// --------------------------------------------------------------------------- + +use base64::Engine as _; + +/// Keys in the cloudInitNoCloud `userData` payload. +/// +/// Carrying the boundary bootstrap artifacts (base64-encoded so the +/// shell-sourceable format is preserved). No gateway material appears in +/// the payload — no sandbox token, no gateway endpoint, no JWT. +pub const PAYLOAD_BOUNDARY_CONFIG: &str = "OPENSHELL_BOUNDARY_CONFIG_B64"; +pub const PAYLOAD_BOUNDARY_CERTIFICATE: &str = "OPENSHELL_BOUNDARY_CERTIFICATE_B64"; +pub const PAYLOAD_BOUNDARY_PRIVATE_KEY: &str = "OPENSHELL_BOUNDARY_PRIVATE_KEY_B64"; +/// The workload uid/gid the boundary runs as (launch-capability-free args). +pub const PAYLOAD_WORKLOAD_UID: &str = "OPENSHELL_WORKLOAD_UID"; +pub const PAYLOAD_WORKLOAD_GID: &str = "OPENSHELL_WORKLOAD_GID"; + +/// Render the cloudInitNoCloud `userData` payload. +/// +/// Delivers the boundary bootstrap artifacts to the guest. The payload is +/// shell-sourceable; the guest init decodes the base64 values, stages +/// the files at the paths written in the `BoundaryConfig`, and runs +/// `openshell-sandbox launch-capability-free `. +/// The uid/gid are derived from the same `ResolvedWorkloadIdentity` the +/// boundary config carries, so they cannot mismatch. +/// +/// Assert (checkpoint 3): no gateway endpoint, no sandbox token, no JWT. +#[must_use] +pub fn render_guest_payload( + artifacts: &crate::sandbox_runtime::BoundaryBootstrapArtifacts, + workload_identity: &ResolvedWorkloadIdentity, +) -> String { + let uid = workload_identity.uid; + let gid = workload_identity.gid; + let b64 = |bytes: &[u8]| base64::engine::general_purpose::STANDARD.encode(bytes); + format!( + "{PAYLOAD_BOUNDARY_CONFIG}={}\n\ + {PAYLOAD_BOUNDARY_CERTIFICATE}={}\n\ + {PAYLOAD_BOUNDARY_PRIVATE_KEY}={}\n\ + {PAYLOAD_WORKLOAD_UID}={uid}\n\ + {PAYLOAD_WORKLOAD_GID}={gid}\n", + b64(&artifacts.config_json), + b64(&artifacts.certificate_pem), + b64(&artifacts.private_key_pem), + ) +} + +#[cfg(test)] +mod payload_tests { + use super::*; + + #[test] + fn guest_payload_carries_no_gateway_material() { + let artifacts = crate::sandbox_runtime::BoundaryBootstrapArtifacts { + config_json: br#"{"boundary_id":"sbx","generation":"g1"}"#.to_vec(), + certificate_pem: b"-----BEGIN CERTIFICATE-----".to_vec(), + private_key_pem: b"-----BEGIN PRIVATE KEY-----".to_vec(), + }; + let identity = ResolvedWorkloadIdentity::new( + 10001, + 10001, + Vec::new(), + "test".to_string(), + "test".to_string(), + ) + .unwrap(); + let payload = render_guest_payload(&artifacts, &identity); + // No gateway material (checkpoint 3). + assert!(!payload.contains("OPENSHELL_ENDPOINT")); + assert!(!payload.contains("OPENSHELL_SANDBOX_TOKEN")); + assert!(!payload.contains("OPENSHELL_SANDBOX_TOKEN_FILE")); + assert!(!payload.contains("OPENSHELL_ENDPOINT=")); + assert!(!payload.contains("grpc_endpoint")); + // The boundary artifacts and identity are present. + assert!(payload.contains(PAYLOAD_BOUNDARY_CONFIG)); + assert!(payload.contains(PAYLOAD_BOUNDARY_CERTIFICATE)); + assert!(payload.contains(PAYLOAD_BOUNDARY_PRIVATE_KEY)); + assert!(payload.contains("OPENSHELL_WORKLOAD_UID=10001")); + assert!(payload.contains("OPENSHELL_WORKLOAD_GID=10001")); + } + + #[test] + fn guest_payload_is_shell_sourceable() { + let artifacts = crate::sandbox_runtime::BoundaryBootstrapArtifacts { + config_json: b"{}".to_vec(), + certificate_pem: b"cert".to_vec(), + private_key_pem: b"key".to_vec(), + }; + let identity = ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + "test".to_string(), + ) + .unwrap(); + let payload = render_guest_payload(&artifacts, &identity); + // Every line is KEY=VALUE (base64 has no spaces or shell metachars). + for line in payload.lines() { + assert!(line.contains('='), "line must be KEY=VALUE: {line}"); + let value = line.split_once('=').unwrap().1; + assert!( + !value.contains(' ') && !value.contains('\'') && !value.contains('"'), + "value must be shell-safe: {value}" + ); + } + } + + pub(super) fn test_launch_authentication() -> SandboxLaunchAuthentication { + use openshell_core::jwt::{ + CredentialEpoch, SecretJwt, SessionRotation, SessionVerificationKey, + SupervisorAuthBundle, + }; + SandboxLaunchAuthentication { + supervisor: SupervisorAuthBundle { + session_id: openshell_core::SandboxSessionId::new(), + runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( + "generation-1", + ) + .unwrap(), + session_rotation: SessionRotation::new(1).unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + gateway_token: SecretJwt::parse("gateway.token.value").unwrap(), + gateway_expires_at: i64::MAX, + sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(), + sandbox_expires_at: i64::MAX, + ssh_host_private_key: None, + traces_otlp_headers: None, + ocsf_otlp_headers: None, + agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, + }, + gateway_id: "gateway-test".to_string(), + verification_keys: vec![SessionVerificationKey { + key_id: "test-key".to_string(), + public_key_pem: b"public-key".to_vec(), + }], + } + } + + #[test] + fn decoded_guest_payload_carries_no_token_values() { + // Decode the real rendered payload (the guest init's view) and + // prove no gateway material survives into it: no token values, no + // JWT fragments, no gateway endpoint — only the boundary config, + // session TLS, and workload uid/gid. + let auth = test_launch_authentication(); + let target = KubeVirtSupervisorTarget::Dedicated { + authority: "os-supervisor-sbx-1.openshell-e2e-ns.svc:5500".to_string(), + address: "10.43.5.10:5500".parse().unwrap(), + supervisor_pod_uid: "supervisor-pod-uid-1".to_string(), + pod_selector: BTreeMap::new(), + port: 5500, + }; + let bootstrap = render_kubevirt_bootstrap( + &auth, + "sbx-1", + "sandbox-1", + "vm-uid-1", + "openshell/sandbox-vm:test", + "np-uid-1", + "1945", + serde_json::json!({}), + &target, + ) + .expect("render"); + let decoded = bootstrap + .guest_payload + .lines() + .filter(|line| line.starts_with("OPENSHELL_BOUNDARY_")) + .filter_map(|line| line.split_once('=').map(|(_, value)| value)) + .map(|value| { + String::from_utf8( + base64::engine::general_purpose::STANDARD + .decode(value) + .expect("boundary artifact values are base64"), + ) + .expect("boundary artifacts are UTF-8") + }) + .collect::>() + .join("\n"); + assert!(!decoded.contains("gateway.token.value")); + assert!(!decoded.contains("sandbox.token.value")); + assert!(!decoded.contains("172.18.0.1")); + assert!(!decoded.contains("grpc_endpoint")); + assert!(!decoded.contains("18085")); + // The boundary artifacts themselves are present: the config (with + // the gateway's public verification key — public material the + // boundary needs) and the session certificate. + assert!(decoded.contains("boundary_id")); + assert!(decoded.contains("public-key")); + assert!(decoded.contains("BEGIN CERTIFICATE")); + } + + /// W3: the rendered descriptor's TLS must be the session material's + /// `server_name` and `trust_anchor_pem` verbatim — the same fields the + /// pod path passes to `SandboxTlsClientConfig`. The trust anchor is + /// the session CA (a distinct certificate from the leaf the guest + /// presents), and it must be non-empty PEM. + #[test] + fn descriptor_tls_uses_session_material_verbatim() { + let auth = test_launch_authentication(); + let session_id = auth.supervisor.session_id; + let supervisor = fence_tests::supervisor(); + let target = KubeVirtSupervisorTarget::Shared { + authority: "openshell-supervisor.openshell.svc:5500".to_string(), + address: "10.43.0.10:5500".parse().unwrap(), + route: "route-1".to_string(), + namespace: "openshell".to_string(), + pod_selector: BTreeMap::new(), + port: 5500, + }; + let _ = supervisor; + let bootstrap = render_kubevirt_bootstrap( + &auth, + "sbx-1", + "sandbox-1", + "vm-uid-1", + "openshell/sandbox-vm:test", + "np-uid-1", + "1945", + serde_json::json!({}), + &target, + ) + .expect("render"); + + let tls = &bootstrap.runtime_descriptor.tls; + // server_name is exactly what generate_sandbox_tls_material assigns + // for this session (sandbox..openshell.internal). + assert_eq!( + tls.server_name, + format!("sandbox.{session_id}.openshell.internal"), + "server_name must be the session material's, verbatim" + ); + // trust_anchor_pem is the session CA — non-empty certificate PEM. + assert!( + !tls.trust_anchor_pem.trim().is_empty(), + "trust anchor must not be empty" + ); + assert!( + tls.trust_anchor_pem.contains("BEGIN CERTIFICATE"), + "trust anchor must be certificate PEM: {}", + tls.trust_anchor_pem + ); + + // The trust anchor is the CA, distinct from the leaf certificate + // the guest presents (the payload's tls.crt): anchoring on the + // chain would break the supervisor's verification. + let payload = &bootstrap.guest_payload; + let cert_line = payload + .lines() + .find(|line| line.starts_with(PAYLOAD_BOUNDARY_CERTIFICATE)) + .expect("payload carries the boundary certificate"); + let cert_b64 = cert_line.split_once('=').unwrap().1; + let cert_pem = String::from_utf8( + base64::engine::general_purpose::STANDARD + .decode(cert_b64) + .expect("certificate decodes"), + ) + .expect("certificate is UTF-8"); + assert_ne!( + tls.trust_anchor_pem, cert_pem, + "trust anchor must be the session CA, not the leaf certificate chain" + ); + } +} + +// --------------------------------------------------------------------------- +// Driver wiring: boundary bootstrap for create and start +// --------------------------------------------------------------------------- + +use crate::sandbox_runtime::{boundary_launch_inputs, render_boundary_bootstrap}; +use openshell_core::jwt::SandboxLaunchAuthentication; + +/// Everything the driver needs to render one generation's boundary +/// bootstrap: the guest payload (cloud-init userData) and the fence +/// expectation (for the start-path re-check). +pub struct KubeVirtBootstrap { + /// The shell-sourceable cloud-init userData delivering the boundary + /// artifacts to the guest. + pub guest_payload: String, + /// The fence UID/resourceVersion the evidence was rendered against. + /// On start, the driver re-reads the fence and compares. + pub fence: ExpectedFence, + /// The runtime descriptor for the shared supervisor's instance + /// registration (W4: `shared_instance_secret`). Its `tls` is the + /// session material's `server_name`/`trust_anchor_pem` verbatim. + pub runtime_descriptor: SandboxRuntimeDescriptor, +} + +/// Render one generation's `KubeVirt` boundary bootstrap. Used by both +/// create (initial) and start (re-render for the new generation). +/// +/// The caller provides the launch authentication (from the gateway), the VM's +/// identity, the observed fence state, and the shared supervisor endpoint. +#[allow(clippy::too_many_arguments)] +#[allow(clippy::similar_names)] +pub fn render_kubevirt_bootstrap( + launch_authentication: &SandboxLaunchAuthentication, + sandbox_id: &str, + sandbox_name: &str, + vm_uid: &str, + guest_image: &str, + fence_uid: &str, + fence_resource_version: &str, + fence_spec: serde_json::Value, + target: &KubeVirtSupervisorTarget, +) -> Result { + let launch_inputs = boundary_launch_inputs(launch_authentication) + .map_err(|error| format!("boundary launch inputs: {error}"))?; + + // Every KubeVirt guest runs its workload as root by design (kernel + // separation is the boundary); the standard constructor refuses + // UID/GID zero, which is the right invariant for every other + // workload. + let workload_identity = ResolvedWorkloadIdentity::new_privileged( + "kubevirt-config".to_string(), + format!("sandbox:{vm_uid}"), + ) + .map_err(|error| format!("workload identity: {error}"))?; + + let spec = KubeVirtBoundarySpec { + boundary_id: sandbox_id.to_string(), + generation: launch_authentication + .supervisor + .runtime_generation + .to_string(), + session_id: launch_inputs.session_id, + session_rotation: launch_inputs.session_rotation, + auth_epoch: launch_inputs.auth_epoch, + gateway_id: launch_inputs.gateway_id, + verification_keys: launch_inputs.verification_keys, + vm_uid: vm_uid.to_string(), + image_identity: guest_image.to_string(), + network_policy_uid: fence_uid.to_string(), + network_policy_resource_version: fence_resource_version.to_string(), + // The supervisor target carries the topology: the per-sandbox + // Service of a dedicated supervisor Pod, or the shared supervisor's + // Service plus this sandbox's route handle. + supervisor: target.clone(), + // W3: the supervisor's TLS client config (how it authenticates the + // boundary). The trust anchor is the session CA (from + // generate_sandbox_tls_material, embedded in the certificate chain); + // the server_name is the sandbox DNS name it assigned to the + // session certificate. + supervisor_tls: SandboxTlsClientConfig { + server_name: launch_inputs.tls.server_name.clone(), + trust_anchor_pem: launch_inputs.tls.trust_anchor_pem.clone(), + }, + workload_identity: workload_identity.clone(), + child_env: HashMap::new(), + }; + let _ = sandbox_name; + + let provisioned = spec + .provision() + .map_err(|error| format!("KubeVirt boundary provisioning: {error}"))?; + + let bootstrap = render_boundary_bootstrap(&provisioned.boundary_config, &launch_inputs.tls) + .map_err(|error| format!("boundary bootstrap rendering: {error}"))?; + + let guest_payload = render_guest_payload(&bootstrap, &workload_identity); + + let fence = ExpectedFence::from_rendered( + fence_spec, + fence_uid.to_string(), + fence_resource_version.to_string(), + ); + + Ok(KubeVirtBootstrap { + guest_payload, + fence, + runtime_descriptor: provisioned.runtime_descriptor, + }) +} + +/// `ApiResource` for `NetworkPolicy` (shared with the fence functions). +pub fn fence_api_resource() -> kube::api::ApiResource { + kube::api::ApiResource { + group: "networking.k8s.io".to_string(), + version: "v1".to_string(), + api_version: "networking.k8s.io/v1".to_string(), + kind: "NetworkPolicy".to_string(), + plural: "networkpolicies".to_string(), + } +} diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index f3a1e419ec..344ea5d7e1 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,6 +6,8 @@ pub mod config; pub mod driver; pub mod grpc; pub mod isolation; +mod kubevirt; +pub mod kubevirt_boundary; mod lifecycle; pub mod otel_tracing; mod resource_admission; @@ -14,9 +16,10 @@ mod shared_supervisor; pub use config::{ DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, KubernetesControlConnection, KubernetesImagePullPolicy, - KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SharedSupervisorConfig, - SupervisorTopology, WorkspaceMode, managed_namespace_prefix, + KubeVirtConfig, KubernetesComputeConfig, KubernetesControlConnection, + KubernetesImagePullPolicy, KubernetesRuntimeMode, KubernetesSandboxRuntimeConfig, + ManagedSshIngressConfig, SharedSupervisorConfig, SupervisorTopology, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index b67accbaca..4f25545a0c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -333,6 +333,8 @@ async fn main() -> Result<()> { .unwrap_or_default(), sandbox_uid: args.sandbox_uid, sandbox_gid: args.sandbox_gid, + runtime: openshell_driver_kubernetes::KubernetesRuntimeMode::default(), + kubevirt: openshell_driver_kubernetes::KubeVirtConfig::default(), }, shutdown_rx, None, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 9ed4ce71e9..37caa4929f 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -21,13 +21,65 @@ use rcgen::{CertificateParams, DnType, IsCa, KeyUsagePurpose}; use k8s_openapi::api::networking::v1::NetworkPolicy; +use crate::KubernetesDriverError; use crate::config::KubernetesControlConnection; +/// Session-scoped inputs every boundary runtime needs from one launch +/// authentication: the identity fields the boundary config and runtime +/// descriptor both carry, the gateway verification keys, and the session +/// TLS material. Extracted so the pod and `KubeVirt` paths cannot drift in +/// how they interpret the gateway's launch bundle. +pub struct BoundaryLaunchInputs { + pub session_id: openshell_core::SandboxSessionId, + pub session_rotation: openshell_core::jwt::SessionRotation, + pub auth_epoch: openshell_core::jwt::CredentialEpoch, + pub gateway_id: String, + pub verification_keys: Vec, + pub tls: openshell_sandbox_backend::boundary_protocol::SandboxTlsMaterial, +} + +/// Interpret the gateway's launch authentication into the shared boundary +/// inputs, generating the session TLS material bound to this launch. +pub fn boundary_launch_inputs( + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, +) -> Result { + let supervisor = &launch_authentication.supervisor; + let tls = openshell_sandbox_backend::boundary_protocol::generate_sandbox_tls_material( + supervisor.session_id, + ) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let verification_keys = launch_authentication + .verification_keys + .iter() + .map(|key| { + String::from_utf8(key.public_key_pem.clone()) + .map(|public_key_pem| GatewayVerificationKey { + key_id: key.key_id.clone(), + public_key_pem, + }) + .map_err(|error| { + KubernetesDriverError::Precondition(format!( + "Kubernetes sandbox verification key is not UTF-8 PEM: {error}" + )) + }) + }) + .collect::, _>>()?; + Ok(BoundaryLaunchInputs { + session_id: supervisor.session_id, + session_rotation: supervisor.session_rotation, + auth_epoch: supervisor.auth_epoch, + gateway_id: launch_authentication.gateway_id.clone(), + verification_keys, + tls, + }) +} + #[cfg(test)] use crate::isolation::BOUNDARY_ROLE_LABEL; use crate::isolation::{ KubernetesSandboxRuntimeNetworkFence, KubernetesSandboxRuntimeNetworkFenceSpec, boundary_service_role, runtime_control_policy, }; +use openshell_sandbox_backend::boundary_protocol::GatewayVerificationKey; pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap"; pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap"; @@ -643,6 +695,50 @@ pub fn supervisor_pod( }) } +/// The three boundary bootstrap artifacts every runtime delivers to the +/// in-workload boundary: the encoded `BoundaryConfig`, the session TLS +/// certificate chain, and its private key. +/// +/// Both the pod path (via a Kubernetes Secret mounted by the workload) and +/// the `KubeVirt` path (via the cloudInitNoCloud bootstrap payload) render +/// these from the same inputs, so their identity claims cannot drift. +pub struct BoundaryBootstrapArtifacts { + /// Encoded `boundary.json`. + pub config_json: Vec, + /// Session TLS certificate chain (`tls.crt`). + pub certificate_pem: Vec, + /// Session TLS private key (`tls.key`). + pub private_key_pem: Vec, +} + +impl std::fmt::Debug for BoundaryBootstrapArtifacts { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The private key must never reach logs or OCSF JSONL; the source + // type (SandboxTlsMaterial) redacts it the same way. + f.debug_struct("BoundaryBootstrapArtifacts") + .field("config_json_len", &self.config_json.len()) + .field("certificate_pem_len", &self.certificate_pem.len()) + .field("private_key_pem", &"") + .finish() + } +} + +/// Render the boundary bootstrap artifacts from a provisioned boundary +/// config and its session TLS material. Shared by every runtime; each +/// delivers the bundle through its own transport. +pub fn render_boundary_bootstrap( + boundary_config: &openshell_sandbox_backend::boundary_protocol::BoundaryConfig, + tls: &openshell_sandbox_backend::boundary_protocol::SandboxTlsMaterial, +) -> Result { + Ok(BoundaryBootstrapArtifacts { + config_json: boundary_config + .encode() + .map_err(|error| error.to_string())?, + certificate_pem: tls.certificate_chain_pem.as_bytes().to_vec(), + private_key_pem: tls.private_key_pem.as_bytes().to_vec(), + }) +} + #[allow(clippy::too_many_arguments)] #[must_use] pub fn sandbox_bootstrap_secret( @@ -795,6 +891,7 @@ pub fn image_pull_secret_copy( #[must_use] pub fn sandbox_owner_reference( + kind: &str, name: &str, uid: &str, api_version: &str, @@ -802,7 +899,7 @@ pub fn sandbox_owner_reference( ) -> OwnerReference { OwnerReference { api_version: api_version.to_string(), - kind: "Sandbox".to_string(), + kind: kind.to_string(), name: name.to_string(), uid: uid.to_string(), controller: controller.then_some(true), @@ -914,7 +1011,7 @@ mod tests { use super::*; fn owner() -> OwnerReference { - sandbox_owner_reference("demo", "uid-1", "agents.x-k8s.io/v1beta1", false) + sandbox_owner_reference("Sandbox", "demo", "uid-1", "agents.x-k8s.io/v1beta1", false) } #[test] @@ -1767,3 +1864,21 @@ mod tests { assert!(material.private_key_pem.contains("PRIVATE KEY")); } } + +#[cfg(test)] +mod boundary_artifacts_tests { + use super::*; + + #[test] + fn boundary_bootstrap_artifacts_debug_redacts_private_key() { + let artifacts = BoundaryBootstrapArtifacts { + config_json: b"{}".to_vec(), + certificate_pem: b"-----BEGIN CERTIFICATE-----\nfake\n".to_vec(), + private_key_pem: b"-----BEGIN PRIVATE KEY-----\nsekrit\n".to_vec(), + }; + let formatted = format!("{artifacts:?}"); + assert!(!formatted.contains("sekrit")); + assert!(!formatted.contains("PRIVATE KEY")); + assert!(formatted.contains("")); + } +} diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index bd328fc0a1..8dd99cd109 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -145,7 +145,10 @@ pub struct BackendDescriptor { /// Validation and normalization run in [`ResolvedWorkloadIdentity::new`] and /// again when a value is decoded from a serialized descriptor: deserialization /// routes through the same constructor, so an invalid identity fails to decode -/// instead of bypassing the contract. +/// instead of bypassing the contract. The exception is the privileged +/// (root-workload) VM class, whose canonical zero identity is only producible +/// by [`ResolvedWorkloadIdentity::new_privileged`]; decoding routes a zero +/// uid through that constructor and requires its canonical form. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(try_from = "RawResolvedWorkloadIdentity")] pub struct ResolvedWorkloadIdentity { @@ -191,6 +194,30 @@ impl ResolvedWorkloadIdentity { resource_digest, }) } + + /// Construct the identity for a privileged (root-workload) sandbox. + /// + /// The standard [`Self::new`] refuses UID or GID zero, which is the + /// right invariant for container and sandboxed-VM workloads: a UID-0 + /// process defeats in-guest enforcement. The privileged VM class + /// deliberately runs the workload as root (kernel separation is the + /// boundary; see the root-workload VM plan), so this constructor + /// accepts zero. It is only for that class: callers must gate it on + /// the workload mode. + pub fn new_privileged(source: String, resource_digest: String) -> Result { + if source.trim().is_empty() || resource_digest.trim().is_empty() { + return Err(BackendError::Descriptor( + "workload identity source and resource digest are required".to_string(), + )); + } + Ok(Self { + uid: 0, + gid: 0, + supplementary_gids: Vec::new(), + source, + resource_digest, + }) + } } /// Unvalidated wire shape of [`ResolvedWorkloadIdentity`], existing only so @@ -209,6 +236,20 @@ impl TryFrom for ResolvedWorkloadIdentity { type Error = BackendError; fn try_from(raw: RawResolvedWorkloadIdentity) -> Result { + if raw.uid == 0 { + // The privileged (root-workload) VM class: a zero uid is only + // producible by `new_privileged`, whose canonical form is gid 0 + // with no supplementary groups. Require that form; anything + // else fails closed. + if raw.gid != 0 || !raw.supplementary_gids.is_empty() { + return Err(BackendError::Descriptor( + "privileged workload identity must be canonical (uid 0, gid 0, no \ + supplementary groups)" + .to_string(), + )); + } + return Self::new_privileged(raw.source, raw.resource_digest); + } Self::new( raw.uid, raw.gid, diff --git a/crates/openshell-isolation-interface/tests/backend_conformance.rs b/crates/openshell-isolation-interface/tests/backend_conformance.rs index 9928b02073..d7f6592bc2 100644 --- a/crates/openshell-isolation-interface/tests/backend_conformance.rs +++ b/crates/openshell-isolation-interface/tests/backend_conformance.rs @@ -246,3 +246,56 @@ async fn port_forward_rejects_non_loopback() { let loopback = LoopbackTarget::new("127.0.0.1".parse().unwrap(), 8080).expect("loopback ok"); assert_eq!(loopback.port(), 8080); } + +#[test] +fn workload_identity_decoding_rejects_non_canonical_zero_forms() { + let privileged = + ResolvedWorkloadIdentity::new_privileged("policy".to_string(), "sha256:test".to_string()) + .unwrap(); + + // The canonical privileged identity round-trips through serialization. + let json = serde_json::to_string(&privileged).unwrap(); + let decoded: ResolvedWorkloadIdentity = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded, privileged); + + // A zero uid with a non-zero gid fails closed instead of decoding. + let forged = serde_json::json!({ + "uid": 0, + "gid": 1000, + "supplementary_gids": [], + "source": "policy", + "resource_digest": "sha256:test", + }); + let error = serde_json::from_value::(forged) + .expect_err("non-canonical zero identity must fail to decode"); + assert!( + error.to_string().contains("canonical"), + "unexpected error: {error}" + ); + + // A zero gid with a non-zero uid still routes through the standard + // constructor, which refuses it. + let forged = serde_json::json!({ + "uid": 1000, + "gid": 0, + "supplementary_gids": [], + "source": "policy", + "resource_digest": "sha256:test", + }); + let error = serde_json::from_value::(forged) + .expect_err("zero gid must fail to decode"); + assert!( + error.to_string().contains("UID or GID zero"), + "unexpected error: {error}" + ); + + // A zero uid with supplementary groups fails closed. + let forged = serde_json::json!({ + "uid": 0, + "gid": 0, + "supplementary_gids": [1000], + "source": "policy", + "resource_digest": "sha256:test", + }); + assert!(serde_json::from_value::(forged).is_err()); +} diff --git a/crates/openshell-ocsf/src/builders/api_activity.rs b/crates/openshell-ocsf/src/builders/api_activity.rs index 7461cc91da..a06a704291 100644 --- a/crates/openshell-ocsf/src/builders/api_activity.rs +++ b/crates/openshell-ocsf/src/builders/api_activity.rs @@ -166,6 +166,14 @@ impl<'a> ApiActivityBuilder<'a> { self } + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + #[must_use] pub fn build(self) -> OcsfEvent { // Only supervisors describe a container on the producing device. diff --git a/crates/openshell-ocsf/src/builders/base.rs b/crates/openshell-ocsf/src/builders/base.rs index c08f1aa0aa..58ce9c120b 100644 --- a/crates/openshell-ocsf/src/builders/base.rs +++ b/crates/openshell-ocsf/src/builders/base.rs @@ -44,6 +44,17 @@ impl<'a> BaseEventBuilder<'a> { self } + /// Mark the event's payload as guest-sourced: the destination, + /// identity, or query data originated from the in-guest boundary (which + /// is untrusted after start in root-workload mode — root can read its + /// memory), not from a supervisor-verified observation. Applied at the + /// supervisor's ingestion of boundary-reported data (network decisions, + /// DNS queries, process reports). + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + #[must_use] pub fn build(self) -> OcsfEvent { let activity_name = self.activity_name.as_deref().unwrap_or("Other"); diff --git a/crates/openshell-ocsf/src/builders/config.rs b/crates/openshell-ocsf/src/builders/config.rs index 906470408d..802e636879 100644 --- a/crates/openshell-ocsf/src/builders/config.rs +++ b/crates/openshell-ocsf/src/builders/config.rs @@ -63,6 +63,14 @@ impl<'a> ConfigStateChangeBuilder<'a> { self } + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + #[must_use] pub fn build(self) -> OcsfEvent { let mut base = BaseEventData::new( diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index fe66c4c796..e29ae8c566 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -92,6 +92,14 @@ impl<'a> DetectionFindingBuilder<'a> { self } + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + /// Add a remediation description. #[must_use] pub fn remediation(mut self, desc: &str) -> Self { diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index e2f85bdab2..ca81b2d466 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -154,6 +154,14 @@ impl<'a, Context> HttpActivityBuilder<'a, Context> { self } + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + #[must_use] pub fn actor_process(mut self, process: crate::objects::Process) -> Self { self.actor = Some(Actor::process(process)); diff --git a/crates/openshell-ocsf/src/builders/network.rs b/crates/openshell-ocsf/src/builders/network.rs index 950cc7e1c8..bdc6a80ddc 100644 --- a/crates/openshell-ocsf/src/builders/network.rs +++ b/crates/openshell-ocsf/src/builders/network.rs @@ -192,6 +192,14 @@ impl<'a, EndpointState> NetworkActivityBuilder<'a, EndpointState> { self } + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } + #[must_use] pub fn actor_process(mut self, process: crate::objects::Process) -> Self { self.actor = Some(Actor::process(process)); @@ -294,3 +302,110 @@ mod tests { assert_eq!(json["is_src_dst_assignment_known"], true); } } + +#[cfg(test)] +mod guest_sourced_tests { + use super::*; + use crate::{ActivityId, SeverityId, StatusId}; + + #[test] + fn guest_sourced_sets_the_data_origin_field() { + use crate::{ActionId, DispositionId, Endpoint}; + let event = NetworkActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain("example.com", 443)) + .guest_sourced() + .message("test") + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert_eq!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .and_then(serde_json::Value::as_str), + Some("guest"), + "the marking must appear as unmapped.data_origin=guest: {json}" + ); + } + + #[test] + fn guest_sourced_appears_in_shorthand_format() { + use crate::{ActionId, DispositionId, Endpoint, StatusId}; + let event = NetworkActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Refuse) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain("example.org", 53)) + .status_detail("policy_dns_ineligible") + .guest_sourced() + .message("Policy DNS refused a name") + .build(); + let shorthand = event.format_shorthand(); + assert!( + shorthand.contains("data_origin:guest"), + "the shorthand format must render the guest-sourced marking: {shorthand}" + ); + } + + #[test] + fn guest_controlled_path_cannot_forge_the_marker() { + // The forgery attempt: a guest embeds "[data_origin:guest]" in a URL + // path. The shorthand renders the marker at a FIXED position (right + // after the severity tag, before any guest-controlled content), so + // the forged text appears LATER in the line and a grep for + // data_origin:guest at the marker position (or in the JSONL + // unmapped field) is not fooled. This test proves the forged path + // text does NOT create a real marking in the JSON. + use crate::{ActionId, DispositionId, Endpoint, StatusId}; + let event = NetworkActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain("example.com", 443)) + .message("CONNECT allowed example.com:443 FORGE[data_origin:guest]-path") + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + // The event is NOT marked: the forged text is in the message, not + // in the unmapped data_origin field. + assert!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .is_none(), + "a guest-controlled path must not create a data_origin marking: {json}" + ); + // The shorthand does not carry the marker tag. + let shorthand = event.format_shorthand(); + assert!( + !shorthand.starts_with("NET:OPEN [INFO] [data_origin:guest]"), + "the marker must not appear at the fixed position for an unmarked event: {shorthand}" + ); + } + + #[test] + fn unmarked_events_have_no_data_origin() { + use crate::{ActionId, DispositionId, Endpoint}; + let event = NetworkActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain("example.com", 443)) + .message("test") + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .is_none(), + "unmarked events must not carry data_origin: {json}" + ); + } +} diff --git a/crates/openshell-ocsf/src/builders/process.rs b/crates/openshell-ocsf/src/builders/process.rs index ae6a6dfe14..7a046ce6bc 100644 --- a/crates/openshell-ocsf/src/builders/process.rs +++ b/crates/openshell-ocsf/src/builders/process.rs @@ -22,6 +22,7 @@ pub struct ProcessActivityBuilder<'a, ActorState = MissingActor> { launch_type: Option, exit_code: Option, message: Option, + unmapped: serde_json::Map, actor_state: std::marker::PhantomData, } @@ -55,6 +56,7 @@ impl<'a> ProcessActivityBuilder<'a, MissingActor> { launch_type: None, exit_code: None, message: None, + unmapped: serde_json::Map::new(), actor_state: std::marker::PhantomData, } } @@ -74,6 +76,7 @@ impl<'a> ProcessActivityBuilder<'a, MissingActor> { launch_type: self.launch_type, exit_code: self.exit_code, message: self.message, + unmapped: self.unmapped, actor_state: std::marker::PhantomData, } } @@ -100,6 +103,21 @@ impl ProcessActivityBuilder<'_, ActorState> { self.exit_code = Some(code); self } + + /// Add an unmapped field. + #[must_use] + pub fn unmapped(mut self, key: &str, value: impl Into) -> Self { + self.unmapped.insert(key.to_string(), value.into()); + self + } + + /// Mark the event's payload as guest-sourced: the data originated + /// from the in-guest boundary (untrusted after start in root-workload + /// mode), not from a supervisor-verified observation. + #[must_use] + pub fn guest_sourced(self) -> Self { + self.unmapped("data_origin", "guest") + } } impl_activity_setter!(ProcessActivityBuilder); @@ -123,6 +141,9 @@ impl ProcessActivityBuilder<'_, HasActor> { ); self.ctx .apply_common_fields(&mut base, self.status, self.message); + if !self.unmapped.is_empty() { + base.unmapped = Some(serde_json::Value::Object(self.unmapped)); + } OcsfEvent::ProcessActivity(ProcessActivityEvent { base, @@ -177,3 +198,195 @@ mod tests { assert_eq!(json["exit_code"], 0); } } + +#[cfg(test)] +mod guest_sourced_tests { + use super::*; + use crate::{ActivityId, SeverityId}; + + #[test] + fn guest_sourced_process_exit_carries_the_marker() { + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(137) + .guest_sourced() + .message("main process exit reported by the boundary: code 137") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert_eq!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .and_then(serde_json::Value::as_str), + Some("guest") + ); + let shorthand = event.format_shorthand(); + assert!( + shorthand.contains("data_origin:guest"), + "the shorthand must render the marker: {shorthand}" + ); + } + + #[test] + fn exec_exit_carries_the_marker_in_root_mode() { + // The exec exit status is boundary-reported; in root mode the + // boundary is untrusted after start, so the event is marked. + use crate::Process; + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(0) + .process(Process::new("sh", 0)) + .guest_sourced() + .message("exec exit reported by the boundary: code 0") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert_eq!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .and_then(serde_json::Value::as_str), + Some("guest"), + "the exec exit event must carry the guest-sourced marking in root mode" + ); + let shorthand = event.format_shorthand(); + assert!( + shorthand.contains("data_origin:guest"), + "the shorthand must render the marker: {shorthand}" + ); + } + + #[test] + fn exec_exit_has_no_marker_in_sandboxed_mode() { + // The exec exit status is boundary-reported but the boundary is + // supervised in sandboxed mode; the event stays unmarked. + use crate::Process; + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(0) + .process(Process::new("sh", 0)) + .message("exec process exited: code 0") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .is_none(), + "the exec exit event must NOT carry the marking in sandboxed mode" + ); + let shorthand = event.format_shorthand(); + assert!( + !shorthand.contains("data_origin:guest"), + "the shorthand must not render the marker: {shorthand}" + ); + } + + #[test] + fn adopted_exec_exit_carries_the_marker_in_root_mode() { + // The adopted-exec exit is boundary-reported; in root mode the + // event is marked (same lifecycle semantics as the exec exit). + use crate::Process; + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(137) + .process(Process::new("python3", 0)) + .guest_sourced() + .message("adopted exec exit reported by the boundary: code 137") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert_eq!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .and_then(serde_json::Value::as_str), + Some("guest"), + "the adopted-exec exit must carry the guest-sourced marking in root mode" + ); + } + + #[test] + fn forged_process_name_cannot_create_a_marker() { + // The PROC shorthand escapes guest-controlled fields (process name, + // command line) with bracket escaping: a name starting with + // "[data_origin:guest] " renders as "\\[data_origin:guest\\] " + // (escaped brackets), visually distinct from the real + // fixed-position marker. The shorthand of an unmarked event must + // NOT start with the marked prefix. + use crate::Process; + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(0) + .process(Process::new("[data_origin:guest] fake-cmd", 0)) + .message("exec process exited: code 0") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + // The event is NOT marked: the forged name is in the process field, + // not in the unmapped data_origin field. + assert!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .is_none(), + "a forged process name must not create a data_origin marking: {json}" + ); + // The shorthand of an unmarked event must NOT start with the + // marked prefix. The escaped brackets make the forged name + // render as \\[data_origin:guest\\], not [data_origin:guest]. + let shorthand = event.format_shorthand(); + assert!( + !shorthand.starts_with("PROC:TERMINATE [INFO] [data_origin:guest]"), + "the forged name must not make the shorthand look marked: {shorthand}" + ); + assert!( + shorthand.contains("\\[data_origin:guest\\]"), + "the forged brackets must be escaped: {shorthand}" + ); + } + + #[test] + fn newline_in_process_name_cannot_start_a_forged_line() { + // A newline in a process name would start a new line in the + // shorthand output, potentially forging a subsequent event. The + // escape quotes the field, preventing this. + use crate::Process; + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(0) + .process(Process::new( + "cmd\nOCSF NET:OPEN [INFO] [data_origin:guest] fake", + 0, + )) + .message("exec process exited: code 0") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let shorthand = event.format_shorthand(); + // The shorthand must not contain a newline (the escaped name is quoted). + assert!( + !shorthand.contains('\n'), + "a newline in the process name must not create a line break: {shorthand:?}" + ); + } + + #[test] + fn unmarked_process_exit_has_no_marker() { + let event = ProcessActivityBuilder::new(crate::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(0) + .message("main process exited: code 0") + .actor_process(Process::new("openshell-sandbox", 1)) + .build(); + let json = serde_json::to_value(&event).expect("serialize"); + assert!( + json.get("unmapped") + .and_then(|u| u.get("data_origin")) + .is_none() + ); + } +} diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index cf052eca8e..30a247deba 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -16,6 +16,29 @@ use std::sync::{Arc, LazyLock, OnceLock}; static OCSF_CTX: OnceLock = OnceLock::new(); +/// Whether events emitted by this process carry guest-sourced payload +/// (`data_origin=guest`). +/// +/// Set once at startup by a supervisor whose launch descriptor marks the +/// boundary as an untrusted guest: every event whose payload was reported +/// by that boundary — process exits, connection identity — is then marked +/// so log readers cannot mistake guest reports for verified observations. +/// Trusted boundaries (container sandboxes, libkrun VM guests) leave it +/// unset. Untrusted guests always run under a dedicated supervisor, so the +/// process-wide flag cannot leak across sandbox instances. +static GUEST_DATA_ORIGIN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Mark every event emitted by this process as guest-sourced. +pub fn set_guest_data_origin(enabled: bool) { + GUEST_DATA_ORIGIN.store(enabled, std::sync::atomic::Ordering::Release); +} + +/// Whether events emitted by this process carry guest-sourced payload. +#[must_use] +pub fn guest_data_origin() -> bool { + GUEST_DATA_ORIGIN.load(std::sync::atomic::Ordering::Acquire) +} + static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext { sandbox_id: String::new(), sandbox_name: String::new(), diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 5b80f29205..b2ffbc46cc 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -92,6 +92,13 @@ fn escape_quoted_field(text: &str) -> String { '\n' => escaped.push_str("\\n"), '\r' => escaped.push_str("\\r"), '\t' => escaped.push_str("\\t"), + // Escape brackets so a guest-controlled value can never begin + // with a context-tag-like [...] sequence — a forged + // [data_origin:guest] prefix must render as + // \[data_origin:guest\], visually distinct from the + // real fixed-position marker. + '[' => escaped.push_str("\\["), + ']' => escaped.push_str("\\]"), character if character.is_control() => { use std::fmt::Write as _; let _ = write!(escaped, "\\u{{{:x}}}", u32::from(character)); @@ -198,6 +205,30 @@ fn unmapped_fields(base: &BaseEventData) -> Vec { .collect() } +/// Render the guest-sourced provenance marker at a fixed position +/// (immediately after the severity tag, before any guest-controlled +/// content). Guest values (URL paths, domain names) appear LATER in the +/// shorthand line, so a guest cannot forge the marker by embedding +/// `[data_origin:guest]` text in a path — the marker is already rendered +/// by then. The JSONL `unmapped.data_origin` field is authoritative. +fn guest_sourced_tag(base: &BaseEventData) -> String { + if base + .unmapped + .as_ref() + .and_then(serde_json::Value::as_object) + .is_some_and(|fields| { + fields + .get("data_origin") + .and_then(serde_json::Value::as_str) + == Some("guest") + }) + { + " [data_origin:guest]".to_string() + } else { + String::new() + } +} + fn unmapped_context(base: &BaseEventData, include_reason: bool) -> String { let mut fields = unmapped_fields(base); @@ -284,6 +315,14 @@ impl OcsfEvent { (false, true) => format!(" {action}"), (false, false) => format!(" {action}{arrow}"), }; + // Unmapped fields (including the guest-sourced provenance + // marker) render after the reason so operators can see the + // data origin alongside the decision. + // The guest-sourced provenance marker renders immediately + // after the severity tag — BEFORE any guest-controlled + // content (URLs, domain names, paths) so a guest cannot + // forge it by embedding the marker text in a path. + let origin_tag = guest_sourced_tag(&e.base); // Most network messages duplicate the structured action and // destination shown above. Transparent TCP correlation is the // exception: its message intentionally carries the logical, @@ -298,7 +337,9 @@ impl OcsfEvent { } else { String::new() }; - format!("NET:{activity} {sev}{detail}{rule_ctx}{reason_ctx}{message_ctx}") + format!( + "NET:{activity} {sev}{origin_tag}{detail}{rule_ctx}{reason_ctx}{message_ctx}" + ) } Self::HttpActivity(e) => { @@ -517,7 +558,15 @@ impl OcsfEvent { Self::ProcessActivity(e) => { let activity = e.base.activity_name.to_uppercase(); - let proc_str = format!("{}({})", e.process.name, e.process.pid); + // Escape guest-controlled fields (process name, command + // line): an unescaped value starting with + // "[data_origin:guest] " would render like a marked event, + // and a newline would start a forged line. + let proc_str = format!( + "{}({})", + escape_quoted_field(&e.process.name), + e.process.pid + ); let exit_ctx = e .exit_code .map(|c| format!(" [exit:{c}]")) @@ -526,10 +575,11 @@ impl OcsfEvent { .process .cmd_line .as_ref() - .map(|c| format!(" [cmd:{c}]")) + .map(|c| format!(" [cmd:{}]", escape_quoted_field(c))) .unwrap_or_default(); - format!("PROC:{activity} {sev} {proc_str}{exit_ctx}{cmd_ctx}") + let origin_tag = guest_sourced_tag(&e.base); + format!("PROC:{activity} {sev}{origin_tag} {proc_str}{exit_ctx}{cmd_ctx}") } Self::DetectionFinding(e) => { diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 8c0ac9c2c5..d62eb3d976 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -995,6 +995,26 @@ pub fn restrictive_default_policy() -> SandboxPolicy { } } +/// The restrictive default policy for a privileged (root-workload) sandbox. +/// +/// Identical network posture to [`restrictive_default_policy`], but with no +/// filesystem, Landlock, or process-identity sections: in-guest enforcement +/// is meaningless against a root workload (the fence and the supervisor's +/// policy evaluation are the boundary), so the default must not promise +/// constraints that cannot be enforced. A policy that authors those sections +/// is rejected by supervisor validation in privileged mode. +#[must_use] +pub fn restrictive_default_policy_privileged() -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: None, + landlock: None, + process: None, + network_policies: HashMap::new(), + network_middlewares: HashMap::default(), + } +} + /// Fill omitted process identity fields with the legacy `sandbox` defaults. /// /// Docker and Podman preserve omission so their supervisors can fall back to diff --git a/crates/openshell-sandbox-backend/src/audit.rs b/crates/openshell-sandbox-backend/src/audit.rs index c94b92d49a..15209ec0b0 100644 --- a/crates/openshell-sandbox-backend/src/audit.rs +++ b/crates/openshell-sandbox-backend/src/audit.rs @@ -6,18 +6,99 @@ use openshell_isolation_interface::contract::BackendError; /// Validate the built-in Linux backend's measured enforcement state. -pub fn validate_native_linux_audit(evidence: &serde_json::Value) -> Result<(), BackendError> { +/// Validate the boundary's audit evidence. +/// +/// The class comes from the TRUSTED launch descriptor's resource claims +/// (never from anything the boundary reports — a guest could otherwise +/// dodge validation by claiming a class). The caller must pass the class +/// it resolved from the descriptor. +/// +/// - Trusted boundary (container or libkrun VM, or unknown): the full +/// capability-free posture (zero caps, `no_new_privs`, seccomp, +/// Landlock, DNS/TCP round trips) is required. +/// - Untrusted `KubeVirt` guest: only what is still meaningful (the boundary +/// honestly reports no caps/Landlock/seccomp because none are enforced). +/// Everything the guest reports after start is advisory and is never +/// treated as proof of enforcement. +pub fn validate_native_linux_audit( + evidence: &serde_json::Value, + privileged: bool, +) -> Result<(), BackendError> { let audit: crate::boundary_protocol::NativeLinuxSandboxAuditEvidence = serde_json::from_value(evidence.clone()).map_err(|error| { BackendError::Confirm(format!("decode Linux sandbox audit evidence: {error}")) })?; - audit.validate() + if privileged { + audit.validate_untrusted_guest() + } else { + audit.validate() + } } #[cfg(test)] mod tests { use super::validate_native_linux_audit; + fn root_evidence() -> serde_json::Value { + // A privileged boundary honestly reports no caps/seccomp/Landlock: + // none are enforced. The kernel release and architecture are + // present (the boundary measured them from the host kernel). + serde_json::json!({ + "capabilities": {"inheritable": 0, "permitted": 0, "effective": 0, + "bounding": 0, "ambient": 0}, + "no_new_privileges": false, + "sandbox_dumpable": false, + "child_dumpable": false, + "core_limit_zero": true, + "native_architecture": "x86_64", + "kernel_release": "6.12.114-0-virt", + "seccomp": {"new_listener": false, "notification_round_trip": false, + "id_validation": false, "addfd_send": false, + "retained_socket_operation": false, + "proc_fd_identity": false, "task_memory_read": false}, + "landlock_abi": 0, + "landlock_allow_deny": false, + "udp_dns_round_trip": false, + "tcp_dns_round_trip": false, + "tcp_allow_round_trip": false, + "tcp_deny_round_trip": false, + "socket_loopback_confinement": false + }) + } + + #[test] + fn root_descriptor_with_privileged_evidence_passes() { + // The privileged mode accepts honest no-evidence reports with + // kernel info present. + let evidence = root_evidence(); + validate_native_linux_audit(&evidence, true).unwrap(); + } + + #[test] + fn sandboxed_descriptor_with_privileged_evidence_fails() { + // The sandboxed mode requires the full capability-free posture; + // a privileged (incomplete) evidence report must be rejected. + // This is the fail-closed direction: a sandboxed guest cannot + // dodge validation by presenting relaxed evidence. + let evidence = root_evidence(); + assert!( + validate_native_linux_audit(&evidence, false).is_err(), + "sandboxed mode must reject privileged evidence" + ); + } + + #[test] + fn root_descriptor_missing_kernel_info_fails() { + // Even in root mode, the kernel release and architecture are + // required (the boundary measured them from the host kernel). + let mut evidence = root_evidence(); + evidence["kernel_release"] = serde_json::json!(""); + assert!( + validate_native_linux_audit(&evidence, true).is_err(), + "root mode must still require kernel release and architecture" + ); + } + #[test] fn default_validator_accepts_complete_linux_evidence() { let evidence = serde_json::json!({ @@ -38,17 +119,17 @@ mod tests { "tcp_allow_round_trip": true, "tcp_deny_round_trip": true, "socket_loopback_confinement": true }); - validate_native_linux_audit(&evidence).unwrap(); + validate_native_linux_audit(&evidence, false).unwrap(); let mut incomplete = evidence; incomplete["no_new_privileges"] = false.into(); - assert!(validate_native_linux_audit(&incomplete).is_err()); + assert!(validate_native_linux_audit(&incomplete, false).is_err()); } #[test] fn default_validator_rejects_unknown_platform_and_incomplete_evidence() { for platform in ["windows_mxc", "unknown", "linux"] { let evidence = serde_json::json!({"platform": platform, "evidence": {}}); - assert!(validate_native_linux_audit(&evidence).is_err()); + assert!(validate_native_linux_audit(&evidence, false).is_err()); } } } diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 724c9e1faa..9fea7166b5 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -159,6 +159,27 @@ impl NativeLinuxSandboxAuditEvidence { )) } } + + /// Validate the evidence an untrusted guest reports: only what is + /// still meaningful. The boundary honestly reports no + /// seccomp/Landlock/`no_new_privs` evidence because none are enforced + /// for an untrusted guest — the security boundary is outside the + /// guest. The kernel release and architecture must still be present + /// (the boundary measured them from the host kernel). Everything + /// the guest reports after start is advisory and is never treated as + /// proof of enforcement. + pub fn validate_untrusted_guest(&self) -> Result<(), BackendError> { + let present = !self.native_architecture.is_empty() && !self.kernel_release.is_empty(); + if present { + Ok(()) + } else { + Err(BackendError::Confirm( + "privileged audit evidence is incomplete: kernel release and \ + architecture are required" + .to_string(), + )) + } + } } /// Ephemeral identity of the supervisor process that owns one sandbox runtime. @@ -1994,3 +2015,214 @@ mod tests { ); } } + +// --------------------------------------------------------------------------- +// VM-guest claim markers +// --------------------------------------------------------------------------- + +/// Resource-claim key marking a libkrun VM guest generation (emitted by +/// `openshell-driver-vm`). +pub const VM_GENERATION_CLAIM: &str = "vm.generation"; + +/// Resource-claim key marking a `KubeVirt` VM guest generation (emitted by +/// the Kubernetes driver's kubevirt runtime). +pub const KUBEVIRT_GENERATION_CLAIM: &str = "kubevirt.generation"; + +/// Whether the boundary's resource claims mark it as a VM guest (libkrun or +/// `KubeVirt`). +/// +/// VM guests get VM policy-identity semantics: the `run_as_user`/ +/// `run_as_group` policy selectors must be `sandbox` or the resolved guest +/// uid/gid, not arbitrary account lookups. +pub fn is_vm_guest(resource_claims: &std::collections::BTreeMap) -> bool { + resource_claims.contains_key(VM_GENERATION_CLAIM) + || resource_claims.contains_key(KUBEVIRT_GENERATION_CLAIM) +} + +/// Whether the trusted launch descriptor's resource claims mark the +/// boundary as an untrusted `KubeVirt` guest. +/// +/// Every `KubeVirt` guest runs its workload as root and its in-guest +/// boundary is untrusted: the guest, including the boundary program, is +/// untrusted as soon as a workload executes, so the supervisor must not +/// trust boundary reports (image policy, binary identity, enforcement +/// evidence) from it. The claim is read only from the TRUSTED descriptor, +/// never from boundary reports. libkrun VM guests (`vm.generation`) keep +/// the trusted-boundary model. +pub fn is_kubevirt_guest(resource_claims: &std::collections::BTreeMap) -> bool { + resource_claims.contains_key(KUBEVIRT_GENERATION_CLAIM) +} + +#[cfg(test)] +mod vm_guest_tests { + use super::*; + use std::collections::BTreeMap; + + fn claims(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn is_kubevirt_guest_keys_on_the_generation_claim() { + // Fail closed: only the kubevirt generation claim marks the + // untrusted guest class; libkrun guests and unmarked boundaries + // keep the trusted-boundary model. + assert!(is_kubevirt_guest(&claims(&[("kubevirt.generation", "g1")]))); + assert!(!is_kubevirt_guest(&claims(&[("vm.generation", "g1")]))); + assert!(!is_kubevirt_guest(&claims(&[]))); + } + + #[test] + fn is_vm_guest_accepts_vm_generation_claim() { + assert!(is_vm_guest(&claims(&[("vm.generation", "g1")]))); + } + + #[test] + fn is_vm_guest_accepts_kubevirt_generation_claim() { + assert!(is_vm_guest(&claims(&[("kubevirt.generation", "g1")]))); + } + + #[test] + fn is_vm_guest_rejects_no_marker() { + assert!(!is_vm_guest(&claims(&[( + "kubernetes.workload_pod_uid", + "x" + )]))); + assert!(!is_vm_guest(&BTreeMap::new())); + } + + #[test] + fn is_vm_guest_accepts_both_markers() { + assert!(is_vm_guest(&claims(&[ + ("vm.generation", "g1"), + ("kubevirt.generation", "g2") + ]))); + } +} + +/// Whether a runtime descriptor matches the provisioned boundary config. +/// +/// Compares every identity field: `boundary_id`, `generation`, +/// `session_id`, `workload_identity`, `resource_claims`, and +/// `provisioning_evidence`. This is the consistency check the boundary +/// runs at open (via `isolation_protocol::validate_open`); drivers and +/// tests use the same function so the comparison cannot drift between +/// implementations. +pub fn descriptor_matches_config( + descriptor: &SandboxRuntimeDescriptor, + config: &BoundaryConfig, +) -> bool { + descriptor.boundary_id == config.boundary_id + && descriptor.generation == config.generation + && descriptor.session_id == config.session_id + && descriptor.workload_identity == config.workload_identity + && descriptor.resource_claims == config.resource_claims + && descriptor.provisioning_evidence == config.provisioning_evidence +} + +#[cfg(test)] +mod descriptor_config_consistency_tests { + use super::*; + use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; + use std::collections::BTreeMap; + + fn descriptor(claims: &BTreeMap) -> SandboxRuntimeDescriptor { + SandboxRuntimeDescriptor { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: SandboxSessionId::new(), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + "test-resource".to_string(), + ) + .unwrap(), + transport: SandboxTransport::Unix { + socket_path: "/unused".into(), + }, + tls: SandboxTlsClientConfig { + server_name: "unused".to_string(), + trust_anchor_pem: String::new(), + }, + host_gateway_ip: None, + resource_claims: claims.clone(), + provisioning_evidence: ProvisioningEvidence::from_enforcement_evidence( + "generation-1", + b"test-evidence", + ) + .unwrap(), + } + } + + fn config(claims: &BTreeMap) -> BoundaryConfig { + BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: SandboxSessionId::new(), + session_rotation: openshell_core::jwt::SessionRotation::new(1).unwrap(), + auth_epoch: openshell_core::jwt::CredentialEpoch::new(1).unwrap(), + gateway_id: "gateway-1".to_string(), + verification_keys: vec![], + listener: BoundaryListener::Unix { + socket_path: "/unused".into(), + tls: SandboxTlsServerConfig { + certificate_chain_path: "/unused".into(), + private_key_path: "/unused".into(), + }, + }, + resource_claims: claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + "test-resource".to_string(), + ) + .unwrap(), + provisioning_evidence: ProvisioningEvidence::from_enforcement_evidence( + "generation-1", + b"test-evidence", + ) + .unwrap(), + child_env: std::collections::HashMap::new(), + } + } + + #[test] + fn matching_descriptor_and_config_pass() { + let claims = BTreeMap::from([("test.claim".to_string(), "value".to_string())]); + // The session_id must match too; build both from the same descriptor's + // session_id. + let d = descriptor(&claims); + let mut c = config(&claims); + c.session_id = d.session_id; + assert!(descriptor_matches_config(&d, &c)); + } + + #[test] + fn different_generation_fails() { + let claims = BTreeMap::new(); + let d = descriptor(&claims); + let mut c = config(&claims); + c.session_id = d.session_id; + c.generation = "generation-2".to_string(); + assert!(!descriptor_matches_config(&d, &c)); + } + + #[test] + fn different_claims_fail() { + let d = descriptor(&BTreeMap::new()); + let mut c = config(&BTreeMap::from([( + "extra".to_string(), + "claim".to_string(), + )])); + c.session_id = d.session_id; + assert!(!descriptor_matches_config(&d, &c)); + } +} diff --git a/crates/openshell-sandbox-backend/src/isolation_protocol.rs b/crates/openshell-sandbox-backend/src/isolation_protocol.rs index 6576d8a24f..010ceaa514 100644 --- a/crates/openshell-sandbox-backend/src/isolation_protocol.rs +++ b/crates/openshell-sandbox-backend/src/isolation_protocol.rs @@ -92,13 +92,7 @@ pub fn validate_open( let descriptor: SandboxRuntimeDescriptor = serde_json::from_slice(&request.driver_descriptor) .map_err(|_| tonic::Status::invalid_argument("invalid OpenShell runtime descriptor"))?; - if descriptor.boundary_id != config.boundary_id - || descriptor.generation != config.generation - || descriptor.session_id != config.session_id - || descriptor.workload_identity != config.workload_identity - || descriptor.resource_claims != config.resource_claims - || descriptor.provisioning_evidence != config.provisioning_evidence - { + if !crate::boundary_protocol::descriptor_matches_config(&descriptor, config) { return Err(tonic::Status::permission_denied( "driver descriptor does not match provisioned boundary", )); diff --git a/crates/openshell-sandbox-backend/src/protobuf.rs b/crates/openshell-sandbox-backend/src/protobuf.rs index 2f0ebc80c4..618da5483a 100644 --- a/crates/openshell-sandbox-backend/src/protobuf.rs +++ b/crates/openshell-sandbox-backend/src/protobuf.rs @@ -352,14 +352,28 @@ impl WireMessage for c::BoundaryConfirmation { let identity = required(v.identity)?; Ok(Self { generation: v.generation, - identity: c::ResolvedWorkloadIdentity::new( - identity.uid, - identity.gid, - identity.supplementary_gids, - identity.source, - identity.resource_digest, - ) - .map_err(invalid)?, + // The wire-format conversion must accept the untrusted-guest + // (root-workload) identity: a `KubeVirt` boundary reports + // uid 0 by design, and the class check (claim-gated, in + // validate_runtime_coordinates) has already run. Using + // new() here would refuse every untrusted-guest confirmation + // at deserialization. + identity: if identity.uid == 0 && identity.gid == 0 { + c::ResolvedWorkloadIdentity::new_privileged( + identity.source, + identity.resource_digest, + ) + .map_err(invalid)? + } else { + c::ResolvedWorkloadIdentity::new( + identity.uid, + identity.gid, + identity.supplementary_gids, + identity.source, + identity.resource_digest, + ) + .map_err(invalid)? + }, session_id: parse(&v.session_id)?, resource_claims: v.resource_claims, provisioning_evidence_digest: parse(&v.provisioning_evidence_digest)?, diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 3f4d751f77..cda14af8ef 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -663,13 +663,28 @@ pub fn validate_runtime_coordinates( )); } let identity = &runtime_descriptor.workload_identity; - let normalized = ResolvedWorkloadIdentity::new( - identity.uid, - identity.gid, - identity.supplementary_gids.clone(), - identity.source.clone(), - identity.resource_digest.clone(), - )?; + // A `KubeVirt` guest runs its workload as root by design, so its + // identity uses the privileged constructor (uid 0, gid 0, no + // supplementary groups). The class is read from the descriptor's + // resource claims (`kubevirt.generation`); everything else — + // including libkrun VM guests — keeps the standard constructor, + // which refuses uid 0 and fails closed. + let untrusted_guest = + crate::boundary_protocol::is_kubevirt_guest(&runtime_descriptor.resource_claims); + let normalized = if untrusted_guest { + ResolvedWorkloadIdentity::new_privileged( + identity.source.clone(), + identity.resource_digest.clone(), + )? + } else { + ResolvedWorkloadIdentity::new( + identity.uid, + identity.gid, + identity.supplementary_gids.clone(), + identity.source.clone(), + identity.resource_digest.clone(), + )? + }; if normalized != *identity { return Err(BackendError::Descriptor( "workload identity is not canonical".into(), @@ -806,7 +821,20 @@ impl BoundBoundary { )); } if self.client.backend_name == crate::BACKEND_NAME { - crate::audit::validate_native_linux_audit(&confirmation.backend_audit)?; + // The per-class evidence check (item 3): the guest class comes + // from the trusted launch descriptor's resource_claims, never + // from the boundary's own report. A trusted boundary (container + // or libkrun VM) requires the full capability-free posture; an + // untrusted KubeVirt guest requires only what is still + // meaningful (kernel release and architecture), because the + // boundary honestly reports no seccomp/Landlock evidence for + // that class. + let untrusted_guest = + crate::boundary_protocol::is_kubevirt_guest(&self.resource_claims); + crate::audit::validate_native_linux_audit( + &confirmation.backend_audit, + untrusted_guest, + )?; } let client = self.client.clone(); // Adopt from a snapshot taken after the promotion: the replaced owner diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 0ccc79af88..f09dedd1ad 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -50,7 +50,7 @@ clap = { workspace = true } miette = { workspace = true } # Unix identity and bootstrap ownership -nix = { workspace = true, features = ["resource", "socket", "feature", "poll"] } +nix = { workspace = true, features = ["resource", "socket", "feature", "poll", "net"] } # TLS crypto provider install (main.rs) rustls = { workspace = true } diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index d95ce62e4c..8bd8024905 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -41,6 +41,11 @@ pub struct LocalBoundaryExec { user_environment: HashMap, runtime: Arc, otlp_relay_active: bool, + /// Whether this executor runs in privileged (root-workload) mode: no + /// Landlock ruleset and no seccomp filter are applied to workload + /// children (they are meaningless against root; the fence and the + /// supervisor's policy are the boundary). + privileged: bool, #[cfg(target_os = "linux")] launcher: crate::linux::workload_launcher::WorkloadLauncher, } @@ -58,6 +63,7 @@ impl LocalBoundaryExec { runtime: Arc, otlp_relay_active: bool, #[cfg(target_os = "linux")] launcher: crate::linux::workload_launcher::WorkloadLauncher, + privileged: bool, ) -> Self { Self { policy, @@ -67,6 +73,7 @@ impl LocalBoundaryExec { user_environment, runtime, otlp_relay_active, + privileged, #[cfg(target_os = "linux")] launcher, } @@ -219,6 +226,12 @@ impl LocalBoundaryExec { &self, workdir: Option<&str>, ) -> Result, BackendError> { + if self.privileged { + // B3c1: no Landlock ruleset for a root workload — it is + // meaningless against root, and the fence plus the + // supervisor's policy are the boundary. + return Ok(None); + } crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); let runtime_read_only = crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); @@ -237,8 +250,14 @@ impl LocalBoundaryExec { #[cfg(target_os = "linux")] let prepared = self.prepare_sandbox(effective_workdir)?; #[cfg(target_os = "linux")] - let child_hardening = crate::linux::child_seccomp::prepare(std::process::id()) - .map_err(|error| BackendError::Process(error.to_string()))?; + let child_hardening = if self.privileged { + // B3c1: no seccomp filter for a root workload — a no-op + // program that sets only the process invariants. + crate::linux::child_seccomp::ChildHardeningProgram::noop() + } else { + crate::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))? + }; crate::pty::install_dedicated_process_group(&mut command); crate::pty::install_pre_exec_no_pty( &mut command, @@ -375,8 +394,14 @@ impl LocalBoundaryExec { #[cfg(target_os = "linux")] let prepared = self.prepare_sandbox(effective_workdir)?; #[cfg(target_os = "linux")] - let child_hardening = crate::linux::child_seccomp::prepare(std::process::id()) - .map_err(|error| BackendError::Process(error.to_string()))?; + let child_hardening = if self.privileged { + // B3c1: no seccomp filter for a root workload — a no-op + // program that sets only the process invariants. + crate::linux::child_seccomp::ChildHardeningProgram::noop() + } else { + crate::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))? + }; crate::pty::install_pre_exec( &mut command, self.policy.clone(), @@ -849,6 +874,7 @@ mod tests { crate::boundary_io::BoundaryRuntimeState::new(), otlp_relay_active, launcher, + false, ) } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 9975d54055..20ceefcff4 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -264,8 +264,15 @@ mod linux { format!("consume boundary config {}: {error}", config_path.display()) })?; crate::process::install_boundary_user_environment(config.child_env.clone())?; - crate::sandbox::apply_supervisor_startup_hardening() - .map_err(|error| format!("install sandbox process prelude: {error}"))?; + // The supervisor prelude (no_new_privs + TSYNC seccomp blocking + // mount/umount/bpf/etc) is the sandboxed boundary's defense. In + // privileged mode it is skipped: the guest is the user's by design, + // and the inherited no_new_privs and filter would defeat setuid + // binaries, container runtimes, and package managers that mount. + if !qualification.privileged { + crate::sandbox::apply_supervisor_startup_hardening() + .map_err(|error| format!("install sandbox process prelude: {error}"))?; + } // Keep orphaned workload descendants in this process tree so // termination can find and kill them, then reap the adopted ones. // PID 1 already receives orphans; elsewhere become a child subreaper. @@ -275,8 +282,7 @@ mod linux { } crate::managed_children::start_orphan_reaper() .map_err(|error| format!("start sandbox orphan reaper: {error}"))?; - let (launcher, listener) = crate::linux::workload_launcher::start() - .map_err(|error| format!("start sandbox workload launcher: {error}"))?; + let protected_control_port = match &config.listener { BoundaryListenerConfig::TlsTcp { address, .. } => Some(address.port()), // Validation requires every supervisor address to share one port. @@ -288,12 +294,56 @@ mod linux { | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } => None, }; - let network_broker = NetworkBroker::start(listener, protected_control_port) - .map_err(|error| format!("start sandbox network broker: {error}"))?; + // The transparent relay's accept loops need the async runtime; the + // broker is otherwise constructed before it exists. let process_runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .map_err(|error| format!("create boundary process runtime: {error}"))?; + let (launcher, network_broker) = if qualification.privileged { + // B3c: the privileged (root-workload) mode launches workload + // children WITHOUT the seccomp-notify filter. Commit 5: the + // transparent nftables relay captures outbound workload + // traffic inside the guest (TCP redirected to a local relay, + // DNS to the loopback resolver) and feeds the SAME Mediate + // accept/decision/bridge flow the seccomp broker uses — the + // supervisor's proxy and policy DNS run unchanged. Deleting + // the ruleset restores direct egress, which the network fence + // blocks (fail-closed: the workload only breaks its own + // connectivity). + let launcher = crate::linux::workload_launcher::start_privileged() + .map_err(|error| format!("start privileged workload launcher: {error}"))?; + let control_endpoints: Vec = match &config.listener { + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { + addresses.clone() + } + // The other listener kinds (unix, vsock, in-guest TCP) + // have no supervisor IP:port to exclude from the + // capture ruleset; the relay is only wired for the + // runtime-initiated TCP transport the KubeVirt driver + // renders. Fall back to the unavailable broker there. + _ => Vec::new(), + }; + let broker = if control_endpoints.is_empty() { + tracing::warn!( + "privileged (root-workload) boundary: the transparent relay needs \ + the runtime-initiated TCP transport; network mediation is unavailable \ + and workload requests are blocked by the fence" + ); + NetworkBroker::unavailable() + .map_err(|error| format!("create unavailable network broker: {error}"))? + } else { + NetworkBroker::start_transparent_relay(&control_endpoints, process_runtime.handle()) + .map_err(|error| format!("start transparent network relay: {error}"))? + }; + (launcher, broker) + } else { + let (launcher, listener) = crate::linux::workload_launcher::start() + .map_err(|error| format!("start sandbox workload launcher: {error}"))?; + let broker = NetworkBroker::start(listener, protected_control_port) + .map_err(|error| format!("start sandbox network broker: {error}"))?; + (launcher, broker) + }; let runtime = Arc::new(BoundaryRuntime::new( config.clone(), process_runtime.handle().clone(), @@ -351,6 +401,54 @@ mod linux { Ok(()) } + /// Mode-gated workload identity validation (pure, testable): uid 0 is + /// allowed ONLY when the claim says "root" (the + /// privileged VM class, entered only through launch-privileged); the + /// sandboxed default and any unknown mode refuse it (fail closed). + /// Conversely, the root mode requires uid/gid zero — a nonzero + /// identity under a root claim is a config error. Each entry point + /// refuses the other mode's config. + /// Refuse provider material (files and environment variables) in the + /// privileged (root-workload) mode — the user's decision: nothing + /// secret enters the guest. This is the boundary-side defence in + /// depth; the supervisor-side refusal is the real guarantee (the + /// request maps are deserialized before this check runs, so a root + /// guest could theoretically observe them in that window). + fn validate_provider_material_privileged( + privileged: bool, + provider_files: &std::collections::HashMap, + provider_env: &std::collections::HashMap, + ) -> Result<(), String> { + if privileged && (!provider_files.is_empty() || !provider_env.is_empty()) { + return Err("provider files and provider environment variables are not \ + supported in privileged (root-workload) mode: the guest runs as \ + root and would be fully exposed to the credentials" + .to_string()); + } + Ok(()) + } + + fn validate_workload_mode_identity( + resource_claims: &std::collections::BTreeMap, + uid: u32, + gid: u32, + ) -> Result<(), String> { + // A `KubeVirt` guest is the untrusted root-workload class: its + // workload identity is uid/gid zero by design. Everything else + // (container boundaries, libkrun VM guests) must be nonzero. + if openshell_sandbox_backend::boundary_protocol::is_kubevirt_guest(resource_claims) { + if uid != 0 || gid != 0 { + return Err( + "untrusted-guest (root-workload) boundary requires UID and GID zero" + .to_string(), + ); + } + } else if uid == 0 || gid == 0 { + return Err("sandbox workload UID and GID must be nonzero".to_string()); + } + Ok(()) + } + fn validate_config(config: &BoundaryConfig) -> Result<(), String> { if config.boundary_id.is_empty() { return Err("boundary ID must not be empty".to_string()); @@ -434,9 +532,11 @@ mod linux { | BoundaryListenerConfig::TlsTcp { .. } | BoundaryListenerConfig::Vsock { .. } => {} } - if config.workload_identity.uid == 0 || config.workload_identity.gid == 0 { - return Err("sandbox workload UID and GID must be nonzero".to_string()); - } + validate_workload_mode_identity( + &config.resource_claims, + config.workload_identity.uid, + config.workload_identity.gid, + )?; Ok(()) } @@ -447,7 +547,7 @@ mod linux { config: &BoundaryConfig, policy: &SandboxPolicyWire, ) -> Result<(), String> { - if !config.resource_claims.contains_key("vm.generation") { + if !openshell_sandbox_backend::boundary_protocol::is_vm_guest(&config.resource_claims) { return Ok(()); } let identity = &config.workload_identity; @@ -3929,6 +4029,18 @@ mod linux { provider_env: std::collections::HashMap, provider_files: std::collections::HashMap, ) -> Response { + // The user's decision: provider files and provider environment + // variables are unsupported in the privileged (root-workload) + // mode — nothing secret enters the guest. A root guest reads + // boundary memory (/proc//mem), so storing the maps even + // without serving them is exposure. + if let Err(error) = validate_provider_material_privileged( + self.qualification.privileged, + &provider_files, + &provider_env, + ) { + return guest_error(BoundaryErrorKind::Unsupported, error); + } let Some(policy) = lock(&self.attached_policy).clone() else { return guest_error( BoundaryErrorKind::Invalid, @@ -4010,6 +4122,7 @@ mod linux { provider_env_revision, provider_env, ca_file_paths, + privileged: self.qualification.privileged, }; let provider_file_count = provider_files.len(); if let Err(error) = self.network_broker.provider_files().replace(provider_files) { @@ -4053,6 +4166,15 @@ mod linux { provider_env: std::collections::HashMap, provider_files: std::collections::HashMap, ) -> Response { + // Same refusal as start_agent: no provider material in + // privileged mode, including on later updates. + if let Err(error) = validate_provider_material_privileged( + self.qualification.privileged, + &provider_files, + &provider_env, + ) { + return guest_error(BoundaryErrorKind::Unsupported, error); + } let process = { let state = lock(&self.state); let RuntimeState::Running(process) = &*state else { @@ -4397,6 +4519,7 @@ mod linux { provider_env_revision: u64, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + privileged: bool, } fn resolve_agent_spec(mut spec: AgentSpecWire) -> Result { @@ -4431,6 +4554,7 @@ mod linux { provider_env_revision, provider_env, ca_file_paths, + privileged, } = launch; debug_assert!(!spec.program.is_empty()); let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); @@ -4449,6 +4573,7 @@ mod linux { spec.interactive, &policy, entrypoint_pid, + privileged, provider_credentials.clone(), provider_env, ca_file_paths, @@ -6746,8 +6871,96 @@ mod linux { .unwrap() } + #[test] + fn provider_material_refused_in_privileged_mode() { + use std::collections::HashMap; + + let files = || { + let mut map = HashMap::new(); + map.insert("secret.txt".to_string(), "data".to_string()); + map + }; + let env = || { + let mut map = HashMap::new(); + map.insert("API_KEY".to_string(), "value".to_string()); + map + }; + let empty = HashMap::new(); + + // Privileged + provider files: refused. + assert!( + validate_provider_material_privileged(true, &files(), &empty).is_err(), + "provider files must be refused in privileged mode" + ); + + // Privileged + provider env: refused. + assert!( + validate_provider_material_privileged(true, &empty, &env()).is_err(), + "provider env must be refused in privileged mode" + ); + + // Privileged + both: refused. + assert!( + validate_provider_material_privileged(true, &files(), &env()).is_err(), + "both provider maps must be refused in privileged mode" + ); + + // Privileged + neither: accepted. + assert!( + validate_provider_material_privileged(true, &empty, &empty).is_ok(), + "empty provider maps are fine in privileged mode" + ); + + // Sandboxed + provider files: accepted (the normal path). + assert!( + validate_provider_material_privileged(false, &files(), &empty).is_ok(), + "provider files are fine in sandboxed mode" + ); + + // Sandboxed + provider env: accepted. + assert!( + validate_provider_material_privileged(false, &empty, &env()).is_ok(), + "provider env is fine in sandboxed mode" + ); + } + + #[test] + fn workload_mode_identity_allows_root_only_for_kubevirt_guests() { + use std::collections::BTreeMap; + + let claims = |kubevirt: bool, vm: bool| { + let mut map = BTreeMap::new(); + if kubevirt { + map.insert("kubevirt.generation".to_string(), "g1".to_string()); + } + if vm { + map.insert("vm.generation".to_string(), "g1".to_string()); + } + map + }; + + // KubeVirt guest + uid 0: passes (the untrusted root-workload class). + assert!(validate_workload_mode_identity(&claims(true, false), 0, 0).is_ok()); + + // KubeVirt guest + nonzero uid: fails (config error). + assert!(validate_workload_mode_identity(&claims(true, false), 1000, 1000).is_err()); + + // libkrun VM guest + uid 0: fails (trusted-boundary invariant). + assert!(validate_workload_mode_identity(&claims(false, true), 0, 0).is_err()); + + // libkrun VM guest + nonzero uid: passes. + assert!(validate_workload_mode_identity(&claims(false, true), 1000, 1000).is_ok()); + + // No claims + uid 0: fails (fail closed). + assert!(validate_workload_mode_identity(&claims(false, false), 0, 0).is_err()); + + // No claims + nonzero uid: passes (the container default). + assert!(validate_workload_mode_identity(&claims(false, false), 1000, 1000).is_ok()); + } + fn test_runtime_qualification() -> crate::RuntimeQualification { crate::RuntimeQualification { + privileged: false, seccomp: openshell_sandbox_backend::boundary_protocol::SeccompEvidence { new_listener: true, notification_round_trip: true, @@ -6924,9 +7137,27 @@ mod linux { let config = &mut Arc::get_mut(&mut boundary) .expect("test boundary has one owner") .config; + config.resource_claims.insert( + openshell_sandbox_backend::boundary_protocol::VM_GENERATION_CLAIM.to_string(), + config.generation.clone(), + ); + (process_runtime, boundary) + } + + fn kubevirt_identity_test_runtime() -> (tokio::runtime::Runtime, Arc) { + let (process_runtime, mut boundary) = vm_identity_test_runtime(); + let config = &mut Arc::get_mut(&mut boundary) + .expect("test boundary has one owner") + .config; + // Swap the libkrun marker for the KubeVirt one; the predicate + // must treat both as VM guests. config .resource_claims - .insert("vm.generation".to_string(), config.generation.clone()); + .remove(openshell_sandbox_backend::boundary_protocol::VM_GENERATION_CLAIM); + config.resource_claims.insert( + openshell_sandbox_backend::boundary_protocol::KUBEVIRT_GENERATION_CLAIM.to_string(), + config.generation.clone(), + ); (process_runtime, boundary) } @@ -6943,6 +7174,33 @@ mod linux { }) } + #[test] + fn kubevirt_claimed_guest_gets_vm_policy_identity_validation() { + let (_runtime, boundary) = kubevirt_identity_test_runtime(); + let uid = boundary.config.workload_identity.uid.to_string(); + let gid = boundary.config.workload_identity.gid.to_string(); + // Matching and omitted selectors pass. + for (user, group) in [ + (None, None), + (Some(uid.as_str()), Some(gid.as_str())), + (Some("sandbox"), Some("sandbox")), + ] { + validate_vm_policy_identity( + &boundary.config, + &vm_identity_test_policy(user, group), + ) + .expect("matching or omitted VM selectors for a KubeVirt guest"); + } + // Conflicting selectors are denied, exactly like a libkrun VM. + let wrong = if uid == "10000" { "10001" } else { "10000" }; + let error = validate_vm_policy_identity( + &boundary.config, + &vm_identity_test_policy(Some(wrong), Some(wrong)), + ) + .expect_err("conflicting VM selectors must be denied"); + assert!(error.contains("conflicts with the resolved workload identity")); + } + #[test] fn vm_policy_identity_checks_independent_selectors() { let (_runtime, mut boundary) = vm_identity_test_runtime(); @@ -7004,7 +7262,7 @@ mod linux { .expect("test boundary has one owner") .config .resource_claims - .remove("vm.generation"); + .remove(openshell_sandbox_backend::boundary_protocol::VM_GENERATION_CLAIM); validate_vm_policy_identity( &boundary.config, &vm_identity_test_policy(Some(wrong_user), Some("image-user")), @@ -10365,6 +10623,7 @@ mod linux { provider_env_revision: 0, provider_env: std::collections::HashMap::new(), ca_file_paths: None, + privileged: false, }, prepared, ) diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index 7b3ec12b16..e443795581 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -37,12 +37,22 @@ pub async fn spawn_workload( interactive: bool, policy: &SandboxPolicy, entrypoint_pid: Arc, + privileged: bool, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, boundary_runtime: Option>, otlp_relay_active: bool, ) -> Result { + // Fail closed: if the process is root but the qualification does not + // say privileged, refuse — a sandboxed boundary running as root + // through any launch bug would silently skip Landlock and seccomp. + if !privileged && nix::unistd::geteuid().as_raw() == 0 { + return Err(miette::miette!( + "sandboxed boundary process is running as root without the \ + privileged qualification; refusing to launch workload children" + )); + } // Driver-selected workspaces are the sandbox identity's home. This keeps // canonical and later exec processes consistent for image WorkingDir and // the managed /sandbox fallback without consulting privileged account @@ -86,6 +96,7 @@ pub async fn spawn_workload( boundary_runtime.clone(), otlp_relay_active, launcher.clone(), + privileged, )); // The delegated workload runs behind OpenShell's mediated network path, @@ -103,6 +114,7 @@ pub async fn spawn_workload( tls_environment_mode, &provider_env, otlp_relay_active, + privileged, ) .wrap_err("spawn delegated workload process")?; #[cfg(not(target_os = "linux"))] @@ -116,6 +128,7 @@ pub async fn spawn_workload( tls_environment_mode, &provider_env, otlp_relay_active, + privileged, )?; entrypoint_pid.store(handle.pid(), Ordering::Release); diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 3c280cc977..9e2c634cd6 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -29,6 +29,8 @@ pub mod main_session; pub mod managed_children; #[cfg(target_os = "linux")] mod network_broker; +#[cfg(target_os = "linux")] +pub mod network_relay; #[cfg(all(target_os = "linux", feature = "perf-harness"))] pub mod perf; #[cfg(target_os = "linux")] @@ -42,8 +44,9 @@ pub mod sandbox; #[cfg(target_os = "linux")] pub mod sftp; -/// Results of actively qualifying the admitted workload runtime before the -/// sandbox consumes protected bootstrap material. +/// Results of actively qualifying the admitted workload runtime. +/// +/// Produced before the sandbox consumes protected bootstrap material. #[cfg(target_os = "linux")] #[derive(Clone, Copy, Debug)] #[allow( @@ -52,6 +55,10 @@ pub mod sftp; )] pub struct RuntimeQualification { pub seccomp: openshell_sandbox_backend::boundary_protocol::SeccompEvidence, + /// Marks the privileged (root-workload) qualification: the in-guest + /// enforcement evidence is advisory, and the seccomp-based network + /// broker and provider files are unavailable. + pub privileged: bool, pub landlock_abi: u32, pub landlock_allow_deny: bool, pub udp_dns_round_trip: bool, @@ -61,6 +68,36 @@ pub struct RuntimeQualification { pub socket_loopback_confinement: bool, } +#[cfg(target_os = "linux")] +impl RuntimeQualification { + /// A qualification for the privileged (root-workload) mode: the + /// enforcement evidence fields are advisory (root can forge them), + /// the seccomp-based network broker is unavailable, and no Landlock + /// is applied. The boundary still runs the authenticated tunnel, + /// exec/PTY/SSH relay, and status reporting. + pub fn privileged() -> Self { + Self { + seccomp: openshell_sandbox_backend::boundary_protocol::SeccompEvidence { + new_listener: false, + notification_round_trip: false, + id_validation: false, + addfd_send: false, + retained_socket_operation: false, + proc_fd_identity: false, + task_memory_read: false, + }, + privileged: true, + landlock_abi: 0, + landlock_allow_deny: false, + udp_dns_round_trip: false, + tcp_dns_round_trip: false, + tcp_allow_round_trip: false, + tcp_deny_round_trip: false, + socket_loopback_confinement: false, + } + } +} + /// Placeholder used when compiling the package on a non-Linux host. /// /// The sandbox binary rejects execution on those hosts before constructing a diff --git a/crates/openshell-sandbox/src/linux/child_seccomp.rs b/crates/openshell-sandbox/src/linux/child_seccomp.rs index 6fc9f67f73..2592a88cf5 100644 --- a/crates/openshell-sandbox/src/linux/child_seccomp.rs +++ b/crates/openshell-sandbox/src/linux/child_seccomp.rs @@ -46,16 +46,42 @@ const CLONE_NAMESPACE_FLAGS: u32 = (libc::CLONE_NEWCGROUP /// A prebuilt child filter that can be installed without heap allocation. pub struct ChildHardeningProgram { instructions: Vec, + /// Whether sets . False only for the + /// privileged (root-workload) noop: is meaningless for + /// root but inherited by every descendant, breaking setuid binaries + /// (sudo, su, ping) for non-root users the root workload creates. + set_no_new_privs: bool, } impl ChildHardeningProgram { + /// An empty program that installs nothing: used for the privileged + /// (root-workload) mode, where no seccomp filter applies to workload + /// children. `install` on an empty program sets only + /// `no_new_privs` and the close-on-exec marks — the sandbox + /// invariants that are still meaningful for root. + pub fn noop() -> Self { + Self { + instructions: Vec::new(), + set_no_new_privs: false, + } + } + /// Install this filter on the calling thread only. /// /// The caller must invoke this from the post-fork child after all /// sandbox-wide TSYNC work and the launcher's `NEW_LISTENER` filter. pub fn install(&mut self) -> io::Result<()> { mark_inherited_descriptors_close_on_exec()?; - set_no_new_privileges()?; + if self.set_no_new_privs { + set_no_new_privileges()?; + } + // The privileged (root-workload) noop program has no instructions: + // the kernel rejects a zero-length seccomp filter with EINVAL, so + // skip the syscall entirely. The process invariants (close-on-exec, + // optionally no_new_privs) are already applied above. + if self.instructions.is_empty() { + return Ok(()); + } let len = u16::try_from(self.instructions.len()).map_err(|_| { io::Error::new( io::ErrorKind::InvalidInput, @@ -250,7 +276,10 @@ pub fn prepare(sandbox_tgid: u32) -> io::Result { } instructions.push(stmt(BPF_RET_K, SECCOMP_RET_ALLOW)); - Ok(ChildHardeningProgram { instructions }) + Ok(ChildHardeningProgram { + instructions, + set_no_new_privs: true, + }) } fn append_unconditional_deny( @@ -600,3 +629,149 @@ mod tests { ); } } + +#[cfg(test)] +mod noop_tests { + use super::ChildHardeningProgram; + + #[test] + fn noop_program_install_skips_the_seccomp_syscall() { + // The kernel rejects a zero-length seccomp filter with EINVAL, so + // noop().install() must return Ok without making the syscall. + // Run in a forked child because install() calls pre-exec-only + // operations (setpgid would fail in the parent). + unsafe { + let pid = libc::fork(); + if pid == 0 { + let mut program = ChildHardeningProgram::noop(); + let result = program.install(); + if result.is_ok() { + std::process::exit(0); + } else { + std::process::exit(1); + } + } + let mut status = 0; + libc::waitpid(pid, std::ptr::addr_of_mut!(status), 0); + assert_eq!( + libc::WEXITSTATUS(status), + 0, + "noop().install() must succeed (the seccomp syscall is skipped for empty programs)" + ); + } + } +} + +#[cfg(test)] +mod privileged_syscall_tests { + /// Read `NoNewPrivs` and `Seccomp` from `/proc/self/status`. + fn proc_status_fields() -> (String, String) { + let status = std::fs::read_to_string("/proc/self/status").expect("read /proc/self/status"); + let no_new_privs = status + .lines() + .find(|line| line.starts_with("NoNewPrivs:")) + .map(|line| line.split_whitespace().nth(1).unwrap_or("?").to_string()) + .unwrap_or_default(); + let seccomp = status + .lines() + .find(|line| line.starts_with("Seccomp:")) + .map(|line| line.split_whitespace().nth(1).unwrap_or("?").to_string()) + .unwrap_or_default(); + (no_new_privs, seccomp) + } + + /// Attempt a syscall in the forked child and report whether it failed + /// with EPERM. Used as a diagnostic only: in restricted environments + /// (dev containers) the environment's own seccomp profile blocks these + /// syscalls, so EPERM here does not implicate our filter. The + /// authoritative check is the Seccomp/NoNewPrivs status, which proves + /// our filter cannot be the blocker. + unsafe fn attempt_syscall(syscall_nr: libc::c_long) -> bool { + let ret = unsafe { libc::syscall(syscall_nr, 0, 0, 0, 0, 0) }; + ret == -1 && std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + } + + #[test] + fn noop_child_has_no_no_new_privs_and_no_seccomp() { + // B7: in privileged mode, the boundary process skips the supervisor + // prelude (no_new_privs + TSYNC seccomp blocking mount, umount2, + // unshare, bpf, perf_event_open, init_module, kexec, ...). The + // noop hardening program installs neither. A forked child must + // report NoNewPrivs: 0 and Seccomp: 0 — with zero seccomp filters + // installed, none of those syscalls can be blocked by our filter + // (they may still fail for kernel or environment reasons, which + // is the documented root-mode posture: the guest kernel is the + // user's by design). + unsafe { + let pid = libc::fork(); + if pid == 0 { + let mut program = super::ChildHardeningProgram::noop(); + program.install().expect("noop install succeeds"); + let (nnp, seccomp) = proc_status_fields(); + // Exit code bit 0: NoNewPrivs/Seccomp status (must be 0/0). + // Bit 1: any of the probe syscalls returned EPERM + // (diagnostic only — environment seccomp may cause this). + let status_fail = !(nnp == "0" && seccomp == "0"); + let eperm_probe = attempt_syscall(libc::SYS_mount) + || attempt_syscall(libc::SYS_umount2) + || attempt_syscall(libc::SYS_unshare) + || attempt_syscall(libc::SYS_bpf) + || attempt_syscall(libc::SYS_perf_event_open) + || attempt_syscall(libc::SYS_init_module); + std::process::exit(i32::from(status_fail) | (i32::from(eperm_probe) << 1)); + } + let mut status = 0; + libc::waitpid(pid, std::ptr::addr_of_mut!(status), 0); + let code = libc::WEXITSTATUS(status); + assert_eq!( + code & 1, + 0, + "noop child must have NoNewPrivs:0 Seccomp:0 (got exit code {code})" + ); + if code & 2 != 0 { + // Not a failure: the probe syscalls returned EPERM, which + // in restricted environments (containers with their own + // seccomp profiles) is expected. Our filter is proven + // absent by the Seccomp:0 check above. + eprintln!("note: probe syscalls returned EPERM (environment seccomp, not ours)"); + } + } + } + + #[test] + fn sandboxed_prelude_still_sets_no_new_privs_and_seccomp() { + // The sandboxed counter-test: apply_supervisor_prelude() must + // still install no_new_privs and the seccomp filter, so sandboxed + // boundary children keep the syscall blocks (mount, bpf, ... + // return EPERM — asserted behaviorally in + // sandbox/linux/seccomp.rs::behavioral_supervisor_prelude_mount_blocked). + // This test asserts the status fields from the child-hardening + // side: the prelude composes with the child program. + unsafe { + let pid = libc::fork(); + if pid == 0 { + if let Err(err) = crate::sandbox::apply_supervisor_startup_hardening() { + let msg = format!("failed to install supervisor prelude: {err}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(2); + } + let (nnp, seccomp) = proc_status_fields(); + if nnp != "1" || seccomp != "2" { + let msg = format!( + "expected NoNewPrivs:1 Seccomp:2 after prelude, got {nnp}/{seccomp}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(3); + } + libc::_exit(0); + } + let mut status = 0; + libc::waitpid(pid, std::ptr::addr_of_mut!(status), 0); + assert_eq!( + libc::WEXITSTATUS(status), + 0, + "sandboxed prelude must set NoNewPrivs:1 Seccomp:2" + ); + } + } +} diff --git a/crates/openshell-sandbox/src/linux/workload_launcher.rs b/crates/openshell-sandbox/src/linux/workload_launcher.rs index 51b3af629e..b2fcd638d3 100644 --- a/crates/openshell-sandbox/src/linux/workload_launcher.rs +++ b/crates/openshell-sandbox/src/linux/workload_launcher.rs @@ -65,6 +65,46 @@ impl WorkloadLauncher { } } +/// Start a privileged workload launcher. +/// +/// The same job-execution thread as [], but WITHOUT the +/// seccomp-notify filter. Workload children spawned from this thread have +/// no syscall interception — socket calls go straight to the kernel, where +/// the network fence (not the broker) is the enforcement point. Used only +/// for the privileged (root-workload) mode. +pub fn start_privileged() -> io::Result { + let (jobs_tx, jobs_rx) = mpsc::sync_channel::(64); + let (ready_tx, ready_rx) = mpsc::sync_channel::>(1); + let alive = Arc::new(AtomicBool::new(true)); + let thread_alive = alive.clone(); + thread::Builder::new() + .name("openshell-workload-launcher".to_string()) + .spawn(move || { + // No seccomp filter: the ready signal confirms the thread is + // running, nothing else. + if ready_tx.send(Ok(())).is_err() { + thread_alive.store(false, Ordering::Release); + return; + } + while let Ok(job) = jobs_rx.recv() { + job(); + } + thread_alive.store(false, Ordering::Release); + }) + .map_err(|error| io::Error::other(format!("start workload launcher thread: {error}")))?; + + ready_rx.recv().map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "workload launcher exited before publishing readiness", + ) + })??; + Ok(WorkloadLauncher { + jobs: jobs_tx, + alive, + }) +} + /// Start the only workload launcher and return its listener to an unfiltered /// sandbox thread. pub fn start() -> io::Result<(WorkloadLauncher, NotificationListener)> { diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 400478a889..ac27db8992 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -43,6 +43,7 @@ const CAPABILITY_PROBE_LAUNCH_SUBCOMMAND: &str = "capability-probe-launch"; const CAPABILITY_SOCKET_CHILD_SUBCOMMAND: &str = "capability-socket-child"; const CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND: &str = "capability-landlock-child"; const CAPABILITY_FREE_LAUNCH_SUBCOMMAND: &str = "launch-capability-free"; +const PRIVILEGED_LAUNCH_SUBCOMMAND: &str = "launch-privileged"; #[cfg(target_os = "linux")] const PROBE_DENIED_TCP_PEER: &str = "203.0.113.1:9"; #[cfg(target_os = "linux")] @@ -204,6 +205,7 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, Qualifi tcp_deny_round_trip: true, }; let qualification = openshell_sandbox::RuntimeQualification { + privileged: false, seccomp: openshell_sandbox_backend::boundary_protocol::SeccompEvidence { new_listener: notification.notification_round_trip(), notification_round_trip: notification.notification_round_trip(), @@ -1479,6 +1481,87 @@ fn launch_capability_free(_args: &[String]) -> Result<()> { )) } +/// Launch the boundary for a privileged (root-workload) sandbox: the same +/// authenticated tunnel, claims, exec/PTY/SSH relay, and network relay as +/// the capability-free launch, but the workload runs as root and the +/// in-guest enforcement (capability drop, `no_new_privs`, Landlock, seccomp) +/// is removed — it is meaningless against root. The security boundary is +/// everything outside the guest: the fence, the supervisor's policy, and +/// the VM boundary itself. The guest is untrusted as soon as a workload +/// executes; guest reports are marked `data_origin=guest`. +#[cfg(target_os = "linux")] +fn launch_privileged(args: &[String]) -> Result<()> { + use miette::Context as _; + + let [bootstrap, _workspace @ ..] = args else { + return Err(miette::miette!( + "usage: openshell-sandbox {PRIVILEGED_LAUNCH_SUBCOMMAND} [WORKSPACE]" + )); + }; + + // Privileged qualification: assert uid 0. The capability-free probe + // (zero caps, no_new_privs) is NOT run — it is the trusted-boundary + // path's invariant; this boundary is the untrusted-guest (root- + // workload) class. + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid != 0 || gid != 0 { + return Err(miette::miette!( + "privileged launch requires root UID and GID, got {uid}:{gid}" + )); + } + let config_bytes = std::fs::read(bootstrap) + .into_diagnostic() + .wrap_err_with(|| format!("read boundary config {bootstrap}"))?; + let config: openshell_sandbox_backend::boundary_protocol::BoundaryConfig = + serde_json::from_slice(&config_bytes) + .into_diagnostic() + .wrap_err("decode boundary config")?; + if !openshell_sandbox_backend::boundary_protocol::is_kubevirt_guest(&config.resource_claims) { + return Err(miette::miette!( + "privileged launch is only for KubeVirt guests (kubevirt.generation claim)" + )); + } + + let log_level = std::env::var(openshell_core::sandbox_env::LOG_LEVEL) + .unwrap_or_else(|_| "warn".to_string()); + run_privileged_boundary(Path::new(bootstrap), &log_level) +} + +/// Run the boundary in privileged (root-workload) mode: the same +/// authenticated tunnel and Sandbox Protocol as the capability-free path, +/// but without the in-guest enforcement. The network broker and provider +/// files are seccomp-based today and are unavailable in this mode; the +/// boundary reports that clearly rather than silently starting a workload +/// with broken networking. +#[cfg(target_os = "linux")] +fn run_privileged_boundary(bootstrap: &Path, log_level: &str) -> Result<()> { + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(log_level)); + let _ = tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new( + openshell_sandbox::container_log::ContainerLog::process().launcher_writer(), + ) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .try_init(); + // A placeholder qualification for the privileged path: the + // enforcement evidence fields are advisory in this mode (root can + // forge them), and the seccomp-based network broker is unavailable + // until the nftables relay (commit 5). + let qualification = openshell_sandbox::RuntimeQualification::privileged(); + openshell_sandbox::run(bootstrap, qualification) +} + +#[cfg(not(target_os = "linux"))] +fn launch_privileged(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "privileged launch is only supported on Linux" + )) +} + #[cfg(not(target_os = "linux"))] fn run_capability_probe() -> Result<()> { Err(miette::miette!( @@ -1848,6 +1931,9 @@ fn main() -> Result<()> { if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_FREE_LAUNCH_SUBCOMMAND) { return launch_capability_free(&raw_args[2..]); } + if raw_args.get(1).map(String::as_str) == Some(PRIVILEGED_LAUNCH_SUBCOMMAND) { + return launch_privileged(&raw_args[2..]); + } let args = BoundaryArgs::parse(); run_boundary(&args.bootstrap, &args.log_level) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 323f1ac715..712dfe039f 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -34,6 +34,12 @@ use tokio::sync::{mpsc, oneshot}; const SOCKET_CAPACITY: usize = 4_096; const SOCKET_FD_HEADROOM: usize = 64; const OPEN_QUEUE_CAPACITY: usize = 256; + +/// The relay broker's admission bound for undecided opens (mirrors the +/// seccomp broker's queue capacity). +pub const fn open_queue_capacity() -> usize { + OPEN_QUEUE_CAPACITY +} const DNS_QUEUE_CAPACITY: usize = 256; const DNS_WORKER_CAPACITY: usize = 256; const DNS_QUERY_TIMEOUT: Duration = Duration::from_secs(10); @@ -101,6 +107,31 @@ impl PendingTcpOpen { self.decide(decision)?.established().await } + /// Construct a pending open for a transparently relayed connection + /// (commit 5, root mode): the destination comes from conntrack's + /// original-destination record instead of an intercepted syscall, and + /// the relayed workload stream is handed to the Mediate bridge when + /// the supervisor allows the open. + pub(crate) fn for_relayed_connection( + destination: SocketAddr, + identity: Result, + socket: NetworkSocketMetadata, + notification_to_queue: Duration, + queued_at: Instant, + decision: std::sync::mpsc::SyncSender, + relay: oneshot::Receiver>, + ) -> Self { + Self { + destination, + identity, + socket, + notification_to_queue, + queued_at, + decision, + relay, + } + } + /// Hand the decision to the broker. Once this returns the decision has /// taken effect; only the relay setup of an allowed open remains. pub(crate) fn decide(self, decision: TcpOpenDecision) -> io::Result { @@ -230,6 +261,15 @@ pub struct NetworkBroker { descriptor_soft_limit: usize, dns_address: SocketAddr, healthy: Arc, + /// True when this broker was explicitly constructed as unavailable for + /// the privileged (root-workload) mode — not a crash. `confirm_healthy` + /// succeeds (the state is designed, not a failure); network requests + /// are still refused because there are no pending channels. + designed_unavailable: bool, + /// The held-alive channel senders for the designed-unavailable mode: + /// keeping them alive keeps the supervisor's Mediate streams open and + /// idle (no hot retry loop). Unused in the sandboxed mode. + _designed_unavailable_senders: (mpsc::Sender, mpsc::Sender), } impl NetworkBroker { @@ -240,6 +280,82 @@ impl NetworkBroker { Self::start_with_dns_address(listener, DNS_RELAY_ADDRESS, protected_control_port) } + /// A broker for the privileged (root-workload) mode, where the + /// seccomp-notify interception is not installed: no pending channels, + /// no DNS relay, marked unhealthy so network requests are refused with + /// a clear error instead of silently bypassing policy. The nftables + /// transparent relay (commit 5) is the replacement mediation path. + /// Construct the transparent relay broker for a privileged + /// (root-workload) boundary (commit 5): the loopback DNS relay + /// (127.0.0.53, same as sandboxed mode), the transparent TCP relay + /// listeners, and the nftables capture ruleset. The pending queues + /// feed the SAME Mediate accept/decision/bridge flow the seccomp + /// broker uses — the supervisor's proxy, policy DNS, and synthetic + /// address mapping run unchanged. + pub(crate) fn start_transparent_relay( + control_endpoints: &[SocketAddr], + runtime: &tokio::runtime::Handle, + ) -> io::Result { + let (pending_tx, pending_rx) = mpsc::channel(OPEN_QUEUE_CAPACITY); + let (pending_dns_tx, pending_dns_rx) = mpsc::channel(DNS_QUEUE_CAPACITY); + let active_opens = Arc::new(AtomicUsize::new(0)); + // The DNS relay is the sandboxed mode's loopback resolver: the + // workload's resolv.conf already points at 127.0.0.53. + let dns_relay = + start_dns_relay_with_admission(DNS_RELAY_ADDRESS, pending_dns_tx.clone(), true) + .map_err(|error| io::Error::other(format!("relay DNS listener: {error}")))?; + // The nftables capture ruleset and the transparent TCP listeners. + crate::network_relay::install_relay_rules( + control_endpoints, + crate::network_relay::TRANSPARENT_RELAY_PORT, + ) + .map_err(|error| io::Error::other(format!("relay nft ruleset: {error}")))?; + crate::network_relay::start_tcp_relay(pending_tx.clone(), active_opens, runtime) + .map_err(|error| io::Error::other(format!("relay TCP listener: {error}")))?; + let broker = Self { + provider_files: crate::provider_files::ProviderFiles::default(), + workload_frozen: Arc::new(AtomicBool::new(false)), + pending: Arc::new(tokio::sync::Mutex::new(pending_rx)), + pending_dns: Arc::new(tokio::sync::Mutex::new(pending_dns_rx)), + registry: Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)), + descriptor_soft_limit: descriptor_soft_limit()?, + dns_address: dns_relay.address, + healthy: Arc::new(AtomicBool::new(true)), + designed_unavailable: false, + // The relay broker IS the mediation source: hold its queue + // senders so the supervisor's Mediate streams stay connected. + _designed_unavailable_senders: (pending_tx, pending_dns_tx), + }; + Ok(broker) + } + + pub(crate) fn unavailable() -> io::Result { + // Keep the channel senders alive (do NOT drop them): the supervisor's + // proxy accept loop and policy DNS open Mediate streams on the + // boundary, and dropping the senders ends those streams immediately, + // causing a hot retry loop (~0.15 core per idle root sandbox). + // Holding the senders keeps the streams open and idle — the proxy + // waits for pending connections that never arrive, which is the + // correct idle behaviour. + let (pending_tx, pending_rx) = mpsc::channel(1); + let (dns_tx, dns_rx) = mpsc::channel(1); + // Leak the senders so they stay alive for the broker's lifetime. + let healthy = Arc::new(AtomicBool::new(false)); + let broker = Self { + provider_files: crate::provider_files::ProviderFiles::default(), + workload_frozen: Arc::new(AtomicBool::new(false)), + pending: Arc::new(tokio::sync::Mutex::new(pending_rx)), + pending_dns: Arc::new(tokio::sync::Mutex::new(dns_rx)), + registry: Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)), + descriptor_soft_limit: descriptor_soft_limit()?, + dns_address: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + healthy, + designed_unavailable: true, + _designed_unavailable_senders: (pending_tx, dns_tx), + }; + Ok(broker) + } + #[cfg(any(test, feature = "perf-harness"))] pub(crate) fn start_for_test(listener: NotificationListener) -> io::Result { Self::start_with_dns_address( @@ -362,6 +478,8 @@ impl NetworkBroker { descriptor_soft_limit, dns_address, healthy, + designed_unavailable: false, + _designed_unavailable_senders: (mpsc::channel(1).0, mpsc::channel(1).0), }) } @@ -405,6 +523,13 @@ impl NetworkBroker { } pub(crate) fn confirm_healthy(&self) -> io::Result<()> { + if self.designed_unavailable { + // The privileged (root-workload) mode explicitly constructed + // this broker as unavailable — it is the designed state, not + // a failure. Network requests are still refused (no pending + // channels); the nftables relay is the future mediation path. + return Ok(()); + } if self.healthy.load(Ordering::Acquire) && self.dns_address.port() != 0 { Ok(()) } else { @@ -456,6 +581,7 @@ fn dns_tcp_accept_loop( tcp_admissions: Arc>>, active_workers: Arc, pending: mpsc::Sender, + admit_unregistered: bool, ) { // Warn once per exhaustion episode plus a bounded re-warn cadence, not on // every backoff retry. @@ -513,8 +639,9 @@ fn dns_tcp_accept_loop( }, }; // One admission authorizes exactly one accepted TCP stream; - // no peer mapping needs to outlive this accept. - if lock(&tcp_admissions).remove(&peer).is_none() { + // no peer mapping needs to outlive this accept. The + // transparent relay admits unregistered sources. + if !admit_unregistered && lock(&tcp_admissions).remove(&peer).is_none() { tracing::warn!(%peer, "dropping DNS stream from unregistered socket"); continue; } @@ -535,6 +662,20 @@ fn dns_tcp_accept_loop( fn start_dns_relay( address: SocketAddr, pending: mpsc::Sender, +) -> io::Result { + start_dns_relay_with_admission(address, pending, false) +} + +/// Start the loopback DNS relay. `admit_unregistered` serves the +/// transparent-relay (root-workload) mode: the seccomp broker that +/// normally registers workload DNS sockets is absent, so queries from any +/// local source are admitted (the identity is the documented-unavailable +/// kernel-write path — root-mode policy does not require binary +/// identity). +fn start_dns_relay_with_admission( + address: SocketAddr, + pending: mpsc::Sender, + admit_unregistered: bool, ) -> io::Result { let (udp, tcp, address) = bind_dns_relay_sockets(address)?; let udp_admissions = Arc::new(Mutex::new(HashMap::new())); @@ -553,7 +694,7 @@ fn start_dns_relay( .spawn(move || { let mut request = vec![0_u8; u16::MAX as usize]; while let Ok((length, peer)) = udp.recv_from(&mut request) { - if !lock(&udp_admissions).contains_key(&peer) { + if !admit_unregistered && !lock(&udp_admissions).contains_key(&peer) { tracing::warn!(%peer, "dropping DNS datagram from unregistered socket"); continue; } @@ -603,6 +744,7 @@ fn start_dns_relay( tcp_admissions, tcp_active_workers, pending, + admit_unregistered, ); }) .map_err(|error| io::Error::other(format!("start TCP DNS relay: {error}")))?; @@ -3678,6 +3820,7 @@ mod tests { Arc::new(Mutex::new(HashMap::new())), Arc::new(AtomicUsize::new(0)), pending, + false, ); let events = capture.json_events(); @@ -3743,6 +3886,7 @@ mod tests { tcp_admissions, Arc::new(AtomicUsize::new(0)), pending_tx, + false, ); }); @@ -3834,6 +3978,7 @@ mod tests { tcp_admissions, Arc::new(AtomicUsize::new(0)), pending_tx, + false, ); }); @@ -3883,3 +4028,28 @@ mod tests { relay.join().expect("accept loop thread panicked"); } } + +#[cfg(test)] +mod unavailable_broker_tests { + use super::NetworkBroker; + + #[test] + fn unavailable_broker_has_real_descriptor_headroom() { + // The exec path calls ensure_descriptor_headroom before every spawn. + // The unavailable() stub must use the real soft limit (not 0), or + // every root-mode exec fails with EMFILE before any process is + // created (the B6 root cause). + let broker = NetworkBroker::unavailable().expect("create unavailable broker"); + broker + .ensure_descriptor_headroom(16) + .expect("the unavailable broker must pass descriptor headroom checks"); + } + + #[test] + fn unavailable_broker_is_healthy_for_confirm() { + let broker = NetworkBroker::unavailable().expect("create unavailable broker"); + broker + .confirm_healthy() + .expect("designed-unavailable broker is healthy"); + } +} diff --git a/crates/openshell-sandbox/src/network_relay.rs b/crates/openshell-sandbox/src/network_relay.rs new file mode 100644 index 0000000000..09525d1785 --- /dev/null +++ b/crates/openshell-sandbox/src/network_relay.rs @@ -0,0 +1,483 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The transparent network relay for privileged (root-workload) sandboxes. +//! +//! In sandboxed mode the seccomp broker intercepts `connect(2)` and the +//! supervisor decides each open over the Mediate control stream. A root +//! workload defeats in-guest interception by design, so the privileged +//! boundary instead installs an nftables DNAT ruleset that transparently +//! captures outbound workload traffic inside the guest: +//! +//! - Outbound TCP (any destination except loopback and the boundary's own +//! supervisor control endpoints) is redirected to a local relay listener. +//! The relay recovers the ORIGINAL destination from conntrack +//! (`SO_ORIGINAL_DST`) — for policy-DNS answers that is the synthetic +//! address the supervisor maps back to the hostname — and feeds the +//! connection into the SAME Mediate accept/decision/bridge flow the +//! seccomp broker uses. The supervisor's proxy enforces policy and +//! carries the bytes. +//! - DNS: the loopback relay at 127.0.0.53 answers via the Mediate DNS +//! stream (policy DNS returns synthetic addresses). Non-loopback UDP/53 +//! is redirected there too, closing the masquerade dnsmasq bypass. +//! - All other UDP is dropped (IPv4 and IPv6): QUIC and arbitrary UDP +//! datagrams are never relayed. This is routing behaviour, not a +//! security control — a root workload can delete the ruleset; the +//! fence outside the guest is the control. +//! - IPv6 TCP is redirected the same way (synthetic AAAA answers). +//! +//! Fail-closed: a root workload that deletes the ruleset restores direct +//! egress through the guest's eth0, which the launcher-side network fence +//! blocks — deleting the rules only breaks its own connectivity. +//! QUIC/UDP (other than DNS) is never relayed and is blocked by the fence. + +use std::fmt::Write as _; +use std::mem::size_of_val; +use std::net::{IpAddr, SocketAddr}; +use std::os::fd::AsRawFd as _; +use std::process::Stdio; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +use tokio::net::TcpListener; + +use crate::network_broker::{NetworkBroker, PendingTcpOpen}; + +use openshell_isolation_interface::contract::{ + BinaryIdentity, NetworkSocketMetadata, ResolveError, TcpOpenDecision, +}; + +/// The port of the transparent TCP relay listener (loopback only). +pub(crate) const TRANSPARENT_RELAY_PORT: u16 = 15500; + +/// The relay ruleset's nftables table name. +const NFT_TABLE: &str = "openshell_relay"; + +/// Install the transparent-relay nftables ruleset (idempotent: any previous +/// instance of the table is destroyed first). The boundary's own control +/// endpoints are excluded so the relay never captures its own supervisor +/// connections. +pub(crate) fn install_relay_rules( + control_endpoints: &[SocketAddr], + relay_port: u16, +) -> std::io::Result<()> { + // One accept rule per supervisor control endpoint (the boundary dials + // them; the relay must not eat its own control stream). + let mut v4_accepts = String::new(); + let mut v6_accepts = String::new(); + for endpoint in control_endpoints { + match endpoint.ip() { + IpAddr::V4(addr) => { + let _ = writeln!( + v4_accepts, + " ip daddr {addr} tcp dport {} accept", + endpoint.port() + ); + } + IpAddr::V6(addr) => { + let _ = writeln!( + v6_accepts, + " ip6 daddr {addr} tcp dport {} accept", + endpoint.port() + ); + } + } + } + let ruleset = format!( + r"destroy table ip {NFT_TABLE} +table ip {NFT_TABLE} {{ + chain output {{ + type nat hook output priority -100; policy accept; +{v4_accepts} ip daddr 127.0.0.0/8 accept + meta l4proto tcp dnat to 127.0.0.1:{relay_port} + udp dport 53 ip daddr != 127.0.0.0/8 dnat to 127.0.0.53 + meta l4proto udp drop + }} +}} +destroy table ip6 {NFT_TABLE} +table ip6 {NFT_TABLE} {{ + chain output {{ + type nat hook output priority -100; policy accept; +{v6_accepts} ip6 daddr ::1/128 accept + meta l4proto tcp dnat to [::1]:{relay_port} + meta l4proto udp drop + }} +}} +" + ); + // The nftables kernel modules are modular in the Alpine -virt kernel: + // load them before applying the ruleset (a missing module surfaces as + // "Operation not supported" from nft). Each child is registered with + // the managed-children registry: the orphan reaper reaps UNREGISTERED + // children process-wide, racing the wait and failing it with ECHILD. + for module in ["nf_tables", "nft_chain_nat", "nft_nat", "ipv6"] { + let _ = spawn_registered( + std::process::Command::new("/usr/sbin/modprobe") + .arg(module) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); + } + // Absolute path: PATH resolution from the static musl binary after the + // verified-boot chroot has proven unreliable; nft is at a fixed + // usr-merged location. + let nft_path = "/usr/sbin/nft"; + if !std::path::Path::new(nft_path).exists() { + return Err(std::io::Error::other(format!( + "{nft_path} not found (PATH={:?})", + std::env::var("PATH").unwrap_or_default() + ))); + } + // Hold the registry guard across spawn+register (see spawn_registered): + // the orphan reaper takes the same guard while deciding whether to + // reap, so it cannot run between the child existing and its + // registration landing. + let mut guard = crate::managed_children::lock(); + let child = std::process::Command::new(nft_path) + .arg("-f") + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map(|child| { + let registered = guard.register(child.id()); + (child, registered) + }); + drop(guard); + let (mut child, registered) = child?; + { + use std::io::Write as _; + let stdin = child + .stdin + .as_mut() + .expect("piped stdin exists before wait"); + if let Err(error) = stdin.write_all(ruleset.as_bytes()) { + // The ruleset write failed after registration: release the + // PID so the reaper can reap the exiting child (otherwise it + // stays registered forever and the wait below never runs). + if let Some(managed) = registered { + crate::managed_children::unregister(managed); + } + return Err(error); + } + } + let output = child.wait_with_output(); + if let Some(managed) = registered { + crate::managed_children::unregister(managed); + } + let output = output?; + if !output.status.success() { + return Err(std::io::Error::other(format!( + "nft -f failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ))); + } + Ok(()) +} + +/// Spawn a subprocess and register it with the managed-children registry +/// before waiting: the boundary's orphan reaper reaps UNREGISTERED children +/// process-wide (it answers the same SIGCHLD), racing any direct wait and +/// failing it with ECHILD. The registry entry is released after the wait. +/// +/// The registry guard (`managed_children::lock()`) is held from before +/// spawn until the PID is registered — the primitive documented for +/// exactly this window. The reaper holds the same guard while deciding +/// whether to reap, so it cannot run between the child existing and its +/// registration landing. (A SIGCHLD mask does NOT close this race: the +/// reaper is a separate thread woken by an eventfd from a process-wide +/// handler, and a process-directed SIGCHLD goes to any thread not +/// blocking it.) +fn spawn_registered( + command: &mut std::process::Command, +) -> std::io::Result { + // Hold the registry guard from before spawn until the PID is + // registered — the primitive documented for exactly this window. + let mut guard = crate::managed_children::lock(); + let spawn_result = command.spawn().map(|child| { + let registered = guard.register(child.id()); + (child, registered) + }); + drop(guard); + let (mut child, registered) = spawn_result?; + let result = child.wait(); + if let Some(managed) = registered { + crate::managed_children::unregister(managed); + } + result +} + +/// Recover the pre-DNAT destination of a transparently redirected +/// connection (the address the workload dialed, e.g. a policy-DNS +/// synthetic address). +/// connection (the address the workload dialed, e.g. a policy-DNS +/// synthetic address). Duplicates the fd so the tokio stream stays owned +/// by the caller. +#[allow( + unsafe_code, + reason = "dup(2) plus from_raw_fd is the standard fd-ownership transfer; both sides are scoped to this function" +)] +fn original_destination(stream: &tokio::net::TcpStream) -> std::io::Result { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; + // SAFETY: dup(2) on a live fd duplicates the descriptor; ownership of + // the copy is taken over by the std stream below. + let duplicated = unsafe { libc::dup(stream.as_raw_fd()) }; + if duplicated < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: the fd is a live, connected TCP socket owned by this scope. + let std_stream = unsafe { std::net::TcpStream::from_raw_fd(duplicated) }; + let result = original_destination_of(&std_stream); + drop(std_stream); + result +} + +fn original_destination_of(std_stream: &std::net::TcpStream) -> std::io::Result { + use nix::sys::socket::{getsockopt, sockopt}; + if std_stream.local_addr()?.is_ipv4() { + let address = getsockopt(std_stream, sockopt::OriginalDst)?; + Ok(SocketAddr::new( + IpAddr::V4(std::net::Ipv4Addr::from( + address.sin_addr.s_addr.to_ne_bytes(), + )), + u16::from_be(address.sin_port), + )) + } else { + let address = getsockopt(std_stream, sockopt::Ip6tOriginalDst)?; + Ok(SocketAddr::new( + IpAddr::V6(std::net::Ipv6Addr::from(address.sin6_addr.s6_addr)), + u16::from_be(address.sin6_port), + )) + } +} + +/// The relay's own binary identity: the boundary executable. Root-mode +/// policy does not require per-binary identity (the workload owns the +/// guest by design); the identity is carried for auditability and TOFU +/// pinning like any other mediated open. +fn relay_identity() -> Result { + // The boundary's own executable identity: root-mode policy does not + // require per-binary identity (the workload owns the guest by design), + // but the identity is carried for auditability and TOFU pinning like + // any other mediated open. + openshell_binary_identity::ProcfsIdentityResolver::for_pid_namespace() + .resolve(std::process::id()) +} + +/// Serve one redirected connection: recover the original destination, +/// register it as a pending open for the supervisor's decision, and on an +/// allow hand the accepted stream to the Mediate bridge. +async fn serve_relayed_connection( + stream: tokio::net::TcpStream, + pending: tokio::sync::mpsc::Sender, + active_opens: Arc, + connected_at: Instant, +) { + let destination = match original_destination(&stream) { + Ok(destination) => destination, + Err(error) => { + tracing::warn!(%error, "transparent relay: no original destination; dropping"); + return; + } + }; + let identity = relay_identity(); + let (decision_tx, decision_rx) = std::sync::mpsc::sync_channel(1); + let (relay_tx, relay_rx) = tokio::sync::oneshot::channel(); + // Mirror the broker's admission control: at most OPEN_QUEUE_CAPACITY + // undecided opens. + if active_opens + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < crate::network_broker::open_queue_capacity()).then_some(current + 1) + }) + .is_err() + { + return; + } + let _slot = ActiveOpenSlot(active_opens); + let queued_at = Instant::now(); + let open = PendingTcpOpen::for_relayed_connection( + destination, + identity, + NetworkSocketMetadata { + socket_cookie: 0, + nonblocking: false, + process_generation: 0, + }, + connected_at.elapsed(), + queued_at, + decision_tx, + relay_rx, + ); + if pending.try_send(open).is_err() { + return; + } + // The supervisor's decision arrives on the sync channel. Allowed: hand + // the workload connection to the Mediate bridge. Denied (or channel + // closed): close the connection — the workload sees a refused + // connection, same as the sandboxed denial. + // + // The wait runs on the blocking pool: a plain recv() would park a + // runtime worker, and the guest VM has as few workers as cores (one), + // so a parked worker would freeze the whole boundary (the cluster + // failure: the supervisor's control stream stopped responding). + let decision = tokio::task::spawn_blocking(move || decision_rx.recv()) + .await + .unwrap_or(Err(std::sync::mpsc::RecvError)); + if decision == Ok(TcpOpenDecision::RelayReady) { + if let Ok(stream) = stream.into_std() { + let _ = relay_tx.send(Ok(stream)); + } + } else { + // Denied: close with an immediate RST (SO_LINGER 0) so the client + // fails fast on the first read/write instead of waiting out its + // timeout. The workload completed the TCP handshake with the local + // relay; the sandboxed denial completes the intercepted connect + // with a refusal, and the RST is the relay-mode equivalent. + reset_stream(&stream); + drop(stream); + } +} + +/// Close a TCP stream with an immediate `RST`: set `SO_LINGER` with a zero +/// timeout, then close. The close converts to an `RST` instead of a `FIN`, +/// so a denied client fails fast on the first read/write. +#[allow( + unsafe_code, + reason = "setsockopt with a plain repr(C) linger struct; the pointer and size are scoped here" +)] +fn reset_stream(stream: &tokio::net::TcpStream) { + let linger = libc::linger { + l_onoff: 1, + l_linger: 0, + }; + // SAFETY: `libc::linger` is a plain repr(C) struct of two scalars; + // `setsockopt` reads exactly its size from the pointer. + let _ = unsafe { + libc::setsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_LINGER, + std::ptr::addr_of!(linger).cast(), + u32::try_from(size_of_val(&linger)).unwrap_or(u32::MAX), + ) + }; + // The caller drops the stream after this returns; the linger option + // converts that close into an RST. +} + +struct ActiveOpenSlot(Arc); + +impl Drop for ActiveOpenSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +/// Start the transparent TCP relay: listeners on the loopback (v4 and v6) +/// relay port feeding redirected connections into the broker's pending +/// queue. +pub(crate) fn start_tcp_relay( + pending: tokio::sync::mpsc::Sender, + active_opens: Arc, + runtime: &tokio::runtime::Handle, +) -> std::io::Result<()> { + // from_std needs the runtime reactor in context; enter the caller's + // handle for the conversions below. + let _guard = runtime.enter(); + let v4 = std::net::TcpListener::bind(("127.0.0.1", TRANSPARENT_RELAY_PORT))?; + v4.set_nonblocking(true)?; + let v4 = TcpListener::from_std(v4)?; + // The IPv6 loopback listener is best effort: the Alpine -virt kernel + // carries IPv6 as a module, and without it there is no IPv6 traffic to + // capture (fail-closed by absence — the fence still blocks any v6 + // egress if the stack appears later). + let v6 = std::net::TcpListener::bind(("::1", TRANSPARENT_RELAY_PORT)) + .and_then(|v6| { + v6.set_nonblocking(true)?; + TcpListener::from_std(v6) + }) + .ok(); + let mut listeners = vec![v4]; + if let Some(v6) = v6 { + listeners.push(v6); + } else { + tracing::warn!( + "transparent relay: no IPv6 loopback (module unavailable); \ + IPv6 TCP is not relayed" + ); + } + for listener in listeners { + let pending = pending.clone(); + let active_opens = active_opens.clone(); + runtime.spawn(async move { + loop { + let Ok((stream, _peer)) = listener.accept().await else { + break; + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let connected_at = Instant::now(); + let pending = pending.clone(); + let active_opens = active_opens.clone(); + tokio::spawn(async move { + serve_relayed_connection(stream, pending, active_opens, connected_at).await; + }); + } + }); + } + Ok(()) +} + +/// Construct the transparent relay broker for a privileged boundary. +/// +/// The loopback DNS relay, the transparent TCP relay, and the nftables +/// capture ruleset. The pending queues feed the SAME Mediate +/// accept/decision/bridge flow the sandboxed seccomp broker uses. +pub fn start_transparent_relay( + control_endpoints: &[SocketAddr], + runtime: &tokio::runtime::Handle, +) -> std::io::Result { + NetworkBroker::start_transparent_relay(control_endpoints, runtime) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn relay_ruleset_installs_and_captures() { + // Requires root + the nftables kernel modules + the nft binary: + // run under `docker run --privileged` (the CI/e2e path). + if !nix::unistd::geteuid().is_root() || !std::path::Path::new("/usr/sbin/nft").exists() { + eprintln!("skipping: needs root + nft (run in the privileged container)"); + return; + } + install_relay_rules(&["10.43.0.1:5500".parse().unwrap()], TRANSPARENT_RELAY_PORT) + .expect("install the relay ruleset"); + let listing = std::process::Command::new("nft") + .args(["list", "table", "ip", NFT_TABLE]) + .output() + .expect("list the ruleset"); + let text = String::from_utf8_lossy(&listing.stdout); + assert!(text.contains("dnat to 127.0.0.1:15500"), "{text}"); + assert!( + text.contains("meta l4proto udp drop"), + "non-DNS UDP must be dropped (routing rule; the fence is the security control): {text}" + ); + assert!( + text.contains("ip daddr 10.43.0.1 tcp dport 5500 accept"), + "{text}" + ); + let listing = std::process::Command::new("nft") + .args(["list", "table", "ip6", NFT_TABLE]) + .output() + .expect("list the v6 ruleset"); + let text = String::from_utf8_lossy(&listing.stdout); + assert!(text.contains("dnat to [::1]:15500"), "{text}"); + // Idempotent re-install. + install_relay_rules(&["10.43.0.1:5500".parse().unwrap()], TRANSPARENT_RELAY_PORT) + .expect("re-install the relay ruleset"); + } +} diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 4967d20086..fc03b264b1 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -497,6 +497,7 @@ impl ProcessHandle { tls_environment_mode: child_env::TlsEnvironmentMode, provider_env: &HashMap, otlp_relay_active: bool, + privileged: bool, ) -> Result { Self::spawn_impl( launcher, @@ -509,6 +510,7 @@ impl ProcessHandle { tls_environment_mode, provider_env, otlp_relay_active, + privileged, ) } @@ -529,6 +531,7 @@ impl ProcessHandle { tls_environment_mode: child_env::TlsEnvironmentMode, provider_env: &HashMap, otlp_relay_active: bool, + privileged: bool, ) -> Result { Self::spawn_impl( program, @@ -540,6 +543,7 @@ impl ProcessHandle { tls_environment_mode, provider_env, otlp_relay_active, + privileged, ) } @@ -556,6 +560,7 @@ impl ProcessHandle { tls_environment_mode: child_env::TlsEnvironmentMode, provider_env: &HashMap, otlp_relay_active: bool, + privileged: bool, ) -> Result { let mut cmd = Command::new(program); cmd.args(args).kill_on_drop(true); @@ -607,17 +612,33 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + if !privileged { + sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + } // Prepare the Landlock ruleset as the workload UID. Inaccessible paths - // are already unavailable to the child and remain omitted. + // are already unavailable to the child and remain omitted. In + // privileged (root-workload) mode there is no Landlock ruleset and no + // child self-protection filter — a no-op program — matching the + // exec path (B3c1/B8): the fence plus the supervisor's policy are + // the boundary, and an inherited no_new_privs/filter would defeat + // setuid binaries, mounts, and container runtimes. #[cfg(target_os = "linux")] let runtime_read_only = ca_runtime_read_only_paths(ca_paths); - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; #[cfg(target_os = "linux")] - let mut child_hardening = crate::linux::child_seccomp::prepare(std::process::id()) - .map_err(|error| miette::miette!("prepare child self-protection filter: {error}"))?; + let prepared_sandbox = if privileged { + None + } else { + prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))? + }; + #[cfg(target_os = "linux")] + let mut child_hardening = if privileged { + crate::linux::child_seccomp::ChildHardeningProgram::noop() + } else { + crate::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| miette::miette!("prepare child self-protection filter: {error}"))? + }; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain // proper terminal control for shells and interactive programs. @@ -713,7 +734,11 @@ impl ProcessHandle { tls_environment_mode: child_env::TlsEnvironmentMode, provider_env: &HashMap, otlp_relay_active: bool, + privileged: bool, ) -> Result { + // Sandbox preparation is Linux-only; the flag is accepted for call + // signature parity so the delegated spawn path stays mode-gated. + let _ = privileged; let mut cmd = Command::new(program); cmd.args(args).kill_on_drop(true); @@ -2101,4 +2126,55 @@ mod tests { } assert!(stdout.contains("PATH=/usr/bin:/bin")); } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn privileged_main_process_has_no_seccomp_or_no_new_privs() { + // B8: the MAIN workload process in privileged (root-workload) mode + // must not inherit Landlock, the child self-protection filter, or + // no_new_privs — before the fix only `sandbox exec` children got + // the root posture and the entrypoint always ran fully sandboxed. + // With Seccomp: 0 and NoNewPrivs: 0 the filter cannot block mount, + // umount2, unshare, bpf, perf_event_open, or init_module (they may + // still fail for kernel or environment reasons). + let launcher = crate::linux::workload_launcher::start_privileged().expect("start launcher"); + let policy = policy_with_process(ProcessPolicy { + run_as_user: None, + run_as_group: None, + }); + let mut handle = ProcessHandle::spawn( + &launcher, + "/bin/sh", + &[ + "-c".to_string(), + "grep -E 'NoNewPrivs:|Seccomp:' /proc/self/status".to_string(), + ], + &ResolvedWorkspace::default(), + false, + &policy, + None, + child_env::TlsEnvironmentMode::FillMissing, + &HashMap::new(), + false, + true, + ) + .expect("spawn privileged main process"); + + // Read the child's /proc/self/status report from its stdout. + let mut buf = String::new(); + if let ProcessIo::Pipes { stdout: out, .. } = handle.take_io() { + use tokio::io::AsyncReadExt; + let mut out = out; + out.read_to_string(&mut buf).await.expect("read stdout"); + } + handle.wait().await.expect("wait for exit"); + assert!( + buf.contains("NoNewPrivs:\t0"), + "privileged main process must report NoNewPrivs:0, got: {buf}" + ); + assert!( + buf.contains("Seccomp:\t0"), + "privileged main process must report Seccomp:0, got: {buf}" + ); + } } diff --git a/crates/openshell-server/src/supervisor_pool.rs b/crates/openshell-server/src/supervisor_pool.rs index 7ee3ad11f2..f8765c382d 100644 --- a/crates/openshell-server/src/supervisor_pool.rs +++ b/crates/openshell-server/src/supervisor_pool.rs @@ -4781,6 +4781,54 @@ mod tests { assert!(fixture.owner("rt-route-x000").await.is_some()); } + #[tokio::test] + async fn a_new_generation_on_a_recorded_route_refuses_with_the_pinned_message() { + a_new_generation_on_a_recorded_route_refuses_with_the_pinned_message_on( + &Fixture::new().await, + ) + .await; + } + + #[tokio::test] + #[ignore = "requires a disposable PostgreSQL database in OPENSHELL_TEST_POSTGRES_URL; run mise run test:rust:postgres"] + async fn postgres_a_new_generation_on_a_recorded_route_refuses_with_the_pinned_message() { + let schema = TestSchema::create("supervisor_pool").await; + a_new_generation_on_a_recorded_route_refuses_with_the_pinned_message_on( + &Fixture::with_store(schema.connect_store().await), + ) + .await; + } + + /// Pins the refusal message the `KubeVirt` driver's + /// `ROUTE_RECORD_RACE_MESSAGE` retry keys on: a launch for a new + /// generation on a route whose record is still Published is refused + /// with exactly this wording. If the message changes, this test fails + /// so the driver's constant is updated with it. + async fn a_new_generation_on_a_recorded_route_refuses_with_the_pinned_message_on( + fixture: &Fixture, + ) { + let pool = fixture.replica("gw-a"); + fixture.member("m1", "p1", false).await; + fixture + .sandbox("sandbox-x", "generation-1", SandboxPhase::Ready) + .await; + pool.place_launch(launch("rt-route-race", "sandbox-x", "generation-1")) + .await + .unwrap(); + let record = fixture.instance("rt-route-race").await.unwrap(); + assert_eq!(record.payload.phase, RecordPhase::Published); + + let refused = pool + .place_launch(launch("rt-route-race", "sandbox-x", "generation-2")) + .await + .unwrap_err(); + assert!( + refused.contains("the sandbox instance route is recorded for another generation"), + "the route-record refusal message changed ({refused}); update \ + ROUTE_RECORD_RACE_MESSAGE in openshell-driver-kubernetes" + ); + } + #[tokio::test] async fn a_failed_admission_keeps_the_record_for_a_retry_with_its_first_material() { a_failed_admission_keeps_the_record_for_a_retry_with_its_first_material_on( diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 72748c8027..4373276f00 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -487,7 +487,26 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { - Self::from_proto_with_pid_and_binary_identity_required(proto, entrypoint_pid, true) + Self::from_proto_with_pid_and_binary_identity(proto, entrypoint_pid, true) + } + + /// Create OPA engine from a typed proto policy with an explicit + /// binary-identity requirement. + /// + /// Privileged (root-workload) sandboxes pass `false`: per-binary + /// identity checks are unenforceable in-guest and would deny every + /// connection the root workload makes. The requirement is a property + /// of the workload mode, not the policy. + pub fn from_proto_with_pid_and_binary_identity( + proto: &ProtoSandboxPolicy, + entrypoint_pid: u32, + require_binary_identity: bool, + ) -> Result { + Self::from_proto_with_pid_and_binary_identity_required( + proto, + entrypoint_pid, + require_binary_identity, + ) } fn from_proto_with_pid_and_binary_identity_required( @@ -812,7 +831,14 @@ impl OpaEngine { registry: Option, commit_credentials: impl FnOnce(), ) -> Result { - let new = Self::from_proto_with_pid(proto, entrypoint_pid)?; + // The new engine inherits the binary-identity requirement from the + // current engine: the workload mode is fixed for the generation and + // cannot change across a policy reload. + let new = Self::from_proto_with_pid_and_binary_identity( + proto, + entrypoint_pid, + self.binary_identity_required(), + )?; let new_engine = new .engine .into_inner() @@ -11832,6 +11858,31 @@ network_policies: assert_eq!(engine.current_generation(), 0); } + #[test] + fn reload_inherits_binary_identity_requirement() { + // The workload mode is fixed for a sandbox generation; a policy + // reload must not silently re-enable per-binary identity checks + // for a privileged (root-workload) engine. + let proto = test_proto(); + let engine = OpaEngine::from_proto_with_pid_and_binary_identity(&proto, 0, false).unwrap(); + assert!(!engine.binary_identity_required()); + engine + .reload_configuration_from_proto_with_pid(&proto, 0, None, || {}) + .expect("reload succeeds"); + assert!( + !engine.binary_identity_required(), + "reload must inherit the privileged binary-identity mode" + ); + + // And a sandboxed engine keeps the requirement across reloads. + let engine = OpaEngine::from_proto(&proto).unwrap(); + assert!(engine.binary_identity_required()); + engine + .reload_configuration_from_proto_with_pid(&proto, 0, None, || {}) + .expect("reload succeeds"); + assert!(engine.binary_identity_required()); + } + #[test] fn configuration_commit_invalidates_old_guards_before_credentials_change() { let proto = test_proto(); diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index ab9c3ace26..230adf24a0 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -694,6 +694,7 @@ fn build_observation_publication_event( .unmapped("policy_generation", record.policy_generation) .unmapped("mapping_generation", record.mapping_generation) .unmapped("mapping_id", record.mapping_id.to_string()) + .guest_sourced() .message(format!( "Policy DNS staged unapproved name {} synthetic={} for TCP policy review", record.normalized_name, record.synthetic_address @@ -739,6 +740,7 @@ fn build_mapping_publication_event(record: &ResolvedEndpointRecord) -> openshell .unmapped("policy_generation", record.policy_generation) .unmapped("mapping_generation", record.mapping_generation) .unmapped("mapping_id", mapping_id) + .guest_sourced() .message(message) .build() } @@ -1133,6 +1135,45 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn non_policy_names_never_reach_the_upstream_resolver() { + // The DNS exfiltration question: a root workload can encode data in + // the query name of a non-policy lookup. The ineligible path must + // answer entirely locally (a staged observation synthetic or an + // Ineligible refusal) — the name must never be sent upstream. + let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let now = Instant::now(); + for name in [ + "review-denied.test", + // Data-encoded subdomains of a non-policy name. + "cGF5bG9hZA.exfil.review-denied.test", + "a.b.c.d.e.f.g.h.i.j.k.unapproved.example", + ] { + let _ = service.answer_query(name, AddressFamily::Ipv4, now).await; + } + assert_eq!( + service.resolver.calls.load(Ordering::SeqCst), + 0, + "non-policy names must never reach the upstream resolver" + ); + // And in fail-closed mode the refusal stays local too. + service + .policy + .enter_fail_closed("invalid candidate") + .unwrap(); + assert!(matches!( + service + .answer_query("review-denied.test", AddressFamily::Ipv4, now) + .await, + Err(PolicyDnsError::Ineligible) + )); + assert_eq!( + service.resolver.calls.load(Ordering::SeqCst), + 0, + "fail-closed non-policy refusals must not query upstream either" + ); + } + #[tokio::test] async fn unknown_name_correlates_without_authorizing_a_port() { let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 9742249119..4a53cedbd4 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1013,6 +1013,7 @@ fn build_staged_transparent_denial_event( .status(StatusId::Failure) .dst_endpoint(transparent_destination_endpoint(destination, mapped_host)) .actor_process(Process::from_bypass(&binary, "-", &ancestors).with_cmd_line(&cmdline)) + .guest_sourced() .message(format!("Transparent TCP denied before relay: {reason}")) .status_detail(status_detail) .build() @@ -1534,6 +1535,7 @@ fn build_transparent_tcp_allow_ocsf_event( "authorization_policy_generation", audit.authorization_policy_generation, ) + .guest_sourced() .message(message) .status_detail("transparent_tcp_allowed"); if let Some(destination) = audit.connected_real_destination { @@ -1623,6 +1625,7 @@ fn emit_transparent_policy_denial( .dst_endpoint(Endpoint::from_domain(host, port)) .src_endpoint_addr(workload.ip(), workload.port()) .actor_process(Process::from_bypass(&binary, &pid, "-")) + .guest_sourced() .firewall_rule("-", "opa") .message(format!("Transparent TCP denied {host}:{port}")) .status_detail(status_detail) @@ -2046,7 +2049,9 @@ fn build_connect_allow_ocsf_event( } else { "CONNECT" }; - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + // The acting-process identity is boundary-reported; for an untrusted + // guest it is guest data, so the event carries the guest-sourced mark. + let mut builder = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Allowed) .disposition(DispositionId::Allowed) @@ -2056,8 +2061,11 @@ fn build_connect_allow_ocsf_event( .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) .firewall_rule(policy, "opa") - .message(format!("{connect_msg} allowed {host}:{port}")) - .build() + .message(format!("{connect_msg} allowed {host}:{port}")); + if openshell_ocsf::ctx::guest_data_origin() { + builder = builder.guest_sourced(); + } + builder.build() } #[allow(clippy::too_many_arguments)] @@ -2073,7 +2081,9 @@ fn build_forward_allow_ocsf_event( cmdline: &str, policy: &str, ) -> openshell_ocsf::OcsfEvent { - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + // The acting-process identity is boundary-reported; for an untrusted + // guest it is guest data, so the event carries the guest-sourced mark. + let mut builder = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::for_http_method(method)) .action(ActionId::Allowed) .disposition(DispositionId::Allowed) @@ -2087,8 +2097,11 @@ fn build_forward_allow_ocsf_event( .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) .firewall_rule(policy, "opa") - .message(format!("FORWARD allowed {method} {host}:{port}{path}")) - .build() + .message(format!("FORWARD allowed {method} {host}:{port}{path}")); + if openshell_ocsf::ctx::guest_data_origin() { + builder = builder.guest_sourced(); + } + builder.build() } fn build_forward_rejection_event( @@ -9010,7 +9023,7 @@ process: {{ run_as_user: sandbox, run_as_group: sandbox }} assert_eq!( mapped.format_shorthand(), format!( - "NET:OPEN [MED] DENIED {}(0) -> blocked.invalid:80 [reason:transparent_tcp_policy_denied]", + "NET:OPEN [MED] [data_origin:guest] DENIED {}(0) -> blocked.invalid:80 [reason:transparent_tcp_policy_denied]", executable_path("/usr/bin/curl").display() ) ); diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 4f6edaac11..4a8861c982 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -95,6 +95,7 @@ pub async fn start_boundary_access( host_key: Option, gateway_home: Option>>, app_installations: Option, + untrusted_guest: bool, ) -> Result { let instance_id = uuid::Uuid::new_v4().to_string(); let terminating = Arc::new(AtomicBool::new(false)); @@ -112,6 +113,7 @@ pub async fn start_boundary_access( port_forward.clone(), main_session.clone(), host_key, + untrusted_guest, ) .await?; @@ -152,6 +154,10 @@ pub async fn start_boundary_access( /// The optional SSH adapter is the only platform-specific access component. #[cfg(unix)] +#[allow( + clippy::too_many_arguments, + reason = "the SSH server setup mirrors the supervisor's access-plane parameters" +)] async fn start_optional_ssh( socket_path: Option, shared_socket: bool, @@ -160,6 +166,7 @@ async fn start_optional_ssh( port_forward: Arc, main_session: Arc, host_key: Option, + untrusted_guest: bool, ) -> Result>> { let Some(socket_path) = socket_path else { return Ok(None); @@ -176,6 +183,7 @@ async fn start_optional_ssh( boundary_exec, Some(main_session), host_key, + untrusted_guest, ) .await { @@ -208,6 +216,10 @@ async fn start_optional_ssh( } #[cfg(not(unix))] +#[allow( + clippy::too_many_arguments, + reason = "the SSH server setup mirrors the supervisor's access-plane parameters" +)] async fn start_optional_ssh( socket_path: Option, _shared_socket: bool, @@ -380,6 +392,7 @@ mod tests { None, None, None, + false, ) .await .expect("access without SSH is portable"); @@ -423,6 +436,7 @@ mod tests { None, None, None, + false, ) .await; assert!( diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index a11eddbd21..acf30f5f98 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -174,6 +174,7 @@ pub async fn run_ssh_server( boundary_exec: Arc, main_session: Option>, host_key: PrivateKey, + untrusted_guest: bool, ) -> Result<()> { let (listener, config, _ca_paths) = match ssh_server_init(&listen_path, &ca_file_paths, shared_socket, host_key) { @@ -214,6 +215,7 @@ pub async fn run_ssh_server( main_session, server_endpoint.clone(), SSH_PEER_TIMEOUT, + untrusted_guest, ) .await { @@ -358,6 +360,7 @@ async fn handle_connection( main_session: Option>, server_endpoint: Arc, peer_timeout: Duration, + untrusted_guest: bool, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -374,7 +377,13 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new(port_forward, boundary_exec, main_session, server_endpoint); + let handler = SshHandler::new( + port_forward, + boundary_exec, + main_session, + server_endpoint, + untrusted_guest, + ); let stream = peer_stream::PeerStream::new(stream, peer_timeout); russh::server::run_stream(config, stream, handler) .await @@ -444,6 +453,10 @@ struct SshHandler { /// This server's endpoint, named on every SSH event it emits. server_endpoint: Arc, channels: HashMap, + /// Whether the sandbox runs the privileged (root-workload) mode: the + /// boundary is untrusted after start, so exec exit codes it reports are + /// guest-sourced data. + untrusted_guest: bool, } impl Drop for SshHandler { @@ -471,6 +484,7 @@ impl SshHandler { boundary_exec: Arc, main_session: Option>, server_endpoint: Arc, + untrusted_guest: bool, ) -> Self { Self { port_forward, @@ -478,6 +492,7 @@ impl SshHandler { main_session, server_endpoint, channels: HashMap::new(), + untrusted_guest, } } } @@ -1356,6 +1371,21 @@ impl SshHandler { spec: openshell_isolation_interface::contract::ExecSpec, ) -> anyhow::Result<()> { use tokio::io::AsyncReadExt; + // Capture the display name and the workload mode before the spec is + // consumed and the task spawns: the exit OCSF event uses both. + // Shell-executed commands carry the command in `shell.command` (the + // `program` field is empty); use the command's first word. + let program_name = if spec.program.is_empty() { + spec.shell + .as_ref() + .and_then(|shell| shell.command.as_deref()) + .and_then(|command| command.split_whitespace().next()) + .unwrap_or("exec") + .to_string() + } else { + spec.program.clone() + }; + let untrusted_guest = self.untrusted_guest; let mut exec = self .boundary_exec @@ -1468,6 +1498,34 @@ impl SshHandler { let Some(code) = code else { return; }; + // The exit code is guest-reported (the boundary reports the + // workload process lifecycle); in root mode the boundary is + // untrusted after start, so the event is marked guest-sourced. + let process = openshell_ocsf::Process::new(&program_name, 0); + if untrusted_guest { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(code.cast_signed()) + .actor_process(openshell_ocsf::Process::new(&program_name, 0)) + .process(process) + .guest_sourced() + .message(format!("exec exit reported by the boundary: code {code}")) + .build() + ); + } else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(code.cast_signed()) + .actor_process(openshell_ocsf::Process::new(&program_name, 0)) + .process(process) + .message(format!("exec process exited: code {code}")) + .build() + ); + } let _ = handle.eof(channel).await; let _ = handle.exit_status_request(channel, code).await; let _ = handle.close(channel).await; @@ -1707,6 +1765,7 @@ mod tests { rpc_test::exec(Arc::new(RejectingExec)), None, server_endpoint, + false, ); if let Ok(session) = russh::server::run_stream(config, stream, handler).await @@ -1847,6 +1906,7 @@ mod tests { Arc::new(ssh_server_endpoint(Path::new( "/run/openshell/test/ssh.sock", ))), + false, ); let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); tokio::spawn(async move { diff --git a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs index 3d8a2b1352..d7182da916 100644 --- a/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs +++ b/crates/openshell-supervisor-process/src/ssh/reconnect_tests.rs @@ -134,6 +134,7 @@ impl Connection { Some(main), server_endpoint, TEST_INTERVAL * 4, + false, ) .await; }); diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index eacec32962..0b2ec0c0d8 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -28,6 +28,13 @@ pub struct LaunchIdentity { pub workload_identity: ResolvedWorkloadIdentity, /// Preserve driver-fixed VM selectors during shared startup and reload. pub vm_policy_identity: Option, + /// Whether the trusted launch descriptor marks the boundary as an + /// untrusted `KubeVirt` guest. Keyed off the descriptor's resource + /// claims, never boundary reports: the class gates supervisor-side + /// policy validation and provider-material refusal. Every `KubeVirt` + /// guest is this class; the guest (including its boundary) is + /// untrusted, and the security boundary is outside it. + pub untrusted_guest: bool, } /// Runtime state remains owned by the supervisor. @@ -156,6 +163,12 @@ impl SelectedBackend { self.identity.vm_policy_identity } + /// Whether the trusted launch descriptor marks an untrusted guest, + /// for supervisor-side policy validation. + pub fn untrusted_guest(&self) -> bool { + self.identity.untrusted_guest + } + /// Delegate image discovery only after shared name and identity checks. /// Callers may retry this read; discovery cannot authorize workload launch. pub async fn discover_policy( @@ -230,13 +243,16 @@ impl BackendSetup for OpenShellBackendSetup { generation: descriptor.generation.clone(), session_id: descriptor.session_id, workload_identity: descriptor.workload_identity.clone(), - vm_policy_identity: descriptor - .resource_claims - .contains_key("vm.generation") - .then_some(super::VmPolicyIdentity { - uid: descriptor.workload_identity.uid, - gid: descriptor.workload_identity.gid, - }), + vm_policy_identity: openshell_sandbox_backend::boundary_protocol::is_vm_guest( + &descriptor.resource_claims, + ) + .then_some(super::VmPolicyIdentity { + uid: descriptor.workload_identity.uid, + gid: descriptor.workload_identity.gid, + }), + untrusted_guest: openshell_sandbox_backend::boundary_protocol::is_kubevirt_guest( + &descriptor.resource_claims, + ), }; let transport = openshell_sandbox_backend::BoundaryTransport::for_descriptor(&descriptor)?; Ok(( @@ -293,6 +309,9 @@ impl BackendSetup for RoutedBackendSetup { session_id: descriptor.session_id, workload_identity: descriptor.workload_identity.clone(), vm_policy_identity: None, + untrusted_guest: openshell_sandbox_backend::boundary_protocol::is_kubevirt_guest( + &descriptor.resource_claims, + ), }; Ok(( identity, diff --git a/crates/openshell-supervisor/src/backend_setup/registration.rs b/crates/openshell-supervisor/src/backend_setup/registration.rs index 978ef65033..3832642436 100644 --- a/crates/openshell-supervisor/src/backend_setup/registration.rs +++ b/crates/openshell-supervisor/src/backend_setup/registration.rs @@ -78,6 +78,9 @@ impl BackendSetup for RegisteredBackendSetup { session_id: self.descriptor.session_id, workload_identity: self.descriptor.workload_identity.clone(), vm_policy_identity: None, + untrusted_guest: openshell_sandbox_backend::boundary_protocol::is_kubevirt_guest( + &self.descriptor.resource_claims, + ), }, Box::new(RegisteredLaunch { name: self.name.clone(), diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 822cf1ae26..cfa114b1e4 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -255,6 +255,7 @@ impl BackendSetup for TestSetup { session_id: self.auth.session_id, workload_identity: identity(), vm_policy_identity: None, + untrusted_guest: false, }, Box::new(TestLaunch { name: self.built_name, @@ -690,6 +691,57 @@ async fn constructed_backend_must_keep_selected_name() { assert!(setup.observed.services.lock().unwrap().upgrade().is_none()); } +#[test] +fn kubevirt_generation_claim_yields_vm_policy_identity() { + use openshell_sandbox_backend::boundary_protocol::{ + SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, + }; + + let setup = TestSetup::new(); + let descriptor = SandboxRuntimeDescriptor { + boundary_id: setup.sandbox_id.clone(), + generation: setup.auth.runtime_generation.to_string(), + session_id: setup.auth.session_id, + workload_identity: ResolvedWorkloadIdentity::new( + 10001, + 10001, + vec![], + "kubevirt".into(), + "vm-uid-1".into(), + ) + .unwrap(), + transport: SandboxTransport::Tcp { + authority: "supervisor.openshell.svc:5500".into(), + addresses: vec![], + }, + tls: SandboxTlsClientConfig { + server_name: "boundary.sandbox.openshell".into(), + trust_anchor_pem: String::new(), + }, + host_gateway_ip: None, + resource_claims: BTreeMap::from([( + openshell_sandbox_backend::boundary_protocol::KUBEVIRT_GENERATION_CLAIM.to_string(), + "generation-1".into(), + )]), + provisioning_evidence: ProvisioningEvidence::from_enforcement_evidence( + "generation-1", + b"test-evidence", + ) + .unwrap(), + }; + let selected = SelectedBackend::select( + &OpenShellBackendSetup, + descriptor.backend_descriptor().unwrap(), + Some(openshell_sandbox_backend::BACKEND_NAME), + Some(&setup.sandbox_id), + &setup.auth, + ) + .unwrap(); + let projected = selected.vm_policy_identity(); + let projected = projected.expect("KubeVirt claims must yield a VM policy identity"); + assert_eq!((projected.uid, projected.gid), (10001, 10001)); +} + #[test] fn standard_setup_decodes_native_descriptor_and_preserves_vm_identity() { use openshell_sandbox_backend::boundary_protocol::{ diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 02736ad184..31fd74c777 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -444,6 +444,7 @@ impl ConfigRuntime { connector: &ctx.middleware_connector, }, ctx.vm_identity, + ctx.untrusted_guest, || { if let Some(prepared) = prepared_provider.as_ref() { ctx.provider_credentials @@ -775,6 +776,20 @@ impl ConfigRuntime { }); } }; + if ctx.untrusted_guest && (!provider.files.is_empty() || !provider.environment.is_empty()) { + // Nothing secret enters the guest: provider material fetched by + // polling is refused supervisor-side for a root workload, + // mirroring startup and session-delivered updates. An entirely + // empty environment stays installable (a fail-closed snapshot). + self.reject_provider_bindings(ctx, desired_identity.clone(), provider) + .await; + let error = "Privileged (root-workload) sandboxes cannot install provider files or environment; detach the provider to repair configuration"; + return Err(ConfigRejection { + code: "privileged_provider_refused", + detail: error.to_string(), + admission_error: error.to_string(), + }); + } if let Ok(prepared) = CapturedProviderEnvironment::prepare(&provider) { return Ok(prepared); } @@ -1120,6 +1135,24 @@ impl ConfigRuntime { true, ) } + Ok(provider) + if ctx.untrusted_guest + && (!provider.files.is_empty() || !provider.environment.is_empty()) => + { + // Nothing secret enters the guest: mid-session provider + // material is refused supervisor-side for a root + // workload, mirroring startup. An entirely empty + // environment stays installable (a fail-closed snapshot). + // The mode comes from the trusted launch descriptor held + // by the poll context. + self.reject_provider_bindings(ctx, desired.clone(), provider) + .await; + config_apply_failure( + "privileged_provider_refused", + "privileged (root-workload) sandboxes cannot install provider files or environment; detach the provider to repair configuration", + true, + ) + } Ok(provider) => match CapturedProviderEnvironment::prepare(&provider) { Ok(prepared) => return Ok(prepared), Err(error) => { @@ -2058,6 +2091,7 @@ mod tests { &startup, &policy, &stable_provider_environment(1, Some("initial")), + false, ) .expect("stable credentials prepare for admitted startup"); assert!(prepared.requires_stable_placeholders()); @@ -2100,7 +2134,7 @@ mod tests { .workload_credential_handle .clear(); startup.provider_env_revision = 2; - assert!(prepare_startup_configuration(&startup, &policy, &invalid).is_err()); + assert!(prepare_startup_configuration(&startup, &policy, &invalid, false).is_err()); let rejected = initial_provider_credentials(invalid, &readiness); assert!(rejected.snapshot().child_env.is_empty()); assert!(rejected.resolver().is_none()); @@ -2577,6 +2611,59 @@ mod tests { let _ = task.await; } + #[tokio::test] + async fn privileged_delivery_refuses_provider_material() { + use openshell_core::proto::{ConfigApplyOutcome, PolicySource}; + + // Nothing secret enters the guest: a privileged (root-workload) + // runtime refuses session-delivered provider material + // supervisor-side, mirroring startup. + let initial = settings_poll_result(Some(proto_policy_fixture()), 1, PolicySource::Sandbox); + let engine = Arc::new( + OpaEngine::from_proto_with_pid_and_binary_identity(&proto_policy_fixture(), 0, false) + .expect("build OPA engine"), + ); + let mut ctx = policy_poll_test_context( + engine, + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.untrusted_guest = true; + let (client, _polls, _reports) = scripted_policy_gateway(); + let mut runtime = started_runtime(&ctx, &initial); + + let mut desired = initial.clone(); + desired.provider_env_revision = 2; + let snapshot = provider_snapshot_for(&desired, "REGION", "west"); + + let result = runtime + .apply_delivered_sandbox(&ctx, &client, desired, snapshot) + .await; + + assert_eq!( + result.outcome, + ConfigApplyOutcome::FailedRetainedLastKnownGood + ); + assert_eq!( + result.failure.as_ref().unwrap().code, + "privileged_provider_refused" + ); + assert_eq!( + result.admission.state, + i32::from(openshell_core::proto::ConfigurationAdmissionState::Rejected) + ); + assert!( + !ctx.provider_credentials + .snapshot() + .child_env + .contains_key("EXTERNAL_TOKEN"), + "no provider environment may reach a root workload" + ); + } + #[tokio::test] async fn provider_poll_preserves_static_references_across_rotation_failure_and_detach() { let engine = Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).unwrap()); @@ -2806,6 +2893,7 @@ mod tests { opa_engine, loaded_policy_origin, vm_identity: None, + untrusted_guest: false, entrypoint_pid: Arc::new(AtomicU32::new(0)), interval_secs: 0, ocsf_enabled: Arc::new(AtomicBool::new(false)), diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index a28e0cf4bf..fc59303e6e 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -222,6 +222,7 @@ impl BackendSetup for HeldBackendSetup { "test-image".into(), )?, vm_policy_identity: None, + untrusted_guest: false, }, Box::new(HeldPreparedBackend(self.probe.clone())), )) diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index fba41bb4e7..d977316034 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -47,8 +47,8 @@ use openshell_supervisor_control::RegistrationTraceContext; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, - DetectionFindingBuilder, DispositionId, EventContext, FindingInfo, OcsfEvent, SeverityId, - StateId, StatusId, ocsf_emit, + DetectionFindingBuilder, DispositionId, EventContext, FindingInfo, OcsfEvent, + ProcessActivityBuilder, SeverityId, StateId, StatusId, ocsf_emit, }; // --------------------------------------------------------------------------- @@ -1200,23 +1200,50 @@ pub async fn run_sandbox_instance( ); } let startup_status = startup.in_scope(openshell_otel::ErrorStatusGuard::current); - let (image_yaml, invalid_image) = selected_backend - .discover_policy(sandbox_bearer.clone(), attach_request_timeout) - .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.discover_policy")) - .await?; - let image_discovery = if invalid_image { - ImagePolicyDiscovery::Invalid - } else if let Some(yaml) = image_yaml { - openshell_policy::parse_sandbox_policy(&yaml) - .map_or(ImagePolicyDiscovery::Invalid, |policy| { - ImagePolicyDiscovery::Policy(Box::new(policy)) - }) - } else { + let vm_policy_identity = selected_backend.vm_policy_identity(); + let untrusted_guest = selected_backend.untrusted_guest(); + // Mark every event this supervisor emits as guest-sourced when the + // boundary is an untrusted guest: process exits, connection identity, + // and any other boundary-reported payload are guest data, not + // verified observations. The dedicated supervisor serves exactly one + // sandbox, so the process-wide mark is exact. + openshell_ocsf::ctx::set_guest_data_origin(untrusted_guest); + // An untrusted guest's boundary must never be asked for policy: the + // guest (including its boundary and anything it plants at + // /etc/openshell/policy.yaml) is untrusted, so its reply cannot + // become the sandbox's enforcement baseline. Discovery is skipped + // entirely; the baseline comes from the gateway's policy or, when + // none is configured, from the restrictive default — and the + // guest-supplied YAML is never synced to the gateway. + let image_discovery = if untrusted_guest { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "disabled") + .message("Image policy discovery disabled: the boundary is an untrusted guest") + .build() + ); ImagePolicyDiscovery::Missing + } else { + let (image_yaml, invalid_image) = selected_backend + .discover_policy(sandbox_bearer.clone(), attach_request_timeout) + .instrument( + tracing::info_span!(parent: &startup, "supervisor.boundary.discover_policy"), + ) + .await?; + if invalid_image { + ImagePolicyDiscovery::Invalid + } else if let Some(yaml) = image_yaml { + openshell_policy::parse_sandbox_policy(&yaml) + .map_or(ImagePolicyDiscovery::Invalid, |policy| { + ImagePolicyDiscovery::Policy(Box::new(policy)) + }) + } else { + ImagePolicyDiscovery::Missing + } }; - let vm_policy_identity = selected_backend.vm_policy_identity(); - // Load policy and initialize OPA engine let sandbox_name_for_agg = sandbox.clone(); let ( @@ -1237,6 +1264,7 @@ pub async fn run_sandbox_instance( &extension_credentials, LocalPolicyIdentity::Required, vm_policy_identity, + untrusted_guest, Some(image_discovery), &RemoteStartupGateway { gateway: gateway.clone(), @@ -1679,6 +1707,7 @@ pub async fn run_sandbox_instance( opa_engine: poll_engine, loaded_policy_origin, vm_identity: vm_policy_identity, + untrusted_guest, entrypoint_pid: poll_pid, interval_secs: poll_interval_secs, ocsf_enabled: poll_ocsf_enabled, @@ -1718,7 +1747,30 @@ pub async fn run_sandbox_instance( }); } - let proxy_exited: Pin + Send>> = if let Some(rx) = networking + // Destructure the confirmed boundary first so the mode claim is + // available for the proxy-exit decision below. + let (confirmed, backend_name) = remote_ready; + + // In the privileged (root-workload) mode the boundary has no mediation + // source by design (the seccomp broker is not installed; the nftables + // relay is the future path). The proxy accept loop would see its Mediate + // streams end immediately and exit, which the lifecycle treats as fatal. + // Skip the proxy-exit signal for that mode — the proxy stays running but + // has nothing to mediate until the relay lands. + // + // The mode is read from the confirmed boundary's confirmation, which is + // equivalent to the trusted launch descriptor because confirm enforces + // confirmation.resource_claims == self.resource_claims (the descriptor's + // claims) — the boundary cannot pick the mode. + let untrusted_guest = confirmed + .confirmation() + .resource_claims + .get("kubevirt.workload_mode") + .map(String::as_str) + == Some("root"); + let proxy_exited: Pin + Send>> = if untrusted_guest { + Box::pin(std::future::pending()) + } else if let Some(rx) = networking .as_mut() .and_then(|n| n.proxy.as_mut()) .and_then(ProxyHandle::take_exit_receiver) @@ -1730,8 +1782,6 @@ pub async fn run_sandbox_instance( Box::pin(std::future::pending()) }; tokio::pin!(proxy_exited); - - let (confirmed, backend_name) = remote_ready; let exit_code = { let running = confirmed .into_boundary() @@ -1754,7 +1804,7 @@ pub async fn run_sandbox_instance( adopted_execs = running.adopted_exec_processes().len(), "Adopted the running agent from the previous supervisor" ); - observe_adopted_execs(running.adopted_exec_processes()); + observe_adopted_execs(running.adopted_exec_processes(), untrusted_guest); } else { info!(backend = %backend_name, "Isolation boundary agent started"); } @@ -1780,6 +1830,7 @@ pub async fn run_sandbox_instance( app_routes .as_ref() .map(app_listener::Routes::installation_source), + untrusted_guest, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.access.start")) .await?; @@ -1907,6 +1958,41 @@ pub async fn run_sandbox_instance( .into_diagnostic() .wrap_err("persist canonical-process completion marker")?; } + // Emit an OCSF event for the main-process exit. The exit code is + // guest-reported (the boundary reports the workload's lifecycle); + // in root mode the boundary is untrusted after start, so the event + // is marked guest-sourced. In sandboxed mode the exit code comes + // from the supervised boundary and stays unmarked. + // The entrypoint binary is the workload the exit code describes; + // set it so the shorthand reads correctly (not "unknown(0)"). + let process_name = + openshell_ocsf::Process::new(&command.first().cloned().unwrap_or_default(), 0); + if untrusted_guest { + openshell_ocsf::ocsf_emit!( + ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(exit_code) + .actor_process(process_name.clone()) + .process(process_name.clone()) + .guest_sourced() + .message(format!( + "main process exit reported by the boundary: code {exit_code}" + )) + .build() + ); + } else { + openshell_ocsf::ocsf_emit!( + ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .exit_code(exit_code) + .actor_process(process_name.clone()) + .process(process_name) + .message(format!("main process exited: code {exit_code}")) + .build() + ); + } if !completion_cancelled && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { @@ -2023,16 +2109,50 @@ fn boundary_exit_code(status: openshell_isolation_interface::contract::BoundaryE /// Exec processes a replacement supervisor adopted outlive their original /// clients; record when each exits so their end is not silent. -fn observe_adopted_execs(processes: &[Arc]) { +fn observe_adopted_execs( + processes: &[Arc], + untrusted_guest: bool, +) { for process in processes { let process = process.clone(); tokio::spawn(async move { match process.wait().await { - Ok(status) => info!( - process_id = %process.process_id(), - ?status, - "Adopted exec process exited" - ), + Ok(status) => { + info!( + process_id = %process.process_id(), + ?status, + "Adopted exec process exited" + ); + // The exit status is guest-reported (the boundary reports + // the adopted exec's lifecycle); in root mode the + // boundary is untrusted after start, so the event is + // marked guest-sourced. + let code = boundary_exit_code(status); + if untrusted_guest { + openshell_ocsf::ocsf_emit!( + ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Informational) + .exit_code(code) + .actor_process(openshell_ocsf::Process::new("adopted-exec", 0,)) + .guest_sourced() + .message(format!( + "adopted exec exit reported by the boundary: code {code}" + )) + .build() + ); + } else { + openshell_ocsf::ocsf_emit!( + ProcessActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Informational) + .exit_code(code) + .actor_process(openshell_ocsf::Process::new("adopted-exec", 0,)) + .message(format!("adopted exec process exited: code {code}")) + .build() + ); + } + } Err(error) => debug!( process_id = %process.process_id(), %error, @@ -2837,6 +2957,67 @@ impl VmPolicyIdentity { } } +/// Reject policy sections that cannot be enforced in the privileged +/// (root-workload) mode. +/// +/// In-guest enforcement is meaningless against a root workload: Landlock, +/// filesystem path rules, and identity selectors are demoted by design, and +/// the fence plus the supervisor's policy evaluation are the boundary. A +/// policy that authors those sections promises constraints that will never +/// hold, so startup and every reload reject it — the operator must remove +/// the unenforceable sections (a network-only policy is fully supported). +/// +/// Keyed off the trusted launch descriptor's mode claim, never a boundary +/// report: a sandboxed boundary cannot dodge its own validation. +fn validate_policy_enforceable_for_untrusted_guest( + policy: &openshell_core::proto::SandboxPolicy, +) -> Result<()> { + let mut unenforceable: Vec<&'static str> = Vec::new(); + if let Some(filesystem) = policy.filesystem.as_ref() + && (filesystem.include_workdir + || !filesystem.read_only.is_empty() + || !filesystem.read_write.is_empty()) + { + unenforceable.push("filesystem (read_only/read_write/include_workdir)"); + } + if let Some(landlock) = policy.landlock.as_ref() + && !landlock.compatibility.is_empty() + { + unenforceable.push("landlock"); + } + if let Some(process) = policy.process.as_ref() + && (!process.run_as_user.is_empty() || !process.run_as_group.is_empty()) + { + unenforceable.push("process (run_as_user/run_as_group)"); + } + // Per-binary selectors are also unenforceable: a root workload cannot be + // pinned to a binary identity in-guest, so an authored binaries list + // promises a restriction that will never hold (it would be silently + // ignored through require_binary_identity=false). + let binary_scoped: Vec<&str> = policy + .network_policies + .iter() + .filter(|(_, rule)| !rule.binaries.is_empty()) + .map(|(name, _)| name.as_str()) + .collect(); + if !binary_scoped.is_empty() { + unenforceable.push("network per-binary selectors (binaries)"); + } + if unenforceable.is_empty() { + Ok(()) + } else { + Err(miette::miette!( + "untrusted-guest (root-workload) sandboxes cannot enforce {} section(s); remove them from the policy — the network fence and supervisor-side policy evaluation are the boundary{}", + unenforceable.join(", "), + if binary_scoped.is_empty() { + String::new() + } else { + format!(" (per-binary rules: {})", binary_scoped.join(", ")) + } + )) + } +} + #[derive(Clone)] struct CapturedProviderEnvironment { credentials: ProviderCredentialState, @@ -2899,6 +3080,7 @@ async fn load_policy( extension_credentials, local_policy_identity, None, + false, None, &RemoteStartupGateway { gateway: openshell_gateway, @@ -2920,6 +3102,7 @@ async fn load_policy_with_gateway( extension_credentials: &openshell_extension_core::ExtensionCredentialStore, local_policy_identity: LocalPolicyIdentity, vm_identity: Option, + untrusted_guest: bool, image_discovery: Option, gateway: &impl StartupGateway, ) -> Result<( @@ -2986,7 +3169,12 @@ async fn load_policy_with_gateway( landlock: config.landlock, process: config.process, }; - enrich_sandbox_baseline_paths(&mut policy); + // Skipped in privileged (root-workload) mode: the enrichment is a + // Landlock accommodation and would inject filesystem sections the + // privileged policy validation rejects (B9). + if !untrusted_guest { + enrich_sandbox_baseline_paths(&mut policy); + } // File mode has no operator-registered middleware to connect. return Ok(( policy, @@ -3065,18 +3253,22 @@ async fn load_policy_with_gateway( .message("Server returned no policy; attempting local discovery") .build() ); - let mut discovered = match &discovery { - ImagePolicyDiscovery::Policy(policy) => *policy.clone(), - ImagePolicyDiscovery::Missing => openshell_policy::restrictive_default_policy(), - ImagePolicyDiscovery::Invalid => { - reject_startup_configuration(gateway, &mut rejection_log, id, &instance_id, &snapshot, "Image policy is invalid; replace the sandbox policy to repair configuration", None).await?; - reconciliation_attempts = 0; - continue; - } + let Some(mut discovered) = + baseline_policy_from_discovery(&discovery, untrusted_guest) + else { + reject_startup_configuration(gateway, &mut rejection_log, id, &instance_id, &snapshot, "Image policy is invalid; replace the sandbox policy to repair configuration", None).await?; + reconciliation_attempts = 0; + continue; }; // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); + // baseline paths from the start. Skipped in privileged mode: + // the enrichment exists to keep Landlock from blocking proxy + // baseline paths, and a root workload has no Landlock — + // injecting filesystem entries would be rejected by the + // privileged policy validation below. + if !untrusted_guest { + enrich_proto_baseline_paths(&mut discovered); + } strip_proto_provider_policy_entries(&mut discovered); // Sync and re-fetch over a single connection to avoid extra // TLS handshakes. @@ -3137,7 +3329,18 @@ async fn load_policy_with_gateway( // a sandbox-sourced policy is written back. The gateway refuses // every sandbox policy write while a global policy is active, so a // global policy keeps the added paths in this process only. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); + // + // Skipped entirely in privileged (root-workload) mode: the + // enrichment exists to keep Landlock from blocking proxy baseline + // paths, a root workload has no Landlock, and a synced-back + // filesystem section would be rejected by the privileged policy + // validation below (B9 — the supervisor was enriching its own + // root policy and then rejecting it). + let enriched = if untrusted_guest { + false + } else { + enrich_proto_baseline_paths(&mut proto_policy) + }; let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched) .filter(|_| snapshot.policy_source == openshell_core::proto::PolicySource::Sandbox); if let Some(sync_policy) = sync_policy { @@ -3226,6 +3429,27 @@ async fn load_policy_with_gateway( reconciliation_attempts = 0; continue; } + // Privileged (root-workload) sandboxes reject unenforceable + // policy sections before any engine or credentials are built: + // filesystem, Landlock, and identity constraints cannot hold + // against a root workload. The mode comes from the trusted + // launch descriptor, not the boundary. + if untrusted_guest + && let Err(error) = validate_policy_enforceable_for_untrusted_guest(&proto_policy) + { + reject_startup_configuration( + gateway, + &mut rejection_log, + id, + &instance_id, + &snapshot, + &error.to_string(), + None, + ) + .await?; + reconciliation_attempts = 0; + continue; + } let provider = grpc_retry("Startup provider environment", || gateway.provider(id)).await?; if provider.provider_env_revision != snapshot.provider_env_revision { @@ -3235,7 +3459,12 @@ async fn load_policy_with_gateway( } debug!("Creating OPA engine from proto policy data"); let (engine, policy, captured_provider_credentials) = - match prepare_startup_configuration(&snapshot, &proto_policy, &provider) { + match prepare_startup_configuration( + &snapshot, + &proto_policy, + &provider, + untrusted_guest, + ) { Ok(prepared) => prepared, Err(error) => { report_initial_policy_failure( @@ -3245,13 +3474,18 @@ async fn load_policy_with_gateway( &error, ) .await; + // Carry the specific failure (privileged section + // rejection, provider refusal, engine build error) + // into the user-visible configuration error rather + // than a generic message. + let diagnostic = render_bounded_rejection_detail(&error.to_string()); reject_startup_configuration( gateway, &mut rejection_log, id, &instance_id, &snapshot, - "Policy or provider environment failed runtime validation", + &diagnostic, None, ) .await?; @@ -3392,6 +3626,35 @@ enum ImagePolicyDiscovery { Policy(Box), } +/// The enforcement baseline to install when the gateway has no policy +/// configured. +/// +/// A trusted boundary's discovered image policy is the baseline. An +/// untrusted guest's "discovered" policy is guest data — anything the +/// guest reports, including YAML planted at `/etc/openshell/policy.yaml` +/// in the guest image — and can never become the baseline: the class uses +/// the restrictive default (no filesystem, Landlock, or identity +/// sections; in-guest enforcement is meaningless for a root workload). +/// Discovery is disabled for the class at startup, and this function +/// keeps the invariant true even if a future path reintroduces discovery. +/// `None` means the discovery was invalid and startup must be rejected. +fn baseline_policy_from_discovery( + discovery: &ImagePolicyDiscovery, + untrusted_guest: bool, +) -> Option { + match discovery { + ImagePolicyDiscovery::Policy(policy) if !untrusted_guest => Some(*policy.clone()), + ImagePolicyDiscovery::Policy(_) | ImagePolicyDiscovery::Missing => { + if untrusted_guest { + Some(openshell_policy::restrictive_default_policy_privileged()) + } else { + Some(openshell_policy::restrictive_default_policy()) + } + } + ImagePolicyDiscovery::Invalid => None, + } +} + fn discover_image_policy() -> ImagePolicyDiscovery { for path in [ openshell_policy::CONTAINER_POLICY_PATH, @@ -3420,6 +3683,7 @@ fn prepare_startup_configuration( snapshot: &openshell_core::grpc_client::SettingsPollResult, policy: &openshell_core::proto::SandboxPolicy, provider: &openshell_core::grpc_client::ProviderEnvironmentResult, + untrusted_guest: bool, ) -> Result<(OpaEngine, SandboxPolicy, ProviderCredentialState)> { if !snapshot.configuration_admitted { return Err(miette::miette!( @@ -3436,7 +3700,27 @@ fn prepare_startup_configuration( "Provider environment revision changed during configuration preparation" )); } - let engine = OpaEngine::from_proto(policy)?; + if untrusted_guest { + // Nothing secret enters the guest: a root workload can read anything + // the in-guest boundary holds, so provider files and environment + // are refused supervisor-side. This is the real guarantee; the + // boundary's own refusal (B3c2) is defence in depth. Repair by + // detaching the provider or replacing the sandbox policy. + if !provider.files.is_empty() || !provider.environment.is_empty() { + return Err(miette::miette!( + "untrusted-guest (root-workload) sandboxes cannot install provider \ + files or environment; detach the provider to repair configuration" + )); + } + } + // Per-binary identity checks are unenforceable for a root workload and + // would deny every connection it makes; the requirement is a property + // of the workload mode, not the policy. + let engine = if untrusted_guest { + OpaEngine::from_proto_with_pid_and_binary_identity(policy, 0, false)? + } else { + OpaEngine::from_proto(policy)? + }; let process_policy = SandboxPolicy::try_from(policy.clone())?; let credentials = prepare_provider_environment(provider)?; Ok((engine, process_policy, credentials)) @@ -3551,6 +3835,17 @@ impl StartupRejectionLog { } } +/// Render a startup rejection detail for the user-visible configuration +/// error: drop control characters and keep at most 512 characters, mirroring +/// the gateway's own configuration diagnostics. +fn render_bounded_rejection_detail(error: &str) -> String { + error + .chars() + .filter(|character| !character.is_control()) + .take(512) + .collect() +} + async fn reject_startup_configuration( gateway: &impl StartupGateway, rejection_log: &mut StartupRejectionLog, @@ -3677,8 +3972,16 @@ async fn reload_gateway_policy_runtime( entrypoint_pid: u32, middleware: MiddlewareReloadContext<'_>, ) -> std::result::Result { - reload_gateway_configuration_runtime(engine, policy, entrypoint_pid, middleware, None, || {}) - .await + reload_gateway_configuration_runtime( + engine, + policy, + entrypoint_pid, + middleware, + None, + false, + || {}, + ) + .await } async fn reload_gateway_configuration_runtime( @@ -3687,6 +3990,7 @@ async fn reload_gateway_configuration_runtime( entrypoint_pid: u32, middleware: MiddlewareReloadContext<'_>, vm_identity: Option, + untrusted_guest: bool, commit_credentials: impl FnOnce(), ) -> std::result::Result { if let (Some(identity), Some(policy)) = (vm_identity, policy) { @@ -3696,6 +4000,15 @@ async fn reload_gateway_configuration_runtime( .validate(policy) .map_err(GatewayRuntimeReloadError::PolicyValidation)?; } + if untrusted_guest + && let Some(policy) = policy + && let Err(error) = validate_policy_enforceable_for_untrusted_guest(policy) + { + // A policy update that authors unenforceable sections must not + // commit: the mode is fixed for the generation, so the update is + // rejected rather than silently dropping the constraints. + return Err(GatewayRuntimeReloadError::PolicyValidation(error)); + } match policy { Some(policy) if middleware.registry_changed => { let registry = (middleware.connector)( @@ -4344,6 +4657,10 @@ struct PolicyPollLoopContext { loaded_policy_origin: LoadedPolicyOrigin, /// Immutable VM overlay identity, also enforced for global policy updates. vm_identity: Option, + /// Whether the trusted launch descriptor selects the privileged + /// (root-workload) mode; enforced for global policy updates and + /// session-delivered configuration. + untrusted_guest: bool, entrypoint_pid: Arc, interval_secs: u64, ocsf_enabled: Arc, @@ -5349,6 +5666,9 @@ network_policies: report_error: Option, pending_snapshot: bool, pending_acceptance: bool, + /// Policies written back through sync, for asserting no privileged + /// policy is enriched and synced. + synced: Arc>>, } impl StartupGateway for TestStartupGateway { @@ -5377,9 +5697,10 @@ network_policies: async fn sync( &self, _sandbox: &str, - _policy: &openshell_core::proto::SandboxPolicy, + policy: &openshell_core::proto::SandboxPolicy, _workspace: &str, ) -> Result { + self.synced.lock().unwrap().push(policy.clone()); self.snapshot("").await } async fn report( @@ -5462,6 +5783,7 @@ network_policies: report_error: None, pending_snapshot: false, pending_acceptance: false, + synced: Arc::default(), }; let active_gateway = gateway.clone(); let handle = tokio::spawn(async move { @@ -5477,6 +5799,7 @@ network_policies: uid: 1000, gid: 1001, }), + false, Some(ImagePolicyDiscovery::Missing), &active_gateway, ) @@ -5569,6 +5892,7 @@ network_policies: uid: 1000, gid: 1001, }), + false, || { committed.store(true, Ordering::SeqCst); }, @@ -5601,6 +5925,7 @@ network_policies: report_error: None, pending_snapshot, pending_acceptance: !pending_snapshot, + synced: Arc::default(), }; let result = timeout( Duration::from_mins(2), @@ -5613,6 +5938,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(ImagePolicyDiscovery::Missing), &gateway, ), @@ -5678,6 +6004,7 @@ network_policies: report_error, pending_snapshot: false, pending_acceptance: false, + synced: Arc::default(), }; let result = timeout( Duration::from_secs(1), @@ -5690,6 +6017,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(ImagePolicyDiscovery::Missing), &gateway, ), @@ -5724,6 +6052,7 @@ network_policies: report_error: None, pending_snapshot: false, pending_acceptance: false, + synced: Arc::default(), }; let active_gateway = gateway.clone(); let handle = tokio::spawn(async move { @@ -5736,6 +6065,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(ImagePolicyDiscovery::Missing), &active_gateway, ) @@ -6069,6 +6399,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(discovery), &startup_gateway, ) @@ -6220,6 +6551,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(discovery), &gateway, ), @@ -6552,6 +6884,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(ImagePolicyDiscovery::Missing), &gateway, ), @@ -6647,6 +6980,7 @@ network_policies: &openshell_extension_core::ExtensionCredentialStore::new(), LocalPolicyIdentity::Required, None, + false, Some(discovery.clone()), &gateway, ), @@ -6733,9 +7067,12 @@ network_policies: openshell_core::proto::PolicySource::Sandbox, ); snapshot.provider_env_revision = 10; - assert!(prepare_startup_configuration(&snapshot, &policy, &startup_provider(11)).is_err()); + assert!( + prepare_startup_configuration(&snapshot, &policy, &startup_provider(11), false) + .is_err() + ); let (_, _, credentials) = - prepare_startup_configuration(&snapshot, &policy, &startup_provider(10)) + prepare_startup_configuration(&snapshot, &policy, &startup_provider(10), false) .expect("matching generation is admitted"); assert_eq!(credentials.revision(), 10); } @@ -6778,8 +7115,9 @@ network_policies: provider .non_secret_environment_keys .push("PROJECT_ID".to_string()); - let (_, _, credentials) = prepare_startup_configuration(&snapshot, &policy, &provider) - .expect("withheld credentials preserve fail-closed startup"); + let (_, _, credentials) = + prepare_startup_configuration(&snapshot, &policy, &provider, false) + .expect("withheld credentials preserve fail-closed startup"); assert_eq!(credentials.revision(), 10); assert!(credentials.snapshot().child_env.contains_key("PROJECT_ID")); } @@ -6796,7 +7134,7 @@ network_policies: let mut provider = startup_provider(0); provider.readiness_reason = ProviderReadinessReason::CredentialInstallFailed; - assert!(prepare_startup_configuration(&snapshot, &policy, &provider).is_err()); + assert!(prepare_startup_configuration(&snapshot, &policy, &provider, false).is_err()); } #[test] @@ -6810,11 +7148,13 @@ network_policies: for _restart in 0..2 { snapshot.configuration_admitted = false; assert!( - prepare_startup_configuration(&snapshot, &policy, &startup_provider(0)).is_err() + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), false) + .is_err() ); snapshot.configuration_admitted = true; assert!( - prepare_startup_configuration(&snapshot, &policy, &startup_provider(0)).is_ok() + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), false) + .is_ok() ); } } @@ -6834,7 +7174,9 @@ network_policies: 1, openshell_core::proto::PolicySource::Sandbox, ); - assert!(prepare_startup_configuration(&snapshot, &policy, &startup_provider(0)).is_err()); + assert!( + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), false).is_err() + ); } fn proto_provenance_policy_fixture() -> openshell_core::proto::SandboxPolicy { @@ -7770,7 +8112,339 @@ network_policies: .contains("previous policy IS active") ); } -} -#[cfg(test)] -mod legacy_startup_regression_tests; + fn network_only_policy() -> openshell_core::proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy( + r#" +version: 1 +network_policies: + web: + name: web + endpoints: + - host: example.com + port: 443 + protocol: tcp +"#, + ) + .expect("parse network-only policy") + } + + #[test] + fn untrusted_guest_ignores_a_planted_guest_policy() { + // A policy "discovered" from the guest — YAML planted at + // /etc/openshell/policy.yaml in the guest image — is guest data. + // For an untrusted guest it can never become the enforcement + // baseline: the restrictive default is used instead, and the + // planted rule is absent. A trusted boundary still uses its + // discovered image policy. + let planted = openshell_policy::parse_sandbox_policy( + r#" +version: 1 +network_policies: + planted: + name: planted + endpoints: + - host: planted.example + port: 443 + protocol: tcp +"#, + ) + .expect("parse the planted policy"); + let discovery = ImagePolicyDiscovery::Policy(Box::new(planted)); + + let baseline = baseline_policy_from_discovery(&discovery, true) + .expect("the untrusted guest still gets a baseline"); + assert_eq!( + baseline, + openshell_policy::restrictive_default_policy_privileged(), + "the planted guest policy must be ignored for an untrusted guest" + ); + assert!( + !baseline.network_policies.contains_key("planted"), + "the planted allow rule must not survive" + ); + + let trusted = baseline_policy_from_discovery(&discovery, false) + .expect("a trusted boundary uses its image policy"); + assert!( + trusted.network_policies.contains_key("planted"), + "a trusted boundary keeps its discovered image policy" + ); + + // No discovery at all: the restrictive default either way. + let missing = baseline_policy_from_discovery(&ImagePolicyDiscovery::Missing, true) + .expect("missing discovery still has a baseline"); + assert_eq!( + missing, + openshell_policy::restrictive_default_policy_privileged() + ); + } + + #[test] + fn untrusted_guest_policy_validation_rejects_unenforceable_sections() { + // filesystem rules + let mut policy = network_only_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + include_workdir: false, + read_only: vec!["/usr".into()], + read_write: vec![], + }); + let error = validate_policy_enforceable_for_untrusted_guest(&policy) + .expect_err("filesystem rules must be rejected"); + assert!(error.to_string().contains("filesystem")); + + // landlock + let mut policy = network_only_policy(); + policy.landlock = Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".into(), + }); + let error = validate_policy_enforceable_for_untrusted_guest(&policy) + .expect_err("landlock must be rejected"); + assert!(error.to_string().contains("landlock")); + + // run_as_user + let mut policy = network_only_policy(); + policy.process = Some(openshell_core::proto::ProcessPolicy { + run_as_user: "nobody".into(), + run_as_group: String::new(), + }); + let error = validate_policy_enforceable_for_untrusted_guest(&policy) + .expect_err("run_as_user must be rejected"); + assert!(error.to_string().contains("run_as_user")); + + // run_as_group + let mut policy = network_only_policy(); + policy.process = Some(openshell_core::proto::ProcessPolicy { + run_as_user: String::new(), + run_as_group: "nogroup".into(), + }); + assert!(validate_policy_enforceable_for_untrusted_guest(&policy).is_err()); + } + + #[test] + fn privileged_policy_validation_accepts_network_only_and_default() { + // A network-only policy is fully supported: the fence and + // supervisor-side evaluation are the boundary. + validate_policy_enforceable_for_untrusted_guest(&network_only_policy()) + .expect("network-only policy must be accepted"); + + // The privileged default policy must pass its own validation. + let default = openshell_policy::restrictive_default_policy_privileged(); + validate_policy_enforceable_for_untrusted_guest(&default) + .expect("privileged default policy must be accepted"); + + // The sandboxed restrictive default must NOT pass: it authors + // filesystem and landlock constraints. + let sandboxed_default = openshell_policy::restrictive_default_policy(); + assert!(validate_policy_enforceable_for_untrusted_guest(&sandboxed_default).is_err()); + } + + #[test] + fn privileged_startup_refuses_provider_files_and_environment() { + // Nothing secret enters the guest: provider material is refused + // supervisor-side for a root workload. + let policy = network_only_policy(); + let snapshot = settings_poll_result( + Some(policy.clone()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + // Provider files + let mut provider = startup_provider(0); + provider + .files + .insert("credentials.json".to_string(), "{}".to_string()); + let error = prepare_startup_configuration(&snapshot, &policy, &provider, true) + .map(|_: (OpaEngine, SandboxPolicy, _)| ()) + .expect_err("provider files must be refused in privileged mode"); + assert!(error.to_string().contains("provider")); + + // Provider environment + let mut provider = startup_provider(0); + provider + .environment + .insert("PROJECT_ID".to_string(), "example-project".to_string()); + provider + .non_secret_environment_keys + .push("PROJECT_ID".to_string()); + let error = prepare_startup_configuration(&snapshot, &policy, &provider, true) + .map(|_: (OpaEngine, SandboxPolicy, _)| ()) + .expect_err("provider environment must be refused in privileged mode"); + assert!(error.to_string().contains("provider")); + + // Empty provider material is fine (nothing to enter the guest). + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), true) + .expect("empty provider material must be accepted"); + + // Sandboxed mode keeps provider material support. + prepare_startup_configuration(&snapshot, &policy, &provider, false) + .expect("sandboxed mode keeps provider environment support"); + } + + #[test] + fn privileged_startup_builds_engine_without_binary_identity_requirement() { + let policy = network_only_policy(); + let snapshot = settings_poll_result( + Some(policy.clone()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + let (engine, _, _) = + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), true) + .expect("privileged startup prepares"); + assert!( + !engine.binary_identity_required(), + "privileged engine must not require per-binary identity" + ); + drop(engine); + + let (engine, _, _) = + prepare_startup_configuration(&snapshot, &policy, &startup_provider(0), false) + .expect("sandboxed startup prepares"); + assert!( + engine.binary_identity_required(), + "sandboxed engine keeps per-binary identity requirements" + ); + } + + #[tokio::test] + async fn privileged_reload_rejects_unenforceable_policy_update() { + // A policy update that authors unenforceable sections must not + // commit in privileged mode. + let engine = Arc::new( + OpaEngine::from_proto_with_pid_and_binary_identity(&network_only_policy(), 0, false) + .expect("build engine"), + ); + let mut policy = network_only_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }); + let result = reload_gateway_configuration_runtime( + &engine, + Some(&policy), + 0, + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, + None, + true, + || {}, + ) + .await; + let error = result.expect_err("unenforceable policy update must be rejected"); + assert!(matches!( + error, + GatewayRuntimeReloadError::PolicyValidation(_) + )); + } + + #[tokio::test] + async fn privileged_startup_keeps_network_policy_unchanged_and_unsynced() { + // B9 regression: the supervisor used to enrich its own root policy + // with baseline filesystem paths, sync the enriched policy back to + // the gateway, and then reject the filesystem section it just + // created. A network-only policy delivered in root mode must stay + // network-only, never sync back, and pass validation. + use openshell_core::proto::{ConfigurationAdmissionState, PolicySource}; + + let policy = network_only_policy(); + assert!( + policy.filesystem.is_none() && policy.landlock.is_none(), + "fixture must be network-only" + ); + let (reports, mut reported) = tokio::sync::mpsc::unbounded_channel(); + let gateway = TestStartupGateway { + desired: Arc::new(std::sync::Mutex::new(settings_poll_result( + Some(policy.clone()), + 1, + PolicySource::Sandbox, + ))), + reports, + reject_next_accept: Arc::new(AtomicBool::new(false)), + snapshot_error: None, + report_error: None, + pending_snapshot: false, + pending_acceptance: false, + synced: Arc::default(), + }; + let synced = gateway.synced.clone(); + let active_gateway = gateway.clone(); + let (loaded, _engine, retained, ..) = timeout( + Duration::from_secs(30), + load_policy_with_gateway( + Some("sandbox-id".to_string()), + Some("sandbox".to_string()), + Some(test_gateway()), + None, + None, + &openshell_extension_core::ExtensionCredentialStore::new(), + LocalPolicyIdentity::Required, + None, + true, + Some(ImagePolicyDiscovery::Missing), + &active_gateway, + ), + ) + .await + .expect("privileged startup with a network-only policy must succeed") + .expect("startup result"); + assert_eq!( + reported.recv().await, + Some(ConfigurationAdmissionState::Pending) + ); + assert_eq!( + reported.recv().await, + Some(ConfigurationAdmissionState::Accepted) + ); + + // The effective policy stays network-only: no enrichment ran. + let retained = retained.expect("retained proto policy"); + assert!( + retained.filesystem.is_none(), + "privileged policy must not be enriched with filesystem sections" + ); + assert!( + loaded.filesystem.read_only.is_empty() && loaded.filesystem.read_write.is_empty(), + "the compiled policy stays network-only" + ); + // Nothing was synced back: an unenriched policy is never written. + assert!( + synced.lock().unwrap().is_empty(), + "privileged startup must not sync an enriched policy back to the gateway" + ); + } + + #[test] + fn privileged_policy_validation_rejects_per_binary_selectors() { + // Per-binary selectors are unenforceable for a root workload and + // must be rejected rather than silently ignored through + // require_binary_identity=false. + let mut policy = network_only_policy(); + let rule = policy.network_policies.get_mut("web").expect("web rule"); + rule.binaries = vec![openshell_core::proto::NetworkBinary { + path: "/usr/bin/busybox".into(), + }]; + let error = validate_policy_enforceable_for_untrusted_guest(&policy) + .expect_err("per-binary selectors must be rejected"); + let message = error.to_string(); + assert!( + message.contains("binaries"), + "message names the section: {message}" + ); + assert!(message.contains("web"), "message names the rule: {message}"); + + // Without binaries the same policy passes. + let mut policy = network_only_policy(); + let rule = policy.network_policies.get_mut("web").expect("web rule"); + rule.binaries = Vec::new(); + validate_policy_enforceable_for_untrusted_guest(&policy) + .expect("network-only policy without binaries passes"); + } +} diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 1b77eb4a8e..208390cd54 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -888,3 +888,15 @@ Validate chart values that Helm would otherwise accept silently. {{- end -}} {{- toYaml $app -}} {{- end -}} + +{{/* +The effective Kubernetes driver runtime, normalized: "kubevirt" (both the +canonical spelling and the "kube-virt" alias) or "pod". Templates compare +against this single value so a values file using either spelling renders +the same Role, RoleBinding, and preflight. +*/}} +{{- define "openshell.kubernetesRuntime" -}} +{{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} +{{- $runtime := get $kubernetesConfig "runtime" | default "" -}} +{{- if or (eq $runtime "kubevirt") (eq $runtime "kube-virt") -}}kubevirt{{- else -}}pod{{- end -}} +{{- end -}} diff --git a/deploy/helm/openshell/templates/agent-sandbox-preflight.yaml b/deploy/helm/openshell/templates/agent-sandbox-preflight.yaml index 83a742f670..d364a49872 100644 --- a/deploy/helm/openshell/templates/agent-sandbox-preflight.yaml +++ b/deploy/helm/openshell/templates/agent-sandbox-preflight.yaml @@ -2,7 +2,7 @@ SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */}} -{{- if .Values.agentSandbox.preflight.enabled }} +{{- if and .Values.agentSandbox.preflight.enabled (ne (include "openshell.kubernetesRuntime" .) "kubevirt") }} {{- $v1beta1 := or (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1") (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1beta1/Sandbox") }} {{- $v1alpha1 := or (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1alpha1") (.Capabilities.APIVersions.Has "agents.x-k8s.io/v1alpha1/Sandbox") }} {{- if not (or $v1beta1 $v1alpha1) }} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 8af91e1901..202feca73a 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -30,6 +30,37 @@ rules: resources: ["persistentvolumeclaims"] verbs: ["get"] {{- end }} + {{- if and (ne $workspaceMode "shared") (eq (include "openshell.kubernetesRuntime" .) "kubevirt") }} + # KubeVirt runtime in managed/operator mode: the same minimal per- + # sandbox verbs the shared-mode Role grants, needed cluster-wide + # because the sandbox namespaces are per-workspace. The driver never + # reads Secrets (create/delete only — the bootstrap Secret is + # immutable per generation), never lists Services, PVCs, or + # NetworkPolicies (companions are derived from the VM's name and + # recorded generation), and the fence reconciler restores drift with + # delete-and-recreate, never patch. + - apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "patch", "watch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get", "delete"] + {{- end }} # Validate projected ServiceAccount tokens during sandbox bootstrap and # internal gateway peer authentication. - apiGroups: diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 85d2e954d4..06f581b3ae 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -3,7 +3,8 @@ {{- $allowDriverConfig := get $kubernetesConfig "allow_driver_config" | default false -}} {{- $supervisorNetwork := .Values.supervisor.network | default dict -}} {{- $additionalCaEnabled := ne (get $supervisorNetwork "additionalCaConfigMapName" | default "") "" -}} -{{- if and (eq $workspaceMode "shared") (or (include "openshell.workspaceResourcesEnabled" .) $additionalCaEnabled) (include "openshell.rbacCreate" .) }} +{{- $kubevirtRuntime := eq (include "openshell.kubernetesRuntime" .) "kubevirt" -}} +{{- if and (eq $workspaceMode "shared") (or (include "openshell.workspaceResourcesEnabled" .) $additionalCaEnabled $kubevirtRuntime) (include "openshell.rbacCreate" .) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 @@ -14,7 +15,51 @@ metadata: labels: {{- include "openshell.labels" . | nindent 4 }} rules: - {{- if include "openshell.workspaceResourcesEnabled" . }} + {{- if $kubevirtRuntime }} + # KubeVirt runtime: the driver provisions kubevirt.io VirtualMachines + # directly (the durable lifecycle handle: created Halted, patched to + # Once, watched for status, digest-pinned after the first boot, deleted + # with the sandbox) and reads the paired VirtualMachineInstances for + # their phase. Listing and watching are the sandbox enumeration APIs; + # every companion is derived from the VM's name and recorded + # generation, so no other resource is ever searched. + - apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["get", "list", "watch"] + # The dedicated supervisor Pod: created gated, patched to release the + # bootstrap scheduling gate, read for status by exact name, watched by + # the failure reconciler, and deleted on stop. + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "patch", "watch"] + # The per-sandbox supervisor Service: created (re-created on start when + # deleted out of band) and resolved by exact name. + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "delete"] + # Bootstrap Secrets are immutable per generation: a start creates the + # new generation's Secret and deletes the previous one by exact name + # with a UID precondition. No get, list, or patch — the driver never + # reads Secrets in the sandbox namespace. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + # The per-sandbox workspace PVC: created once, read by exact name on + # teardown, deleted by exact name with a UID precondition. + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "delete"] + # The egress fence, the namespace-wide launcher isolation floor, and + # the control-ingress policy: created, verified by exact name (get), + # and deleted on teardown or drift. The reconciler restores drift with + # delete-and-recreate, never patch. + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get", "delete"] + {{- else if include "openshell.workspaceResourcesEnabled" . }} {{- if $allowDriverConfig }} # Metadata-only admission of caller-selected PVCs requires get permission. - apiGroups: [""] diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index ab487477c1..d084ba49c0 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -2,7 +2,8 @@ {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- $supervisorNetwork := .Values.supervisor.network | default dict -}} {{- $additionalCaEnabled := ne (get $supervisorNetwork "additionalCaConfigMapName" | default "") "" -}} -{{- if and (eq $workspaceMode "shared") (include "openshell.rbacCreate" .) (or (include "openshell.workspaceResourcesEnabled" .) $additionalCaEnabled) }} +{{- $kubevirtRuntime := eq (include "openshell.kubernetesRuntime" .) "kubevirt" -}} +{{- if and (eq $workspaceMode "shared") (include "openshell.rbacCreate" .) (or (include "openshell.workspaceResourcesEnabled" .) $additionalCaEnabled $kubevirtRuntime) }} # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/tests/agent_sandbox_preflight_test.yaml b/deploy/helm/openshell/tests/agent_sandbox_preflight_test.yaml index 45ab2e76b9..0db07f44f3 100644 --- a/deploy/helm/openshell/tests/agent_sandbox_preflight_test.yaml +++ b/deploy/helm/openshell/tests/agent_sandbox_preflight_test.yaml @@ -10,3 +10,21 @@ tests: asserts: - failedTemplate: errorPattern: "Agent Sandbox is required but neither agents.x-k8s.io/v1beta1 nor agents.x-k8s.io/v1alpha1 is served" + + - it: does not block a KubeVirt-only install without the Agent Sandbox CRDs + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kubevirt + asserts: + - hasDocuments: + count: 0 + + - it: does not block a KubeVirt-only install with the kube-virt alias + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kube-virt + asserts: + - hasDocuments: + count: 0 diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 309b10fc0e..07312cefec 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -221,3 +221,59 @@ tests: - equal: path: metadata.name value: openshell-node-reader-my-namespace + + - it: grants the least-privilege KubeVirt rules in managed mode (cluster-wide) + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kubevirt + workspace_mode: managed + asserts: + - hasDocuments: + count: 1 + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["get", "list", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + # H4: never get/list on Secrets in a cluster-wide role. + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["list"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + + - it: grants no KubeVirt rules in shared mode + asserts: + - notContains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] diff --git a/deploy/helm/openshell/tests/rbac_test.yaml b/deploy/helm/openshell/tests/rbac_test.yaml index 3698465e27..646a47192e 100644 --- a/deploy/helm/openshell/tests/rbac_test.yaml +++ b/deploy/helm/openshell/tests/rbac_test.yaml @@ -93,3 +93,166 @@ tests: asserts: - hasDocuments: count: 2 + + - it: grants the least-privilege KubeVirt runtime rules when the kubevirt runtime is configured + template: templates/role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kubevirt + server: + sandboxNamespace: sandbox-ns + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: sandbox-ns + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["get", "list", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["create", "delete", "get", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "delete"] + # H4: bootstrap Secrets are immutable per generation — create and + # delete only, never get/list/patch. + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "delete"] + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "get", "delete"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["list"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["patch"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - notContains: + path: rules + content: + apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + + - it: grants the least-privilege KubeVirt rules with the kube-virt alias + template: templates/role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kube-virt + server: + sandboxNamespace: sandbox-ns + asserts: + - hasDocuments: + count: 1 + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] + + - it: emits the sandbox RoleBinding when the kubevirt runtime is configured + template: templates/rolebinding.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + runtime: kubevirt + server: + sandboxNamespace: sandbox-ns + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: sandbox-ns + - equal: + path: roleRef.name + value: openshell-sandbox + + - it: keeps the pod-path rules when the kubevirt runtime is not configured + template: templates/role.yaml + asserts: + - contains: + path: rules + content: + apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes", "sandboxes/status"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - notContains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + + - it: keeps the pod-path rules when the kubevirt runtime is not configured + template: templates/role.yaml + asserts: + - contains: + path: rules + content: + apiGroups: ["agents.x-k8s.io"] + resources: ["sandboxes", "sandboxes/status"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - notContains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachines"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 9e02467ab9..c8889a99b2 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1494,7 +1494,6 @@ sa_token_ttl_secs = 3600 # Optional SPIFFE Workload API socket mounted into sandbox pods for dynamic # provider token grants. Use an absolute path under a dedicated directory; # shared roots such as /run, /var, /tmp, and /etc are rejected. -# Supervisor-to-gateway auth still uses gateway JWTs. provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.sock" # Explicit sandbox UID/GID for the supervisor container securityContext and # PVC init container. When unset, the driver auto-detects from OpenShift SCC @@ -1511,6 +1510,45 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +# Runtime substrate for sandboxes (proof of concept). "pod" (default) +# provisions `agents.x-k8s.io` Sandbox CRs reconciled into pods by the +# upstream Agent Sandbox controller. "kubevirt" provisions `kubevirt.io/v1` +# VirtualMachine objects directly — the KubeVirt controller replaces the +# Agent Sandbox controller, and each sandbox boots a purpose-built guest +# image whose init stages the boundary and runs `openshell-sandbox`. The +# kubevirt runtime currently supports only `workspace_mode = "shared"` and +# only the dedicated supervisor topology: one supervisor Pod per sandbox +# (which requires `grpc_endpoint` to be reachable from those Pods); the +# gateway-shared supervisor is rejected. Every KubeVirt guest runs its +# workload as root and its in-guest boundary is untrusted: the security +# boundary is the NetworkPolicy fence, the supervisor's policy, and the +# driver's Kubernetes checks — guest configuration (the nftables relay, +# the DNS relay) is required for networking to work and is never relied +# on for security. Provider files and provider env vars are unsupported — +# nothing secret enters the guest. See `e2e/kubevirt/README.md` for the +# PoC workflow, guest-image build, and known gaps. +# runtime = "kubevirt" + +# guest_image is the containerDisk providing the bootable guest filesystem +# (Alpine + supervisor + kernel, built by `e2e/kubevirt/build-guest-image.sh`); +# it is NOT an arbitrary sandbox OCI image. cpu_cores and memory are the +# defaults when a sandbox requests no CPU/memory of its own: a sandbox's +# requests and limits map onto the VM (the effective CPU rounded up to +# whole vCPUs the guest sees, the effective memory as the guest's RAM, +# and the requests/limits on the virt-launcher container). GPU requests +# are rejected — use the pod runtime for GPU workloads. +# workspace_pvc attaches a per-sandbox PVC at /sandbox (virtio-fs), +# surviving stop/start. Guest egress is fenced by a per-sandbox +# NetworkPolicy to the sandbox's supervisor only — no DNS, no gateway: the +# boundary's in-guest DNS relay forwards through the supervisor's network +# mediation. No gateway token or endpoint reaches the guest. +# [openshell.drivers.kubernetes.kubevirt] +# guest_image = "openshell/sandbox-vm:dev" +# guest_image_pull_policy = "IfNotPresent" +# cpu_cores = 1 +# memory = "1Gi" +# workspace_pvc = true + [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = true gateway_namespace = "openshell" diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index d16938da01..d5a6d6e485 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -285,6 +285,8 @@ On Linux with a non-root gateway, the sandbox can create files in a read-write m The Kubernetes driver runs sandboxes as pods in a sandbox namespace. It requires the [Agent Sandbox](https://github.com/kubernetes-sigs/agent-sandbox) controller. Install the gateway with the Helm chart; see [Kubernetes setup](/kubernetes/setup). +The driver also has an experimental `runtime = "kubevirt"` mode that runs each sandbox as a KubeVirt virtual machine instead of a pod, with a per-sandbox egress fence and a purpose-built guest image. The sandbox API and security model are unchanged. See the [KubeVirt runtime settings](/how-it-works/gateways/configuration) and `e2e/kubevirt/README.md` in the source tree for the workflow, guest-image build, and current limits. + The cluster CNI must enforce `NetworkPolicy` in every sandbox namespace. Without it, sandbox pods can bypass OpenShell network policy. diff --git a/e2e/kubevirt/README.md b/e2e/kubevirt/README.md new file mode 100644 index 0000000000..fa72739aee --- /dev/null +++ b/e2e/kubevirt/README.md @@ -0,0 +1,551 @@ +# KubeVirt compute-driver proof of concept + +This directory contains the proof of concept for running OpenShell sandboxes as +**KubeVirt VirtualMachines** through the Kubernetes compute driver, instead of +`agents.x-k8s.io` Sandbox CRs reconciled into pods. It is built on current +`main`'s **isolation boundary architecture** (an in-workload +`openshell-sandbox` boundary plus its supervisor; see +`crates/openshell-sandbox-backend`) using the **Option A** boundary delivery: +the pod path's boundary artifacts ride the cloud-init channel into the guest. +Every KubeVirt guest runs its workload as **root**, and the guest — +including the in-guest `openshell-sandbox` boundary — is **untrusted**: the +security boundary is everything outside it (the NetworkPolicy fence, the +supervisor's policy, and the driver's Kubernetes checks). The runtime +supports one supervisor topology: **a dedicated supervisor Pod per sandbox** +(the gateway-shared supervisor is rejected for this runtime). + +What this branch demonstrates, on a local k3d cluster with nested KVM: + +1. `openshell sandbox create` reaches **Ready**: the driver creates the + `VirtualMachine` (Halted first, so its UID is captured for the claims), + the supervisor generation — a dedicated supervisor Pod with its + per-sandbox Service and bootstrap Secret — the egress-fence + NetworkPolicy, the bootstrap Secret, and the workspace PVC; the guest + boots, its boundary dials the supervisor over the runtime connection, + and the supervisor session establishes. +2. **Exec in the VM** as root; the workload environment carries no gateway + material (no token, no endpoint). +3. **L4/L7 network policy** enforced by the supervisor: allowed and + denied raw-TCP connects, and HTTPS GET allowed / POST denied through the + L7 proxy — the GET proves the per-sandbox proxy CA is delivered into the + guest. +4. **Egress fencing**: the virt-launcher pod's NetworkPolicy admits only + the supervisor — there is no DNS allowance: direct DNS to the + kube-dns Service or the CoreDNS pods is blocked, and the guest's + resolver is the boundary's loopback relay (`127.0.0.53`), which + answers policy-approved names synthetically (non-policy names never + reach an upstream resolver). Direct egress from the guest is blocked + by the pod-network fence; the in-guest nftables relay steers traffic + through the supervisor's mediation so policy applies — it is functional + routing, not a security control. +5. **Stop/start**: `Halted`/`Once` run-strategy mapping with the VM UID and + fence UID stable across generations; start re-renders the boundary + bootstrap for the new generation and provisions a fresh supervisor Pod + whose new UID the re-rendered bootstrap claims. +6. **Fence verification** on every start: the expected fence (UID + rendered + spec) is persisted on the VM; a spec edit fails start with a drift + Precondition, a label-only edit is accepted. +7. A guest whose runtime exits powers off (no kernel panic, no hung QEMU): + the VMI completes and the driver maps it to the terminal `VMFailed` + condition; a main-process exit surfaces `Error` with the exit code. +8. `sandbox delete` removes the VM, companions, and fence with zero + leftovers. + +Everything here is explicitly **proof-of-concept quality**: single shared +workspace mode, no GPU, no migration handling, and no support for arbitrary +OCI sandbox images as VM disks (the guest image is purpose-built). See +[Known gaps](#known-gaps--follow-ups). + +## How it works + +``` +┌──────────────┐ kube-rs (kubeconfig) ┌─────────────────────────┐ +│ openshell- │◀═══ assignment stream ═══▶│ kvpoc k3d cluster │ +│ gateway │ (TLS, supervisor token) │ │ +│ (host, TLS) │ │ kubevirt.io/v1 │ +│ │ kube-rs │ VirtualMachine │ +└──────┬───────┘──────────────────────────▶└─▶ VMI ─▶ virt-launcher + │ pod ─▶ QEMU/KVM guest + │ routed runtime connection (TLS) (Alpine + openshell- + │ ┌──────────────────────┐ sandbox boundary) + ╚════════▶│ supervisor │ ▲ + │ (dedicated: per- │ │ fence NetworkPolicy: + │ sandbox Pod+Service │ │ launcher egress → only + │ in kvpoc-sandboxes; │ │ this sandbox's supervisor + │ gateway: Helm-shared │ │ + │ Deployment+Service) │ │ + └──────────────────────┘ │ + │ + VM volumes: │ + rootdisk containerDisk (boot ext4 + kernel/initrd) + cloudinit bootstrap Secret (cloudInitNoCloud ISO) + workspace PVC (virtio-fs, mounted at /sandbox) +``` + +- **Driver change** (`crates/openshell-driver-kubernetes`): a + `runtime = "kubevirt"` mode in `[openshell.drivers.kubernetes]` that + provisions one dedicated supervisor Pod per sandbox (reusing the pod + path's `supervisor_pod`, `boundary_service`, + `supervisor_bootstrap_secret`, and control-policy renderers with + `VirtualMachine` owner references); the gateway-shared supervisor + topology is rejected for this runtime. The driver creates + `VirtualMachine` objects directly (the KubeVirt controller replaces the + Agent Sandbox controller) and maps + get/list/watch/stop/start/delete onto VM/VMI state. See `src/kubevirt.rs`, + `src/kubevirt_boundary.rs`, and the branches in `src/driver.rs`. +- **Boundary delivery (Option A)**: the driver renders the pod path's + boundary artifacts (`boundary.json` + session TLS, from the same + `render_boundary_bootstrap` shared helper) into a shell-sourceable + cloud-init `userData` (base64 lines, no gateway material). The guest init + stages them on a tmpfs and runs + `openshell-sandbox launch-privileged `; the boundary dials the + per-sandbox `os-supervisor-` Service of its dedicated generation. +- **Guest image** (`build-guest-image.sh`): Alpine + static-musl + `openshell-sandbox` + the init script, booted via direct kernel boot. The + guest's resolver is the boundary's loopback DNS relay (`127.0.0.53`). +- **Readiness** stays gateway-owned: the driver reports VM/VMI conditions + only; the sandbox becomes `Ready` when the supervisor session connects. + +## Files + +| File | Purpose | +|---|---| +| `build-guest-image.sh` | Builds the `kvpoc/openshell-sandbox-vm:dev` containerDisk (Alpine rootfs + static musl supervisor + guest init + kernel, repacked initramfs, KubeVirt's `container-disk` helper). | + +| `run-probes.sh` | Root-workload attack-surface probes (run via `demo.sh --probes`): in-guest reset via the reboot syscall and the iTCO watchdog, a fake qemu-guest-agent over the virtio-serial channel, virtiofsd privileged-operation surface (chown past the idmap, mknod, setuid), and the launcher-netns listener audit. Outcomes are recorded as findings for the RFC; only posture invariants (the fake agent must not affect the sandbox; a reset must not pass unnoticed) are asserted. | +| `port-io.c` | Static-musl port-I/O helper for the P2 probe — compiled at probe time in a docker alpine container and staged into the guest through the exec channel; it does NOT ship in the product guest image. `/dev/port` is byte-granular and QEMU drops byte writes to dword/word emulated registers (PCI config address, TCO counters), so the probe arms the iTCO watchdog with real `OUT` instructions through this helper. | + +| `guest-init.sh` | Guest PID-1 init (baked into the image). | +| `initramfs-init.sh` | Boot-stage initramfs init: mounts the read-only containerDisk root, stacks the writable tmpfs overlay, bind-pins the guest init + boundary binary from the image, and switch_roots. | +| `demo.sh` | Starts a host gateway (TLS) against the `kvpoc` cluster and registers it as the CLI's mTLS gateway; `--demo` then runs `run-demo.sh` against it. | +| `run-demo.sh` | The full acceptance demo (10 steps, no failure tolerance): create → Ready, fence check, exec, env scrub, L4/L7 policy, bypass blocked, stop/start with stable VM/fence UIDs (a rotated and re-claimed supervisor Pod UID), fence drift verification, main-process exit → Error, delete with zero leftovers. | +| `demo-policy.yaml` | The acceptance policy: L4 raw-TCP allow/deny and L7 HTTPS GET-allow/POST-deny, network rules only (filesystem/landlock/process sections and `binaries` lists are rejected for the untrusted-guest class — unenforceable in-guest). **Relay deny semantics**: the transparent relay completes the TCP handshake locally before the supervisor's decision, so handshake-only tools (`nc -z`, `curl --connect-timeout` without a transfer) cannot observe a relay denial — the connection is accepted, then reset on first read/write. The demo uses data-level checks (wget) for deny assertions. This is not an enforcement gap: zero upstream connections are made for denied destinations (verified with host-side listeners). | + +## Reproducing + +Prerequisites: a Linux host with KVM (`/dev/kvm`), docker, k3d, kubectl, +helm, and this repo built (`cargo build --release -p openshell-gateway -p +openshell-cli`; the guest image build compiles the supervisor itself as +static musl). + +E2e prerequisites (discovered the hard way): + +- `OPENSHELL_BIN=$CARGO_TARGET_DIR/debug/openshell` — the sandbox_lifecycle + tests spawn the CLI via this env var (default is `target/debug/openshell` + relative to the repo, which misses when `CARGO_TARGET_DIR` is redirected) +- `util-linux-script` installed — the TTY tests run the CLI under a PTY via + `/usr/bin/script`; on minimal Fedora installs this is a separate subpackage +- `docker pull public.ecr.aws/docker/library/python:3.13-slim` — the + `driver_config_volume` suite builds an OCI test image from this base + +```shell +# 1. Cluster: k3d with /dev/kvm passed to the node and working DNS. +# (KubeVirt >= 1.9's containerDisk flow needs the Kubernetes ImageVolume +# feature, so use k8s >= 1.33 — k3s v1.35.x is known good. The static +# resolv.conf works around docker handing the k3d node an unreachable +# stub resolver on hosts that run systemd-resolved.) +printf 'nameserver 10.43.0.10\nnameserver 1.1.1.1\noptions ndots:0\n' > /tmp/kvpoc-resolv.conf +K3D_FIX_DNS=false k3d cluster create kvpoc \ + --image rancher/k3s:v1.35.9-k3s1 \ + --volume /dev/kvm:/dev/kvm@server:0 \ + --volume /tmp/kvpoc-resolv.conf:/etc/resolv.conf@server:0 \ + --k3s-arg "--disable=traefik@server:0" + +# If virt-handler crash-loops with "Failed to create an inotify watcher: +# too many open files", raise the host limit: +# sudo sysctl -w fs.inotify.max_user_instances=1024 + +# 2. KubeVirt (v1.9.1 known good) with virtio-fs PVC sharing enabled. +kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/v1.9.1/kubevirt-operator.yaml +kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/v1.9.1/kubevirt-cr.yaml +kubectl patch kubevirt kubevirt -n kubevirt --type merge \ + -p '{"spec":{"configuration":{"developerConfiguration":{"featureGates":["EnableVirtioFsStorageVolumes"]}}}}' +kubectl wait --for=condition=Available deployment/virt-api -n kubevirt --timeout=600s + +# 3. Guest image, then the shared supervisor image (both from this branch). +e2e/kubevirt/build-guest-image.sh +k3d image import kvpoc/openshell-sandbox-vm:dev -c kvpoc +mise run build:docker:supervisor # builds openshell/supervisor:dev (podman) +# import it into the cluster's docker+k3d (docker load from a podman save): +podman save openshell/supervisor:dev -o /tmp/supervisor.tar +docker load -i /tmp/supervisor.tar +docker tag localhost/openshell/supervisor:dev kvpoc/openshell-supervisor:dev +k3d image import kvpoc/openshell-supervisor:dev -c kvpoc + +# 4. Run the demo (builds the gateway + CLI, starts the gateway, runs the +# full acceptance, tears the sandbox down). The supervisor topology is +# dedicated (one Pod per sandbox, created by the driver from the locally +# imported kvpoc/openshell-supervisor:dev image); no cluster-side +# supervisor is needed. +e2e/kubevirt/demo.sh --demo + +# 5b. (Optional) prove the chart's RBAC is sufficient: render the chart's +# ServiceAccount and Roles for the kubevirt runtime, mint a kubeconfig +# from the gateway ServiceAccount token, and run the gateway on it +# (the demo's own kubectl scaffolding keeps the ambient KUBECONFIG): +# The serviceaccount template renders the gateway SA first (namespace-less) +# and then the sandbox SA (pinned to the sandbox namespace, which -n +# openshell would reject — the demos do not need it): take the first +# document only. +helm template kvpoc deploy/helm/openshell -n openshell \ + --set agentSandbox.preflight.enabled=false \ + --set gatewayConfig."openshell\.drivers\.kubernetes"."runtime"=kubevirt \ + --set server.sandboxNamespace=kvpoc-sandboxes \ + -s templates/serviceaccount.yaml \ + | sed -n '1,/^---$/p' \ + | kubectl apply -n openshell -f - +helm template kvpoc deploy/helm/openshell -n openshell \ + --set agentSandbox.preflight.enabled=false \ + --set gatewayConfig."openshell\.drivers\.kubernetes"."runtime"=kubevirt \ + --set server.sandboxNamespace=kvpoc-sandboxes \ + -s templates/role.yaml -s templates/rolebinding.yaml \ + -s templates/clusterrole.yaml -s templates/clusterrolebinding.yaml \ + -s templates/peer-role.yaml \ + | kubectl apply -f - +SA=kvpoc-openshell # match the chart's gateway ServiceAccount name (release-openshell) +TOKEN="$(kubectl create token "${SA}" -n openshell --duration=2h)" +SERVER="$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.server}')" +CA="$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')" +cat > /tmp/kvpoc-sa-kubeconfig <-boot- +persistentvolumeclaim/default--kvpoc-demo-workspace Bound 2Gi RWO local-path +pod/os-supervisor-50f1d5b4-... 1/1 Running +supervisor Pod UID matches the kubevirt.supervisor_pod_uid claim (e7c477ac-...) +=== 2/10 fence NetworkPolicy and its podSelector === +fence: openshell-kubevirt-fence-50f1d5b4-... (uid c411e924-...) +exactly one virt-launcher pod matches the fence podSelector +fence egress selects exactly this sandbox's supervisor Pod +=== 3/10 exec inside the VM === +Linux (none) 6.12.117-0-virt #1-Alpine SMP PREEMPT_DYNAMIC x86_64 Linux +uid=0(root) gid=0(root) +init +workload runs as root (uid 0) — the untrusted boundary relays for it +workspace writable by the workload (idmapped virtiofs) +=== 4/10 no gateway material in the workload environment === +environment clean (no BOUNDARY/PRIVATE/TOKEN/OPENSHELL_ENDPOINT vars) +=== 5/10 L4 allow/deny and L7 HTTPS GET allow / POST deny === +the nftables transparent relay carries these checks (workload TCP +is captured to the boundary relay, policy-enforced at the supervisor; +deleting the rules restores direct egress, which the fence blocks — +fail-closed). +L4 ALLOW: connected +L4 DENY: connect to example.org:80 blocked (OCSF DENIED confirmed) +...(Example Domain page over HTTPS)... +L7 GET over HTTPS allowed (per-sandbox proxy CA trusted in the guest) +L7 POST denied (HTTP 403) +=== 6/10 direct egress from the guest is blocked === +relay rules deleted by root; direct egress must be fenced: +direct egress and raw DNS blocked by the fence (fail-closed) +fence check: launcher pod -> 1.1.1.1:443 refused (Connection refused) +fence check: launcher pod -> supervisor 10.43.191.239:5500 reachable (selective) +=== 7/10 stop -> start -> Ready with VM and fence UIDs unchanged === +✓ Stopped sandbox kvpoc-demo +supervisor Pod deleted by stop +✓ Started sandbox kvpoc-demo +VM uid unchanged (84861a38-...); fence uid unchanged (c411e924-...) +workspace data survived stop/start and stays writable +supervisor Pod UID rotated and re-claimed (81c3987e-...) +one current-generation instance Secret (previous generation pruned) +=== 8/10 fence verification: spec drift fails start, label-only edit passes === +✓ Stopped sandbox kvpoc-demo +spec drift rejected: ... spec drifted from the rendered shape (podSelector, policyTypes, or egress rules changed) +M1 reconciler re-fenced the drifted fence (spec restored) +✓ Started sandbox kvpoc-demo +label-only fence edit accepted (resourceVersion moved, spec unchanged) +=== 9/10 main-process exit -> Error === + Last exit code: 137 +main process exit observed through the boundary session (Error, exit code 137) +=== 10/10 delete -> zero leftover managed objects === +✓ Sandbox kvpoc-demo deletion accepted; cleanup is pending +supervisor Pod gone after delete +zero leftover vm/vmi/pod/svc/secret/pvc/netpol for sandbox 50f1d5b4-... +demo complete (dedicated supervisor topology, root workload): Ready, +exec as root, clean environment, L4/L7 policy through the transparent +relay, bypass blocked, stop/start, fence verification, dead-runtime +Error, and clean teardown all verified +``` + +Stop semantics: stop waits for the VM's ready flag to clear; the VMI and +virt-launcher pod can take another ~30 seconds to terminate, during which +the fence (owner-referenced to the VM) still applies. + +Boundary-port ingress (L2): the control-ingress policy +(`os-control-`) selects only the pair's supervisor Pod, so no pod — +forged labels or not — can reach the virt-launcher on the boundary port; +the fence's deny-all ingress is the whole story for the launcher. +Residual, inherent to `NetworkPolicy` label selection: a pod that forges +the pair's workload labels can still REACH the supervisor's boundary +port at the network level. The actual boundary is the supervisor's +pinned TLS plus the descriptor/claims check (a forged pod cannot +authenticate), and namespace RBAC — the gateway's `ServiceAccount` should +be the only identity allowed to create pods in the sandbox namespace +(the chart's RBAC delivers the gateway side; step 5b proves it). + +Start semantics: the plan is decided from the run strategy, the VM status, +and the paired VMI's phase. A halted VM boots with a single `Once` patch +(`PatchOnce`). A live or booting VMI makes start idempotent +(`AlreadyRunning`). A terminal VMI under `Once` is bounced through +`Halted` first. The create holds the VM `Halted` while it captures the UID +and builds the companions (fence, Secret, instance registration) — during +that window the VM carries the `openshell.ai/kubevirt-create-in-flight` +annotation so the watch reports `Bootstrapping` (Provisioning) rather than +`Suspended` (Stopped, which would withdraw the just-registered instance). + +### The demo policy and the busybox binary-identity gotcha + +`demo-policy.yaml` names `/usr/bin/busybox` in its `binaries` lists, not +`/usr/bin/nc` or `/usr/bin/wget`: busybox applets all execute as +`/usr/bin/busybox`, so a binary-scoped rule must name the resolved +executable path, and naming busybox grants every applet the file-system +policy allows to run. This is the KubeVirt guest image's version of the +binary-identity caveat from the first review. + +Run `e2e/kubevirt/demo.sh` (without `--demo`) to keep the gateway in the +foreground and drive the CLI by hand; it registers gateway `kvpoc` +(`https://127.0.0.1:18085`, mTLS) as the CLI's active gateway. + +### Inspecting a running sandbox + +```shell +kubectl get vm,vmi,pod,secret,pvc -n kvpoc-sandboxes +# serial console (guest boot + boundary logs): +kubectl exec -n kvpoc-sandboxes $(kubectl get pod -l kubevirt.io=virt-launcher \ + -n kvpoc-sandboxes -o jsonpath='{.items[0].metadata.name}') -c compute -- \ + cat /var/run/kubevirt-private/$(kubectl get vmi default--kvpoc-demo \ + -n kvpoc-sandboxes -o jsonpath='{.metadata.uid}')/virt-serial0-log | tail -30 +``` + +## Teardown + +```shell +# remove sandbox resources (if any) +kubectl delete vm,secret,pvc,netpol -n kvpoc-sandboxes --all +# remove the shared supervisor +kubectl delete namespace openshell +# remove the whole cluster +k3d cluster delete kvpoc +``` + +The demo gateway state lives in `.cache/gateway-kubevirt/` (sqlite DB + TLS +material); delete it to fully reset. + +## Known gaps / follow-ups + +- **Root-workload trust model**: the guest is untrusted as soon as an + agent/sandbox workload executes for the first time. Everything that + matters is enforced outside the guest: the VM boundary, the egress + fence (NetworkPolicy), the supervisor's policy, and guest-reported + data marked `data_origin=guest` in OCSF. Guest configuration — the + nftables relay, the seccomp broker's steering, the config-disk + eject — is required for networking to function and is never relied + on for security: root in the guest can change all of it. Nothing + in-guest is a trust anchor — root in the guest can + change anything, including the base, kernel, and bootloader (writes + land in the RAM overlay; the containerDisk base is read-only at the + device level, and every boot starts fresh from the image). There is + deliberately no dm-verity / verified boot: the in-guest integrity of + an untrusted guest has no security value under this model. +- **What the outside boundary relies on**: the fence is only as good + as the CNI's NetworkPolicy enforcement (k3d's built-in flannel + + policy layer here), and the boundary channel is + only as isolated as the dedicated supervisor topology — root mode + requires it precisely so a hostile guest cannot reach another + tenant's supervisor. The P1 boundary-handle check (a fresh boundary + attaching to a live generation ends it) and the R3 halt of an ended + root VM are the controls for a guest that resets in place (including + a kexec'd custom kernel): the reset ends the generation, the VM is + halted, and the re-read bootstrap material is useless for an ended + generation. +- **The bootstrap config disk is read once, then ejected (hygiene, + not a control)**: the cloud-init disk (`/dev/vdb`) carries the full + boundary bootstrap, including the boundary's private key and launch + authentication. The guest init reads it exactly once and removes the + PCI function backing it. **Root can undo the eject**: writing 1 to + `/sys/bus/pci/rescan` re-enumerates the disk (QEMU/KubeVirt still + have it attached — cloudInitNoCloud volumes cannot be hot-unplugged + in KubeVirt today), and the user-data is readable again. The + eject is therefore hygiene (the disk is not sitting attached and + trivially readable for the whole session), not a control: the key's + exposure to root is the same as through the boundary's own memory + (ptrace), and the blast radius is its own sandbox. The L1 probe + asserts the eject and then demonstrates the rescan limit. A guest + reset re-attaches the device too, but the pristine boot re-reads it + only within the boot flow (before any workload code runs), and the + fresh boundary attaching to the live generation ends it (P1) and the + VM halts (R3). Next step (not built here): deliver the bootstrap + through a host-side one-shot channel — e.g. a hotpluggable data + volume the driver detaches from the VM once the boundary is Ready, + or an enrollment flow over the supervisor channel — so no + guest-side eject is needed. +- **Guest console logs are guest output, never authoritative**: lines + root writes to `/dev/ttyS0` appear verbatim in the launcher pod's + `guest-console-log` container. Console output is spoofable by the + guest; it is diagnostic only. The authoritative record of sandbox + behavior is the supervisor's OCSF event stream (with guest-reported + events marked `data_origin=guest`). +- **Surfaces exposed to the guest** (probed in `run-probes.sh`, inputs + for the RFC): the iTCO watchdog (root can arm and fire it; the + resulting reset is covered by reset detection + the halt), the QEMU + guest-agent virtio-serial channel (attached unconditionally; a + host-driven parser exploit is the upstream risk — the disable ask is + filed), masquerade DHCP (udp 67 in the launcher netns, + guest-reachable), and virtiofsd (root can mount the share without + the idmap; chown/mknod are refused and setuid grants no privilege). +- **The root disk**: a plain writable guest root. The containerDisk + carries the pristine base (read-only device); the initramfs stacks a + tmpfs overlay, so the workload can write anywhere in `/` at runtime. + Across stop and start the root is **wiped** — every boot re-reads the + image and starts fresh; only the workspace PVC (`/mnt/workspace`, + per-sandbox) persists. This is the recommended choice: the guest is + untrusted after the first workload executes, so persisted guest state + has no security value, and stale mutated state would poison later + boots of the same sandbox (a rebooting root workload would inherit + its own tampering). The workspace PVC itself is persistent, + root-writable input to every later session of the sandbox — treat + anything read from it as untrusted (it is attacker-controlled + storage). +- **Image identity (digest pinning, option A)**: the first boot uses the + configured `guest_image` tag (the digest does not exist until something + pulls it — the documented first-generation window). Once the launcher + Pod has pulled the guest image, the driver pins the VM spec's + containerDisk and kernelBoot images to the resolved + `repo@sha256:` on its 30-second reconcile tick, so + starts after a tag move boot and claim the same digest, and a new + create after the retag gets the new image. Side-loaded images (the + k3d/ctr import path this demo uses) report a bare `sha256:` config + digest with no repository, which is not pullable: those are never + pinned — the tag stays and the sandbox surfaces an + `ImagePin: False (NoRepositoryDigest)` condition. Registry-backed + guest images (pulled from a real registry) pin cleanly. +- An unpullable guest image maps the sandbox to `Error` only after two + minutes of pull backoff (`ImagePullBackOff` with a stale VMI + not-ready condition): kubelet retries with exponential backoff, so an + earlier verdict would flip slow-but-working registry pulls to `Error`. + Below the threshold the state stays transient. +- The dedicated topology creates the namespace-wide supervisor egress + policy (`openshell-sandbox-supervisors`, the supervisor half of the pod + path's workload fence) once per namespace; it is shared infrastructure + and is deliberately not deleted with any one sandbox. The per-sandbox + virt-launcher fence plays the workload role, so the namespace-wide + workload policy is not created for KubeVirt sandboxes. +- Single (shared) workspace mode; managed/operator modes are rejected. +- The chart's sandbox Role (and the managed-mode ClusterRole) now carry + the least-privilege KubeVirt rules: Secrets create/delete only (the + bootstrap Secret is immutable per generation — `-boot-` + — and its exact name and UID are recorded on the VM, so the driver + never reads, lists, or patches a Secret), no listing of pods, Services, + PVCs, or NetworkPolicies (companions are derived from the VM's name and + recorded generation), and the fence reconciler restores drift with + delete-and-recreate. The PoC gateway itself still runs on the host with + the cluster-admin kubeconfig; step 5b proves the chart's RBAC is + sufficient. +- Networking: masquerade pod network only; the guest clamps its MTU to 1400 + in `guest-init.sh` because the k3d flannel-vxlan pod network is 1450. +- No GPU passthrough (a GPU request is rejected at admission with a clear + message — use the pod runtime for GPU workloads), no + live-migration/VMI-replacement semantics. +- The sandbox's CPU and memory requests and limits map onto the VM: the + effective CPU (limit, else request, else `cpu_cores`) rounded up to + whole vCPUs the guest sees, the effective memory as the guest's RAM, + and the requests/limits verbatim on the virt-launcher container (with + the pod path's mirroring of an omitted request to the limit). Resource + admission runs exactly as in the pod path, and a limit below its + request or memory below the safe guest minimum (256Mi) is rejected at + admission. +- A create interrupted between the Halted VM creation and its `Once` patch + leaves the VM with the in-flight annotation; after ten minutes the driver + maps it to a terminal create timeout (public `Error`), so it is visible + and stoppable even when the gateway was restarted from a clean database + and its own provisioning-deadline reclaim has no record to act on (the + stop clears the annotation). +- Upstream context: kubernetes-sigs/agent-sandbox#1157 proposes a + `runtimeBackend: VirtualMachine` Sandbox CR extension (unmerged POC fork). + If that lands, the driver could switch from direct VM creation back to the + Sandbox CR with a VM workload. + +## Claims model (option A, user decision) + +The KubeVirt boundary claims model follows the VM driver's approach: + +- **Claims are a consistency check, not the anchor:** the boundary's + confirmation claims (the VM UID, the fence UID and resourceVersion, + the supervisor Pod UID) bind the guest's session to the objects the + driver created, so a mismatch is detected. They are reported by the + untrusted guest, so they can never *prove* anything on their own: the + anchor is the driver's own Kubernetes reads — the VM, fence, and + supervisor objects the driver get/list/patches through the API — and + the fence the CNI enforces at the launcher pod. A guest that lies + about a claim fails the consistency check (and is rejected); a guest + that tells the truth gains nothing. +- **Claimed:** `kubevirt.vm_uid` (the VirtualMachine's real UID — the + create builds the VM `Halted` first, captures its UID, and only then + renders the boundary claims), `kubevirt.generation`, and + `kubevirt.image_identity`. The dedicated topology additionally claims + `kubevirt.supervisor_pod_uid` — the dedicated supervisor Pod's UID, + mirroring the pod path's `kubernetes.supervisor_pod_uid` and re-bound on + every start (each generation gets a fresh Pod). No control-policy UID is + claimed in either topology. +- **Evidence:** the virt-launcher egress NetworkPolicy's UID and + resourceVersion, projected against the generation via + `ProvisioningEvidence::from_enforcement_evidence`. The expected fence + (UID + rendered spec) is persisted on the VM at create; every start + re-reads the NetworkPolicy and fails when the fence was deleted, + replaced (UID changed), or its spec drifted. A resourceVersion-only + change is accepted and re-captured. +- **Why the virt-launcher pod UID cannot be claimed:** KubeVirt creates + that pod asynchronously from the VM template, so its UID does not exist + when the boundary config is rendered. + +Who verifies the fence: + +- the driver attests it by re-reading the NetworkPolicy each generation + (`verify_fence` / `check_fence`), and the watch re-verifies every running + sandbox on a 30-second interval, re-creating a drifted, replaced, or + deleted fence from the persisted expectation and surfacing a + `FenceRestored` condition; +- a namespace-wide default-deny NetworkPolicy + (`openshell-kubevirt-launcher-isolation`) selects every virt-launcher pod + in the sandbox namespace, so isolation never depends on a per-sandbox + object's lifecycle — the per-sandbox fence and control policy only add + allowances on top of that floor; +- inherent limitation: Kubernetes NetworkPolicy is additive, so an + out-of-band policy that selects the launcher pod and allows more egress + reopens egress, and a start still passes (the verification checks this + driver's fence, not every policy in the namespace). The default-deny + floor and the reconciler narrow the window to deliberate action; policy + governance over the namespace is the production answer; +- because the expected fence spec is re-rendered from the *current* + supervisor Service selector and port, a Helm upgrade that changes the + supervisor's labels or port makes every existing KubeVirt sandbox fail + start with a spec-drift Precondition error — fail-closed by design, so + stop and restart sandboxes after such an upgrade; +- the boundary checks the evidence's generation binding + (`contract.rs:254`); +- descriptor/config consistency is enforced at + `isolation_protocol.rs:100` and on confirmation (`runtime.rs:757`); +- neither the boundary nor the supervisor can independently query the + cluster. diff --git a/e2e/kubevirt/build-guest-image.sh b/e2e/kubevirt/build-guest-image.sh new file mode 100755 index 0000000000..b76f575644 --- /dev/null +++ b/e2e/kubevirt/build-guest-image.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build the KubeVirt guest containerDisk for the OpenShell kubevirt runtime +# mode PoC (see e2e/kubevirt/README.md). +# +# Produces a local image `kvpoc/openshell-sandbox-vm:dev` containing: +# /disk/disk.qcow2 bootable ext4 rootfs (Alpine + openshell-sandbox +# supervisor + the guest init from guest-init.sh) +# /boot/vmlinuz-virt guest kernel (direct kernel boot) +# /boot/initramfs-virt guest initramfs: the boot init +# (initramfs-init.sh) mounts the containerDisk +# rootfs read-only, stacks a writable tmpfs +# overlay, and switch_roots. The guest root is +# writable at runtime and fresh from the image on +# every boot — the guest is untrusted as soon as a +# workload executes, so nothing in-guest is a +# trust anchor. +# +# Usage: e2e/kubevirt/build-guest-image.sh [image-tag] +# (then: k3d image import -c kvpoc) + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +IMAGE_TAG="${1:-kvpoc/openshell-sandbox-vm:dev}" +WORKDIR="$(mktemp -d /tmp/kvpoc-guest.XXXXXX)" +ROOTFS="${WORKDIR}/rootfs" +CARGO_TARGET="${CARGO_TARGET_DIR:-${REPO_ROOT}/target}" +# The guest is Alpine (musl): the supervisor must be the fully static musl +# build (same requirement as the published supervisor image, see +# .github/workflows/rust-native-build.yml). +SUPERVISOR="${CARGO_TARGET}/x86_64-unknown-linux-musl/release/openshell-sandbox" + +KUBEVIRT_VERSION="${KUBEVIRT_VERSION:-v1.9.1}" + +command -v qemu-img >/dev/null || { echo "qemu-img is required" >&2; exit 1; } +# The build stages ~360 MB under WORKDIR; always remove it on exit unless +# KEEP_WORKDIR is set for debugging. +cleanup() { + if [ -z "${KEEP_WORKDIR:-}" ] && [ -n "${WORKDIR:-}" ] && [ -d "${WORKDIR}" ]; then + rm -rf "${WORKDIR}" 2>/dev/null || true + fi +} +trap cleanup EXIT +# Always invoke cargo build: it is incremental, and a bare [ -x ] check +# silently ships a stale binary whenever a shared CARGO_TARGET_DIR already +# holds one from another checkout (which is exactly how the old-branch +# supervisor ended up in an image built from current main). +(cd "${REPO_ROOT}" && cargo build --release -p openshell-sandbox --target x86_64-unknown-linux-musl) +echo "==> guest supervisor: $("${SUPERVISOR}" --version)" + +echo "==> workdir ${WORKDIR}" + +# 1. Assemble the Alpine rootfs in a container. All shell setup runs in the +# initial `docker run`; the container exits when done and `docker cp` +# stages the host-built binaries into the stopped container's filesystem. +docker rm -f kvpoc-guest-rootfs >/dev/null 2>&1 || true +docker run --name kvpoc-guest-rootfs alpine:3.22 sh -c ' + set -eu + apk add --no-cache linux-virt e2fsprogs iproute2 nftables iptables bash >/dev/null + mkdir -p /opt/openshell /sandbox-seed /sandbox /usr/local/bin + chown 10001:10001 /sandbox /sandbox-seed + grep -q "^sandbox:" /etc/passwd || + printf "sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/sh\n" >> /etc/passwd + grep -q "^sandbox:" /etc/group || + printf "sandbox:x:10001:\n" >> /etc/group +' +docker cp "${REPO_ROOT}/e2e/kubevirt/guest-init.sh" kvpoc-guest-rootfs:/opt/openshell/init +docker cp "${SUPERVISOR}" kvpoc-guest-rootfs:/usr/local/bin/openshell-sandbox +docker export kvpoc-guest-rootfs -o "${WORKDIR}/rootfs.tar" +docker rm kvpoc-guest-rootfs >/dev/null + +cp "${REPO_ROOT}/e2e/kubevirt/idmap-mount.c" "${WORKDIR}/idmap-mount.c" +cp "${REPO_ROOT}/e2e/kubevirt/initramfs-init.sh" "${WORKDIR}/initramfs-init.sh" + +# 2. Unpack the exported rootfs, usr-merge it, render the ext4 disk, and +# repack the initramfs — all as root inside a container. An unprivileged +# `tar -xpf` cannot chown, so the rootfs (and with it the ext4 image +# built by `mkfs.ext4 -d`) would be owned by the build uid instead of +# root; the container keeps every filesystem step ownership-correct +# without requiring root on the host. +docker run --rm -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ + -v "${WORKDIR}:/work:Z" alpine:3.22 sh -c ' + set -eu + apk add --no-cache e2fsprogs gcc musl-dev linux-headers >/dev/null + + mkdir -p /work/rootfs + tar -xpf /work/rootfs.tar -C /work/rootfs + + # The idmapped-mount helper (static musl): remounts the virtiofs + # workspace share with an idmap so the workload uid can write through + # the unprivileged virtiofsd. + cc -static -O2 -o /work/rootfs/usr/local/bin/idmap-mount /work/idmap-mount.c + chmod 0755 /work/rootfs/usr/local/bin/openshell-sandbox /work/rootfs/opt/openshell/init + + # Usr-merge /bin and /sbin into /usr: the supervisor default Landlock + # policy allows exec under /usr (glibc images are usr-merged; Alpine is + # not, so busybox in /bin would be denied with EACCES). + merge_into() { + local src="$1" dest="$2" + mkdir -p "${dest}" + cp -a "${src}/." "${dest}/" + rm -rf "${src}" + ln -s "usr/$(basename "${dest}")" "${src}" + } + merge_into /work/rootfs/bin /work/rootfs/usr/bin + merge_into /work/rootfs/sbin /work/rootfs/usr/sbin + + # Render the ext4 rootfs disk (700 MiB: Alpine base + kernel + + # supervisor with headroom for runtime). + mkfs.ext4 -F -q -d /work/rootfs /work/disk.raw 700m + + # The boot initramfs: our init (mount + overlay + switch_root). The + # stock Alpine initramfs is unpacked, its init replaced, and repacked. + mkdir -p /work/initrd + cd /work/initrd + zcat /work/rootfs/boot/initramfs-virt | cpio -idm --quiet + cp /work/initramfs-init.sh init + chmod 0755 init + find . | cpio -o -H newc --quiet | gzip -9 > /work/initramfs-virt + + # Hand the workdir back to the invoking user: the docker build context + # and the cleanup trap run unprivileged on the host. + chown -R "${HOST_UID}:${HOST_GID}" /work +' + +# 3. Compress to qcow2 and stage the kernel artifacts. +qemu-img convert -c -O qcow2 "${WORKDIR}/disk.raw" "${WORKDIR}/disk.qcow2" +cp "${ROOTFS}/boot/vmlinuz-virt" "${WORKDIR}/vmlinuz-virt" + +# 5. Assemble the containerDisk image. KubeVirt >= 1.9 runs each +# containerDisk image as a virt-launcher init container executing +# /container-disk-binary/usr/bin/container-disk, so the helper binary +# (taken from the matching virt-launcher image) must ship inside the +# containerDisk image itself. +docker pull "quay.io/kubevirt/virt-launcher:${KUBEVIRT_VERSION}" >/dev/null 2>&1 || true +docker rm -f kvpoc-vl-ref >/dev/null 2>&1 || true +docker create --name kvpoc-vl-ref "quay.io/kubevirt/virt-launcher:${KUBEVIRT_VERSION}" >/dev/null +docker cp kvpoc-vl-ref:/usr/bin/container-disk "${WORKDIR}/container-disk" +docker rm kvpoc-vl-ref >/dev/null + +cat > "${WORKDIR}/Dockerfile" < built ${IMAGE_TAG}" +docker image inspect "${IMAGE_TAG}" --format 'size: {{.Size}} bytes' +echo "==> import into the cluster with: k3d image import ${IMAGE_TAG} -c kvpoc" diff --git a/e2e/kubevirt/demo-policy.yaml b/e2e/kubevirt/demo-policy.yaml new file mode 100644 index 0000000000..9b5ac0266c --- /dev/null +++ b/e2e/kubevirt/demo-policy.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Acceptance policy for e2e/kubevirt/run-demo.sh. +# +# Every KubeVirt guest runs its workload as root and its boundary is +# untrusted, so the policy carries network rules only: +# +# - No filesystem_policy, landlock, or process sections. In-guest +# enforcement is meaningless against a root workload (the fence and +# supervisor-side policy evaluation are the boundary), and the +# supervisor rejects an untrusted-guest policy that authors them. +# - No binaries lists. Per-binary identity is unenforceable in-guest for +# a root workload, so the supervisor REJECTS any policy that authors +# them (require_binary_identity=false would otherwise silently ignore +# the restriction); they are omitted here. +# - L4: one.one.one.one:443 is the only raw-TCP allowance; the demo dials +# example.org:80 and expects a denial. +# - L7: on example.com:443 (HTTPS) GET / is allowed and POST / is denied. +version: 1 + +network_policies: + l4_allowed: + name: l4-allowed + endpoints: + - host: one.one.one.one + port: 443 + protocol: tcp + l7_https: + name: l7-https + endpoints: + - host: example.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: / + deny_rules: + - method: POST + path: / diff --git a/e2e/kubevirt/demo.sh b/e2e/kubevirt/demo.sh new file mode 100755 index 0000000000..82d17e24a8 --- /dev/null +++ b/e2e/kubevirt/demo.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Start a standalone OpenShell gateway on the host whose Kubernetes compute +# driver runs in `runtime = "kubevirt"` mode against the local `kvpoc` k3d +# cluster (see e2e/kubevirt/README.md for the full PoC walkthrough). +# +# Mirrors tasks/scripts/gateway-vm.sh (the libkrun VM driver dev gateway) but +# provisions KubeVirt VirtualMachines instead. The gateway runs on the host +# with the k3d kubeconfig; sandbox guests reach it over the k3d docker bridge +# (http://:). +# +# Prerequisites (see README.md): +# - kvpoc k3d cluster with KubeVirt deployed and the guest image imported +# - e2e/kubevirt/build-guest-image.sh has been run +# +# Usage: e2e/kubevirt/demo.sh [--demo | --probes] +# --demo after the gateway starts, run the create/exec/teardown demo +# (requires the gateway to stay in the foreground; the demo runs +# against it from a background shell) +# --probes after the gateway starts, run the root-workload attack-surface +# probes (e2e/kubevirt/run-probes.sh): in-guest reset (reboot +# syscall + iTCO watchdog), fake qemu-guest-agent, virtiofsd +# privileged operations, and the launcher-netns listener audit + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PORT="${OPENSHELL_SERVER_PORT:-18085}" +GATEWAY_NAME="kvpoc" +STATE_DIR="${ROOT}/.cache/gateway-kubevirt" +LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" +CARGO_TARGET="${CARGO_TARGET_DIR:-${ROOT}/target}" +GATEWAY_BIN="${CARGO_TARGET}/release/openshell-gateway" +OPENSHELL_CLI="${CARGO_TARGET}/release/openshell" +SANDBOX_NAMESPACE="kvpoc-sandboxes" +GUEST_IMAGE="${OPENSHELL_KVPOC_GUEST_IMAGE:-kvpoc/openshell-sandbox-vm:dev}" + +RUN_DEMO=0 +RUN_PROBES=0 +case "${1:-}" in + --demo) RUN_DEMO=1 ;; + --probes) RUN_PROBES=1 ;; +esac +# Every KubeVirt guest is the root workload class (the only class), and +# the only supported topology is the dedicated supervisor (one Pod per +# sandbox): the config validation rejects the gateway-shared supervisor +# for the kubevirt runtime. + +# The k3d docker bridge gateway is the host address guests use to reach the +# gateway (k3d prints it at cluster creation; derive it from the network). +CLUSTER="${OPENSHELL_KVPOC_CLUSTER:-kvpoc}" +# The bridge gateway is the host address guests use to reach the gateway. +# Docker and podman report it with different JSON shapes; try both. +BRIDGE_IP="$(docker network inspect "k3d-${CLUSTER}" \ + -f '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || true)" +if [ -z "${BRIDGE_IP}" ]; then + BRIDGE_IP="$(podman network inspect "k3d-${CLUSTER}" \ + -f '{{(index .Subnets 0).Gateway}}' 2>/dev/null || true)" +fi +[ -n "${BRIDGE_IP}" ] || { echo "could not resolve the k3d-${CLUSTER} bridge gateway IP" >&2; exit 1; } +GRPC_ENDPOINT="https://${BRIDGE_IP}:${PORT}" + +echo "==> Building openshell-gateway and openshell CLI" +cargo build --release -p openshell-gateway -p openshell-cli + +TLS_DIR="${STATE_DIR}/tls" +SUPERVISOR_CA_SECRET="${OPENSHELL_KVPOC_SUPERVISOR_CA_SECRET:-kvpoc-server-tls}" +PKI_DIR="${STATE_DIR}/tls/pki" +if [ ! -f "${PKI_DIR}/server/tls.crt" ]; then + mkdir -p "${PKI_DIR}" + "${GATEWAY_BIN}" generate-certs --output-dir "${PKI_DIR}" \ + --server-san 127.0.0.1 --server-san "${BRIDGE_IP}" \ + --server-san localhost --server-san "${GATEWAY_NAME}" \ + >/dev/null +fi +# The dedicated supervisor Pods live in the sandbox namespace and +# verify the gateway through this CA (client_tls_secret_name). +kubectl create namespace "${SANDBOX_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "${SANDBOX_NAMESPACE}" create secret generic "${SUPERVISOR_CA_SECRET}" \ + --from-file=ca.crt="${PKI_DIR}/ca.crt" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +if [ ! -f "${TLS_DIR}/jwt/signing.pem" ]; then + echo "==> Generating local gateway credentials" + "${GATEWAY_BIN}" generate-certs --output-dir "${TLS_DIR}" +fi + +mkdir -p "${STATE_DIR}" +# The demo is single-shot: drop any sandbox records left by a previous run +# so `sandbox create --name kvpoc-demo` is reproducible. Nothing durable is +# kept in this gateway. +rm -f "${STATE_DIR}"/gateway.db* +# Topology-dependent config fragments: the gateway topology selects the +# Helm-shared supervisor; the dedicated default (matching the pod path's +# default) runs one supervisor Pod per sandbox with the locally imported +# supervisor image and the gateway-CA client-TLS Secret synced into the +# sandbox namespace above. +DEDICATED_KEYS=$(cat <"${CONFIG_PATH}" <"${GATEWAY_DIR}/metadata.json" <"${CONFIG_HOME}/openshell/active_gateway" + +# Sandboxes render into a dedicated namespace. +kubectl create namespace "${SANDBOX_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +cleanup() { + if [ "${RUN_DEMO}" = "1" ] && [ -n "${GATEWAY_PID:-}" ]; then + kill "${GATEWAY_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo +echo "Starting KubeVirt-mode Kubernetes gateway..." +echo " gateway: ${GATEWAY_NAME} (http://127.0.0.1:${PORT})" +echo " guest view: ${GRPC_ENDPOINT}" +echo " namespace: ${SANDBOX_NAMESPACE}" +echo " guest disk: ${GUEST_IMAGE}" +echo + +# The gateway may run on a restricted kubeconfig (e.g. one minted from the +# chart's gateway ServiceAccount, to prove the Helm RBAC is sufficient): +# set OPENSHELL_KVPOC_GATEWAY_KUBECONFIG. The demo's own kubectl calls keep +# the ambient KUBECONFIG (they are demo scaffolding: namespace and CA +# Secret setup). +KUBECONFIG="${OPENSHELL_KVPOC_GATEWAY_KUBECONFIG:-${KUBECONFIG}}" \ + OPENSHELL_DRIVERS=kubernetes "${GATEWAY_BIN}" \ + --config "${CONFIG_PATH}" \ + --port "${PORT}" \ + --log-level "${LOG_LEVEL}" \ + --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" & +GATEWAY_PID=$! + +# Wait for the gateway to accept requests. +for _ in $(seq 1 60); do + if curl -sf "http://127.0.0.1:$((PORT + 1))/healthz" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +if [ "${RUN_DEMO}" = "1" ] || [ "${RUN_PROBES}" = "1" ]; then + if [ "${RUN_PROBES}" = "1" ]; then + "${ROOT}/e2e/kubevirt/run-probes.sh" || { + kill "${GATEWAY_PID}" 2>/dev/null || true + exit 1 + } + else + "${ROOT}/e2e/kubevirt/run-demo.sh" || { + kill "${GATEWAY_PID}" 2>/dev/null || true + exit 1 + } + fi + kill "${GATEWAY_PID}" 2>/dev/null || true + trap - EXIT + wait "${GATEWAY_PID}" 2>/dev/null || true +else + wait "${GATEWAY_PID}" +fi diff --git a/e2e/kubevirt/guest-init.sh b/e2e/kubevirt/guest-init.sh new file mode 100644 index 0000000000..c4d9ba1060 --- /dev/null +++ b/e2e/kubevirt/guest-init.sh @@ -0,0 +1,337 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# KubeVirt guest init for OpenShell sandboxes (proof of concept). +# +# Runs as PID 1 (kernel arg init=/opt/openshell/init inside the purpose-built +# containerDisk from e2e/kubevirt/build-guest-image.sh). Modeled on the libkrun +# init (crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh) but +# for KubeVirt hardware: +# +# /dev/vda rootfs (containerDisk, read-only at the device level; +# the initramfs mounts the read-only lower and +# stacks a tmpfs overlay as the writable root — +# in-session writes are RAM and every boot starts +# fresh from the image) +# /dev/vdb config disk (cloudInitNoCloud ISO rendered from the driver's +# bootstrap Secret; shell-sourceable user-data) +# workspace virtio-fs share backed by the per-sandbox PVC (filesystem +# PVCs are not exposed as block disks by KubeVirt), mounted at +# /sandbox and seeded from the image on first boot +# +# After setup it runs the static openshell-sandbox boundary as a child +# process; PID 1 stays the init so it can power off the guest when the +# boundary exits. + +set -eu + +# Every exit of PID 1 must power the guest off: a dead init means either a +# kernel panic (QEMU serves a dead kernel, the VM reports ready forever) or +# a hung guest. The trap makes any `set -e` failure path, and the script's +# normal end, end the VMI so the driver maps the sandbox to Error. The +# normal poweroff path below fires the trap too; poweroff -f is idempotent +# enough at that point (the machine is already going down). +trap 'ts "init exiting (rc=$?); powering off"; sync; poweroff -f' EXIT + +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export HOME=/root +export TERM=xterm + +BOOT_START=$(date +%s 2>/dev/null || echo 0) +ts() { + now=$(date +%s 2>/dev/null || echo 0) + echo "[kvpoc-init $((now - BOOT_START))s] $*" +} + +mount_fs() { + mount -t proc proc /proc 2>/dev/null || true + mount -t sysfs sysfs /sys 2>/dev/null || true + # The Alpine initramfs does not mount devtmpfs on the real root, so /dev + # is the image's static directory (no ptmx/null). The supervisor's exec + # relay needs PTY allocation, which requires /dev/ptmx from devtmpfs. + mount -t devtmpfs devtmpfs /dev 2>/dev/null || true + mkdir -p /dev/pts /dev/shm + mount -t devpts -o gid=5,mode=620 devpts /dev/pts 2>/dev/null || true + [ -e /dev/ptmx ] || ln -sf pts/ptmx /dev/ptmx + mount -t tmpfs tmpfs /run 2>/dev/null || true + mount -t tmpfs tmpfs /tmp 2>/dev/null || true + # On the verified boot the root is the tmpfs overlay (already rw). + # On the legacy plain boot the initramfs mounted the containerDisk + # read-only and the remount makes it writable for the VMI's lifetime. + mount -o remount,rw / 2>/dev/null || true +} + +configure_network() { + ip link set lo up 2>/dev/null || true + ip link set eth0 up + # The k3d pod network runs flannel-vxlan (MTU 1450); a guest eth0 at the + # default 1500 black-holes full-size TCP segments — small unary RPCs work + # but large stream frames (SSH KEX, gRPC session) stall. Clamp below the + # launcher-side MTU before DHCP so the advertised MSS fits. + ip link set dev eth0 mtu 1400 2>/dev/null || true + # KubeVirt masquerade networking runs dnsmasq in the virt-launcher pod: + # DHCP hands out the pod-network address plus cluster DNS. + if udhcpc -i eth0 -q -n -t 30 -T 2 2>/dev/null; then + ts "network configured via DHCP (mtu $(cat /sys/class/net/eth0/mtu 2>/dev/null || echo '?'))" + else + ts "WARNING: DHCP failed; supervisor will not reach the gateway" + fi + # The workload's resolver is the boundary's loopback DNS relay + # (127.0.0.53), which forwards queries through the mediated control + # stream — the workload cannot reach the cluster DNS directly (the + # seccomp broker denies it, mirroring the libkrun VM driver's init). + # DHCP (or its default script) may have written the cluster DNS into + # /etc/resolv.conf; overwrite it. + printf 'nameserver 127.0.0.53\noptions timeout:2 attempts:2\n' \ + > /etc/resolv.conf 2>/dev/null || \ + ts "WARNING: could not point /etc/resolv.conf at the loopback DNS relay" +} + +read_bootstrap_config() { + mkdir -p /config + if ! mount -t iso9660 -o ro /dev/vdb /config 2>/dev/null; then + mount -o ro /dev/vdb /config 2>/dev/null || { + ts "FATAL: cannot mount bootstrap config disk" + return 1 + } + fi + # The bootstrap payload is the shell-sourceable file rendered by the + # Kubernetes driver's kubevirt runtime mode from the bootstrap Secret. + # The Secret key is KubeVirt's cloudInitNoCloud convention (`userData`); + # the file name inside the generated NoCloud ISO is the cloud-init + # datasource convention (`user-data`). Accept either. + if [ -f /config/user-data ]; then + # shellcheck disable=SC1091 + # `set -a` exports every assignment while sourcing so the exec'd + # supervisor inherits them (the libkrun path passes these as a real + # VM environment; here they arrive as a file). + set -a + . /config/user-data + set +a + elif [ -f /config/userData ]; then + # shellcheck disable=SC1091 + set -a + . /config/userData + set +a + else + ts "FATAL: no bootstrap payload on config disk" + ls -la /config || true + return 1 + fi + ts "bootstrap config loaded (sandbox ${OPENSHELL_SANDBOX:-unknown})" + umount /config + # L1 (bootstrap config disk hygiene): read the cloud-init disk + # exactly once, then remove the PCI function backing it so the disk + # is not sitting attached and trivially readable for the whole + # session. This is HYGIENE, NOT A CONTROL: root in the guest can + # undo it (writing 1 to /sys/bus/pci/rescan re-enumerates the disk + # — QEMU/KubeVirt still have it attached, and cloudInitNoCloud + # volumes cannot be hot-unplugged in KubeVirt today), and the + # boundary key is equally exposed through the boundary's own memory + # (ptrace). The blast radius is its own sandbox. virtio-blk has no + # SCSI-style per-device `delete`, so the ejection removes the PCI + # function (resolved from the sysfs device path — never a hardcoded + # address). A guest reset re-attaches the device, but the pristine + # boot re-reads it only within the boot flow (before any workload + # code runs), and a fresh boundary attaching to the live generation + # ends it (P1's handle check) and the VM is halted (R3). Failing to + # eject is fatal — it signals a broken boot environment rather + # than silently leaving the disk attached. + local vdb_pci="" + vdb_pci="$(basename "$(dirname "$(readlink -f /sys/block/vdb/device 2>/dev/null)" 2>/dev/null)" 2>/dev/null)" || true + case "${vdb_pci}" in + *:*:*.*) ;; + *) vdb_pci="" ;; + esac + if [ -n "${vdb_pci}" ] && [ -w "/sys/bus/pci/devices/${vdb_pci}/remove" ]; then + if ! echo 1 > "/sys/bus/pci/devices/${vdb_pci}/remove" 2>/dev/null; then + ts "FATAL: cannot eject the bootstrap config disk (pci ${vdb_pci})" + return 1 + fi + ts "bootstrap config disk ejected (one-shot read, pci ${vdb_pci})" + else + ts "FATAL: no eject handle for the bootstrap config disk" + return 1 + fi +} + +mount_workspace() { + mkdir -p /sandbox + # The workspace PVC arrives as a virtio-fs share (filesystem PVCs are not + # exposed as block disks by KubeVirt). virtiofs is a kernel module in the + # Alpine -virt kernel. + modprobe virtiofs 2>/dev/null || true + if ! mount -t virtiofs workspace /sandbox 2>/dev/null; then + ts "WARNING: workspace virtio-fs share not available" + return 0 + fi + # The share arrives owned by the launcher fsGroup (virtiofsd runs as + # that uid, unprivileged, and cannot switch credentials to the guest + # workload uid): idmap the mount so the workload uid/gid and the + # on-disk fsGroup translate into each other, then seed as the + # workload user (root is unmapped through the idmap). + share_gid="$(stat -c %g /sandbox 2>/dev/null || echo 0)" + if [ -x /usr/local/bin/idmap-mount ] \ + && [ "${share_gid}" != "0" ] && [ "${share_gid}" != "10001" ] \ + && /usr/local/bin/idmap-mount /sandbox "${share_gid}" 10001; then + ts "workspace idmapped (on-disk id ${share_gid} <-> guest 10001)" + if [ ! -f /sandbox/.openshell-seeded ] && [ -d /sandbox-seed ]; then + su -s /bin/sh sandbox -c \ + 'cp -a /sandbox-seed/. /sandbox/ && touch /sandbox/.openshell-seeded' \ + 2>/dev/null || true + fi + # Adopt files left by pre-idmap generations (created as other ids). + su -s /bin/sh sandbox -c 'chown -R 10001:10001 /sandbox' 2>/dev/null || true + else + ts "WARNING: workspace idmap unavailable; writes as the workload uid will fail" + if [ ! -f /sandbox/.openshell-seeded ] && [ -d /sandbox-seed ]; then + cp -a /sandbox-seed/. /sandbox/ 2>/dev/null || true + touch /sandbox/.openshell-seeded 2>/dev/null || true + fi + chown -R 10001:10001 /sandbox 2>/dev/null || true + fi + ts "workspace PVC mounted (virtio-fs)" +} + +main() { + ts "OpenShell KubeVirt guest init starting" + mount_fs + if [ -c /dev/ptmx ]; then + ts "/dev/ptmx present" + else + ts "WARNING: /dev/ptmx missing — exec PTYs will fail" + fi + configure_network + # The capability-free sandbox owns a loopback-only DNS relay on + # 127.0.0.53:53. Guest init grants the low port before handing control + # to the zero-capability workload UID (mirrors the libkrun VM driver's + # init and the container runtimes' sysctl). + if ! echo 0 > /proc/sys/net/ipv4/ip_unprivileged_port_start; then + ts "FATAL: failed to permit the unprivileged DNS relay to bind port 53" + return 1 + fi + read_bootstrap_config + mount_workspace + ts "starting openshell-sandbox" + # Stage the boundary bootstrap artifacts from the cloud-init payload + # (base64 in the sourced userData) at the absolute paths written in the + # BoundaryConfig, mode 0600, then launch the boundary. The boundary + # deletes its config and TLS material after loading them. + if [ -n "${OPENSHELL_BOUNDARY_CONFIG_B64:-}" ]; then + # L1: capture the workload identity from the payload BEFORE + # staging unsets it; the same values drive both the chown in + # staging and the launch arguments. They come from the same + # ResolvedWorkloadIdentity the boundary config carries + # (render_guest_payload), so there is no second source of truth. + local launch_uid launch_gid + launch_uid="${OPENSHELL_WORKLOAD_UID}" + launch_gid="${OPENSHELL_WORKLOAD_GID}" + if [ -z "${launch_uid}" ] || [ -z "${launch_gid}" ]; then + ts "FATAL: payload is missing the workload identity" + return 1 + fi + stage_boundary_bootstrap + # G2: assert the environment is clean before launching the child. + # The boundary inherits PID 1's env; any B64 or token var here + # would leak into the boundary and its workload children. + if env | grep -q "BOUNDARY_.*_B64\|OPENSHELL_SANDBOX_TOKEN"; then + ts "FATAL: key material still in environment after staging" + return 1 + fi + ts "environment verified clean (no B64 or token vars)" + # Run the boundary as a child (not exec) so PID 1 can power off on + # exit (see the wait + poweroff below). Every KubeVirt guest is + # the untrusted root-workload class: the workload runs as uid 0 + # and the boundary launches in privileged mode (in-guest + # enforcement is meaningless against root — the fence, the + # supervisor's policy, and the driver's checks are the boundary). + # The identity must be exactly 0/0; anything else is a tampered + # or mismatched payload. + if [ "${launch_uid}" != "0" ] || [ "${launch_gid}" != "0" ]; then + ts "FATAL: KubeVirt guests run as root; payload identity ${launch_uid}:${launch_gid} is invalid" + return 1 + fi + # The boundary binary comes from the initramfs bind path when + # staged (the initramfs pins the image inode at + # /run/openshell/openshell-sandbox); otherwise the rootfs copy is + # used. The pin is a freshness measure (the boundary starts from + # image bytes), not a trust boundary — a root workload can unmount + # anything in-guest. + BOUNDARY_BIN=/usr/local/bin/openshell-sandbox + if [ -x /run/openshell/openshell-sandbox ]; then + BOUNDARY_BIN=/run/openshell/openshell-sandbox + ts "boundary from the verified trusted path (${BOUNDARY_BIN})" + fi + ts "launching privileged (root-workload) boundary" + "${BOUNDARY_BIN}" launch-privileged \ + /.openshell/state/bootstrap/boundary.json & + SANDBOX_PID=$! + else + # No boundary payload: the gateway cannot have created this VM + # (create requires launch authentication and renders the boundary + # payload), so this is an invalid or tampered bootstrap. + ts "FATAL: no boundary payload in bootstrap config (OPENSHELL_BOUNDARY_CONFIG_B64 unset)" + return 1 + fi + # `wait` returns the child's exit status, and this script runs under + # `set -e`: an unguarded failing wait would exit PID 1 (kernel panic) + # before the poweroff below. + SANDBOX_RC=0 + wait "${SANDBOX_PID}" || SANDBOX_RC=$? + ts "openshell-sandbox exited with code ${SANDBOX_RC}; powering off" + sync + # -f skips the shutdown script hierarchy (there is none here): the + # kernel powers the machine off immediately, QEMU exits, and the VMI + # reaches Succeeded so the driver maps the sandbox to Error. + poweroff -f + # poweroff -f does not return; sleep forever as a fallback so PID 1 + # never exits into a kernel panic if the syscall fails. + while true; do sleep 3600; done +} + +# Decode the base64 boundary artifacts from the cloud-init payload and +# stage them at the paths the BoundaryConfig expects. The boundary reads +# and DELETES these files as the zero-capability workload uid:gid, so: +# - /.openshell/state is a tmpfs (G3: the key never lands on the +# rootfs block device); +# - the files and directory are chowned to the workload uid:gid (G1: the +# boundary can read and unlink them); +# - the base64 environment variables are unset after staging (G2: they +# never leak into the boundary or workload children). +stage_boundary_bootstrap() { + local uid gid + uid="${OPENSHELL_WORKLOAD_UID}" + gid="${OPENSHELL_WORKLOAD_GID}" + + # G3: tmpfs so the key material never touches the containerDisk. + mkdir -p /.openshell/state/bootstrap + mount -t tmpfs -o mode=0700,nosuid,nodev tmpfs /.openshell/state + mkdir -p /.openshell/state/bootstrap + chmod 0700 /.openshell/state /.openshell/state/bootstrap + + printf '%s' "${OPENSHELL_BOUNDARY_CONFIG_B64}" | base64 -d \ + > /.openshell/state/bootstrap/boundary.json + printf '%s' "${OPENSHELL_BOUNDARY_CERTIFICATE_B64}" | base64 -d \ + > /.openshell/state/bootstrap/tls.crt + printf '%s' "${OPENSHELL_BOUNDARY_PRIVATE_KEY_B64}" | base64 -d \ + > /.openshell/state/bootstrap/tls.key + chmod 0600 \ + /.openshell/state/bootstrap/boundary.json \ + /.openshell/state/bootstrap/tls.crt \ + /.openshell/state/bootstrap/tls.key + + # G1: the boundary reads and deletes these as the workload uid. + chown -R "${uid}:${gid}" /.openshell/state + + # G2: the base64 key material must not survive into the boundary's + # environment or any workload child. + unset OPENSHELL_BOUNDARY_CONFIG_B64 \ + OPENSHELL_BOUNDARY_CERTIFICATE_B64 \ + OPENSHELL_BOUNDARY_PRIVATE_KEY_B64 \ + OPENSHELL_WORKLOAD_UID OPENSHELL_WORKLOAD_GID + + ts "boundary bootstrap staged on tmpfs (uid=${uid}, gid=${gid})" +} + +main diff --git a/e2e/kubevirt/idmap-mount.c b/e2e/kubevirt/idmap-mount.c new file mode 100644 index 0000000000..e2ddc48680 --- /dev/null +++ b/e2e/kubevirt/idmap-mount.c @@ -0,0 +1,210 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// idmap-mount: remount the filesystem at with an idmap mapping +// on-disk uid/gid <-> mount-visible uid/gid : files stored +// as appear owned by , and processes running as +// create files stored as . +// +// KubeVirt shares filesystem PVCs through virtiofsd, which runs as the +// launcher pod's user (uid/gid 107 by default, runAsNonRoot, no +// capabilities): it cannot switch credentials to an arbitrary guest uid, so +// every guest write EPERMs. An idmapped mount translates the share's +// on-disk ownership (the launcher's fsGroup) into the guest workload's +// uid/gid in both directions: the workload (uid ) creates files as +// the on-disk uid — the one credential virtiofsd can act as — and +// sees them as its own. +// +// The mapping needs a user namespace owned by a privileged process (guest +// root): a child unshares CLONE_NEWUSER and the parent writes its +// uid_map/gid_map (" 1>"). The mount is then idmapped +// through a detached clone re-attached over the original, so no other +// consumer of the path sees an intermediate state. +// +// Build (static musl, as part of the guest image): +// cc -static -O2 -o idmap-mount idmap-mount.c +// +// Usage: idmap-mount (exit 0 on success) + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef OPEN_TREE_CLONE +#define OPEN_TREE_CLONE 1 +#endif +#ifndef OPEN_TREE_CLOEXEC +#define OPEN_TREE_CLOEXEC O_CLOEXEC +#endif +#ifndef MOVE_MOUNT_F_EMPTY_PATH +#define MOVE_MOUNT_F_EMPTY_PATH 0x00000004 +#endif +#ifndef MOVE_MOUNT_T_EMPTY_PATH +#define MOVE_MOUNT_T_EMPTY_PATH 0x00000010 +#endif + +static int mount_setattr_wrapper(int dfd, const char *path, unsigned int flags, + struct mount_attr *attr, size_t size) { + return (int)syscall(SYS_mount_setattr, dfd, path, flags, attr, size); +} + +static int open_tree_wrapper(int dfd, const char *path, unsigned int flags) { + return (int)syscall(SYS_open_tree, dfd, path, flags); +} + +static int move_mount_wrapper(int from_dfd, const char *from_path, int to_dfd, + const char *to_path, unsigned int flags) { + return (int)syscall(SYS_move_mount, from_dfd, from_path, to_dfd, to_path, flags); +} + +static int write_proc_file(pid_t child, const char *file, const char *content) { + char path[64]; + snprintf(path, sizeof(path), "/proc/%d/%s", child, file); + int fd = open(path, O_WRONLY); + if (fd < 0) { + return -1; + } + ssize_t len = (ssize_t)strlen(content); + if (write(fd, content, (size_t)len) != len) { + close(fd); + return -1; + } + close(fd); + return 0; +} + +static int write_map(pid_t child, const char *file, unsigned int inner, + unsigned int outer) { + // Two entries: the workload <-> fs mapping, plus an identity mapping + // for root. An inode whose owner is unmapped fails directory-permission + // checks entirely on idmapped mounts (the share root is root-owned), so + // root must stay mapped even though the workload never acts as it. + char content[96]; + snprintf(content, sizeof(content), "%u %u 1\n0 0 1\n", inner, outer); + return write_proc_file(child, file, content); +} + +int main(int argc, char **argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + const char *path = argv[1]; + unsigned int fs_id = (unsigned int)strtoul(argv[2], NULL, 10); + unsigned int user_id = (unsigned int)strtoul(argv[3], NULL, 10); + if (fs_id == 0 || user_id == 0 || fs_id == user_id) { + fprintf(stderr, "%s: refusing a trivial or root mapping\n", argv[0]); + return 2; + } + + // A child creates the user namespace and signals readiness through a + // pipe; this process (guest root) holds the privilege to write its + // maps, and the namespace must stay alive while the mount references + // it (the child parks until killed). + int readiness[2]; + if (pipe(readiness) < 0) { + perror("pipe"); + return 1; + } + pid_t child = fork(); + if (child < 0) { + perror("fork"); + return 1; + } + if (child == 0) { + close(readiness[0]); + if (unshare(CLONE_NEWUSER) < 0) { + (void)!write(readiness[1], "U", 1); + _exit(1); + } + // Ready: the parent can now write the maps. + (void)!write(readiness[1], "R", 1); + pause(); + _exit(0); + } + close(readiness[1]); + char ready = 0; + if (read(readiness[0], &ready, 1) != 1 || ready != 'R') { + fprintf(stderr, "%s: creating the user namespace failed: %s\n", argv[0], + ready == 'U' ? "unshare(CLONE_NEWUSER) not permitted" : strerror(errno)); + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 1; + } + // The mount's idmap reads: namespace uid == kernel uid + // — the on-disk id is the namespace side, the process id the + // kernel side. + if (write_map(child, "uid_map", fs_id, user_id) < 0) { + fprintf(stderr, "%s: writing uid_map failed: %s\n", argv[0], strerror(errno)); + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 1; + } + if (write_proc_file(child, "setgroups", "deny") < 0 || + write_map(child, "gid_map", fs_id, user_id) < 0) { + fprintf(stderr, "%s: writing gid_map failed: %s (setgroups must be denied)\n", + argv[0], strerror(errno)); + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 1; + } + + char namespace_path[64]; + snprintf(namespace_path, sizeof(namespace_path), "/proc/%d/ns/user", child); + int userns_fd = open(namespace_path, O_RDONLY | O_CLOEXEC); + if (userns_fd < 0) { + fprintf(stderr, "%s: opening the user namespace failed: %s\n", argv[0], + strerror(errno)); + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 1; + } + + // Clone the mount detached, idmap it, and move it back over the + // original path. + int tree_fd = open_tree_wrapper(AT_FDCWD, path, OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC); + if (tree_fd < 0) { + fprintf(stderr, "%s: open_tree(%s) failed: %s\n", argv[0], path, strerror(errno)); + kill(child, SIGKILL); + return 1; + } + struct mount_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.attr_set = MOUNT_ATTR_IDMAP; + attr.userns_fd = (unsigned int)userns_fd; + if (mount_setattr_wrapper(tree_fd, "", AT_EMPTY_PATH, &attr, sizeof(attr)) < 0) { + fprintf(stderr, "%s: mount_setattr(IDMAP) failed: %s\n", argv[0], strerror(errno)); + kill(child, SIGKILL); + return 1; + } + int target_fd = open(path, O_PATH | O_CLOEXEC); + if (target_fd < 0) { + fprintf(stderr, "%s: open(%s) failed: %s\n", argv[0], path, strerror(errno)); + kill(child, SIGKILL); + return 1; + } + if (move_mount_wrapper(tree_fd, "", target_fd, "", + MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH) < 0) { + fprintf(stderr, "%s: move_mount failed: %s\n", argv[0], strerror(errno)); + kill(child, SIGKILL); + return 1; + } + + // The idmapped mount holds a reference to the user namespace; the + // parked child can go. + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 0; +} diff --git a/e2e/kubevirt/initramfs-init.sh b/e2e/kubevirt/initramfs-init.sh new file mode 100644 index 0000000000..66403579b6 --- /dev/null +++ b/e2e/kubevirt/initramfs-init.sh @@ -0,0 +1,100 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# OpenShell KubeVirt initramfs init (boot stage). +# +# This script is packed into the initramfs (e2e/kubevirt/build-guest-image.sh) +# and runs as PID 1 BEFORE the real root is mounted. It assembles the root +# filesystem and execs the guest init, which continues as PID 1. +# +# Boot plan (root-workload VM PoC): +# +# /dev/vda containerDisk ext4 carrying the rootfs. The containerDisk is +# read-only at the device level (KubeVirt containerDisks are +# read-only volumes), so the base image can never be modified +# in place — every boot reads the image as built. +# overlay tmpfs upper + work over the read-only lower: the workload +# gets a writable / in RAM, and every boot starts fresh from +# the image. Nothing in the guest is a trust anchor: the guest +# is untrusted as soon as a workload executes, so persisted +# guest state has no security value and stale mutated state +# would only poison later boots of the same sandbox. +# /run/ the guest init and the boundary binary are bind-mounted from +# openshell the image lower (inode-pinned, read-only) so the boundary +# always starts from image bytes, but this is a freshness +# measure, not a trust boundary: root in the guest can unmount +# or bypass anything in-guest. The real boundary is outside +# the guest (the fence, the supervisor's policy, the VM). + +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# The stock initramfs exposes only /bin/sh; install every busybox applet +# symlink so insmod/sleep/mount/switch_root resolve as commands. +/bin/busybox --install -s /bin + +fatal() { + echo "[boot-init FATAL] $*" >&2 + echo "[boot-init FATAL] powering off" >&2 + sync + poweroff -f + while true; do sleep 3600; done +} + +mount -t proc proc /proc 2>/dev/null || true +mount -t sysfs sysfs /sys 2>/dev/null || true +mount -t devtmpfs devtmpfs /dev 2>/dev/null || true + +# The stock initramfs's kmod modprobe resolves the module tree: the +# virtio block device carrying the rootfs. +modprobe virtio_blk 2>/dev/null || true +modprobe virtio_net 2>/dev/null || true +modprobe overlay 2>/dev/null || true +[ -e /dev/vda ] \ + || fatal "/dev/vda missing (virtio_blk did not load?)" + +# Mount the containerDisk rootfs read-only (containerDisks are read-only +# volumes; noload skips journal replay on the pristine image). +mkdir -p /lower +mount -t ext4 -o ro,noload /dev/vda /lower \ + || fatal "mounting the guest rootfs failed" + +# The writable root: a tmpfs-backed overlay over the read-only lower. +# NOTE: upper/work are created AFTER the tmpfs is mounted on /ovl — the +# pre-mount mkdir is hidden by the tmpfs, so the overlay's upperdir and +# workdir must exist inside it. +mkdir -p /ovl/upper /ovl/work /sysroot +mount -t tmpfs -o mode=0755,nosuid tmpfs /ovl +mkdir -p /ovl/upper /ovl/work +mount -t overlay overlay \ + -o "lowerdir=/lower,upperdir=/ovl/upper,workdir=/ovl/work" /sysroot \ + || fatal "overlayfs mount failed" + +# Bind paths: pin the guest init and boundary binaries from the image +# lower into the new root so the boundary starts from image bytes (the +# workload may shadow /opt/openshell/init and +# /usr/local/bin/openshell-sandbox in the overlay; /run/openshell/* keep +# resolving to the pinned inodes until the workload tears them down — +# a freshness measure, not a trust boundary). +mkdir -p /sysroot/run/openshell +# A bind mount needs an existing target file; the bound inode is what +# resolves afterwards. +touch /sysroot/run/openshell/init /sysroot/run/openshell/openshell-sandbox +mount --bind /lower/opt/openshell/init /sysroot/run/openshell/init \ + || fatal "bind-mounting the guest init failed" +mount --bind /lower/usr/local/bin/openshell-sandbox \ + /sysroot/run/openshell/openshell-sandbox \ + || fatal "bind-mounting the boundary binary failed" + +# The overlay holds references to the lower and /ovl superblocks; moving +# both mounts into the new root keeps them alive across switch_root (which +# otherwise frees the initramfs). +mkdir -p /sysroot/run/lower /sysroot/run/ovl +mount --move /lower /sysroot/run/lower || fatal "moving the lower mount failed" +mount --move /ovl /sysroot/run/ovl || fatal "moving the overlay tmpfs failed" + +# /dev, /proc, /sys move with switch_root's -c or land via the guest init's +# own mounts; busybox switch_root moves them when they are mounted at the +# standard paths. +echo "[boot-init] switching to the overlay root" +exec switch_root /sysroot /run/openshell/init diff --git a/e2e/kubevirt/port-io.c b/e2e/kubevirt/port-io.c new file mode 100644 index 0000000000..a8902d1ec4 --- /dev/null +++ b/e2e/kubevirt/port-io.c @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Minimal x86 port-I/O and MMIO helper for the e2e probes (static musl +// build, compiled at probe time and staged into the guest; it does not +// ship in the guest image). +// +// /dev/port performs byte-granular IN/OUT, but emulated registers like the +// PCI config address (0xcf8) and the ICH9 TCO counters need word/dword +// access — byte writes are silently dropped by the emulation. The guest +// busybox also has no devmem applet, so chipset MMIO (the RCBA/GCS +// register) needs a mapping. This helper performs real OUT instructions +// after iopl(3) and maps physical pages through /dev/mem. +// +// Usage: +// port-io read8|read16|read32 -> hex value on stdout +// port-io write8|write16|write32 +// port-io mem-read [width] width: 8|16|32 (default 32) +// port-io mem-write [width] +// +// Ports, addresses, and values are accepted in decimal or 0x-hex. Exits +// non-zero on argument, iopl, or mmap failure. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long parse_num(const char *text) { + return strtoul(text, NULL, 0); +} + +static int fail_usage(void) { + fprintf(stderr, + "usage: port-io read8|read16|read32 \n" + " port-io write8|write16|write32 \n" + " port-io mem-read [width]\n" + " port-io mem-write [width]\n"); + return 2; +} + +static int mem_access(int argc, char **argv, const char *op) { + if (argc < 3) { + return fail_usage(); + } + unsigned long addr = parse_num(argv[2]); + long page = (long)(addr & ~(unsigned long)0xfff); + size_t offset_in_page = (size_t)(addr & 0xfff); + int width = 32; + if (strcmp(op, "mem-write") == 0) { + if (argc < 4) { + return fail_usage(); + } + if (argc >= 5) { + width = (int)parse_num(argv[4]); + } + } else if (argc >= 4) { + width = (int)parse_num(argv[3]); + } + int fd = open("/dev/mem", O_RDWR | O_SYNC); + if (fd < 0) { + fprintf(stderr, "open /dev/mem failed: %s\n", strerror(errno)); + return 1; + } + volatile void *map = + mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, (off_t)page); + if (map == MAP_FAILED) { + fprintf(stderr, "mmap /dev/mem at 0x%lx failed: %s\n", + (unsigned long)page, strerror(errno)); + close(fd); + return 1; + } + volatile void *reg = (volatile char *)map + offset_in_page; + int rc = 0; + if (strcmp(op, "mem-read") == 0) { + switch (width) { + case 8: + printf("0x%02x\n", *(volatile uint8_t *)reg); + break; + case 16: + printf("0x%04x\n", *(volatile uint16_t *)reg); + break; + case 32: + printf("0x%08x\n", *(volatile uint32_t *)reg); + break; + default: + fprintf(stderr, "bad width %d\n", width); + rc = 2; + } + } else { + unsigned long value = parse_num(argv[3]); + switch (width) { + case 8: + *(volatile uint8_t *)reg = (uint8_t)value; + break; + case 16: + *(volatile uint16_t *)reg = (uint16_t)value; + break; + case 32: + *(volatile uint32_t *)reg = (uint32_t)value; + break; + default: + fprintf(stderr, "bad width %d\n", width); + rc = 2; + } + } + munmap((void *)map, 0x1000); + close(fd); + return rc; +} + +int main(int argc, char **argv) { + if (argc < 3) { + return fail_usage(); + } + if (strncmp(argv[1], "mem-", 4) == 0) { + return mem_access(argc, argv, argv[1]); + } + if (iopl(3) != 0) { + fprintf(stderr, "iopl(3) failed: %s\n", strerror(errno)); + return 1; + } + const char *op = argv[1]; + unsigned long port = parse_num(argv[2]); + if (port > 0xffff) { + fprintf(stderr, "port out of range: %lu\n", port); + return 2; + } + + if (strcmp(op, "read8") == 0) { + printf("0x%02x\n", inb((unsigned short)port)); + return 0; + } + if (strcmp(op, "read16") == 0) { + printf("0x%04x\n", inw((unsigned short)port)); + return 0; + } + if (strcmp(op, "read32") == 0) { + printf("0x%08x\n", inl((unsigned short)port)); + return 0; + } + if (argc < 4) { + return fail_usage(); + } + unsigned long value = parse_num(argv[3]); + if (strcmp(op, "write8") == 0) { + outb((uint8_t)value, (unsigned short)port); + return 0; + } + if (strcmp(op, "write16") == 0) { + outw((uint16_t)value, (unsigned short)port); + return 0; + } + if (strcmp(op, "write32") == 0) { + outl((uint32_t)value, (unsigned short)port); + return 0; + } + return fail_usage(); +} diff --git a/e2e/kubevirt/run-demo.sh b/e2e/kubevirt/run-demo.sh new file mode 100755 index 0000000000..3c2f7f7009 --- /dev/null +++ b/e2e/kubevirt/run-demo.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Full acceptance run for the KubeVirt-mode Kubernetes compute driver +# against a running kvpoc gateway (started by e2e/kubevirt/demo.sh with the +# shared supervisor deployed; see e2e/kubevirt/README.md). Every step fails +# the demo on error — there is no Ready-wait tolerance. +# +# 1. create (with demo-policy.yaml) reaching Ready +# 2. fence NetworkPolicy exists, and exactly one virt-launcher pod +# matches its podSelector +# 3. exec inside the VM (uname, id, PID 1) +# 4. no gateway material in the workload environment +# 5. L4 allow (one.one.one.one:443) and deny (example.org:80); +# L7 HTTPS on example.com: GET allowed, POST denied (403) — the GET +# proves the per-sandbox proxy CA was delivered into the guest +# 6. direct egress from the guest to a raw IP is blocked +# 7. stop -> start -> Ready, with the VM UID and fence UID unchanged +# 8. fence verification: a spec edit fails start with drift; a +# label-only edit is accepted +# 9. main-process exit -> Error (exit code 137) +# 10. delete -> zero leftover managed objects +# +# Requires the `openshell` CLI built at $CARGO_TARGET_DIR/release/openshell. +# The topology is dedicated (one supervisor Pod per sandbox — the only +# supported KubeVirt topology) and the workload is root (the only KubeVirt +# workload class). The steps verify the supervisor Pod's UID is the +# boundary claim, that the fence selects only that Pod, that stop/start +# rotates it, and that delete removes it. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CARGO_TARGET="${CARGO_TARGET_DIR:-${ROOT}/target}" +OPENSHELL_CLI="${CARGO_TARGET}/release/openshell" +SANDBOX_NAMESPACE="${OPENSHELL_KVPOC_NAMESPACE:-kvpoc-sandboxes}" +SUPERVISOR_NAMESPACE="${OPENSHELL_KVPOC_SUPERVISOR_NAMESPACE:-openshell}" +DEMO_POLICY="${ROOT}/e2e/kubevirt/demo-policy.yaml" +NAME="kvpoc-demo" +VM_SELECTOR="openshell.ai/sandbox-name=${NAME}" +FENCE_PREFIX="openshell-kubevirt-fence-" + +step() { echo; echo "=== $* ==="; } +fail() { echo "FAIL: $*" >&2; exit 1; } + +# The single EXIT trap. A second `trap ... EXIT` later in the script would +# REPLACE this one (bash keeps one EXIT trap), which is exactly how the +# failure teardown was silently lost before — keep every exit handling +# here. +# +# On failure, tear down the sandbox: without this the VM, VMI, and +# supervisor Pod are left running after the demo's gateway exits, and only +# a manual `kubectl delete vm` cascades them away. The CLI delete runs +# first (while the gateway is still up); the labelled kubectl deletions +# are the gateway-independent backstop — owner references cascade from +# the VM, and the labelled companions cover what the VM does not own. +# On every exit (success or failure), remove the scratch directory. +demo_exit_trap() { + local rc=$? + if [ "${rc}" -ne 0 ]; then + echo >&2 + echo "demo failed (rc=${rc}); tearing down sandbox '${NAME}'..." >&2 + "${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true + kubectl delete vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --wait=false --ignore-not-found >/dev/null 2>&1 || true + kubectl delete pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --ignore-not-found >/dev/null 2>&1 || true + echo "teardown complete" >&2 + fi + rm -rf "${WORKDIR:-}" + exit "${rc}" +} +trap demo_exit_trap EXIT + +WORKDIR="$(mktemp -d /tmp/kvpoc-demo-XXXXXX)" + +sandbox_phase() { + "${OPENSHELL_CLI}" sandbox get "${NAME}" 2>/dev/null | grep -a "Phase:" | head -1 \ + | sed 's/^ *//' || true +} + +require_phase() { + local expected="$1" got + got="$(sandbox_phase)" + [ "${got}" = "Phase: ${expected}" ] \ + || fail "expected phase ${expected}, got '${got:-none}'" +} + +# Wait for the sandbox to reach the expected phase. The create uses --detach +# (a non-detached create attaches to the main process, and the demo's main +# process is `sleep infinity`), so readiness is polled through `sandbox get`. +wait_for_phase() { + local expected="$1" timeout_secs="${2:-300}" deadline got + deadline=$(( $(date +%s) + timeout_secs )) + while [ "$(date +%s)" -lt "${deadline}" ]; do + got="$(sandbox_phase)" + case "${got}" in + "Phase: ${expected}") return 0 ;; + "Phase: Error") fail "sandbox reached Error while waiting for ${expected}" ;; + esac + sleep 5 + done + fail "timed out waiting for phase ${expected} (got '${got:-none}')" +} + +vm_jsonpath() { + kubectl get vm -l "${VM_SELECTOR}" -n "${SANDBOX_NAMESPACE}" \ + -o "jsonpath=$1" 2>/dev/null +} + +fence_name() { + local sandbox_id + sandbox_id="$(vm_jsonpath '{.items[0].metadata.labels.openshell\.ai/sandbox-id}')" + [ -n "${sandbox_id}" ] || fail "VM carries no sandbox-id label" + echo "${FENCE_PREFIX}${sandbox_id}" +} + +# The exec channel can report a benign teardown error after the output was +# delivered completely ("exec output could not be delivered intact", exit +# 74); each step's content assertions decide success, so only that code is +# swallowed. The workload's own exit codes (e.g. a denied connect) still +# propagate. +# Assert the supervisor logged an OCSF NET:OPEN DENIED line for the +# destination, so a network failure cannot pass as a policy deny. The log +# source is topology-dependent: the shared supervisor Deployment, or this +# sandbox's dedicated supervisor Pod. +supervisor_logs() { + kubectl logs -n "${SANDBOX_NAMESPACE}" "pod/${SUPERVISOR_POD}" --since=5m 2>/dev/null +} + +supervisor_denied() { + local destination="$1" + supervisor_logs | grep -a "OCSF NET:OPEN.*DENIED.*${destination}" +} + +exec_in_sandbox() { + "${OPENSHELL_CLI}" sandbox exec "${NAME}" -- "$@" || { + local rc=$? + [ "${rc}" = "74" ] || return "${rc}" + return 0 + } +} + +step "1/10 create reaching Ready (boundary dials the dedicated supervisor)" +"${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true +# A previous run whose gateway record was already deleted can leave an +# orphaned VM behind (name collision on create); clear it and its +# labelled companions directly. +if kubectl get vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" --no-headers 2>/dev/null | grep -q .; then + echo "clearing an orphaned VM from a previous run" + kubectl delete vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" --wait=true >/dev/null 2>&1 || true +fi +if kubectl get pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --no-headers 2>/dev/null | grep -q .; then + echo "clearing orphaned labelled companions from a previous run:" + kubectl get pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --no-headers 2>/dev/null | awk '{print " " $1}' | sort -u +fi +kubectl delete pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" --ignore-not-found >/dev/null 2>&1 || true +sleep 5 +"${OPENSHELL_CLI}" sandbox create --detach --name "${NAME}" \ + --policy "${DEMO_POLICY}" \ + -- /bin/sleep infinity +wait_for_phase Ready +echo "objects:" +kubectl get vm,vmi,netpol,secret,pvc -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + || fail "managed objects missing after create" +SBX_ID="$(vm_jsonpath '{.items[0].metadata.labels.openshell\.ai/sandbox-id}')" +VM_UID="$(vm_jsonpath '{.items[0].metadata.uid}')" +[ -n "${VM_UID}" ] || fail "VM has no UID" +# The dedicated supervisor Pod's name derives from the sandbox id +# (SandboxRuntimeNames::new lowercases it). +SBX_LC="$(printf '%s' "${SBX_ID}" | tr '[:upper:]' '[:lower:]')" +SUPERVISOR_POD="os-supervisor-${SBX_LC}" +# The bootstrap Secret is generation-scoped (-boot-, the +# generation is opaque), so it is found by label. +BOOTSTRAP_SECRET="$(kubectl get secrets -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID},openshell.ai/managed-by=openshell" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" \ + || fail "could not find the sandbox's bootstrap Secret by label" +[ -n "${BOOTSTRAP_SECRET}" ] || fail "no labelled bootstrap Secret found" +case "${BOOTSTRAP_SECRET}" in + default--${NAME}-boot-*) ;; + *) fail "unexpected bootstrap Secret name ${BOOTSTRAP_SECRET}" ;; +esac + +echo "dedicated supervisor generation:" +kubectl get pod,svc,secret -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID},openshell.ai/component=supervisor" \ + || fail "dedicated supervisor objects missing after create" +SUPERVISOR_POD_UID="$(kubectl get pod -n "${SANDBOX_NAMESPACE}" "${SUPERVISOR_POD}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null)" \ + || fail "dedicated supervisor Pod ${SUPERVISOR_POD} not found" +[ -n "${SUPERVISOR_POD_UID}" ] || fail "supervisor Pod has no UID" +# The boundary config delivered to the guest claims the supervisor Pod's +# UID (kubevirt.supervisor_pod_uid); the rendered claim must be the live +# Pod's UID. +CLAIMED_UID="$(kubectl get secret -n "${SANDBOX_NAMESPACE}" "${BOOTSTRAP_SECRET}" \ + -o jsonpath='{.data.userData}' | base64 -d \ + | grep "^OPENSHELL_BOUNDARY_CONFIG_B64=" | cut -d= -f2- | base64 -d \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["resource_claims"]["kubevirt.supervisor_pod_uid"])')" \ + || fail "could not extract kubevirt.supervisor_pod_uid from the guest bootstrap" +[ "${CLAIMED_UID}" = "${SUPERVISOR_POD_UID}" ] \ + || fail "boundary claim kubevirt.supervisor_pod_uid (${CLAIMED_UID}) does not match the supervisor Pod UID (${SUPERVISOR_POD_UID})" +echo "supervisor Pod UID matches the kubevirt.supervisor_pod_uid claim (${SUPERVISOR_POD_UID})" + +step "2/10 fence NetworkPolicy and its podSelector" +FENCE="$(fence_name)" +FENCE_UID="$(kubectl get netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null)" \ + || fail "fence ${FENCE} not found" +[ -n "${FENCE_UID}" ] || fail "fence has no UID" +echo "fence: ${FENCE} (uid ${FENCE_UID})" +LAUNCHER_PODS="$(kubectl get pods -n "${SANDBOX_NAMESPACE}" \ + -l kubevirt.io=virt-launcher,openshell.ai/sandbox-id="${SBX_ID}" \ + --no-headers 2>/dev/null | wc -l)" +[ "${LAUNCHER_PODS}" = "1" ] \ + || fail "expected exactly one launcher pod matching the fence selector, got ${LAUNCHER_PODS}" +echo "exactly one virt-launcher pod matches the fence podSelector" +# The fence's egress target is the pair's supervisor-role selector: +# exactly this sandbox's supervisor Pod (same namespace, no +# namespaceSelector). +FENCE_PODS="$(kubectl get pods -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/boundary-pair=${SBX_LC},openshell.ai/boundary-role=supervisor" \ + --no-headers 2>/dev/null | wc -l)" +[ "${FENCE_PODS}" = "1" ] \ + || fail "expected exactly one supervisor pod in the fence egress selector, got ${FENCE_PODS}" +echo "fence egress selects exactly this sandbox's supervisor Pod" + +step "3/10 exec inside the VM" +exec_in_sandbox sh -c 'uname -a; id; cat /proc/1/comm' | tee ${WORKDIR}/exec.log +grep -q "uid=0(root) gid=0(root)" ${WORKDIR}/exec.log \ + || fail "workload does not run as root" +echo "workload runs as root (uid 0) — the untrusted boundary relays for it" +grep -q "^init$" ${WORKDIR}/exec.log || fail "PID 1 is not the guest init" + +# H2: the workspace PVC must be writable by the workload (the idmapped +# virtiofs mount; in root mode the root workload writes through the same +# idmap and files show as 10001 on the host side); a marker file proves it +# and is re-read after stop/start. +exec_in_sandbox sh -c 'echo workspace-ok > /sandbox/.kvpoc-demo-marker' \ + || fail "the workload cannot write to the workspace PVC" +echo "workspace writable by the workload (idmapped virtiofs)" + +step "4/10 no gateway material in the workload environment" +if exec_in_sandbox sh -c \ + 'env | grep -iE "BOUNDARY|PRIVATE|TOKEN|OPENSHELL_ENDPOINT"' \ + > ${WORKDIR}/env.log 2>&1; then + cat ${WORKDIR}/env.log >&2 + fail "gateway material found in the workload environment" +fi +echo "environment clean (no BOUNDARY/PRIVATE/TOKEN/OPENSHELL_ENDPOINT vars)" + +step "5/10 L4 allow/deny and L7 HTTPS GET allow / POST deny" +echo "the nftables transparent relay carries these checks (workload TCP" +echo "is captured to the boundary relay, policy-enforced at the supervisor;" +echo "deleting the rules restores direct egress, which the fence blocks —" +echo "fail-closed)." +exec_in_sandbox sh -c \ + 'nc -z -w 10 one.one.one.one 443 && echo "L4 ALLOW: connected"' \ + | tee ${WORKDIR}/l4.log +grep -q "L4 ALLOW: connected" ${WORKDIR}/l4.log \ + || fail "L4 connect to the allowed host was denied" +# Data-level denial check: the transparent relay completes the TCP +# handshake locally before the supervisor's decision, so handshake-only +# tools (nc -z, curl --connect-timeout without a transfer) cannot see a +# denial — the connection is accepted then reset on first read/write. A +# data transfer must fail instead. This is not an enforcement gap: zero +# upstream connections are made for denied destinations. +if exec_in_sandbox sh -c 'wget -q -T 8 -O /dev/null http://example.org/'; then + fail "L4 data transfer to a non-allowed host succeeded" +fi +supervisor_denied "example.org:80" \ + || fail "no matching OCSF NET:OPEN DENIED line for example.org:80" +echo "L4 DENY: connect to example.org:80 blocked (OCSF DENIED confirmed)" +exec_in_sandbox sh -c 'wget -q -T 20 -O - https://example.com/' \ + | tee ${WORKDIR}/l7get.log +grep -q "Example Domain" ${WORKDIR}/l7get.log \ + || fail "L7 GET over HTTPS did not return the page (CA delivery or policy broken)" +echo "L7 GET over HTTPS allowed (per-sandbox proxy CA trusted in the guest)" +exec_in_sandbox sh -c 'wget -q -T 20 -O /dev/null --post-data=x https://example.com/' \ + > ${WORKDIR}/l7post.log 2>&1 \ + && fail "L7 POST was not denied" || true +grep -q "403" ${WORKDIR}/l7post.log \ + || { cat ${WORKDIR}/l7post.log >&2; fail "L7 POST denial was not a policy 403"; } +echo "L7 POST denied (HTTP 403)" + +step "6/10 direct egress from the guest is blocked" +# The fail-closed test the posture requires: root deletes the relay's +# nftables rules (restoring direct egress), and the fence still blocks +# everything — deleting the rules only breaks the workload's own +# connectivity. Raw DNS to the masquerade dnsmasq is blocked too (the +# UDP redirect died with the table, and the fence admits nothing). +exec_in_sandbox nft delete table ip openshell_relay \ + || fail "root could not delete the relay ruleset" +echo "relay rules deleted by root; direct egress must be fenced:" +if exec_in_sandbox sh -c 'wget -q -T 8 -O /dev/null http://1.1.1.1/'; then + fail "direct egress to 1.1.1.1 escaped the fence after rule deletion" +fi +if exec_in_sandbox sh -c 'nslookup one.one.one.one 10.0.2.3'; then + fail "raw DNS to the masquerade dnsmasq bypassed the fence" +fi +echo "direct egress and raw DNS blocked by the fence (fail-closed)" + +# N4: prove the FENCE (not only the boundary) blocks direct egress: from +# the launcher pod's network namespace — outside the guest, with no +# boundary or broker — a raw connect to an external address must be +# refused, while the supervisor Service must remain reachable. +LAUNCHER="$(kubectl get pods -n "${SANDBOX_NAMESPACE}" \ + -l kubevirt.io=virt-launcher,openshell.ai/sandbox-id="${SBX_ID}" \ + -o jsonpath='{.items[0].metadata.name}')" +[ -n "${LAUNCHER}" ] || fail "no virt-launcher pod found" +# The fence's selective control target: this sandbox's dedicated +# supervisor Service. +SUPERVISOR_IP="$(kubectl get svc -n "${SANDBOX_NAMESPACE}" \ + "${SUPERVISOR_POD}" -o jsonpath='{.spec.clusterIP}' 2>/dev/null)" +[ -n "${SUPERVISOR_IP}" ] || fail "supervisor Service has no ClusterIP" +kubectl exec -n "${SANDBOX_NAMESPACE}" "${LAUNCHER}" -c compute -- \ + timeout 8 openssl s_client -connect 1.1.1.1:443 \ + > ${WORKDIR}/fence-bypass.log 2>&1 || true +grep -q "Connection refused" ${WORKDIR}/fence-bypass.log \ + || { cat ${WORKDIR}/fence-bypass.log >&2; \ + fail "raw TCP from the launcher pod to 1.1.1.1:443 was not refused by the fence"; } +echo "fence check: launcher pod -> 1.1.1.1:443 refused (Connection refused)" +# The selective control: the supervisor Service must remain reachable. The +# connect itself is expected to fail the TLS handshake (the supervisor +# awaits its boundary, not an s_client), so success is judged ONLY by the +# absence of "Connection refused" — the previous `&&` formulation could +# never fail because a refused connect fails the exec and short-circuits. +# A fresh ClusterIP's first new connections can race the endpoint +# programming and be refused once; retry briefly before declaring the +# fence too narrow. +CONTROL_OK=1 +for _ in 1 2 3 4 5; do + kubectl exec -n "${SANDBOX_NAMESPACE}" "${LAUNCHER}" -c compute -- \ + timeout 8 openssl s_client -connect "${SUPERVISOR_IP}:5500" \ + > ${WORKDIR}/fence-control.log 2>&1 || true + if ! grep -q "Connection refused" ${WORKDIR}/fence-control.log; then + CONTROL_OK=0 + break + fi + sleep 3 +done +if [ "${CONTROL_OK}" != "0" ]; then + cat ${WORKDIR}/fence-control.log >&2 + fail "launcher pod -> supervisor Service connect was refused (fence too narrow?)" +fi +echo "fence check: launcher pod -> supervisor ${SUPERVISOR_IP}:5500 reachable (selective)" + +step "7/10 stop -> start -> Ready with VM and fence UIDs unchanged" +"${OPENSHELL_CLI}" sandbox stop "${NAME}" +require_phase Stopped +# Stop deletes the supervisor Pod, exactly as the pod path's stop +# deletes both Pods. +sleep 5 +if kubectl get pod -n "${SANDBOX_NAMESPACE}" "${SUPERVISOR_POD}" >/dev/null 2>&1; then + fail "supervisor Pod still present after stop" +fi +echo "supervisor Pod deleted by stop" +"${OPENSHELL_CLI}" sandbox start "${NAME}" +wait_for_phase Ready +VM_UID_AFTER="$(vm_jsonpath '{.items[0].metadata.uid}')" +FENCE_UID_AFTER="$(kubectl get netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null)" +[ "${VM_UID_AFTER}" = "${VM_UID}" ] || fail "VM UID changed across stop/start" +[ "${FENCE_UID_AFTER}" = "${FENCE_UID}" ] || fail "fence UID changed across stop/start" +echo "VM uid unchanged (${VM_UID}); fence uid unchanged (${FENCE_UID})" +# H2: the workspace marker survived the stop/start cycle. +MARKER="$(exec_in_sandbox sh -c 'cat /sandbox/.kvpoc-demo-marker' 2>/dev/null)" \ + || fail "the workspace marker could not be read after restart" +[ "${MARKER}" = "workspace-ok" ] \ + || fail "the workspace marker did not survive stop/start (got '${MARKER}')" +exec_in_sandbox sh -c 'touch /sandbox/.kvpoc-demo-marker' \ + || fail "the workspace is not writable after restart" +echo "workspace data survived stop/start and stays writable" +# A dedicated start provisions a fresh supervisor generation: a new Pod +# with a new UID, claimed by the re-rendered boundary bootstrap. +NEW_SUPERVISOR_POD_UID="$(kubectl get pod -n "${SANDBOX_NAMESPACE}" "${SUPERVISOR_POD}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null)" \ + || fail "no supervisor Pod after start" +[ -n "${NEW_SUPERVISOR_POD_UID}" ] || fail "restarted supervisor Pod has no UID" +[ "${NEW_SUPERVISOR_POD_UID}" != "${SUPERVISOR_POD_UID}" ] \ + || fail "supervisor Pod UID did not rotate across stop/start" +# The start created a NEW generation's bootstrap Secret (-boot-) +# and deleted the old one; re-resolve the current Secret by label. +BOOTSTRAP_SECRET="$(kubectl get secrets -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID},openshell.ai/managed-by=openshell" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" \ + || fail "could not find the restarted sandbox's bootstrap Secret by label" +[ -n "${BOOTSTRAP_SECRET}" ] || fail "no labelled bootstrap Secret after start" +CLAIMED_UID="$(kubectl get secret -n "${SANDBOX_NAMESPACE}" "${BOOTSTRAP_SECRET}" \ + -o jsonpath='{.data.userData}' | base64 -d \ + | grep "^OPENSHELL_BOUNDARY_CONFIG_B64=" | cut -d= -f2- | base64 -d \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["resource_claims"]["kubevirt.supervisor_pod_uid"])')" \ + || fail "could not extract kubevirt.supervisor_pod_uid after start" +[ "${CLAIMED_UID}" = "${NEW_SUPERVISOR_POD_UID}" ] \ + || fail "re-rendered claim (${CLAIMED_UID}) does not match the new supervisor Pod UID (${NEW_SUPERVISOR_POD_UID})" +echo "supervisor Pod UID rotated and re-claimed (${NEW_SUPERVISOR_POD_UID})" +SUPERVISOR_POD_UID="${NEW_SUPERVISOR_POD_UID}" +INSTANCE_SECRETS="$(kubectl get secret -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID},openshell.ai/component=supervisor-bootstrap" \ + --no-headers 2>/dev/null | wc -l)" +[ "${INSTANCE_SECRETS}" = "1" ] \ + || fail "expected one current instance Secret, got ${INSTANCE_SECRETS}" +echo "one current-generation instance Secret (previous generation pruned)" + +step "8/10 fence verification: spec drift fails start, label-only edit passes" +"${OPENSHELL_CLI}" sandbox stop "${NAME}" +kubectl patch netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" --type=json \ + -p '[{"op":"add","path":"/spec/egress/-","value":{}}]' >/dev/null +if "${OPENSHELL_CLI}" sandbox start "${NAME}" > ${WORKDIR}/drift.log 2>&1; then + fail "start succeeded despite fence spec drift" +fi +grep -q "spec drifted" ${WORKDIR}/drift.log \ + || { cat ${WORKDIR}/drift.log >&2; fail "start failure was not the drift error"; } +echo "spec drift rejected: $(grep -a 'drifted' ${WORKDIR}/drift.log | head -1 | tr -s ' ')" +# Remove the drift idempotently. The M1 reconciler may re-fence (restore +# the rendered spec) before this script removes its own drift entry, and +# the previous test+remove patch then failed on the missing path ("The +# request is invalid"). Wait briefly for the reconciler's restore — which +# also demonstrates M1 drift repair live — and only patch the drift away +# ourselves if the reconciler has not acted yet. +fence_drift_present() { + # The demo's drift is a trailing empty egress entry. + [ "$(kubectl get netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" \ + -o jsonpath='{.spec.egress}' 2>/dev/null | python3 -c \ + 'import json,sys +e = json.load(sys.stdin) +print(len(e) >= 2 and e[-1] == {})')" = "True" ] +} +RECONCILED=1 +DRIFT_DEADLINE=$(( $(date +%s) + 60 )) +while fence_drift_present; do + if [ "$(date +%s)" -ge "${DRIFT_DEADLINE}" ]; then + RECONCILED=0 + break + fi + sleep 3 +done +if [ "${RECONCILED}" = "1" ]; then + echo "M1 reconciler re-fenced the drifted fence (spec restored)" +else + # The reconciler has not acted; remove the demo's own drift entry. + # test/remove need the concrete index (the `-` append suffix is only + # valid for `add`), so compute the last egress position first — + # fence_drift_present already confirmed it is the empty drift entry. + EGRESS_LEN="$(kubectl get netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" \ + -o jsonpath='{.spec.egress}' 2>/dev/null | python3 -c \ + 'import json,sys; print(len(json.load(sys.stdin)))')" + LAST_IDX=$(( EGRESS_LEN - 1 )) + kubectl patch netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" --type=json \ + -p "[{\"op\":\"test\",\"path\":\"/spec/egress/${LAST_IDX}\",\"value\":{}},{\"op\":\"remove\",\"path\":\"/spec/egress/${LAST_IDX}\"}]" \ + >/dev/null \ + || fail "could not remove the demo's drift entry from the fence" + echo "removed the demo's drift entry (reconciler had not re-fenced yet)" +fi +kubectl label netpol -n "${SANDBOX_NAMESPACE}" "${FENCE}" \ + kvpoc-demo-label-edit=1 --overwrite >/dev/null +"${OPENSHELL_CLI}" sandbox start "${NAME}" +wait_for_phase Ready +echo "label-only fence edit accepted (resourceVersion moved, spec unchanged)" + +step "9/10 main-process exit -> Error" +exec_in_sandbox sh -c 'pkill -9 sleep' || true +DEADLINE=$(( $(date +%s) + 120 )) +while [ "$(date +%s)" -lt "${DEADLINE}" ]; do + PHASE="$(sandbox_phase)" + [ "${PHASE}" = "Phase: Error" ] && break + sleep 5 +done +require_phase Error +"${OPENSHELL_CLI}" sandbox get "${NAME}" | grep -a "exit code" \ + || fail "Error phase carries no main-process exit code" +echo "main process exit observed through the boundary session (Error, exit code 137)" + +step "10/10 delete -> zero leftover managed objects" +"${OPENSHELL_CLI}" sandbox delete "${NAME}" +# The leftover set includes pods and services: the dedicated topology's +# supervisor Pod and per-sandbox Service carry the sandbox-id label (the +# shared topology has none, so the count stays complete either way). +DEADLINE=$(( $(date +%s) + 180 )) +while [ "$(date +%s)" -lt "${DEADLINE}" ]; do + LEFT="$(kubectl get vm,vmi,pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID}" --no-headers 2>/dev/null | wc -l)" + [ "${LEFT}" = "0" ] && break + sleep 5 +done +LEFT="$(kubectl get vm,vmi,pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "openshell.ai/sandbox-id=${SBX_ID}" --no-headers 2>/dev/null | wc -l)" +[ "${LEFT}" = "0" ] || fail "${LEFT} managed objects left after delete" +if kubectl get pod -n "${SANDBOX_NAMESPACE}" "${SUPERVISOR_POD}" >/dev/null 2>&1; then + fail "supervisor Pod still present after delete" +fi +echo "supervisor Pod gone after delete" +echo "zero leftover vm/vmi/pod/svc/secret/pvc/netpol for sandbox ${SBX_ID}" + +echo +echo "demo complete (dedicated supervisor topology, root workload): Ready," +echo "exec as root, clean environment, L4/L7 policy through the transparent" +echo "relay, bypass blocked, stop/start, fence verification, dead-runtime" +echo "Error, and clean teardown all verified" diff --git a/e2e/kubevirt/run-probes.sh b/e2e/kubevirt/run-probes.sh new file mode 100755 index 0000000000..9c854c6849 --- /dev/null +++ b/e2e/kubevirt/run-probes.sh @@ -0,0 +1,802 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Root-workload attack-surface probes for the KubeVirt compute-driver PoC. +# +# Each probe exercises an attack surface a root guest owns by design and +# records what the control plane observes. The probes document evidence for +# the RFC; outcomes that are design inputs (not pass/fail) are recorded as +# FINDINGS. Assertions are limited to invariants the approved posture +# requires. +# +# P1 in-guest reset: the reboot syscall (a root workload can always +# reboot its own kernel). Records how the control plane observes the +# reset (phase, reason, VMI/supervisor identity) — the input for the +# reset-detection design (item 7). +# P2 the iTCO watchdog: root can arm it (no configurable action; the +# device is attached unconditionally). Records the reset path from +# hardware rather than the syscall. +# P3 fake qemu-guest-agent: the virtio-serial guest-agent channel is +# attached unconditionally; a root guest can run a lying agent. +# Verifies KubeVirt surfaces the lies in VMI status AND that nothing +# in OpenShell consumes them (the sandbox must be unaffected). +# P4 virtiofsd against a root guest: the workspace share's privileged +# operation surface (chown past the idmap, mknod, setuid) — what the +# share admits and what reaches the PVC host-side. +# P5 launcher-netns listener audit: what listens in the launcher pod's +# network namespace and what the fence admits from it. +# +# Usage: e2e/kubevirt/demo.sh --probes (root workload mode is forced) +# +# The script creates its own sandbox (kvpoc-probe) with the root policy and +# tears it down on exit. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CARGO_TARGET="${CARGO_TARGET_DIR:-${ROOT}/target}" +OPENSHELL_CLI="${CARGO_TARGET}/release/openshell" +SANDBOX_NAMESPACE="${OPENSHELL_KVPOC_NAMESPACE:-kvpoc-sandboxes}" + +# Every KubeVirt guest is the root class by definition: the probes +# exercise what a root workload can do to its own guest. +DEMO_POLICY="${ROOT}/e2e/kubevirt/demo-policy.yaml" +NAME="kvpoc-probe" +VM_SELECTOR="openshell.ai/sandbox-name=${NAME}" +FENCE_PREFIX="openshell-kubevirt-fence-" +VMI_NAME="default--${NAME}" +WORKDIR="$(mktemp -d /tmp/kvpoc-probe-XXXXXX)" + +step() { echo; echo "=== $* ==="; } +fail() { echo "FAIL: $*" >&2; exit 1; } +note() { echo "NOTE: $*" >&2; } + +# Single EXIT trap: teardown on failure (CLI delete first while the gateway +# is up, then labelled kubectl deletions as the gateway-independent +# backstop), scratch removal on every exit. +probe_exit_trap() { + local rc=$? + if [ "${rc}" -ne 0 ] && [ "${PROBE_CREATED}" = "1" ]; then + echo >&2 + echo "probe failed (rc=${rc}); tearing down sandbox '${NAME}'..." >&2 + "${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true + kubectl delete vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --wait=false --ignore-not-found >/dev/null 2>&1 || true + kubectl delete pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --ignore-not-found >/dev/null 2>&1 || true + echo "teardown complete" >&2 + fi + rm -rf "${WORKDIR:-}" + exit "${rc}" +} +trap probe_exit_trap EXIT +PROBE_CREATED=0 + +sandbox_phase() { + "${OPENSHELL_CLI}" sandbox get "${NAME}" 2>/dev/null | grep -a "Phase:" | head -1 \ + | sed 's/^ *//' || true +} + +wait_for_phase() { + local expected="$1" timeout_secs="${2:-300}" deadline got + deadline=$(( $(date +%s) + timeout_secs )) + while [ "$(date +%s)" -lt "${deadline}" ]; do + got="$(sandbox_phase)" + case "${got}" in + "Phase: ${expected}") return 0 ;; + esac + sleep 5 + done + fail "timed out waiting for phase ${expected} (got '${got:-none}')" +} + +exec_in_sandbox() { + "${OPENSHELL_CLI}" sandbox exec "${NAME}" -- "$@" || { + local rc=$? + [ "${rc}" = "74" ] || return "${rc}" + return 0 + } +} + +vm_jsonpath() { + kubectl get vm -l "${VM_SELECTOR}" -n "${SANDBOX_NAMESPACE}" \ + -o "jsonpath=$1" 2>/dev/null +} + +supervisor_pod_name() { + local sandbox_id sbx_lc + sandbox_id="$(vm_jsonpath '{.items[0].metadata.labels.openshell\.ai/sandbox-id}')" + sbx_lc="$(printf '%s' "${sandbox_id}" | tr '[:upper:]' '[:lower:]')" + echo "os-supervisor-${sbx_lc}" +} + +launcher_pod_name() { + kubectl get pods -n "${SANDBOX_NAMESPACE}" \ + -l kubevirt.io=virt-launcher,openshell.ai/sandbox-name="${NAME}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null +} + +# Summed container restart counts: virt-launcher (this sandbox's launcher +# pod) and virt-handler (the node's daemonset pod that owns the launcher). +launcher_pod_restarts() { + kubectl get pod -n "${SANDBOX_NAMESPACE}" "$(launcher_pod_name)" \ + -o jsonpath='{.status.containerStatuses[*].restartCount}' 2>/dev/null \ + | tr ' ' '\n' | awk '{s+=$1} END {print s+0}' +} + +virt_handler_restarts() { + local node + node="$(kubectl get pod -n "${SANDBOX_NAMESPACE}" "$(launcher_pod_name)" \ + -o jsonpath='{.spec.nodeName}' 2>/dev/null)" + kubectl get pods -n kubevirt -l kubevirt.io=virt-handler \ + --field-selector "spec.nodeName=${node}" \ + -o jsonpath='{.items[0].status.containerStatuses[*].restartCount}' 2>/dev/null \ + | tr ' ' '\n' | awk '{s+=$1} END {print s+0}' +} + +# --------------------------------------------------------------------------- +step "setup: create a root sandbox reaching Ready" + +"${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true +kubectl delete vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --ignore-not-found >/dev/null 2>&1 || true +kubectl delete pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --ignore-not-found >/dev/null 2>&1 || true +sleep 5 + +"${OPENSHELL_CLI}" sandbox create --detach --name "${NAME}" \ + --policy "${DEMO_POLICY}" \ + -- /bin/sleep infinity +PROBE_CREATED=1 +wait_for_phase Ready +exec_in_sandbox id | tee "${WORKDIR}/id.log" +grep -q "uid=0(root)" "${WORKDIR}/id.log" || fail "probe sandbox is not root mode" +echo "probe sandbox Ready (root)" + +# --------------------------------------------------------------------------- +step "L1: the bootstrap config disk eject — and its guest-side limit" + +# The cloud-init disk (/dev/vdb) carries the full boundary bootstrap +# (private key + launch auth). The guest init reads it once and removes +# the PCI function (hygiene: the disk is not sitting attached for the +# whole session). Assert the eject happened, then demonstrate the +# limit: as root, a PCI rescan re-enumerates the disk — QEMU/KubeVirt +# still have it attached, so the guest-side eject is not a control. +# The key's exposure to root is the same as through the boundary's +# memory (ptrace); the blast radius is its own sandbox. +if exec_in_sandbox test -b /dev/vdb 2>/dev/null; then + fail "L1: /dev/vdb is still attached in the running root guest — the eject did not happen" +fi +if exec_in_sandbox sh -c 'ls /sys/block/vdb 2>/dev/null' | grep -q vdb; then + fail "L1: /sys/block/vdb still exists in the running root guest — the eject did not happen" +fi +echo "PASS: the bootstrap config disk was ejected after the one-shot read (no /dev/vdb, no /sys/block/vdb)" + +# The limit, demonstrated: as root, rescan the PCI bus and record that +# the config disk comes back (kernel-level re-enumeration is the +# authoritative check; the /dev node and the mount are attempted and +# recorded when they succeed). This documents that the eject is +# hygiene, not a control. +exec_in_sandbox sh -c 'echo 1 > /sys/bus/pci/rescan' 2>/dev/null +sleep 3 +RESCAN_KERNEL=0 +if exec_in_sandbox sh -c 'cat /proc/partitions' 2>/dev/null | grep -q vdb; then + RESCAN_KERNEL=1 +fi +RESCAN_MOUNTED=0 +if exec_in_sandbox sh -c 'mkdir -p /mnt/l1 && mount -t iso9660 -o ro /dev/vdb /mnt/l1 2>/dev/null && ls /mnt/l1 2>/dev/null | grep -q user-data && umount /mnt/l1' 2>/dev/null; then + RESCAN_MOUNTED=1 +fi +if [ "${RESCAN_KERNEL}" = "1" ] && [ "${RESCAN_MOUNTED}" = "1" ]; then + echo "FINDING/LIMIT: a root PCI rescan re-enumerates the ejected config disk and it mounts read-only with the user-data intact — the guest-side eject is hygiene, not a control; the key's exposure to root is the same as through the boundary's memory. The host-side fix (a hotpluggable data volume or another one-shot delivery channel, since cloudInitNoCloud cannot be hot-unplugged) is a recorded next step." +elif [ "${RESCAN_KERNEL}" = "1" ]; then + echo "FINDING/LIMIT: a root PCI rescan re-enumerates the ejected config disk at the kernel level (vdb returns in /proc/partitions; the reviewer verified it mounts and the user-data is readable as root) — the guest-side eject is hygiene, not a control. The host-side fix (a hotpluggable data volume or another one-shot delivery channel) is a recorded next step." +else + echo "NOTE: the PCI rescan did not re-enumerate the config disk in this run (unexpected under the known limit — record it)" +fi + +# --------------------------------------------------------------------------- +step "P5: launcher-netns listener audit (run first: it needs the live pair)" + +LAUNCHER="$(launcher_pod_name)" +[ -n "${LAUNCHER}" ] || fail "no virt-launcher pod found" +SUPERVISOR_POD="$(supervisor_pod_name)" +echo "launcher: ${LAUNCHER}" +echo "supervisor: ${SUPERVISOR_POD}" + +# Listeners in the launcher pod's network namespace (the netns the guest's +# masquerade traffic lands in). /proc/net/tcp is hex-encoded +# little-endian-address:big-endian-port; the audit lists the decoded ports. +echo "launcher netns listeners (/proc/net/tcp, /proc/net/tcp6, /proc/net/udp, /proc/net/udp6):" +# Decode without gawk extensions: the port field is hex big-endian; POSIX +# awk converts it with CONVFMT via +"0x" arithmetic where supported, so do +# the decode in the pipeline's shell instead. virtiofsd (unix socket + +# vhost device) and QEMU's guest-facing dataplane do not listen on TCP; +# what remains in this netns is the masquerade and anything the launcher +# itself runs. +# TCP listeners have state 0A (LISTEN) in /proc/net/tcp; UDP sockets carry +# state 07 (unconnected) — a bound-but-unconnected udp socket IS the +# listener shape. Filter per protocol or the DHCP server is missed. +for proto in tcp tcp6 udp udp6; do + state="0A" + case "${proto}" in udp*) state="07" ;; esac + kubectl exec -n "${SANDBOX_NAMESPACE}" "${LAUNCHER}" -c compute -- \ + sh -c "cat /proc/net/${proto} 2>/dev/null" \ + | awk -v st="${state}" '$4 == st { split($2, a, ":"); print a[2], $10 }' \ + | sort -u \ + | while read -r hexport inode; do + port="$(( 0x${hexport} ))" + echo " ${proto} listening port ${port} (inode ${inode})" + done +done | tee "${WORKDIR}/listeners.log" || true +# The masquerade DHCP server (KubeVirt's dnsmasq) listens on udp 0.0.0.0:67 +# and is reachable from the guest over the tap — a guest-reachable parser. +grep -q " udp listening port 67 " "${WORKDIR}/listeners.log" \ + && echo "FINDING: the masquerade DHCP server (udp 67) listens in the launcher netns — guest-reachable, named in the RFC alongside the guest agent" \ + || echo "NOTE: no udp 67 listener found in the launcher netns" + +# Reachability from the launcher netns (the fence's egress applies to this +# pod): the fence must admit ONLY the supervisor service. Probe the +# standard cluster-internal targets and record refused/connected. +# The fence's allow target (the supervisor Service) is judged like the +# demo's N4 check: anything other than "Connection refused" means the +# connect reached the Service (the supervisor awaits its boundary, not an +# s_client, so the TLS handshake never completes and a timeout is the +# expected reachable signal). Every other target must be refused. +probe_tcp() { + local target="$1" label="$2" log + log="${WORKDIR}/tcp-$(echo "${target}" | tr ':/' '--').log" + if kubectl exec -n "${SANDBOX_NAMESPACE}" "${LAUNCHER}" -c compute -- \ + timeout 5 openssl s_client -connect "${target}" \ + > "${log}" 2>&1; then + echo " ${label} (${target}): CONNECTED" + elif grep -q "Connection refused" "${log}" 2>/dev/null; then + echo " ${label} (${target}): refused by the fence" + else + echo " ${label} (${target}): reachable (no refusal; handshake incomplete is expected for non-TLS-peer targets)" + fi +} +KUBEDNS_IP="$(kubectl get svc -n kube-system kube-dns \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" +K8S_API_IP="$(kubectl get svc -n default kubernetes \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" +NODE_IP="$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true)" +echo "fence reachability from the launcher netns:" +SUPERVISOR_SVC_IP="$(kubectl get svc -n "${SANDBOX_NAMESPACE}" \ + "${SUPERVISOR_POD}" -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" +[ -n "${SUPERVISOR_SVC_IP}" ] && probe_tcp "${SUPERVISOR_SVC_IP}:5500" "supervisor Service (expected: the fence's allow target)" +[ -n "${KUBEDNS_IP}" ] && probe_tcp "${KUBEDNS_IP}:53" "kube-dns (cluster DNS)" +[ -n "${K8S_API_IP}" ] && probe_tcp "${K8S_API_IP}:443" "Kubernetes API service" +[ -n "${NODE_IP}" ] && probe_tcp "${NODE_IP}:10250" "node kubelet" +probe_tcp "1.1.1.1:443" "external internet" + +# The guest-reachable parser probe: the masquerade DHCP server (dnsmasq in +# the launcher netns) parses what the guest sends over the tap. Send +# malformed and oversized DHCP datagrams from the ROOT guest and assert +# virt-launcher stays healthy and the sandbox is unaffected. +GUEST_GW="$(exec_in_sandbox sh -c \ + 'ip route 2>/dev/null | sed -n "s/^default via \([0-9.]*\).*/\1/p" | head -1' 2>/dev/null || true)" +[ -n "${GUEST_GW}" ] || GUEST_GW="10.0.2.2" +echo "guest-side DHCP parser probe (targeting ${GUEST_GW}:67):" +LAUNCHER_RESTARTS_BEFORE="$(launcher_pod_restarts)" +exec_in_sandbox sh -c \ + "printf 'MALFORMED-DHCP-PROBE-NOT-BOOTP' | nc -u -w 2 ${GUEST_GW} 67" \ + >/dev/null 2>&1 || true +exec_in_sandbox sh -c \ + "head -c 60000 /dev/urandom | nc -u -w 2 ${GUEST_GW} 67" \ + >/dev/null 2>&1 || true +sleep 10 +LAUNCHER_RESTARTS_AFTER="$(launcher_pod_restarts)" +echo "virt-launcher restarts: ${LAUNCHER_RESTARTS_BEFORE} -> ${LAUNCHER_RESTARTS_AFTER}" +[ "${LAUNCHER_RESTARTS_AFTER}" = "${LAUNCHER_RESTARTS_BEFORE}" ] \ + || fail "virt-launcher restarted from malformed/oversized guest DHCP datagrams" +PHASE_AFTER_DHCP="$(sandbox_phase)" +[ "${PHASE_AFTER_DHCP}" = "Phase: Ready" ] \ + || fail "sandbox left Ready after malformed/oversized guest DHCP datagrams" +exec_in_sandbox true || fail "exec broken after malformed/oversized guest DHCP datagrams" +echo "PASS: virt-launcher stayed healthy and the sandbox is unaffected by malformed/oversized DHCP from the root guest" +echo "P5 recorded (evidence above; the fence is expected to admit only the supervisor Service)" + +# --------------------------------------------------------------------------- +step "P3: fake qemu-guest-agent (the channel is attached unconditionally)" + +# Find the guest-agent virtio-serial port and start a lying responder: it +# answers the standard QGA commands with fabricated data. KubeVirt polls +# the agent through this channel and surfaces the answers in VMI status. +# Find the port via the sysfs name (the stable reference; the /dev node +# may not exist yet if the virtio_console driver has not created it — +# mdev -s can force device creation), falling back to the /dev listing. +GA_PORT="$(exec_in_sandbox sh -c ' + for n in /sys/class/virtio-ports/*/name; do + [ -f "${n}" ] || continue + grep -q "org.qemu.guest_agent" "${n}" 2>/dev/null \\ + && basename "$(dirname "${n}")" && break + done +' 2>/dev/null | head -1 || true)" +if [ -z "${GA_PORT}" ]; then + # Force device-node creation, then retry the /dev listing. + exec_in_sandbox sh -c 'mdev -s 2>/dev/null; true' >/dev/null 2>&1 || true + GA_PORT="$(exec_in_sandbox sh -c 'ls /dev/virtio-ports/ 2>/dev/null' | head -1 || true)" +fi +if [ -z "${GA_PORT}" ]; then + echo "FINDING: no virtio-ports device in the guest — the agent channel is absent" +else + echo "guest-agent port: /dev/virtio-ports/${GA_PORT}" + + # The responder runs as a background exec session (it must keep the + # port open). It first completes the QGA handshake so KubeVirt proceeds + # past the version check (guest-sync echoes the request id; guest-info + # advertises a supported schema version), then answers the polling + # commands with deliberately identifiable lies. + # The responder holds the port with one read-write fd (the port is + # single-open: a second open gets EBUSY) and answers on that fd. The + # open itself drives KubeVirt's agent lifecycle: opening fires the + # connect event (the agent poller starts), closing ends it. + "${OPENSHELL_CLI}" sandbox exec "${NAME}" -- sh -c " + exec 3<>/dev/virtio-ports/${GA_PORT} || { echo 'port open failed'; exit 1; } + while IFS= read -r line <&3; do + printf '%s\n' \"\$line\" >> /tmp/ga-rx.log + case \"\$line\" in + *guest-sync-delimited*|*guest-sync*) + id=\"\$(printf '%s' \"\$line\" | sed -n 's/.*\"id\":\([0-9]*\).*/\1/p')\" + printf '{\"return\": %s}\n' \"\${id:-1}\" >&3 ;; + *guest-info*) + printf '{\"return\": {\"version\": 1, \"supported_commands\": [{\"enabled\": true, \"name\": \"guest-ping\", \"success-response\": true}, {\"enabled\": true, \"name\": \"guest-get-osinfo\", \"success-response\": true}, {\"enabled\": true, \"name\": \"guest-get-host-name\", \"success-response\": true}, {\"enabled\": true, \"name\": \"guest-get-users\", \"success-response\": true}, {\"enabled\": true, \"name\": \"guest-get-fsinfo\", \"success-response\": true}]}}\n' >&3 ;; + *guest-ping*) + printf '{\"return\": {}}\n' >&3 ;; + *guest-get-host-name*) + printf '{\"return\": {\"host-name\": \"totally-honest-host\"}}\n' >&3 ;; + *guest-get-osinfo*) + printf '{\"return\": {\"kernel-release\": \"6.6.6-fake\", \"machine\": \"x86_64\", \"name\": \"FakeOS\", \"pretty-name\": \"FakeOS 99 (Liar Edition)\", \"version\": \"99\", \"version-id\": \"99\", \"id\": \"fakeos\"}}\n' >&3 ;; + *guest-get-users*) + printf '{\"return\": [{\"user\": \"definitely-real-user\", \"login-time\": 1.0}]}\n' >&3 ;; + *guest-get-fsinfo*) + printf '{\"return\": [{\"name\": \"disk-fake\", \"mountpoint\": \"/fake\", \"type\": \"fakefs\", \"disk\": []}]}\n' >&3 ;; + *) + printf '{\"return\": {}}\n' >&3 ;; + esac + done + " > "${WORKDIR}/fake-agent.log" 2>&1 & + FAKE_AGENT_PID=$! + echo "fake agent running (pid ${FAKE_AGENT_PID})" + + # Give KubeVirt time to complete the handshake and poll the agent. + sleep 45 + echo "VMI status after the fake agent answered:" + kubectl get vmi -n "${SANDBOX_NAMESPACE}" "${VMI_NAME}" -o json \ + | python3 -c ' +import json, sys +vmi = json.load(sys.stdin) +status = vmi.get("status", {}) +for c in status.get("conditions", []): + print(" condition %s=%s" % (c.get("type"), c.get("status"))) +print(" guestOSInfo: %s" % json.dumps(status.get("guestOSInfo", {}))) +' | tee "${WORKDIR}/vmi-agent-status.log" + + # Diagnostics: what the host actually sent down the channel. + echo "host->guest requests received by the fake agent:" + exec_in_sandbox sh -c 'cat /tmp/ga-rx.log 2>/dev/null' \ + | tee "${WORKDIR}/ga-rx.log" | head -10 || echo " (nothing received)" + + if grep -q "FakeOS" "${WORKDIR}/vmi-agent-status.log"; then + echo "FINDING (expected): KubeVirt surfaces the fake agent's lies in VMI status (osinfo/name/fsinfo reach the VMI object)" + elif grep -q "AgentConnected=True" "${WORKDIR}/vmi-agent-status.log"; then + echo "FINDING: KubeVirt connected to the fake agent (AgentConnected) but the OS-info lies did not surface within the window" + else + echo "NOT EXERCISABLE in this environment: the guest-agent parser cannot be" + echo "reached. The channel DOES connect (a read-write guest open flips the" + echo "host-side port state to connected and starts KubeVirt's agent poller," + echo "which attempts guest-sync every 5s), but ZERO bytes traverse" + echo "host->guest even with a live guest reader on the open port — verified" + echo "on both kvpoc and kvreview with raw readers. The break is in the" + echo "virtualization stack's chardev->virtqueue data path, not in the probe" + echo "mechanics. The RFC records the guest-agent parser as a NAMED," + echo "UNTESTED root-mode surface; the mitigation is the upstream ask for a" + echo "supported way to disable the channel." + fi + + # The OpenShell-side invariant is still observable and holds: nothing in + # OpenShell consumes agent data. The sandbox must be unaffected: same + # phase, exec still works. (Recorded as an observation, not a PASS of the + # parser probe — nothing on the host read the replies.) + PHASE_AFTER_AGENT="$(sandbox_phase)" + [ "${PHASE_AFTER_AGENT}" = "Phase: Ready" ] \ + || fail "sandbox left Ready while only the fake agent ran (something consumed agent data?)" + exec_in_sandbox true || fail "exec broken while the fake agent ran" + echo "OBSERVED (OpenShell-side only): the sandbox is unaffected by the fake agent (Ready, exec works) — no host-side parser was exercised" + + # Hostile phase: malformed and oversized replies. The user's decision + # (option B) requires virt-launcher and virt-handler to stay healthy + # while the guest-side agent misbehaves. Replace the responder with a + # hostile one: invalid JSON first, then a 10 MiB single-line reply. + kill "${FAKE_AGENT_PID}" 2>/dev/null || true + wait "${FAKE_AGENT_PID}" 2>/dev/null || true + sleep 2 + LAUNCHER_RESTARTS_BEFORE="$(launcher_pod_restarts)" + VIRT_HANDLER_RESTARTS_BEFORE="$(virt_handler_restarts)" + # NOTE: with the host->guest data path dead, the guest->host replies at + # most reach QEMU's chardev flow control (writes block once the buffer + # fills) — no host-side parser reads them. The health assertion below is + # therefore an OpenShell/cluster stability observation, not a parser + # fuzz; the parser itself is NOT EXERCISABLE here (see the handshake + # note above). + # The oversized write is bounded: virtio-serial applies host-side flow + # control, so a guest write with no host consumer BLOCKS (itself a + # finding — the guest cannot flood the host unboundedly). A 20s bound + # proves the oversized payload was offered without hanging the probe. + # The CLI-level timeout bounds the whole session: once the virtio port + # buffer fills (no host consumer), EVERY further guest write blocks — + # including small ones — so the guest-side timeout alone cannot free the + # session. Killing the CLI closes the relay, and the boundary reaps the + # exec child when the channel ends. + timeout 60 "${OPENSHELL_CLI}" sandbox exec "${NAME}" -- sh -c " + exec 3<>/dev/virtio-ports/${GA_PORT} || { echo 'port open failed'; exit 1; } + # Malformed: not JSON at all. + printf 'NOT JSON AT ALL {{{\n' >&3 + sleep 3 + # Oversized: a single 10 MiB reply line. The write blocks once the + # port buffer fills (virtio-serial host flow control) — recorded as + # a finding, the session is bounded by the CLI timeout. + printf '{\"return\": \"' >&3 + head -c 10485760 /dev/zero | tr '\\0' 'A' >&3 + printf '\"}\n' >&3 + sleep 5 + " > "${WORKDIR}/hostile-agent.log" 2>&1 || true + grep -q "port open failed" "${WORKDIR}/hostile-agent.log" 2>/dev/null \ + && echo "NOTE: the hostile agent could not open the port (single-open contention with the poller window)" \ + || echo "hostile replies delivered (or bounded by flow control); session closed" + sleep 10 + LAUNCHER_RESTARTS_AFTER="$(launcher_pod_restarts)" + VIRT_HANDLER_RESTARTS_AFTER="$(virt_handler_restarts)" + echo "virt-launcher restarts: ${LAUNCHER_RESTARTS_BEFORE} -> ${LAUNCHER_RESTARTS_AFTER}" + echo "virt-handler restarts: ${VIRT_HANDLER_RESTARTS_BEFORE} -> ${VIRT_HANDLER_RESTARTS_AFTER}" + [ "${LAUNCHER_RESTARTS_AFTER}" = "${LAUNCHER_RESTARTS_BEFORE}" ] \ + || fail "virt-launcher restarted from malformed/oversized guest-agent replies" + [ "${VIRT_HANDLER_RESTARTS_AFTER}" = "${VIRT_HANDLER_RESTARTS_BEFORE}" ] \ + || fail "virt-handler restarted from malformed/oversized guest-agent replies" + PHASE_AFTER_HOSTILE="$(sandbox_phase)" + [ "${PHASE_AFTER_HOSTILE}" = "Phase: Ready" ] \ + || fail "sandbox left Ready after hostile guest-agent replies" + exec_in_sandbox true || fail "exec broken after hostile guest-agent replies" + echo "OBSERVED (stability only): virt-launcher and virt-handler stayed healthy through malformed/oversized guest-agent replies — the host-side parser was not exercised (see above)" +fi + +# --------------------------------------------------------------------------- +step "P4: virtiofsd privileged-operation surface (root guest)" + +# 1. Can root remount the share RAW (no idmap) and chown past the idmap? +exec_in_sandbox sh -c 'mkdir -p /mnt/raw && mount -t virtiofs workspace /mnt/raw' \ + > "${WORKDIR}/rawmount.log" 2>&1 \ + && echo "FINDING: root CAN mount the virtiofs share without the idmap (/mnt/raw)" \ + || echo "the raw remount was refused" +if ! grep -qi "refused\|denied\|Permission" "${WORKDIR}/rawmount.log" 2>/dev/null; then + cat "${WORKDIR}/rawmount.log" 2>/dev/null | head -2 || true +fi + +# 2. chown to an arbitrary uid through the idmapped /sandbox (mapped range +# only) and on the raw mount (if it exists). +echo "chown probe through the idmapped mount:" +exec_in_sandbox sh -c 'touch /sandbox/p4-chown && chown 1234:1234 /sandbox/p4-chown; echo rc=$?; stat -c "guest sees %u:%g" /sandbox/p4-chown' \ + | tee "${WORKDIR}/chown-idmap.log" +if mountpoint_ok="$(exec_in_sandbox sh -c 'mountpoint -q /mnt/raw && echo yes' 2>/dev/null)" && [ "${mountpoint_ok}" = "yes" ]; then + echo "chown probe on the RAW mount:" + exec_in_sandbox sh -c 'touch /mnt/raw/p4-chown-raw && chown 1234:1234 /mnt/raw/p4-chown-raw; echo rc=$?; stat -c "guest sees %u:%g" /mnt/raw/p4-chown-raw' \ + | tee "${WORKDIR}/chown-raw.log" +fi + +# 3. mknod a device node on the share (would persist on the PVC across +# generations if admitted). +echo "mknod probe:" +exec_in_sandbox sh -c 'mknod /sandbox/p4-dev c 1 3; echo rc=$?; ls -l /sandbox/p4-dev 2>/dev/null' \ + | tee "${WORKDIR}/mknod.log" || true + +# 4. setuid bit on a copied binary (would persist on the PVC). +echo "setuid probe:" +exec_in_sandbox sh -c 'cp /bin/busybox /sandbox/p4-su && chmod 4755 /sandbox/p4-su; echo rc=$?; ls -l /sandbox/p4-su' \ + | tee "${WORKDIR}/setuid.log" || true + +# 5. Host-side truth: what actually landed on the PVC (k3d local-path +# provisioning on the node container). +# The workspace PVC name is deterministic: -workspace. +PVC_NAME="default--${NAME}-workspace" +K3D_NODE="k3d-$(kubectl config current-context | sed 's/^k3d-//' | cut -d- -f1-2)-server-0" +if [ -n "${PVC_NAME}" ] && docker inspect "${K3D_NODE}" >/dev/null 2>&1; then + echo "host-side PVC contents (${PVC_NAME} on ${K3D_NODE}):" + docker exec "${K3D_NODE}" sh -c \ + "ls -ln /var/lib/rancher/k3s/storage/ | grep '${PVC_NAME}' || true" | head -2 + docker exec "${K3D_NODE}" sh -c \ + "ls -ln /var/lib/rancher/k3s/storage/pvc-*_${SANDBOX_NAMESPACE}_${PVC_NAME}/* 2>/dev/null" \ + | grep -E "p4-" || echo " (no p4-* files visible host-side)" +else + echo "NOTE: host-side PVC inspection skipped (PVC ${PVC_NAME:-none} or node ${K3D_NODE} unavailable)" +fi +echo "P4 recorded (evidence above; the RFC documents the share's admitted surface)" + +# --------------------------------------------------------------------------- +step "P1: in-guest reset via the reboot syscall" + +VMI_UID_BEFORE="$(kubectl get vmi -n "${SANDBOX_NAMESPACE}" "${VMI_NAME}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" +SUP_POD_UID_BEFORE="$(kubectl get pod -n "${SANDBOX_NAMESPACE}" "$(supervisor_pod_name)" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" +echo "before: VMI uid=${VMI_UID_BEFORE:-none} supervisor pod uid=${SUP_POD_UID_BEFORE:-none}" + +echo "issuing: reboot -f (root workload reboots its own kernel)" +# The exec session dies mid-stream when the guest resets; a non-zero rc +# here is the expected outcome, not a failure. +exec_in_sandbox reboot -f > "${WORKDIR}/reboot.log" 2>&1 || true +sleep 10 + +# Record what the control plane observes for up to 5 minutes: the phase, +# the reason, whether the VMI identity survived, whether the supervisor +# generation rotated. +echo "observing the control plane for up to 300s..." +RESET_OBSERVED="" +DEADLINE=$(( $(date +%s) + 300 )) +while [ "$(date +%s)" -lt "${DEADLINE}" ]; do + PHASE="$(sandbox_phase)" + VMI_UID_NOW="$(kubectl get vmi -n "${SANDBOX_NAMESPACE}" "${VMI_NAME}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" + case "${PHASE}" in + "Phase: Error"|"Phase: Stopped") + RESET_OBSERVED="${PHASE}" + break ;; + esac + # A same-uid VMI back at Ready means the guest rebooted in place and + # everything reconnected — record it and keep watching briefly. + if [ "${PHASE}" = "Phase: Ready" ] && [ -n "${VMI_UID_BEFORE}" ] \ + && [ "${VMI_UID_NOW}" != "${VMI_UID_BEFORE}" ]; then + RESET_OBSERVED="Ready-with-new-VMI" + break + fi + sleep 5 +done + +VMI_UID_AFTER="$(kubectl get vmi -n "${SANDBOX_NAMESPACE}" "${VMI_NAME}" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" +SUP_POD_UID_AFTER="$(kubectl get pod -n "${SANDBOX_NAMESPACE}" "$(supervisor_pod_name)" \ + -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" +echo "after: phase='${RESET_OBSERVED:-unresolved (${PHASE:-none})}'" +echo " VMI uid: ${VMI_UID_BEFORE:-none} -> ${VMI_UID_AFTER:-none}" +echo " supervisor pod uid: ${SUP_POD_UID_BEFORE:-none} -> ${SUP_POD_UID_AFTER:-none}" +"${OPENSHELL_CLI}" sandbox get "${NAME}" 2>/dev/null | grep -aE "Phase:|Ready:|exit code" | head -3 || true +# The detection mechanism: guest init re-runs after the warm reset, re-stages +# the SAME bootstrap (same key) from the cloud-init disk, and launches a +# fresh privileged boundary that reconnects to the live generation. The +# supervisor ends the sandbox when the fresh boundary attaches — the handle +# changed during launch recovery. Record the supervisor's own reason line. +SUP_EXIT_REASON="$(kubectl logs -n "${SANDBOX_NAMESPACE}" \ + "$(supervisor_pod_name)" --tail=200 2>/dev/null \ + | grep -a "attachment denied" | tail -1 || true)" +if [ -n "${SUP_EXIT_REASON}" ]; then + echo " supervisor exit reason: ${SUP_EXIT_REASON}" + echo "PASS mechanism: the reset is detected by the supervisor's boundary-handle check on launch recovery (a fresh boundary attaching to the live generation ends it)" +else + echo "NOTE: the supervisor's attachment-denied line was not captured (log cadence); the phase flip above is the detection evidence" +fi +# RFC caveat: root can read the boundary's memory, learn the handle, and +# present it after a reset. Acceptable: root already holds the key and +# handle in-session (it can impersonate its own boundary without +# rebooting). The fence plus the supervisor's policy make post-reset +# CODE trusted. + +# The invariant: a reset must not pass unnoticed. Whatever terminal or +# observed state results, the sandbox must not silently continue the SAME +# session as if nothing happened: either the phase flipped away from Ready +# (detected), or the VMI identity changed (new boot observed). +if [ -n "${RESET_OBSERVED}" ] && [ "${RESET_OBSERVED}" != "Ready" ]; then + echo "PASS: the guest reset was observed by the control plane (${RESET_OBSERVED})" +else + echo "FINDING: the control plane did not clearly observe the reset (phase '${PHASE:-none}', VMI ${VMI_UID_BEFORE:-none} -> ${VMI_UID_AFTER:-none}); record for the reset-detection design (item 7)" +fi + +# --------------------------------------------------------------------------- +step "P2: the iTCO watchdog (root can arm it; no configurable action)" + +# Compile the port-io helper for the guest (static musl). The product guest +# image does not ship it — this is a probe-time tool. +echo "compiling the port-io helper (static musl, docker alpine)..." +docker run --rm \ + -v "${ROOT}/e2e/kubevirt:/src:ro,Z" \ + -v "${WORKDIR}:/out:Z" \ + alpine:3.22 sh -c ' + apk add --no-cache gcc musl-dev linux-headers >/dev/null 2>&1 + cc -static -O2 -o /out/port-io /src/port-io.c + ' || fail "could not compile the port-io helper" +[ -s "${WORKDIR}/port-io" ] || fail "the port-io helper did not build" +echo "port-io built: $(wc -c < "${WORKDIR}/port-io") bytes" + +# Stage it into the guest through the exec channel (base64 over stdin). +stage_port_io() { + local sandbox="$1" + base64 < "${WORKDIR}/port-io" \ + | "${OPENSHELL_CLI}" sandbox exec "${sandbox}" -- sh -c \ + 'base64 -d > /tmp/port-io && chmod +x /tmp/port-io && /tmp/port-io read16 0x60' \ + > /dev/null 2>&1 +} + +# Recreate the sandbox (P1 left it in a terminal or ambiguous state). +"${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true +kubectl delete vm -n "${SANDBOX_NAMESPACE}" -l "${VM_SELECTOR}" \ + --ignore-not-found >/dev/null 2>&1 || true +kubectl delete pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --ignore-not-found >/dev/null 2>&1 || true +sleep 5 +"${OPENSHELL_CLI}" sandbox create --detach --name "${NAME}" \ + --policy "${DEMO_POLICY}" \ + -- /bin/sleep infinity +wait_for_phase Ready +stage_port_io "${NAME}" || fail "could not stage the port-io helper into the guest" + +# The reset watch must distinguish a RESET from the sandbox's own young +# boot (P2's sandbox is recreated right after P1's reboot): settle first +# (uptime >= 60s), then arm, then watch for a uptime DROP. +echo "waiting for the guest to settle (uptime >= 60s) before arming..." +while :; do + UPTIME_AT_ARM="$(exec_in_sandbox sh -c 'cut -d. -f1 /proc/uptime' 2>/dev/null || echo 0)" + [ "${UPTIME_AT_ARM}" -ge 60 ] && break + sleep 5 +done + +# The stock linux-virt kernel ships iTCO_wdt.ko but cannot bind it (lpc_ich +# is not built), so /dev/watchdog is absent. A root workload can still load +# its own lpc_ich build, or program the TCO timer directly through +# /dev/port. The probe tries, in order: the module, the device node, and a +# direct port-I/O arm of the TCO timer. RFC wording: "not exposed by the +# stock kernel; reachable by root; covered by reset detection (P1) and +# the fresh-per-boot image". +# +# The port-io helper is a PROBE TOOL and does not ship in the guest image: +# /dev/port is byte-granular and QEMU drops byte writes to the dword/word +# emulated registers (the PCI config address 0xcf8 and the TCO counters), +# so the arm needs real OUT instructions from a compiled binary. The probe +# compiles it (static musl, in a docker alpine container like the image +# build) and stages it into the guest through the exec channel. +MODPROBE_RC="$(exec_in_sandbox sh -c 'modprobe iTCO_wdt 2>&1; echo rc=$?' || true)" +echo "modprobe iTCO_wdt: ${MODPROBE_RC}" +WD_PRESENT="$(exec_in_sandbox sh -c 'ls -l /dev/watchdog* 2>/dev/null || echo absent')" +echo "watchdog device: ${WD_PRESENT}" +ARMED=0 +if ! echo "${WD_PRESENT}" | grep -q "absent"; then + # Opening /dev/watchdog starts it; closing WITHOUT the magic 'V' leaves + # it armed, and the iTCO resets the machine when it expires. + echo "arming: open + non-magic close of /dev/watchdog" + exec_in_sandbox sh -c 'exec 3>/dev/watchdog' || true + ARMED=1 +else + echo "no /dev/watchdog after modprobe (lpc_ich is not built for linux-virt; the module cannot bind)" + # Direct port-I/O arm through the guest's port-io helper (static musl, + # shipped in the image). /dev/port CANNOT be used: it is byte-granular, + # and QEMU drops byte writes to the dword/word emulated registers (the + # PCI config address 0xcf8 and the TCO counters) — verified live: dd + # writes through /dev/port never latch. + # The ACPI PMBASE is discovered from the ICH9 LPC's PCI config (bus 0, + # device 0x1f, function 0, offset 0x40 = PMBA) via 0xCF8/0xCFC — never + # hardcoded, so a different QEMU default cannot misdirect the writes. + # TCO base = PMBASE + 0x60 (TCO_RLD at +0x00) + # TCO1_CNT = PMBASE + 0x68 (bit 11: TCO_TMR_HLT) + # On ICH9 the TCO reset gate is GCS.NO_REBOOT in the RCBA chipset + # config space (RCBA+0x3410, bit 5). In every settled run so far the + # timer fired but the gate held; the probe now OPENS the gate first — + # read the RCBA base from the ICH9 LPC PCI config (offset 0xf0), clear + # NO_REBOOT through /dev/mem (busybox devmem) — then clears TMR_HLT and + # kicks TCO_RLD. Either outcome is RFC evidence: a reset is covered by + # reset detection (P1); a held gate documents the emulated wiring. + PORT_ARM="$(exec_in_sandbox sh -c ' + PIO=/tmp/port-io + [ -x "${PIO}" ] || { echo "no port-io helper in the guest image"; exit 9; } + # PCI config read: 0xcf8 <- 0x8000f840 (00:1f.0 offset 0x40), read 0xcfc. + "${PIO}" write32 0xcf8 0x8000f840 + pmba=$("${PIO}" read32 0xcfc) + pmba=$(( pmba & 0xfffffff0 )) + case "${pmba}" in + 0x0|0xffffffff) echo "PMBA readback implausible (${pmba}); no LPC?"; exit 7 ;; + esac + # The RCBA base: 00:1f.0 offset 0xf0 (bit 0 enables the region). + "${PIO}" write32 0xcf8 0x8000f8f0 + rcba=$("${PIO}" read32 0xcfc) + rcba=$(( rcba & 0xffffc000 )) + gcs_addr=$(( rcba + 0x3410 )) + # The guest busybox has no devmem applet; the port-io helper maps + # the page through /dev/mem (also surfacing any strict-devmem + # policy that blocks chipset MMIO). + gcs_before=$("${PIO}" mem-read "${gcs_addr}" 16 2>/dev/null || echo 0xread-failed) + # Clear NO_REBOOT (bit 5) through the MMIO mapping. + "${PIO}" mem-write "${gcs_addr}" $(( gcs_before & ~0x20 )) 16 >/dev/null 2>&1 || true + gcs_after=$("${PIO}" mem-read "${gcs_addr}" 16 2>/dev/null || echo 0xread-failed) + cnt=$("${PIO}" read16 $(( pmba + 0x68 ))) + echo "PMBASE=${pmba} RCBA=0x$(printf %x "${rcba}") GCS=${gcs_before} -> ${gcs_after} (bit5 noreboot=$(( (gcs_after >> 5) & 1 ))) TCO1_CNT=${cnt} (bit11 halt=$(( (cnt >> 11) & 1 )))" + # Arm: clear the halt bit and kick the reload register. + "${PIO}" write16 $(( pmba + 0x68 )) $(( cnt & ~0x800 )) + "${PIO}" write16 $(( pmba + 0x60 )) 0 + after=$("${PIO}" read16 $(( pmba + 0x68 ))) + echo "after arm: TCO1_CNT=${after} (bit11 halt=$(( (after >> 11) & 1 )))" + [ $(( (after >> 11) & 1 )) = 0 ] \ + && echo "armed: NO_REBOOT cleared, TCO_TMR_HLT cleared, TCO_RLD kicked (timer running)" + ' 2>&1 || true)" + echo "port-I/O arm: ${PORT_ARM}" + echo "${PORT_ARM}" | grep -q "armed:" && ARMED=1 +fi +if [ "${ARMED}" = "1" ]; then + echo "watching for the watchdog reset (up to 120s, uptime at arm ${UPTIME_AT_ARM}s)..." + WD_RESET="" + DEADLINE=$(( $(date +%s) + 120 )) + PHASE_AT_ARM="$(sandbox_phase)" + while [ "$(date +%s)" -lt "${DEADLINE}" ]; do + PHASE="$(sandbox_phase)" + [ "${PHASE}" != "${PHASE_AT_ARM}" ] && { WD_RESET="${PHASE}"; break; } + # A guest-side check: the uptime drops when the machine resets. + UPTIME="$(exec_in_sandbox sh -c 'cut -d. -f1 /proc/uptime' 2>/dev/null || echo 0)" + [ "${UPTIME}" -lt "${UPTIME_AT_ARM}" ] && { WD_RESET="guest-rebooted(uptime=${UPTIME}s, was ${UPTIME_AT_ARM}s)"; break; } + sleep 5 + done + # Whether or not the machine reset, read the TCO status flags: the + # timer FIRING is provable even when QEMU's reset gating holds. + TCO_STATUS="$(exec_in_sandbox sh -c ' + PIO=/tmp/port-io + "${PIO}" write32 0xcf8 0x8000f840 + pmba=$(( $("${PIO}" read32 0xcfc) & 0xfffffff0 )) + [ "${pmba}" -gt 0 ] || pmba=0x600 + echo "TCO1_STS=$("${PIO}" read16 $(( pmba + 0x64 ))) TCO2_STS=$("${PIO}" read16 $(( pmba + 0x66 )))" + ' 2>/dev/null || true)" + echo "TCO status after the window: ${TCO_STATUS:-unavailable}" + if [ -n "${WD_RESET}" ]; then + # P2(a): prove the reset ENDS the generation, like P1 — the same + # handle-check line and Phase Error, not just a guest-side uptime + # observation. The phase flip takes ~25s (the fresh boundary must + # attach and be denied first), so poll rather than sample once. + P2_PHASE="" + P2_DEADLINE=$(( $(date +%s) + 120 )) + while [ "$(date +%s)" -lt "${P2_DEADLINE}" ]; do + P2_PHASE="$(sandbox_phase)" + [ "${P2_PHASE}" = "Phase: Error" ] && break + sleep 5 + done + P2_HANDLE_LINE="$(kubectl logs -n "${SANDBOX_NAMESPACE}" \ + "$(supervisor_pod_name)" --tail=200 2>/dev/null \ + | grep -a "attachment denied" | tail -1 || true)" + echo "control plane after the watchdog reset: phase='${P2_PHASE}'" + [ -n "${P2_HANDLE_LINE}" ] \ + && echo " supervisor exit reason: ${P2_HANDLE_LINE}" \ + || echo " (no attachment-denied line captured; log cadence)" + if [ "${P2_PHASE}" = "Phase: Error" ]; then + echo "PASS/recorded: the iTCO watchdog reset the guest (guest side: ${WD_RESET}); the control plane ended the generation (Phase Error) — covered by reset detection (P1) and the fresh-per-boot image" + else + echo "FINDING: the watchdog reset the guest (${WD_RESET}) but the control plane reports '${P2_PHASE}'; record for the reset-detection design (item 7)" + fi + elif echo "${TCO_STATUS}" | grep -q "TCO1_STS=0x0008"; then + # TCO1_STS bit 3 = TCO_TIMEOUT (the timer fired); TCO2_STS bits + # SECOND_TO_STS/BOOT_STS record the double timeout. No settled run + # has demonstrated an actual machine reset: the record for the RFC + # is "root can arm the TCO and make it fire; the reset gate is + # what it is on this wiring" — either way covered by reset + # detection (P1) and trusted boot. + echo "FINDING: the TCO timer FIRED (TCO1_STS TIMEOUT, ${TCO_STATUS#*TCO1_STS=0x0008}) — root arms and fires it. This run: $( [ -n "${WD_RESET}" ] && echo "the machine RESET (${WD_RESET})" || echo "the machine did NOT reset (uptime continuous, phase unchanged); the reset gate held" ). Either outcome is covered by reset detection (P1) and the fresh-per-boot image; the RFC records the demonstrated state (armable, firing, reset outcome as observed)" + else + echo "FINDING: the watchdog did not observably fire within 120s and no status flags were readable; the RFC records the surface as reachable-by-root regardless — covered by reset detection (P1) and the fresh-per-boot image" + fi +else + echo "FINDING: the TCO could not be armed from the stock guest (no /dev/watchdog, port-I/O arm unavailable or readback implausible); the surface is still reachable by root with a custom kernel module — covered by reset detection (P1) and the fresh-per-boot image" +fi + +# --------------------------------------------------------------------------- +step "teardown" + +"${OPENSHELL_CLI}" sandbox delete "${NAME}" >/dev/null 2>&1 || true +DEADLINE=$(( $(date +%s) + 180 )) +while [ "$(date +%s)" -lt "${DEADLINE}" ]; do + LEFT="$(kubectl get vm,vmi,pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --no-headers 2>/dev/null | wc -l)" + [ "${LEFT}" = "0" ] && break + sleep 5 +done +LEFT="$(kubectl get vm,vmi,pod,svc,secret,pvc,netpol -n "${SANDBOX_NAMESPACE}" \ + -l "${VM_SELECTOR}" --no-headers 2>/dev/null | wc -l)" +[ "${LEFT}" = "0" ] || fail "${LEFT} managed objects left after delete" +PROBE_CREATED=0 +echo "zero leftovers for sandbox probe ${NAME}" + +echo +echo "probes complete. Evidence in ${WORKDIR} is removed by the exit trap;" +echo "the findings above are the RFC inputs."