Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 57 additions & 65 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4814,75 +4814,64 @@ impl KubernetesComputeDriver {
.await?;
let selector = self.sandbox_lookup_selector(sandbox_id);
let lp = ListParams::default().labels(&selector);
let (kube_name, obj_namespace, _workspace, preconditions, pod_name, placement) =
match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await {
Ok(Ok(list)) => {
if let Some(obj) = list.items.into_iter().next() {
match obj.metadata.name.clone() {
Some(name) => {
let ns = obj
.metadata
.namespace
.clone()
.unwrap_or_else(|| self.config.namespace.clone());
let ws = obj
.metadata
.labels
.as_ref()
.and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned())
.unwrap_or_default();
// An unreadable placement leaves any instance to reconciliation.
let placement = supervisor_placement(&obj).ok();
let pc = Preconditions {
uid: obj.metadata.uid,
resource_version: obj.metadata.resource_version,
};
let pod_name = obj
.metadata
.annotations
.as_ref()
.and_then(|annotations| {
annotations.get(SANDBOX_POD_NAME_ANNOTATION)
})
.cloned()
.unwrap_or_else(|| name.clone());
(name, ns, ws, pc, pod_name, placement)
}
None => return Ok(false),
}
} else {
debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)");
return Ok(false);
}
}
Ok(Err(err)) => {
warn!(
sandbox_id = %sandbox_id,
error = %err,
"Failed to list sandbox for deletion from Kubernetes"
);
return Err(err.to_string());
}
Err(_elapsed) => {
warn!(
sandbox_id = %sandbox_id,
timeout_secs = KUBE_API_TIMEOUT.as_secs(),
"Timed out listing sandbox for deletion from Kubernetes"
);
return Err(format!(
"timed out after {}s waiting for Kubernetes API",
KUBE_API_TIMEOUT.as_secs()
));
}
};
let list = match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await {
Ok(Ok(list)) => list,
Ok(Err(err)) => {
warn!(
sandbox_id = %sandbox_id,
error = %err,
"Failed to list sandbox for deletion from Kubernetes"
);
return Err(err.to_string());
}
Err(_elapsed) => {
warn!(
sandbox_id = %sandbox_id,
timeout_secs = KUBE_API_TIMEOUT.as_secs(),
"Timed out listing sandbox for deletion from Kubernetes"
);
return Err(format!(
"timed out after {}s waiting for Kubernetes API",
KUBE_API_TIMEOUT.as_secs()
));
}
};
let Some(obj) = list.items.into_iter().next() else {
debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)");
return Ok(false);
};
// An unreadable placement leaves any instance to reconciliation.
let placement = supervisor_placement(&obj).ok();
let kube_name = obj.metadata.name.ok_or("Sandbox CR has no name")?;
let obj_namespace = obj
.metadata
.namespace
.unwrap_or_else(|| self.config.namespace.clone());
let uid = obj
.metadata
.uid
.filter(|uid| !uid.is_empty())
.ok_or_else(|| format!("Sandbox CR {obj_namespace}/{kube_name} has no UID"))?;
let pod_name = obj
.metadata
.annotations
.as_ref()
.and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION))
.cloned()
.unwrap_or_else(|| kube_name.clone());

let delete_api = self
.supported_agent_sandbox_api(self.client.clone(), &obj_namespace)
.await?;
let dp = DeleteParams::default().preconditions(preconditions);
// Delete this identity, not this revision. A controller status write
// must not block deletion, and a same-name replacement has another UID.
let dp = DeleteParams::default().preconditions(Preconditions {
uid: Some(uid),
resource_version: None,
});
match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await {
Ok(Ok(_response)) => {
info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes");
info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Kubernetes accepted Sandbox CR deletion");
// The workload is going away, so its sandbox instance cannot
// stop it gracefully; end the instance without delaying the
// delete.
Expand Down Expand Up @@ -4923,11 +4912,14 @@ impl KubernetesComputeDriver {
}
Ok(true)
}
Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => {
debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)");
Ok(Err(KubeError::Api(err))) if err.code == 404 => {
debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)");
Ok(false)
}
Ok(Err(err)) => {
// In particular, a 409 is not proof of absence. Returning
// false would let the gateway release the name without a
// deletion event or an independent lookup.
warn!(
sandbox_id = %sandbox_id,
error = %err,
Expand Down
101 changes: 101 additions & 0 deletions crates/openshell-driver-kubernetes/src/lifecycle_tests.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,107 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

mod sandbox_deletion {
use super::*;
use http_body_util::BodyExt as _;

async fn delete_after_concurrent_change(current_uid: &'static str) -> Result<bool, String> {
use http::{Method, StatusCode};
use serde_json::json;

const CR: &str = "/apis/agents.x-k8s.io/v1beta1/namespaces/openshell/sandboxes";
const NAMED_CR: &str =
"/apis/agents.x-k8s.io/v1beta1/namespaces/openshell/sandboxes/default--demo";
const POD: &str = "/api/v1/namespaces/openshell/pods/default--demo";

let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_attempts = attempts.clone();
let service = tower::service_fn(move |request: http::Request<kube::client::Body>| {
let attempts = attempts.clone();
async move {
let (code, body) = match (request.method(), request.uri().path()) {
(&Method::GET, CR) => (
StatusCode::OK,
json!({
"apiVersion": "agents.x-k8s.io/v1beta1",
"kind": "SandboxList",
"items": [{
"apiVersion": "agents.x-k8s.io/v1beta1",
"kind": "Sandbox",
"metadata": {
"name": "default--demo",
"namespace": "openshell",
"uid": "original",
"resourceVersion": "41"
}
}]
}),
),
(&Method::DELETE, NAMED_CR) => {
attempts.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let bytes = request.into_body().collect().await.unwrap().to_bytes();
let params: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
let conditions = &params["preconditions"];
assert_eq!(conditions["uid"], "original");
// The controller wrote version 42 after the list. A
// same-name replacement also has a different UID.
if conditions["uid"] != current_uid
|| (!conditions["resourceVersion"].is_null()
&& conditions["resourceVersion"] != "42")
{
(
StatusCode::CONFLICT,
json!({
"status": "Failure", "reason": "Conflict", "code": 409,
"message": "deletion precondition failed"
}),
)
} else {
(
StatusCode::OK,
json!({
"apiVersion": "v1", "kind": "Status", "status": "Success"
}),
)
}
}
(&Method::GET, POD | NAMED_CR) if current_uid == "original" => (
StatusCode::NOT_FOUND,
json!({"status": "Failure", "reason": "NotFound", "code": 404}),
),
_ => panic!("unexpected Kubernetes request: {request:?}"),
};
Ok::<_, std::convert::Infallible>(kube_test_response(code, body))
}
});
let (mut driver, _, _) = scripted_driver(KubernetesComputeConfig::default(), vec![]);
driver.client = Client::new(service, "openshell");
driver
.sandbox_api_version
.set(SANDBOX_VERSION_V1BETA1)
.expect("set test Sandbox API version");
let result = driver.delete_sandbox("sandbox-id").await;
assert_eq!(
observed_attempts.load(std::sync::atomic::Ordering::Relaxed),
1
);
result
}

#[tokio::test]
async fn delete_sandbox_survives_a_controller_status_update() {
assert!(delete_after_concurrent_change("original").await.unwrap());
}

#[tokio::test]
async fn delete_sandbox_conflict_does_not_delete_replacement_or_claim_absence() {
let error = delete_after_concurrent_change("replacement")
.await
.unwrap_err();
assert!(error.contains("deletion precondition failed"), "{error}");
}
}

mod lifecycle_reconciliation {
use super::*;

Expand Down
Loading