Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion crates/openshell-driver-kubernetes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -266,13 +266,22 @@ fails closed.
Shared supervisors receive final access tokens through their authenticated
sessions. No SPIFFE socket is mounted in a shared supervisor or workload Pod.

The topology does not support upstream proxies.
- **Upstream proxy.** The gateway snapshots the operator's proxy settings,
credential Secret from the sandbox namespace, and CA bundle in the
generation's private launch record. Pool members stage the material in the
instance's private directory and use the same CONNECT path as dedicated
supervisors. Recovery and handoff reuse the record. Restart the sandbox
generation to pick up proxy changes. Upgrade the gateway and every pool
member before enabling these settings. The Helm chart grants the gateway
`get` on the configured credential Secret name in the sandbox namespace.

With a live replacement supervisor and a takeover-capable runtime, the gateway
can move a paused sandbox generation to the replacement using a signed takeover
grant. Recovery preserves workload processes and memory, but proxied connections
and interactive sessions must reconnect. If recovery exceeds the configured
deadline, the runtime terminates the workload. It cannot recover a lost workload
Pod or node. See [supervisor recovery](../../docs/kubernetes/high-availability.mdx#recover-from-a-supervisor-failure).

Shared sandbox instances export no supervisor OTLP traces or OCSF events and do
not relay agent traces; the driver warns at startup when the gateway configures
any of them.
Expand Down
28 changes: 13 additions & 15 deletions crates/openshell-driver-kubernetes/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -586,9 +586,6 @@ impl KubernetesComputeConfig {
Ok(())
}

/// The shared supervisor serves only sandbox instances it can isolate: it
/// holds no per-sandbox workload identity, upstream proxy credential, or
/// proxy CA.
pub fn validate_supervisor_topology(&self) -> Result<(), String> {
match self.supervisor_topology {
None => {
Expand All @@ -610,15 +607,6 @@ impl KubernetesComputeConfig {
{
return Err("supervisor.app_listener.port conflicts with shared_supervisor.runtime_port".into());
}
if self.https_proxy.is_some()
|| self.proxy_auth_secret_name.is_some()
|| self.proxy_ca_bundle.is_some()
{
return Err(
"supervisor_topology=gateway does not support upstream proxy settings"
.into(),
);
}
Ok(())
}
}
Expand Down Expand Up @@ -1232,17 +1220,27 @@ mod tests {
.is_err()
);
}
// A shared supervisor holds no per-sandbox workload identity or proxy credential.
let with_proxy = KubernetesComputeConfig {
https_proxy: Some("http://proxy.example:3128".to_string()),
..gateway(Some(shared()))
};
assert!(with_proxy.validate_supervisor_topology().is_err());
assert!(with_proxy.validate_supervisor_topology().is_ok());
assert!(with_proxy.validate_upstream_proxy_config().is_ok());
let with_credentials = KubernetesComputeConfig {
proxy_auth_secret_name: Some("egress-auth".to_string()),
proxy_auth_secret_key: Some("credentials".to_string()),
proxy_auth_allow_insecure: Some(true),
..with_proxy
};
assert!(with_credentials.validate_supervisor_topology().is_ok());
assert!(with_credentials.validate_upstream_proxy_config().is_ok());
let with_proxy_ca = KubernetesComputeConfig {
https_proxy: Some("https://proxy.example:3130".to_string()),
proxy_ca_bundle: Some("/etc/proxy/ca.pem".to_string()),
..gateway(Some(shared()))
};
assert!(with_proxy_ca.validate_supervisor_topology().is_err());
assert!(with_proxy_ca.validate_supervisor_topology().is_ok());
assert!(with_proxy_ca.validate_upstream_proxy_config().is_ok());
let workspace = KubernetesComputeConfig {
supervisor_topology: Some(SupervisorTopology::Workspace),
..KubernetesComputeConfig::default()
Expand Down
150 changes: 150 additions & 0 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3518,6 +3518,65 @@ impl KubernetesComputeDriver {
})
}

/// Snapshot proxy credentials from the workload namespace into the same
/// private generation record as its route and trust, including on takeover.
async fn shared_upstream_proxy_launch(
&self,
namespace: &str,
ca_bundle: Option<Vec<u8>>,
) -> Result<Option<openshell_supervisor_control::UpstreamProxyLaunch>, KubernetesDriverError>
{
let Some(url) = &self.config.https_proxy else {
return Ok(None);
};
let credential = match (
self.config.proxy_auth_secret_name.as_deref(),
self.config.proxy_auth_secret_key.as_deref(),
) {
(None, None) => None,
(Some(name), Some(key)) => {
let secret = Api::<Secret>::namespaced(self.client.clone(), namespace)
.get(name)
.await
.map_err(KubernetesDriverError::from_kube)?;
let value = secret
.data
.and_then(|mut data| data.remove(key))
.ok_or_else(|| {
KubernetesDriverError::Precondition(
"proxy credential Secret is missing its configured key".to_string(),
)
})?;
Some(String::from_utf8(value.0).map_err(|_| {
KubernetesDriverError::Precondition(
"proxy credential Secret must contain UTF-8 user:pass".to_string(),
)
})?)
}
_ => {
return Err(KubernetesDriverError::Precondition(
"proxy_auth_secret_name and proxy_auth_secret_key must be set together".into(),
));
}
};
let proxy = openshell_supervisor_control::UpstreamProxyLaunch {
url: url.clone(),
no_proxy: self.config.no_proxy.clone(),
credential,
auth_allow_insecure: self.config.proxy_auth_allow_insecure.unwrap_or(false),
connect_by_hostname: self.config.proxy_connect_by_hostname.unwrap_or(false),
ca_bundle_pem: ca_bundle.map(String::from_utf8).transpose().map_err(|_| {
KubernetesDriverError::Precondition(
"proxy_ca_bundle must contain UTF-8 PEM".to_string(),
)
})?,
};
proxy
.validate()
.map_err(|error| KubernetesDriverError::Precondition(error.to_string()))?;
Ok(Some(proxy))
}

/// Create one generation's gated supervisor Pod and return its UID.
#[allow(clippy::too_many_arguments, clippy::similar_names)]
async fn create_sandbox_runtime_supervisor(
Expand Down Expand Up @@ -4015,6 +4074,9 @@ impl KubernetesComputeDriver {
.map_err(KubernetesDriverError::from_kube)?;
}
GenerationLaunchTarget::Shared { shared, route, .. } => {
let upstream_proxy = self
.shared_upstream_proxy_launch(namespace, staged.upstream_proxy_ca_bundle)
.await?;
self.create_generation_image_pull_secrets(
namespace,
names,
Expand All @@ -4029,6 +4091,7 @@ impl KubernetesComputeDriver {
provisioned.runtime_descriptor,
launch_authentication,
proxy_ca,
upstream_proxy,
)?;
// The coordinator persists the launch material with the
// generation before it assigns the instance, so a restarted
Expand Down Expand Up @@ -5889,6 +5952,7 @@ fn shared_instance_registration(
runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor,
launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication,
proxy_ca: crate::sandbox_runtime::ProxyCaMaterial,
upstream_proxy: Option<openshell_supervisor_control::UpstreamProxyLaunch>,
) -> Result<SandboxInstanceRegistration, KubernetesDriverError> {
Ok(SandboxInstanceRegistration {
route: route.to_string(),
Expand Down Expand Up @@ -5922,6 +5986,7 @@ fn shared_instance_registration(
certificate_pem: proxy_ca.certificate_pem,
private_key_pem: openshell_supervisor_control::SecretPem::new(proxy_ca.private_key_pem),
},
upstream_proxy,
trace_context: RegistrationTraceContext::current(),
})
}
Expand Down Expand Up @@ -9134,6 +9199,7 @@ fn spawn_namespace_file_watcher(
#[cfg(test)]
mod tests {
use super::*;
use k8s_openapi::ByteString;

/// Serve Pod GETs over one reusable HTTP/1 connection per client,
/// delaying a request for Pod `slow-<millis>` by that many milliseconds.
Expand Down Expand Up @@ -13587,6 +13653,90 @@ mod tests {
(driver, steps, bodies)
}

#[tokio::test]
async fn shared_proxy_launch_snapshots_credentials_from_each_sandbox_namespace() {
let ca = generate_proxy_ca_material().unwrap().certificate_pem;
for (namespace, path) in [
(
"workspace-a",
"/api/v1/namespaces/workspace-a/secrets/egress-auth",
),
(
"workspace-b",
"/api/v1/namespaces/workspace-b/secrets/egress-auth",
),
] {
let credential = format!("{namespace}:private-password");
let secret = Secret {
data: Some(BTreeMap::from([(
"credentials".to_string(),
ByteString(credential.as_bytes().to_vec()),
)])),
..Secret::default()
};
let config = KubernetesComputeConfig {
https_proxy: Some("https://proxy.example:3130".to_string()),
proxy_auth_secret_name: Some("egress-auth".to_string()),
proxy_auth_secret_key: Some("credentials".to_string()),
..KubernetesComputeConfig::default()
};
let (driver, pending, _) = scripted_driver(
config,
vec![(
http::Method::GET,
path,
kube_test_response(http::StatusCode::OK, serde_json::to_value(secret).unwrap()),
)],
);
let proxy = driver
.shared_upstream_proxy_launch(namespace, Some(ca.as_bytes().to_vec()))
.await
.unwrap()
.unwrap();
assert_eq!(proxy.credential.as_deref(), Some(credential.as_str()));
assert_eq!(proxy.ca_bundle_pem.as_deref(), Some(ca.as_str()));
assert!(pending.lock().unwrap().is_empty());
}
}

#[tokio::test]
async fn shared_proxy_launch_refuses_missing_or_invalid_credentials() {
let path = "/api/v1/namespaces/workspace-a/secrets/egress-auth";
let invalid = [
None,
Some(vec![0xff]),
Some(b"private-password".to_vec()),
Some(vec![b'x'; 4097]),
];
for credential in invalid {
let secret = Secret {
data: credential
.map(|value| BTreeMap::from([("credentials".to_string(), ByteString(value))])),
..Secret::default()
};
let config = KubernetesComputeConfig {
https_proxy: Some("https://proxy.example:3130".to_string()),
proxy_auth_secret_name: Some("egress-auth".to_string()),
proxy_auth_secret_key: Some("credentials".to_string()),
..KubernetesComputeConfig::default()
};
let (driver, _, _) = scripted_driver(
config,
vec![(
http::Method::GET,
path,
kube_test_response(http::StatusCode::OK, serde_json::to_value(secret).unwrap()),
)],
);
let error = driver
.shared_upstream_proxy_launch("workspace-a", None)
.await
.unwrap_err()
.to_string();
assert!(!error.contains("private-password"), "{error}");
}
}

fn pod_owner(name: &str, uid: &str) -> OwnerReference {
OwnerReference {
api_version: "v1".to_string(),
Expand Down
2 changes: 1 addition & 1 deletion crates/openshell-supervisor-control/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,6 @@ pub use pool_limits::{
};
pub use registration::{
INSTANCE_REMOVAL_GRACE, MAX_LAUNCH_MATERIAL_BYTES, ProxyCaMaterial, RegistrationError,
RegistrationTraceContext, SandboxInstanceRegistration, SecretPem,
RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, UpstreamProxyLaunch,
};
pub use status::{SandboxInstancePhaseReport, SandboxInstanceStatusReport};
Loading
Loading