From f0a5731a2bf186bf27665288e0932ffc71bf6c7b Mon Sep 17 00:00:00 2001 From: ashwind Date: Thu, 5 Nov 2026 00:20:41 -0800 Subject: [PATCH 01/10] fix(windows): end sandbox processes when the gateway is killed When the MXC driver runs inside the gateway, a hard-killed gateway left its sandbox processes (wxc-exec, openshell-supervisor and openshell-windows-sandbox) running with no owner. They also kept the gateway's listening socket open, so the port stayed bound and a new gateway failed to start with "Only one usage of each socket address" (os error 10048). There were two causes, so there are two changes: - The gateway now puts itself in a kill-on-close job object before it spawns anything. Every sandbox process joins the job, and Windows ends them all when the gateway process exits for any reason, including a hard kill. If the job cannot be created the gateway logs a warning and continues. - mio creates listening sockets with plain socket(), which leaves the handle inheritable, and child processes are started with handle inheritance enabled, so each sandbox process held a copy of the socket. The gateway, health and metrics listeners now clear the inherit flag right after bind. Either change releases the port on its own; the job object also removes the orphaned processes. Tests: unit tests check that the process is in a kill-on-close job and that a bound gateway listener is not inheritable and still accepts connections. On a native Windows 11 x64 host with three running sandboxes, hard-killing the gateway left 9 processes and an unbindable port in 5 of 5 runs before this change. With it, no process survived, the port could be bound and a new gateway started in 3 of 3 runs on the final build, and in 5 of 5 runs on an earlier build that differed only by two lint-driven edits. The existing MXC end-to-end scenarios give the same results with and without the change. Signed-off-by: ashwind --- Cargo.lock | 2 + crates/openshell-gateway/Cargo.toml | 1 + crates/openshell-gateway/src/lib.rs | 2 + crates/openshell-gateway/src/main.rs | 6 + .../openshell-gateway/src/process_lifetime.rs | 114 ++++++++++++++++++ crates/openshell-server/Cargo.toml | 4 + .../openshell-server/src/gateway_listener.rs | 88 ++++++++++++++ crates/openshell-server/src/lib.rs | 4 +- 8 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 crates/openshell-gateway/src/process_lifetime.rs diff --git a/Cargo.lock b/Cargo.lock index a438d9a878..8a7b6fc48f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4643,6 +4643,7 @@ dependencies = [ "tonic", "tower", "tracing", + "windows", ] [[package]] @@ -5011,6 +5012,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "windows", "wiremock", "x509-parser", ] diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index 17b9c959e3..f164c64a64 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -38,6 +38,7 @@ tracing = { workspace = true, optional = true } [target.'cfg(target_os = "windows")'.dependencies] openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true } +windows = { workspace = true, features = ["Win32_System_JobObjects"] } [features] default = ["telemetry", "in-tree-compute-drivers"] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 463edd3bc0..64861e8314 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -16,6 +16,8 @@ compile_error!( build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`" ); +pub mod process_lifetime; + #[cfg(all(not(target_os = "windows"), feature = "compute-driver-vm"))] mod vm; diff --git a/crates/openshell-gateway/src/main.rs b/crates/openshell-gateway/src/main.rs index 85d0611867..cc635aa568 100644 --- a/crates/openshell-gateway/src/main.rs +++ b/crates/openshell-gateway/src/main.rs @@ -3,6 +3,12 @@ #[tokio::main] async fn main() -> miette::Result<()> { + // Must run before anything is spawned so every sandbox process joins the job. + if let Err(error) = openshell_gateway::process_lifetime::kill_children_on_exit() { + eprintln!( + "warning: could not tie sandbox processes to the gateway lifetime; they will outlive it: {error}" + ); + } openshell_server::cli::run_cli_with_compute_drivers( openshell_gateway::install_default_compute_drivers(), ) diff --git a/crates/openshell-gateway/src/process_lifetime.rs b/crates/openshell-gateway/src/process_lifetime.rs new file mode 100644 index 0000000000..ee620acbf9 --- /dev/null +++ b/crates/openshell-gateway/src/process_lifetime.rs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Tie the lifetime of every sandbox process to the gateway process. +//! +//! With the MXC driver built into the gateway, the gateway starts `wxc-exec`, +//! `openshell-supervisor` and `openshell-windows-sandbox` for each sandbox. If +//! the gateway is killed (crash, `taskkill`, logoff), those processes used to +//! survive with no owner and, because they inherited the gateway's listening +//! socket, kept its port bound so a new gateway could not start. +//! +//! A Windows *job object* with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` fixes the +//! first half: the gateway puts itself in the job, every child it spawns +//! inherits membership, and Windows ends all members when the last handle to +//! the job closes, which happens when the gateway process ends for any reason. +//! (The listening sockets are also made non-inheritable; see +//! `openshell_server::gateway_listener`.) + +/// Put the current process into a new kill-on-close job object. +/// +/// Call this first thing in `main`, before anything is spawned. The job handle +/// is deliberately never closed: it must stay open for as long as the process +/// lives, and closing it (which only happens when the process exits) is what +/// triggers the clean-up. On other platforms this does nothing. +/// +/// # Errors +/// +/// Returns a description of the failing Win32 call. The caller decides whether +/// that is fatal; nested jobs are supported on Windows 8 and later, so this +/// normally only fails on unusual hosts. +pub fn kill_children_on_exit() -> Result<(), String> { + #[cfg(windows)] + { + windows_impl::kill_children_on_exit() + } + #[cfg(not(windows))] + { + Ok(()) + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 job object calls. +mod windows_impl { + use std::mem::size_of; + + use windows::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, + SetInformationJobObject, + }; + use windows::Win32::System::Threading::GetCurrentProcess; + use windows::core::PCWSTR; + + pub(super) fn kill_children_on_exit() -> Result<(), String> { + // SAFETY: plain Win32 calls with valid arguments; the info struct + // outlives the call and `size_of` matches its type. + unsafe { + let job = CreateJobObjectW(None, PCWSTR::null()) + .map_err(|e| format!("CreateJobObjectW failed: {e}"))?; + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + (&raw const info).cast(), + u32::try_from(size_of::()) + .map_err(|e| format!("job info size: {e}"))?, + ) + .map_err(|e| format!("SetInformationJobObject failed: {e}"))?; + AssignProcessToJobObject(job, GetCurrentProcess()) + .map_err(|e| format!("AssignProcessToJobObject failed: {e}"))?; + // `job` is intentionally leaked (see the module docs). + let _ = job; + } + Ok(()) + } +} + +#[cfg(all(test, windows))] +#[allow(unsafe_code)] // Win32 job object query. +mod tests { + use std::mem::size_of; + + use windows::Win32::System::JobObjects::{ + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JobObjectExtendedLimitInformation, QueryInformationJobObject, + }; + + #[test] + fn puts_the_process_in_a_kill_on_close_job() { + super::kill_children_on_exit().expect("job object"); + + // A null job handle asks about the job the calling process is in; with + // nested jobs that is the innermost one, i.e. the one just created. + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + // SAFETY: `info` outlives the call and the size matches its type. + unsafe { + QueryInformationJobObject( + None, + JobObjectExtendedLimitInformation, + (&raw mut info).cast(), + u32::try_from(size_of::()).unwrap(), + None, + ) + } + .expect("QueryInformationJobObject"); + assert_ne!( + info.BasicLimitInformation.LimitFlags.0 & JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE.0, + 0, + "the process job must kill its members when closed" + ); + } +} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index d3bed5daff..e8b35997d3 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -116,6 +116,10 @@ x509-parser = "0.16" arc-swap = "1" notify = "8" +[target.'cfg(target_os = "windows")'.dependencies] +# Clears the handle-inherit flag on the gateway's listening sockets (see gateway_listener.rs). +windows = { workspace = true } + [features] default = ["telemetry"] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index afc2f4a763..23a5e8a99f 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -5,6 +5,8 @@ use openshell_core::{Error, Result}; use std::net::SocketAddr; use tokio::net::TcpListener; use tracing::info; +#[cfg(windows)] +use tracing::warn; pub struct BoundGatewayListener { pub listener: TcpListener, @@ -16,6 +18,7 @@ pub async fn bind_gateway_listener(address: SocketAddr) -> Result Result HANDLE { + // A SOCKET is a pointer-sized handle value; this is an integer-to-pointer cast. + HANDLE(listener.as_raw_socket() as *mut c_void) + } + + pub(super) fn disable_handle_inheritance(listener: &TcpListener) -> windows::core::Result<()> { + // SAFETY: `listener` owns a live socket handle for the whole call. + unsafe { SetHandleInformation(handle_of(listener), HANDLE_FLAG_INHERIT.0, HANDLE_FLAGS(0)) } + } + + #[cfg(test)] + pub(super) fn is_inheritable(listener: &TcpListener) -> bool { + use windows::Win32::Foundation::GetHandleInformation; + + let mut flags = 0u32; + // SAFETY: `listener` owns a live socket handle and `flags` outlives the call. + unsafe { GetHandleInformation(handle_of(listener), &raw mut flags) } + .expect("GetHandleInformation"); + flags & HANDLE_FLAG_INHERIT.0 != 0 + } +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + + #[tokio::test] + async fn bound_gateway_listener_is_not_inheritable() { + let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + assert!( + !windows_impl::is_inheritable(&bound.listener), + "child processes would inherit the gateway's listening socket" + ); + } + + #[tokio::test] + async fn listener_still_accepts_connections_after_disabling_inheritance() { + let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + let client = tokio::net::TcpStream::connect(bound.address); + let (accepted, connected) = tokio::join!(bound.listener.accept(), client); + accepted.expect("accept"); + connected.expect("connect"); + } +} diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 1390f61a11..8b88d25410 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -89,7 +89,7 @@ pub(crate) fn install_jsonwebtoken_crypto_provider() { } use compute::ComputeRuntime; -use gateway_listener::{BoundGatewayListener, bind_gateway_listener}; +use gateway_listener::{BoundGatewayListener, bind_gateway_listener, disable_handle_inheritance}; pub use grpc::OpenShellService; pub use http::{health_router, http_router, metrics_router, service_http_router}; @@ -854,6 +854,7 @@ pub(crate) async fn run_server( "failed to bind health port {health_bind_address}: {e}" )) })?; + disable_handle_inheritance(&health_listener, "health"); info!(address = %health_bind_address, "Health server listening"); // `health_router` returns immediately; the listener serves // `Initializing → 503` until the background monitor publishes the @@ -880,6 +881,7 @@ pub(crate) async fn run_server( "failed to bind metrics port {metrics_bind_address}: {e}", )) })?; + disable_handle_inheritance(&metrics_listener, "metrics"); info!(address = %metrics_bind_address, "Metrics server listening"); tokio::spawn(async move { if let Err(e) = axum::serve( From 6d3d2ce10856cee33d289c19d98c41c2923c05d7 Mon Sep 17 00:00:00 2001 From: ashwind Date: Mon, 9 Nov 2026 22:55:46 +0530 Subject: [PATCH 02/10] fix(windows): rely on mio 1.2.1 for non-inheritable listener sockets Addresses review feedback on #5299. mio 1.2.1 creates Windows sockets with WSA_FLAG_NO_HANDLE_INHERIT (1.2.0 used plain socket(), which is inheritable), so a sandbox process the gateway spawns can no longer keep a copy of the gateway's listening socket and hold its port after a gateway crash. Update mio from 1.2.0 to 1.2.1 and remove the Win32 wrapper that cleared the inherit flag after bind, together with the windows dependency it added to openshell-server; windows is now a test-only dependency there. The listener tests stay, so a regression to an inheritable socket still fails the build. Cargo.lock changes only the mio entry (version and checksum); its dependency list is unchanged and tokio 1.52.1 requires mio ^1.2.0. Tests: on Windows x64, bound_gateway_listener_is_not_inheritable and listener_accepts_connections pass without the wrapper. With three sandboxes running, hard-killing the gateway with the job object disabled leaves the nine sandbox processes alive but the port can be bound and a new gateway starts (2 of 2 runs), so the mio update alone frees the port. With the job object enabled no process survives and the port can be bound (3 of 3 runs). cargo fmt, cargo check --locked and clippy with -D warnings pass for openshell-gateway and openshell-server on Linux. Refs #5298 Signed-off-by: ashwind --- Cargo.lock | 4 +- crates/openshell-server/Cargo.toml | 8 +-- .../openshell-server/src/gateway_listener.rs | 72 ++++--------------- crates/openshell-server/src/lib.rs | 4 +- 4 files changed, 22 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a7b6fc48f..ec3286b914 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3849,9 +3849,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "log", diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index e8b35997d3..83e5f491ff 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -116,10 +116,6 @@ x509-parser = "0.16" arc-swap = "1" notify = "8" -[target.'cfg(target_os = "windows")'.dependencies] -# Clears the handle-inherit flag on the gateway's listening sockets (see gateway_listener.rs). -windows = { workspace = true } - [features] default = ["telemetry"] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). @@ -150,5 +146,9 @@ wiremock = "0.6" # `testing` provides InMemorySpanExporter, so span assertions do not need a # collector, a network hop, or a flush barrier. opentelemetry_sdk = { workspace = true, features = ["testing"] } +# Windows-only tests read the inherit flag of the gateway's listening socket. +[target.'cfg(target_os = "windows")'.dev-dependencies] +windows = { workspace = true } + [lints] workspace = true diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 23a5e8a99f..3ebfc3ec02 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -5,8 +5,6 @@ use openshell_core::{Error, Result}; use std::net::SocketAddr; use tokio::net::TcpListener; use tracing::info; -#[cfg(windows)] -use tracing::warn; pub struct BoundGatewayListener { pub listener: TcpListener, @@ -18,7 +16,6 @@ pub async fn bind_gateway_listener(address: SocketAddr) -> Result Result HANDLE { - // A SOCKET is a pointer-sized handle value; this is an integer-to-pointer cast. - HANDLE(listener.as_raw_socket() as *mut c_void) - } - - pub(super) fn disable_handle_inheritance(listener: &TcpListener) -> windows::core::Result<()> { - // SAFETY: `listener` owns a live socket handle for the whole call. - unsafe { SetHandleInformation(handle_of(listener), HANDLE_FLAG_INHERIT.0, HANDLE_FLAGS(0)) } - } + use windows::Win32::Foundation::{GetHandleInformation, HANDLE, HANDLE_FLAG_INHERIT}; - #[cfg(test)] - pub(super) fn is_inheritable(listener: &TcpListener) -> bool { - use windows::Win32::Foundation::GetHandleInformation; + use super::*; + fn is_inheritable(listener: &TcpListener) -> bool { + let handle = HANDLE(listener.as_raw_socket() as *mut c_void); let mut flags = 0u32; // SAFETY: `listener` owns a live socket handle and `flags` outlives the call. - unsafe { GetHandleInformation(handle_of(listener), &raw mut flags) } - .expect("GetHandleInformation"); + unsafe { GetHandleInformation(handle, &raw mut flags) }.expect("GetHandleInformation"); flags & HANDLE_FLAG_INHERIT.0 != 0 } -} - -#[cfg(all(test, windows))] -mod tests { - use super::*; #[tokio::test] async fn bound_gateway_listener_is_not_inheritable() { @@ -95,13 +53,13 @@ mod tests { .await .unwrap(); assert!( - !windows_impl::is_inheritable(&bound.listener), + !is_inheritable(&bound.listener), "child processes would inherit the gateway's listening socket" ); } #[tokio::test] - async fn listener_still_accepts_connections_after_disabling_inheritance() { + async fn listener_accepts_connections() { let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap()) .await .unwrap(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 8b88d25410..1390f61a11 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -89,7 +89,7 @@ pub(crate) fn install_jsonwebtoken_crypto_provider() { } use compute::ComputeRuntime; -use gateway_listener::{BoundGatewayListener, bind_gateway_listener, disable_handle_inheritance}; +use gateway_listener::{BoundGatewayListener, bind_gateway_listener}; pub use grpc::OpenShellService; pub use http::{health_router, http_router, metrics_router, service_http_router}; @@ -854,7 +854,6 @@ pub(crate) async fn run_server( "failed to bind health port {health_bind_address}: {e}" )) })?; - disable_handle_inheritance(&health_listener, "health"); info!(address = %health_bind_address, "Health server listening"); // `health_router` returns immediately; the listener serves // `Initializing → 503` until the background monitor publishes the @@ -881,7 +880,6 @@ pub(crate) async fn run_server( "failed to bind metrics port {metrics_bind_address}: {e}", )) })?; - disable_handle_inheritance(&metrics_listener, "metrics"); info!(address = %metrics_bind_address, "Metrics server listening"); tokio::spawn(async move { if let Err(e) = axum::serve( From f7f563232b77aabee49674a94a30ea70bcd830b7 Mon Sep 17 00:00:00 2001 From: ashwind Date: Fri, 13 Nov 2026 23:03:42 +0530 Subject: [PATCH 03/10] fix(windows): give each sandbox generation its own kill-on-close job Addresses review feedback on #5299. The gateway-wide job object lived in the gateway binary, so the cleanup was tied to whichever process hosts the driver and would not survive moving the driver out of the gateway. It also left stop and delete killing only the two direct children (openshell-supervisor and wxc-exec), not the boundary and workload tree beneath them. Move the ownership into the MXC driver. run_lifecycle_inner creates one job object with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE per sandbox generation and spawns the supervisor and wxc-exec inside it. Each process is created suspended, assigned to the job, then resumed, so it cannot start children outside the job. If a process cannot be assigned to its job, spawning fails and so does sandbox creation. After the runtime pair exits (stop, delete, or either process dying) the job is terminated and dropped, which ends the rest of the tree; if the driver's host process dies, Windows closes the job handle and does the same. Remove the gateway-level job (process_lifetime.rs, its call in main.rs and the windows dependency it added to openshell-gateway). The non-inheritable listener sockets from mio 1.2.1 are unchanged. Tests: on Windows x64, cargo nextest for the workspace passes (6666 of 6666), the 82 openshell-driver-mxc tests pass, and cargo fmt and clippy with -D warnings are clean. New tests in job.rs cover: a spawned process runs after resume and terminate ends it; dropping the job ends the tree; a failed spawn is reported; assignment failure fails the spawn; hard-killing the process that owns the job ends the root and its grandchild. With a fresh release gateway and three sandboxes, hard-killing the gateway leaves no sandbox process, the port can be bound and a new gateway starts. Deleting sandboxes one at a time removes exactly that sandbox's supervisor, wxc-exec, boundary and workload processes. Refs #5298 Signed-off-by: ashwind --- Cargo.lock | 1 - crates/openshell-driver-mxc/Cargo.toml | 2 +- crates/openshell-driver-mxc/src/driver.rs | 18 +- crates/openshell-driver-mxc/src/job.rs | 350 ++++++++++++++++++ crates/openshell-driver-mxc/src/lib.rs | 2 + crates/openshell-driver-mxc/src/mxc.rs | 6 +- crates/openshell-gateway/Cargo.toml | 1 - crates/openshell-gateway/src/lib.rs | 2 - crates/openshell-gateway/src/main.rs | 6 - .../openshell-gateway/src/process_lifetime.rs | 114 ------ 10 files changed, 373 insertions(+), 129 deletions(-) create mode 100644 crates/openshell-driver-mxc/src/job.rs delete mode 100644 crates/openshell-gateway/src/process_lifetime.rs diff --git a/Cargo.lock b/Cargo.lock index ec3286b914..2ae6b05c07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4643,7 +4643,6 @@ dependencies = [ "tonic", "tower", "tracing", - "windows", ] [[package]] diff --git a/crates/openshell-driver-mxc/Cargo.toml b/crates/openshell-driver-mxc/Cargo.toml index d16d40da45..a34f3c5eb3 100644 --- a/crates/openshell-driver-mxc/Cargo.toml +++ b/crates/openshell-driver-mxc/Cargo.toml @@ -37,7 +37,7 @@ uuid = { workspace = true } openshell-mxc-boundary = { path = "../openshell-mxc-boundary" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } -windows = { workspace = true } +windows = { workspace = true, features = ["Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects"] } [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index b2ae367bfd..cf5e39e730 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -757,7 +757,13 @@ async fn run_lifecycle_inner( // ProcessContainer launch so unrelated local processes cannot squat it // during the more expensive host-side setup. drop(proxy_reservation); - let mut supervisor = spawn_supervisor(context, &descriptor_path, &auth_bundle_path, &main)?; + // One kill-on-close job per generation owns the supervisor and the whole + // boundary tree. Every early return below drops it, which ends whatever was + // already started. + let job = crate::job::GenerationJob::new() + .map_err(|error| format!("create sandbox job object: {error}"))?; + let mut supervisor = + spawn_supervisor(context, &descriptor_path, &auth_bundle_path, &main, &job)?; let sandbox_command = encode_windows_command_line(&[ context.config.sandbox_binary_path.clone(), "--bootstrap".to_string(), @@ -795,6 +801,7 @@ async fn run_lifecycle_inner( process, Some(network), mapped.ui, + &job, ) .await .map_err(|error| format!("start MXC ProcessContainer: {error}"))?; @@ -838,6 +845,10 @@ async fn run_lifecycle_inner( } drop(startup_guard); let result = monitor_runtime_pair(boundary, supervisor, shutdown_rx).await; + // Whatever ended the pair, end the rest of the tree (workload, conhost, + // grandchildren) that `Child::kill` on the two roots does not reach. + job.terminate(); + drop(job); let _ = terminated_tx.send(true); if let Ok(mut attribution) = context.attribution.lock() { attribution.forget(&context.sandbox.id); @@ -859,9 +870,10 @@ fn spawn_supervisor( descriptor_path: &Path, auth_bundle_path: &Path, main: &openshell_core::sandbox_env::MainProcessConfig, + job: &crate::job::GenerationJob, ) -> Result { - supervisor_command(context, descriptor_path, auth_bundle_path, main)? - .spawn() + let mut command = supervisor_command(context, descriptor_path, auth_bundle_path, main)?; + job.spawn(&mut command) .map_err(|error| format!("start host openshell-supervisor: {error}")) } diff --git a/crates/openshell-driver-mxc/src/job.rs b/crates/openshell-driver-mxc/src/job.rs new file mode 100644 index 0000000000..dee1aa3c2d --- /dev/null +++ b/crates/openshell-driver-mxc/src/job.rs @@ -0,0 +1,350 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One kill-on-close job object per sandbox generation. +//! +//! A generation's process tree is `openshell-supervisor`, `wxc-exec`, the +//! in-container `openshell-windows-sandbox` and the workload with its +//! children. Putting the root processes in a job with +//! `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` makes Windows end the whole tree when +//! the last handle to the job closes: on stop/delete (explicit +//! [`GenerationJob::terminate`]), when the lifecycle task ends, or when the +//! driver's host process dies for any reason. Children inherit job membership +//! and `BREAKAWAY_OK` is not set, so nothing can leave the job. +//! +//! The job lives with the driver rather than with whichever process hosts it, +//! so the cleanup survives moving the driver out of the gateway. + +use std::io; +use std::mem::size_of; + +use tokio::process::{Child, Command}; +use windows::Win32::Foundation::{CloseHandle, HANDLE}; +use windows::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, TH32CS_SNAPTHREAD, THREADENTRY32, Thread32First, Thread32Next, +}; +use windows::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, + SetInformationJobObject, TerminateJobObject, +}; +use windows::Win32::System::Threading::{OpenThread, ResumeThread, THREAD_SUSPEND_RESUME}; +use windows::core::PCWSTR; + +const CREATE_SUSPENDED: u32 = 0x0000_0004; + +/// Owns a kill-on-close job object. Dropping it ends every member process. +pub struct GenerationJob { + handle: HANDLE, +} + +// SAFETY: a job object handle is a process-wide kernel handle with no thread +// affinity; every operation used here is thread-safe. +#[allow(unsafe_code)] +unsafe impl Send for GenerationJob {} +#[allow(unsafe_code)] +unsafe impl Sync for GenerationJob {} + +#[allow(unsafe_code)] // Win32 job object calls. +impl GenerationJob { + pub fn new() -> io::Result { + // SAFETY: plain Win32 calls with valid arguments; `info` outlives the + // call and the size matches its type. No security attributes are + // passed, so the handle is not inheritable and children cannot keep + // the job open. + unsafe { + let handle = CreateJobObjectW(None, PCWSTR::null()).map_err(io::Error::from)?; + let job = Self { handle }; + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject( + job.handle, + JobObjectExtendedLimitInformation, + (&raw const info).cast(), + u32::try_from(size_of::()) + .map_err(io::Error::other)?, + ) + .map_err(io::Error::from)?; + Ok(job) + } + } + + /// Spawn `command` as a member of this job, with no window in which it can + /// run (or start children) outside it. + /// + /// The process is created suspended, assigned to the job, then resumed. + /// If it cannot be assigned it is killed and an error is returned: a + /// sandbox whose processes could outlive the driver must not start. + pub fn spawn(&self, command: &mut Command) -> io::Result { + command.creation_flags(CREATE_SUSPENDED); + let mut child = command.spawn()?; + match self.adopt_and_resume(&child) { + Ok(()) => Ok(child), + Err(error) => { + // Still suspended, so it has run no code. `kill_on_drop` + // retries if this kill fails. + let _ = child.start_kill(); + Err(error) + } + } + } + + fn adopt_and_resume(&self, child: &Child) -> io::Result<()> { + let pid = child + .id() + .ok_or_else(|| io::Error::other("child exited before it could be assigned to a job"))?; + let raw = child + .raw_handle() + .ok_or_else(|| io::Error::other("child handle unavailable"))?; + // SAFETY: `raw` is the live process handle owned by `child`. + unsafe { AssignProcessToJobObject(self.handle, HANDLE(raw)) }.map_err(|error| { + io::Error::other(format!("assign process {pid} to its sandbox job: {error}")) + })?; + resume_process(pid) + } + + /// End every process in the job now. + pub fn terminate(&self) { + // SAFETY: valid job handle. An already-empty job is not an error worth + // reporting. + let _ = unsafe { TerminateJobObject(self.handle, 1) }; + } +} + +impl Drop for GenerationJob { + #[allow(unsafe_code)] + fn drop(&mut self) { + // Closing the last handle triggers KILL_ON_JOB_CLOSE. + // SAFETY: `handle` came from `CreateJobObjectW` and is closed once. + let _ = unsafe { CloseHandle(self.handle) }; + } +} + +/// Resume every thread of a process created with `CREATE_SUSPENDED`. +#[allow(unsafe_code)] // Toolhelp and thread calls. +fn resume_process(pid: u32) -> io::Result<()> { + // SAFETY: the snapshot handle is closed on every path; `entry` outlives the + // calls and `dwSize` is set as the API requires. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0).map_err(io::Error::from)?; + let mut resumed = 0_u32; + let mut result = Ok(()); + let mut entry = THREADENTRY32 { + dwSize: u32::try_from(size_of::()).map_err(io::Error::other)?, + ..Default::default() + }; + let mut more = Thread32First(snapshot, &raw mut entry).is_ok(); + while more { + if entry.th32OwnerProcessID == pid { + match OpenThread(THREAD_SUSPEND_RESUME, false, entry.th32ThreadID) { + Ok(thread) => { + if ResumeThread(thread) == u32::MAX { + result = Err(io::Error::last_os_error()); + } else { + resumed += 1; + } + let _ = CloseHandle(thread); + } + Err(error) => result = Err(io::Error::from(error)), + } + } + more = Thread32Next(snapshot, &raw mut entry).is_ok(); + } + let _ = CloseHandle(snapshot); + result?; + if resumed == 0 { + return Err(io::Error::other(format!( + "no thread of process {pid} could be resumed" + ))); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::process::Stdio; + use std::time::Duration; + + use super::*; + + fn sleeper() -> Command { + let mut command = Command::new("cmd.exe"); + command + .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + command + } + + #[tokio::test] + async fn spawned_process_runs_and_is_killed_by_terminate() { + let job = GenerationJob::new().expect("job"); + let mut child = job.spawn(&mut sleeper()).expect("spawn in job"); + // Resumed: it must still be running (not stuck suspended, not exited). + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + child.try_wait().unwrap().is_none(), + "child should be running" + ); + job.terminate(); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("child ended after terminate") + .unwrap(); + assert!(!status.success()); + } + + #[tokio::test] + async fn dropping_the_job_kills_the_whole_tree() { + let job = GenerationJob::new().expect("job"); + // cmd.exe starts ping.exe: a grandchild that inherits membership. + let mut child = job.spawn(&mut sleeper()).expect("spawn in job"); + tokio::time::sleep(Duration::from_millis(300)).await; + drop(job); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("child ended after the job closed") + .unwrap(); + // Closing the job ends members with exit code 0, so only that the + // process ended (not how) can be asserted. + let _ = status; + } + + /// Fail closed: if the process cannot be assigned to the job, `spawn` + /// returns an error (and kills the still-suspended child) instead of + /// handing back a process that could outlive the driver. + #[tokio::test] + async fn assignment_failure_fails_the_spawn() { + // An invalid job handle makes `AssignProcessToJobObject` fail. + let job = GenerationJob { + handle: HANDLE::default(), + }; + let error = job.spawn(&mut sleeper()).expect_err("must fail closed"); + assert!( + error.to_string().contains("assign process"), + "unexpected error: {error}" + ); + std::mem::forget(job); // do not CloseHandle the invalid handle + } + + /// Set (by the test below) on the re-run of this test binary that plays the + /// role of the driver's host process. + const HOST_ENV: &str = "OPENSHELL_JOB_TEST_HOST"; + + #[allow(unsafe_code)] + fn is_alive(pid: u32) -> bool { + use windows::Win32::Foundation::WAIT_TIMEOUT; + use windows::Win32::System::Threading::{ + OpenProcess, PROCESS_SYNCHRONIZE, WaitForSingleObject, + }; + // SAFETY: plain Win32 calls; the handle is closed before returning. + unsafe { + let Ok(handle) = OpenProcess(PROCESS_SYNCHRONIZE, false, pid) else { + return false; // no such process any more + }; + let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT; + let _ = CloseHandle(handle); + alive + } + } + + #[allow(unsafe_code)] + fn child_pids(parent: u32) -> Vec { + use windows::Win32::System::Diagnostics::ToolHelp::{ + PROCESSENTRY32, Process32First, Process32Next, TH32CS_SNAPPROCESS, + }; + // SAFETY: the snapshot is closed before returning; `entry.dwSize` is set. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0).unwrap(); + let mut entry = PROCESSENTRY32 { + dwSize: u32::try_from(size_of::()).unwrap(), + ..Default::default() + }; + let mut found = Vec::new(); + let mut more = Process32First(snapshot, &raw mut entry).is_ok(); + while more { + if entry.th32ParentProcessID == parent { + found.push(entry.th32ProcessID); + } + more = Process32Next(snapshot, &raw mut entry).is_ok(); + } + let _ = CloseHandle(snapshot); + found + } + } + + /// Plays the driver host: owns a job, starts a process tree in it, reports + /// the PIDs, then waits to be killed. A no-op in a normal test run. + #[tokio::test] + async fn hard_kill_host_helper() { + if std::env::var_os(HOST_ENV).is_none() { + return; + } + let job = GenerationJob::new().expect("job"); + let child = job.spawn(&mut sleeper()).expect("spawn in job"); + let root = child.id().expect("pid"); + tokio::time::sleep(Duration::from_millis(700)).await; // let cmd start ping + let grandchildren = child_pids(root) + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + println!("TREE {root} {grandchildren}"); + tokio::time::sleep(Duration::from_mins(2)).await; + drop(job); + } + + /// The property the job exists for: when the process that owns the job is + /// hard-killed (no destructors, no cleanup code), Windows still ends every + /// process in the tree, grandchildren included. + #[tokio::test] + async fn hard_killing_the_job_owner_ends_the_whole_tree() { + use std::io::{BufRead as _, BufReader}; + if std::env::var_os(HOST_ENV).is_some() { + return; // inside the helper run + } + let mut host = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "job::tests::hard_kill_host_helper", "--nocapture"]) + .env(HOST_ENV, "1") + .stdout(Stdio::piped()) + .spawn() + .expect("start helper host"); + let mut lines = BufReader::new(host.stdout.take().unwrap()).lines(); + let tree: Vec = loop { + let line = lines.next().expect("helper exited early").unwrap(); + if let Some(rest) = line.strip_prefix("TREE ") { + break rest + .split([' ', ',']) + .filter(|p| !p.is_empty()) + .map(|p| p.parse().unwrap()) + .collect(); + } + }; + assert!( + tree.len() >= 2, + "expected the root and at least one grandchild, got {tree:?}" + ); + assert!(tree.iter().all(|&pid| is_alive(pid)), "tree should be live"); + + host.kill().expect("hard-kill the host"); + host.wait().unwrap(); + + for _ in 0..100 { + if tree.iter().all(|&pid| !is_alive(pid)) { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("processes survived the hard kill of their job owner: {tree:?}"); + } + + #[tokio::test] + async fn a_failed_spawn_is_reported_not_swallowed() { + let job = GenerationJob::new().expect("job"); + let mut command = Command::new("definitely-not-a-real-binary.exe"); + assert!(job.spawn(&mut command).is_err()); + } +} diff --git a/crates/openshell-driver-mxc/src/lib.rs b/crates/openshell-driver-mxc/src/lib.rs index 9b0850b456..fda0ea1af2 100644 --- a/crates/openshell-driver-mxc/src/lib.rs +++ b/crates/openshell-driver-mxc/src/lib.rs @@ -24,6 +24,8 @@ mod grpc; #[cfg(target_os = "windows")] mod isolation; #[cfg(target_os = "windows")] +mod job; +#[cfg(target_os = "windows")] mod mxc; #[cfg(target_os = "windows")] mod policy; diff --git a/crates/openshell-driver-mxc/src/mxc.rs b/crates/openshell-driver-mxc/src/mxc.rs index f252493e2d..1a6c025bab 100644 --- a/crates/openshell-driver-mxc/src/mxc.rs +++ b/crates/openshell-driver-mxc/src/mxc.rs @@ -325,6 +325,7 @@ impl WxcExecInvoker { /// default-deny, so a write to any ungranted path is denied by the OS. /// /// **Stdout is raw agent output; the exit code is the agent's own exit code.** + #[allow(clippy::too_many_arguments)] pub async fn run_oneshot( &self, container_id: &str, @@ -333,6 +334,7 @@ impl WxcExecInvoker { process: MxcProcess, network: Option, ui: Option, + job: &crate::job::GenerationJob, ) -> Result { let config = oneshot_config_json( container_id, @@ -386,7 +388,9 @@ impl WxcExecInvoker { } info!(container_id = %container_id, "wxc-exec one-shot processContainer spawn"); - let child = cmd.spawn()?; + // Spawned inside the generation's job so the boundary and workload tree + // cannot outlive the driver; fails closed if the job cannot adopt it. + let child = job.spawn(&mut cmd)?; Ok(child) } } diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index f164c64a64..17b9c959e3 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -38,7 +38,6 @@ tracing = { workspace = true, optional = true } [target.'cfg(target_os = "windows")'.dependencies] openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true } -windows = { workspace = true, features = ["Win32_System_JobObjects"] } [features] default = ["telemetry", "in-tree-compute-drivers"] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 64861e8314..463edd3bc0 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -16,8 +16,6 @@ compile_error!( build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`" ); -pub mod process_lifetime; - #[cfg(all(not(target_os = "windows"), feature = "compute-driver-vm"))] mod vm; diff --git a/crates/openshell-gateway/src/main.rs b/crates/openshell-gateway/src/main.rs index cc635aa568..85d0611867 100644 --- a/crates/openshell-gateway/src/main.rs +++ b/crates/openshell-gateway/src/main.rs @@ -3,12 +3,6 @@ #[tokio::main] async fn main() -> miette::Result<()> { - // Must run before anything is spawned so every sandbox process joins the job. - if let Err(error) = openshell_gateway::process_lifetime::kill_children_on_exit() { - eprintln!( - "warning: could not tie sandbox processes to the gateway lifetime; they will outlive it: {error}" - ); - } openshell_server::cli::run_cli_with_compute_drivers( openshell_gateway::install_default_compute_drivers(), ) diff --git a/crates/openshell-gateway/src/process_lifetime.rs b/crates/openshell-gateway/src/process_lifetime.rs deleted file mode 100644 index ee620acbf9..0000000000 --- a/crates/openshell-gateway/src/process_lifetime.rs +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Tie the lifetime of every sandbox process to the gateway process. -//! -//! With the MXC driver built into the gateway, the gateway starts `wxc-exec`, -//! `openshell-supervisor` and `openshell-windows-sandbox` for each sandbox. If -//! the gateway is killed (crash, `taskkill`, logoff), those processes used to -//! survive with no owner and, because they inherited the gateway's listening -//! socket, kept its port bound so a new gateway could not start. -//! -//! A Windows *job object* with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` fixes the -//! first half: the gateway puts itself in the job, every child it spawns -//! inherits membership, and Windows ends all members when the last handle to -//! the job closes, which happens when the gateway process ends for any reason. -//! (The listening sockets are also made non-inheritable; see -//! `openshell_server::gateway_listener`.) - -/// Put the current process into a new kill-on-close job object. -/// -/// Call this first thing in `main`, before anything is spawned. The job handle -/// is deliberately never closed: it must stay open for as long as the process -/// lives, and closing it (which only happens when the process exits) is what -/// triggers the clean-up. On other platforms this does nothing. -/// -/// # Errors -/// -/// Returns a description of the failing Win32 call. The caller decides whether -/// that is fatal; nested jobs are supported on Windows 8 and later, so this -/// normally only fails on unusual hosts. -pub fn kill_children_on_exit() -> Result<(), String> { - #[cfg(windows)] - { - windows_impl::kill_children_on_exit() - } - #[cfg(not(windows))] - { - Ok(()) - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] // Win32 job object calls. -mod windows_impl { - use std::mem::size_of; - - use windows::Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, - JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, - SetInformationJobObject, - }; - use windows::Win32::System::Threading::GetCurrentProcess; - use windows::core::PCWSTR; - - pub(super) fn kill_children_on_exit() -> Result<(), String> { - // SAFETY: plain Win32 calls with valid arguments; the info struct - // outlives the call and `size_of` matches its type. - unsafe { - let job = CreateJobObjectW(None, PCWSTR::null()) - .map_err(|e| format!("CreateJobObjectW failed: {e}"))?; - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - (&raw const info).cast(), - u32::try_from(size_of::()) - .map_err(|e| format!("job info size: {e}"))?, - ) - .map_err(|e| format!("SetInformationJobObject failed: {e}"))?; - AssignProcessToJobObject(job, GetCurrentProcess()) - .map_err(|e| format!("AssignProcessToJobObject failed: {e}"))?; - // `job` is intentionally leaked (see the module docs). - let _ = job; - } - Ok(()) - } -} - -#[cfg(all(test, windows))] -#[allow(unsafe_code)] // Win32 job object query. -mod tests { - use std::mem::size_of; - - use windows::Win32::System::JobObjects::{ - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JobObjectExtendedLimitInformation, QueryInformationJobObject, - }; - - #[test] - fn puts_the_process_in_a_kill_on_close_job() { - super::kill_children_on_exit().expect("job object"); - - // A null job handle asks about the job the calling process is in; with - // nested jobs that is the innermost one, i.e. the one just created. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - // SAFETY: `info` outlives the call and the size matches its type. - unsafe { - QueryInformationJobObject( - None, - JobObjectExtendedLimitInformation, - (&raw mut info).cast(), - u32::try_from(size_of::()).unwrap(), - None, - ) - } - .expect("QueryInformationJobObject"); - assert_ne!( - info.BasicLimitInformation.LimitFlags.0 & JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE.0, - 0, - "the process job must kill its members when closed" - ); - } -} From 3e19fbe4e8f75ffa06d5bc25e689650c63f647b5 Mon Sep 17 00:00:00 2001 From: ashwind Date: Fri, 13 Nov 2026 23:03:42 +0530 Subject: [PATCH 04/10] test(windows): add gateway-kill scenario to the MXC e2e runner Add a default process_container scenario to run-mxc-e2e.ps1 that checks sandbox cleanup when the gateway dies without warning. It creates a sandbox with a long-running marked workload, records the process tree under that scenario's own gateway (other sandboxes on the host are never counted or touched), hard-kills the gateway, and requires every recorded process to end within 30 s and a replacement gateway to bind the same port. Document the scenario in the driver README and update the scenario-count wording in the README and the build-openshell-mxc-windows skill. Tests: on Windows x64 with the per-sandbox job, the scenario passes (6 sandbox processes ended, port rebound). Against a gateway built without the job object it fails, with all 6 processes, including the workload, still running. The runner's existing port probe (GetActiveTcpListeners) does not see the listener on the test machine (issue #11), so both runs used a copy of the runner with a loopback connect probe; the probe in the repository is unchanged. Refs #5298 Signed-off-by: ashwind --- .../build-openshell-mxc-windows/SKILL.md | 2 +- crates/openshell-driver-mxc/README.md | 13 ++- .../examples/run-mxc-e2e.ps1 | 83 ++++++++++++++++++- 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index ee688314bd..93d025a17e 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -476,7 +476,7 @@ variable by default). Require both a nonempty response and the OpenShell proxy CA issuer; never replace these assertions with an insecure TLS request. This fixture permits unavailable CRL information using best-effort Schannel revocation for the ephemeral private CA; this is not revocation qualification. This -scenario is not part of the default seven offline-capable E2E scenarios. +scenario is not part of the default offline-capable E2E scenarios. The default `host-service` scenario runs the HTTP fixture on host `127.0.0.1` and a dynamic port, then checks two nonce-bound replies from direct curl requests diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 7a4c834af0..3ad6d86b06 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -349,7 +349,7 @@ explicitly loads the boundary-injected CA bundle and records the peer certificat success requires a nonempty response and the OpenShell proxy CA issuer. It does not install a host trust root or disable CA/hostname verification. The fixture uses best-effort Schannel revocation because the ephemeral private CA has no CRL -distribution point; it does not qualify revocation enforcement. The default seven E2E +distribution point; it does not qualify revocation enforcement. The default E2E scenarios remain independent of Internet access. The `forwarding` scenario starts a TCP listener inside MXC, then verifies two @@ -373,12 +373,21 @@ host service is checked before launch and stopped after testing. Select it with `-Scenario host-service`; no Internet access, LAN address, or firewall changes are required. This verifies direct localhost connectivity. +The default `gateway-kill` scenario checks sandbox cleanup when the gateway dies +without warning. It creates a sandbox with a long-running workload, records the +sandbox process tree under that gateway, hard-kills the gateway, and requires +every recorded process to end and the gateway port to be bindable again (a +replacement gateway must start on it). Only processes under the scenario's own +gateway are tracked, so other sandboxes on the host are left alone. It relies on +each sandbox generation's kill-on-close job object; no workload outlives the +gateway. Select it with `-Scenario gateway-kill`. + The `ws-agent.yaml` and `openclaw-gateway.yaml` policies preserve the Windows branch's workload grants. `inference.yaml` is a template: replace its filesystem, curl executable, inference host, and port placeholders before use. Authorize curl, which owns the network socket, not its cmd launcher. These policies have parser/mapper regression coverage; they do not add live WebSocket, OpenClaw, -or provider-backed inference scenarios to the seven-scenario E2E runner. +or provider-backed inference scenarios to the E2E runner. With `-KeepRunning`, the runner restores the caller's environment but retains the disposable CLI/runtime directory for the live gateway and prints its path. diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 0b5709d7d4..ce64131c99 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -37,6 +37,8 @@ # forwarding - two TCP request/reply exchanges through OpenShell ingress. # service-forwarding - named HTTP service routing and endpoint deletion. # host-service - workload reaches an HTTP service on host 127.0.0.1. +# gateway-kill - hard-killing the gateway ends every sandbox process and +# frees its port. processcontainer only. # https-ca - opt-in outbound HTTPS using the injected proxy CA. [CmdletBinding()] @@ -545,6 +547,23 @@ function Test-ServiceForwarding([string]$sandboxName, [string]$readyFile, [strin # Detect an agent *launch* failure (binary not found / not implemented) vs a # legitimate policy denial. Used to avoid false-passing a deny scenario when the # agent never actually ran. +# All descendants of a process, found by walking ParentProcessId links in one +# snapshot. Orphans keep their ParentProcessId, so the walk is still correct +# for a tree whose root has since died, but it must run while the root lives. +function Get-DescendantPids([int]$rootPid) { + $all = @(Get-CimInstance Win32_Process) + $found = New-Object System.Collections.Generic.HashSet[int] + $queue = New-Object System.Collections.Generic.Queue[int] + $queue.Enqueue($rootPid) + while ($queue.Count -gt 0) { + $parent = $queue.Dequeue() + foreach ($p in $all) { + if ($p.ParentProcessId -eq $parent -and $found.Add([int]$p.ProcessId)) { $queue.Enqueue([int]$p.ProcessId) } + } + } + return @($found) +} + function Launch-Failed([string]$gwText) { if ($null -eq $gwText) { return $false } return ($gwText -match 'CreateProcessW failed error:2' ` @@ -790,10 +809,15 @@ try { Name = 'host-service'; PolicyFile = Join-Path $policyDir 'host-service.yaml' SandboxId = 'host'; Backends = 'process_container'; Kind = 'host-service' Description = 'workload connects directly to an HTTP service on host 127.0.0.1' + }, + @{ + Name = 'gateway-kill'; PolicyFile = Join-Path $policyDir 'fs-rw.yaml' + SandboxId = 'gk'; Backends = 'process_container'; Kind = 'gateway-kill' + Description = 'hard-killing the gateway ends every sandbox process and frees its port' } ) - # External connectivity is deliberately opt-in; the default seven scenarios + # External connectivity is deliberately opt-in; the default scenarios # remain runnable without Internet access. Use the same authenticated # gateway launch path as every other workload, not a bare driver fixture. if ($Scenario -eq 'https-ca') { @@ -916,6 +940,11 @@ try { } else { $command = @($cmdExe, "/c", "echo denied 1> $denied") } + } elseif ($sc.Kind -eq 'gateway-kill') { + # A long-running workload carrying a unique marker, so its process + # can be identified without matching unrelated cmd.exe instances. + $killNonce = [guid]::NewGuid().ToString('N') + $command = @($cmdExe, '/d', '/c', "for /l %i in (1,1,2000000000) do @rem $killNonce") } elseif ($sc.Kind -eq 'host-service') { Remove-Item -Path (Join-Path $DemoDir 'host-service.*') -Force -ErrorAction SilentlyContinue $batch = Join-Path $DemoDir 'host-service.cmd' @@ -990,7 +1019,7 @@ try { "--policy", [string]$policyUsed, "--driver-config-json", $driverConfig ) - $createArguments += if ($sc.Kind -eq 'host-service') { '--detach' } else { '--no-tty' } + $createArguments += if ($sc.Kind -in @('host-service', 'gateway-kill')) { '--detach' } else { '--no-tty' } $createResult = Invoke-NativeCaptured $cli $createArguments $createOut = $createResult.Output $createExitCode = $createResult.ExitCode @@ -1014,6 +1043,53 @@ try { Bad "host-service: $($_.Exception.Message)" $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } } + } elseif ($sc.Kind -eq 'gateway-kill') { + try { + if ($createExitCode -ne 0) { throw "sandbox creation failed: $createOutStr" } + # Wait for the whole sandbox tree to exist, then record it. + # Only processes under THIS gateway (plus the marked workload) + # are tracked, so unrelated MXC sandboxes on the host are never + # counted or touched. + $victims = @{} + $deadline = (Get-Date).AddSeconds(60) + do { + Start-Sleep -Milliseconds 500 + $victims = @{} + foreach ($id in (Get-DescendantPids $gw.Id)) { + $proc = Get-Process -Id $id -ErrorAction SilentlyContinue + if ($proc) { $victims[$id] = $proc.ProcessName } + } + foreach ($proc in @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID })) { + $victims[[int]$proc.ProcessId] = $proc.Name + } + $names = @($victims.Values | ForEach-Object { $_ -replace '\.exe$', '' }) + $ready = ($names -contains 'wxc-exec') -and ($names -contains 'openshell-supervisor') -and (@(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID }).Count -gt 0) + } while (-not $ready -and (Get-Date) -lt $deadline) + if (-not $ready) { throw "sandbox process tree did not come up (saw: $(($victims.Values) -join ', '))" } + Info "tracking $($victims.Count) sandbox processes: $((($victims.GetEnumerator() | Sort-Object Name | ForEach-Object { "$($_.Value)($($_.Key))" })) -join ', ')" + + Info "hard-killing gateway pid $($gw.Id)" + Stop-Process -Id $gw.Id -Force + $gw.WaitForExit() + $survivors = @() + $deadline = (Get-Date).AddSeconds(30) + do { + Start-Sleep -Milliseconds 500 + $survivors = @($victims.Keys | Where-Object { Get-Process -Id $_ -ErrorAction SilentlyContinue }) + } while ($survivors.Count -gt 0 -and (Get-Date) -lt $deadline) + if ($survivors.Count -gt 0) { + $left = ($survivors | ForEach-Object { "$($victims[$_])($_)" }) -join ', ' + throw "sandbox processes survived the gateway's death: $left" + } + # The port must be free again: a replacement gateway must bind it. + $gw = Start-Gw + Info "replacement gateway pid $($gw.Id) bound port $Port" + Ok "gateway-kill: all $($victims.Count) sandbox processes ended; port rebound" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'PASS'; Reason = "$($victims.Count) sandbox processes ended, port rebound" } + } catch { + Bad "gateway-kill: $($_.Exception.Message)" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } + } } elseif ($sc.Kind -eq 'socket-owner') { try { if (-not (Wait-File $sc.ControlTarget 30)) { throw "parent cmd did not launch: $createOutStr" } @@ -1208,6 +1284,9 @@ admission and execution, not network enforcement. Forwarding proves managed ingress works, not that direct host ingress is blocked. Host-service verifies two exact direct HTTP replies from a service bound to host 127.0.0.1, with host-side request evidence. It checks localhost connectivity. +Gateway-kill creates a sandbox, hard-kills the gateway, and requires every process +that was under that gateway (and the marked workload) to end and the port to be +bindable by a replacement gateway. "@ Set-Content -Path (Join-Path $resultDir "summary.txt") -Value $summary -Encoding UTF8 Write-Host $summary -ForegroundColor ($(if ($verdict -eq "PASS") { "Green" } else { "Red" })) From a0939469e341eaa3dd99705c2806b03cb126dde1 Mon Sep 17 00:00:00 2001 From: ashwind Date: Fri, 13 Nov 2026 23:43:56 +0530 Subject: [PATCH 05/10] test(windows): add gateway-restart scenario to the MXC e2e runner Addresses review feedback on #5299: cover a graceful gateway shutdown and restart with persisted state, not only a hard kill, and record what happens to the workload and the sandbox record. Add a default process_container scenario to run-mxc-e2e.ps1. It runs the gateway on a SQLite file, creates a sandbox with a long-running marked workload, sends a real Ctrl-C, and requires the gateway to log a graceful shutdown and exit and every recorded sandbox process to end. It then restarts the gateway on the same database and requires the sandbox record to still be listed, its workload not to be running (MXC has no restart recovery: gateway_manages_lifecycle is false, so the gateway's shutdown sweep and startup recovery are skipped), and the record to be deletable with its name reusable. The scenario pins this no-recovery behavior and needs updating when recovery is added. Start-Gw takes an optional database URL (default stays in-memory) and clears an inherited "ignore Ctrl-C" setting, which a session such as ssh can pass to the gateway and which would otherwise stop it from ever seeing a graceful-stop request. gateway-kill now shares the new Wait-SandboxTree and Wait-TreeGone helpers with the new scenario. Tests: on Windows x64 the scenario passes (6 sandbox processes ended on Ctrl-C, gateway exit code 0, record persisted in phase Provisioning and not recovered, delete and re-create work). gateway-kill, fs-rw and fs-readonly still pass after the refactor. A graceful stop ends the workloads with or without the per-generation job (checked against a gateway built without it), so this scenario documents graceful behavior; the job is what covers a hard kill. The runner's existing port probe (GetActiveTcpListeners) does not see the listener on the test machine (issue #11), so the runs used a copy of the runner with a loopback connect probe; the probe in the repository is unchanged. Refs #5298 Signed-off-by: ashwind --- crates/openshell-driver-mxc/README.md | 12 ++ .../examples/run-mxc-e2e.ps1 | 198 +++++++++++++++--- 2 files changed, 177 insertions(+), 33 deletions(-) diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 3ad6d86b06..605052f5df 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -382,6 +382,18 @@ gateway are tracked, so other sandboxes on the host are left alone. It relies on each sandbox generation's kill-on-close job object; no workload outlives the gateway. Select it with `-Scenario gateway-kill`. +The default `gateway-restart` scenario covers a graceful gateway stop with +persisted state. It runs the gateway on a SQLite file, creates a sandbox with a +long-running workload, sends a real Ctrl-C, and requires the gateway to log a +graceful shutdown and exit, every recorded sandbox process to end, and a restart +on the same database to start. After the restart the sandbox record must still be +listed (the observed phase is `Provisioning` and does not change), its workload +must not be running (MXC has no restart recovery), and the record must be +deletable with its name reusable. The scenario pins today's no-recovery behavior, +so it needs updating when recovery is added. A graceful stop ends the workloads +with or without the per-generation job; the job is what covers a hard kill. +Select it with `-Scenario gateway-restart`. + The `ws-agent.yaml` and `openclaw-gateway.yaml` policies preserve the Windows branch's workload grants. `inference.yaml` is a template: replace its filesystem, curl executable, inference host, and port placeholders before use. Authorize diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index ce64131c99..b507db710e 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -39,6 +39,10 @@ # host-service - workload reaches an HTTP service on host 127.0.0.1. # gateway-kill - hard-killing the gateway ends every sandbox process and # frees its port. processcontainer only. +# gateway-restart - a graceful gateway stop (Ctrl-C) ends every sandbox process; +# after a restart on the same database the sandbox record is +# still there but its workload is not recovered, and the +# record can be deleted and its name reused. processcontainer only. # https-ca - opt-in outbound HTTPS using the injected proxy CA. [CmdletBinding()] @@ -317,8 +321,12 @@ function Test-PortListening { return @([Net.NetworkInformation.IPGlobalProperties]::GetIPGlobalProperties().GetActiveTcpListeners() | Where-Object { $_.Port -eq $Port }).Count -gt 0 } -function Start-Gw { +function Start-Gw([string]$DbUrl = "sqlite::memory:") { Remove-Item $gwLog, $gwErrLog -Force -ErrorAction SilentlyContinue + # A parent that ignores Ctrl-C (some ssh/service sessions do) passes that on to + # the gateway, which would then never see a graceful-stop request. Clear it for + # this process so the gateway starts with the default handling. + [void][OpenShellMxcCtrlC]::EnableCtrlC() # Ephemeral in-memory DB: this is a test harness, so it must NOT write sandbox # records to the persistent default store (%LOCALAPPDATA%\openshell\gateway\ # openshell.db). Without this, sandbox names persist across gateway restarts @@ -329,7 +337,7 @@ function Start-Gw { # config path containing a space gets split and the gateway's arg parser rejects it. $env:OPENSHELL_GATEWAY_CONFIG = $toml $p = Start-Process -FilePath $gateway ` - -ArgumentList @("--disable-tls", "--db-url", "sqlite::memory:", "--port", $Port, "--log-level", "info") ` + -ArgumentList @("--disable-tls", "--db-url", $DbUrl, "--port", $Port, "--log-level", "info") ` -WorkingDirectory $here -PassThru -WindowStyle Hidden ` -RedirectStandardOutput $gwLog -RedirectStandardError $gwErrLog $deadline = (Get-Date).AddSeconds(30) @@ -339,6 +347,7 @@ function Start-Gw { throw "gateway exited early (code $($p.ExitCode)). See $gwLog." } if (Test-PortListening) { + $null = $p.Handle # keep the handle so ExitCode is available later return $p } Start-Sleep -Milliseconds 400 @@ -547,6 +556,90 @@ function Test-ServiceForwarding([string]$sandboxName, [string]$readyFile, [strin # Detect an agent *launch* failure (binary not found / not implemented) vs a # legitimate policy denial. Used to avoid false-passing a deny scenario when the # agent never actually ran. +if (-not ("OpenShellMxcCtrlC" -as [type])) { + Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; + +public static class OpenShellMxcCtrlC +{ + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + static extern bool SetConsoleCtrlHandler(IntPtr handler, bool add); + + // Restore default Ctrl-C handling for this process (and what it starts). + public static bool EnableCtrlC() { return SetConsoleCtrlHandler(IntPtr.Zero, false); } +} +'@ +} + +# Deliver a real Ctrl-C to another console process. Attaching to its console +# detaches the caller from its own, so this runs in a throwaway PowerShell. +function Send-CtrlC([int]$ProcessId) { + $script = @" +Add-Type -TypeDefinition @' +using System; using System.Runtime.InteropServices; +public static class SendCtrlC { + [DllImport("kernel32.dll", SetLastError=true)] static extern bool FreeConsole(); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool AttachConsole(uint pid); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool SetConsoleCtrlHandler(IntPtr h, bool add); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool GenerateConsoleCtrlEvent(uint ev, uint grp); + public static string Run(uint pid) { + FreeConsole(); + if (!AttachConsole(pid)) return "AttachConsole failed: " + Marshal.GetLastWin32Error(); + SetConsoleCtrlHandler(IntPtr.Zero, true); + bool ok = GenerateConsoleCtrlEvent(0, 0); + int err = Marshal.GetLastWin32Error(); + FreeConsole(); + return ok ? "sent" : "GenerateConsoleCtrlEvent failed: " + err; + } +} +'@ +[SendCtrlC]::Run($ProcessId) +"@ + $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($script)) + $result = Invoke-NativeCaptured "powershell.exe" @("-NoProfile", "-NonInteractive", "-EncodedCommand", $encoded) 30 + # An encoded command also emits CLIXML progress noise on stderr; the verdict is + # the one line that starts with one of the helper's results. + $verdict = @($result.Output | ForEach-Object { "$_".Trim() } | Where-Object { $_ -match '^(sent|AttachConsole failed|GenerateConsoleCtrlEvent failed)' }) + if ($verdict.Count -eq 0) { return "no result from the Ctrl-C helper (exit $($result.ExitCode))" } + return $verdict[0] +} + +# Wait until a sandbox's whole process tree exists, then return it as +# @{ pid = name }. Only processes under $gwProc, plus the process whose command +# line carries $nonce, are included, so unrelated MXC sandboxes on the host are +# never counted or touched. Throws if the tree does not come up. +function Wait-SandboxTree($gwProc, [string]$nonce) { + $victims = @{} + $deadline = (Get-Date).AddSeconds(60) + do { + Start-Sleep -Milliseconds 500 + $victims = @{} + foreach ($id in (Get-DescendantPids $gwProc.Id)) { + $proc = Get-Process -Id $id -ErrorAction SilentlyContinue + if ($proc) { $victims[$id] = $proc.ProcessName } + } + $marked = @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$nonce*" -and $_.ProcessId -ne $PID }) + foreach ($proc in $marked) { $victims[[int]$proc.ProcessId] = $proc.Name } + $names = @($victims.Values | ForEach-Object { $_ -replace '\.exe$', '' }) + $ready = ($names -contains 'wxc-exec') -and ($names -contains 'openshell-supervisor') -and ($marked.Count -gt 0) + } while (-not $ready -and (Get-Date) -lt $deadline) + if (-not $ready) { throw "sandbox process tree did not come up (saw: $(($victims.Values) -join ', '))" } + return $victims +} + +# Processes from a recorded tree that are still alive after waiting up to +# $seconds for them to end. +function Wait-TreeGone($victims, [int]$seconds = 30) { + $deadline = (Get-Date).AddSeconds($seconds) + do { + Start-Sleep -Milliseconds 500 + $survivors = @($victims.Keys | Where-Object { Get-Process -Id $_ -ErrorAction SilentlyContinue }) + } while ($survivors.Count -gt 0 -and (Get-Date) -lt $deadline) + return ($survivors | ForEach-Object { "$($victims[$_])($_)" }) +} + # All descendants of a process, found by walking ParentProcessId links in one # snapshot. Orphans keep their ParentProcessId, so the walk is still correct # for a tree whose root has since died, but it must run while the root lives. @@ -814,6 +907,11 @@ try { Name = 'gateway-kill'; PolicyFile = Join-Path $policyDir 'fs-rw.yaml' SandboxId = 'gk'; Backends = 'process_container'; Kind = 'gateway-kill' Description = 'hard-killing the gateway ends every sandbox process and frees its port' + }, + @{ + Name = 'gateway-restart'; PolicyFile = Join-Path $policyDir 'fs-rw.yaml' + SandboxId = 'gr'; Backends = 'process_container'; Kind = 'gateway-restart' + Description = 'graceful gateway stop ends sandbox processes; the record persists but is not recovered' } ) @@ -940,7 +1038,7 @@ try { } else { $command = @($cmdExe, "/c", "echo denied 1> $denied") } - } elseif ($sc.Kind -eq 'gateway-kill') { + } elseif ($sc.Kind -in @('gateway-kill', 'gateway-restart')) { # A long-running workload carrying a unique marker, so its process # can be identified without matching unrelated cmd.exe instances. $killNonce = [guid]::NewGuid().ToString('N') @@ -1003,7 +1101,11 @@ try { # policyUsed already lives in the result bundle and is the exact # rendered policy passed to OpenShell. - $gw = Start-Gw + # gateway-restart needs state that survives the gateway; every other + # scenario keeps the ephemeral in-memory database. + $persistedDb = Join-Path $resultDir "gateway.$($sc.Name).db" + $gwDbUrl = if ($sc.Kind -eq 'gateway-restart') { "sqlite:${persistedDb}?mode=rwc" } else { "sqlite::memory:" } + $gw = Start-Gw $gwDbUrl Info "gateway pid $($gw.Id)" Register-Cli @@ -1019,7 +1121,7 @@ try { "--policy", [string]$policyUsed, "--driver-config-json", $driverConfig ) - $createArguments += if ($sc.Kind -in @('host-service', 'gateway-kill')) { '--detach' } else { '--no-tty' } + $createArguments += if ($sc.Kind -in @('host-service', 'gateway-kill', 'gateway-restart')) { '--detach' } else { '--no-tty' } $createResult = Invoke-NativeCaptured $cli $createArguments $createOut = $createResult.Output $createExitCode = $createResult.ExitCode @@ -1047,39 +1149,15 @@ try { try { if ($createExitCode -ne 0) { throw "sandbox creation failed: $createOutStr" } # Wait for the whole sandbox tree to exist, then record it. - # Only processes under THIS gateway (plus the marked workload) - # are tracked, so unrelated MXC sandboxes on the host are never - # counted or touched. - $victims = @{} - $deadline = (Get-Date).AddSeconds(60) - do { - Start-Sleep -Milliseconds 500 - $victims = @{} - foreach ($id in (Get-DescendantPids $gw.Id)) { - $proc = Get-Process -Id $id -ErrorAction SilentlyContinue - if ($proc) { $victims[$id] = $proc.ProcessName } - } - foreach ($proc in @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID })) { - $victims[[int]$proc.ProcessId] = $proc.Name - } - $names = @($victims.Values | ForEach-Object { $_ -replace '\.exe$', '' }) - $ready = ($names -contains 'wxc-exec') -and ($names -contains 'openshell-supervisor') -and (@(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID }).Count -gt 0) - } while (-not $ready -and (Get-Date) -lt $deadline) - if (-not $ready) { throw "sandbox process tree did not come up (saw: $(($victims.Values) -join ', '))" } + $victims = Wait-SandboxTree $gw $killNonce Info "tracking $($victims.Count) sandbox processes: $((($victims.GetEnumerator() | Sort-Object Name | ForEach-Object { "$($_.Value)($($_.Key))" })) -join ', ')" Info "hard-killing gateway pid $($gw.Id)" Stop-Process -Id $gw.Id -Force $gw.WaitForExit() - $survivors = @() - $deadline = (Get-Date).AddSeconds(30) - do { - Start-Sleep -Milliseconds 500 - $survivors = @($victims.Keys | Where-Object { Get-Process -Id $_ -ErrorAction SilentlyContinue }) - } while ($survivors.Count -gt 0 -and (Get-Date) -lt $deadline) + $survivors = @(Wait-TreeGone $victims 30) if ($survivors.Count -gt 0) { - $left = ($survivors | ForEach-Object { "$($victims[$_])($_)" }) -join ', ' - throw "sandbox processes survived the gateway's death: $left" + throw "sandbox processes survived the gateway's death: $($survivors -join ', ')" } # The port must be free again: a replacement gateway must bind it. $gw = Start-Gw @@ -1090,6 +1168,57 @@ try { Bad "gateway-kill: $($_.Exception.Message)" $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } } + } elseif ($sc.Kind -eq 'gateway-restart') { + # Documents today's behavior (MXC has no restart recovery): a graceful + # gateway stop ends the workloads, the persisted record is left + # behind, and nothing restarts it. Revisit the "not recovered" check + # when recovery is added. + try { + if ($createExitCode -ne 0) { throw "sandbox creation failed: $createOutStr" } + $victims = Wait-SandboxTree $gw $killNonce + Info "tracking $($victims.Count) sandbox processes: $((($victims.GetEnumerator() | Sort-Object Name | ForEach-Object { "$($_.Value)($($_.Key))" })) -join ', ')" + + Info "sending Ctrl-C to gateway pid $($gw.Id)" + $sent = Send-CtrlC $gw.Id + if ($sent -ne 'sent') { throw "could not send Ctrl-C to the gateway: $sent" } + if (-not $gw.WaitForExit(60000)) { throw 'gateway did not exit within 60 s of Ctrl-C (no graceful shutdown)' } + Info "gateway exited (exit code $($gw.ExitCode))" + if (-not (Select-String -Path $gwLog, $gwErrLog -Pattern 'Shutdown signal received' -Quiet -ErrorAction SilentlyContinue)) { + throw 'gateway exited without logging a graceful shutdown (it was not a graceful stop)' + } + $survivors = @(Wait-TreeGone $victims 30) + if ($survivors.Count -gt 0) { + throw "sandbox processes survived a graceful gateway stop: $($survivors -join ', ')" + } + Copy-Item $gwLog "$gwLog.first" -Force -ErrorAction SilentlyContinue + Copy-Item $gwErrLog "$gwErrLog.first" -Force -ErrorAction SilentlyContinue + + # Restart on the same database: the record persists; nothing recovers it. + $gw = Start-Gw $gwDbUrl + Info "restarted gateway pid $($gw.Id) on the same database" + $listed = Invoke-NativeCaptured $cli @('sandbox', 'list') + $row = @($listed.Output | Where-Object { $_ -match [regex]::Escape($sandboxName) }) | Select-Object -First 1 + if (-not $row) { throw "persisted sandbox record '$sandboxName' is missing after restart: $($listed.Output -join ' ')" } + Info "record after restart: $(($row -replace '\s+', ' ').Trim())" + Start-Sleep -Seconds 10 + $running = @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID }) + if ($running.Count -gt 0) { throw "workload was running after restart ($($running.Count) process(es)); recovery behavior changed, update this scenario and the Windows docs" } + + # The stale record must be manageable: delete it, then reuse the name. + $deleted = Invoke-NativeCaptured $cli @('sandbox', 'delete', $sandboxName) + if ($deleted.ExitCode -ne 0) { throw "deleting the stale record failed: $($deleted.Output -join ' ')" } + $listed = Invoke-NativeCaptured $cli @('sandbox', 'list') + if (@($listed.Output | Where-Object { $_ -match [regex]::Escape($sandboxName) }).Count -gt 0) { throw 'record still listed after delete' } + $again = Invoke-NativeCaptured $cli $createArguments + if ($again.ExitCode -ne 0) { throw "re-creating '$sandboxName' failed: $($again.Output -join ' ')" } + $null = Wait-SandboxTree $gw $killNonce + + Ok "gateway-restart: graceful stop ended $($victims.Count) processes; record persisted, not recovered; delete and re-create work" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'PASS'; Reason = "graceful stop ended $($victims.Count) processes; record persisted, not recovered, name reusable" } + } catch { + Bad "gateway-restart: $($_.Exception.Message)" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } + } } elseif ($sc.Kind -eq 'socket-owner') { try { if (-not (Wait-File $sc.ControlTarget 30)) { throw "parent cmd did not launch: $createOutStr" } @@ -1286,7 +1415,10 @@ Host-service verifies two exact direct HTTP replies from a service bound to host 127.0.0.1, with host-side request evidence. It checks localhost connectivity. Gateway-kill creates a sandbox, hard-kills the gateway, and requires every process that was under that gateway (and the marked workload) to end and the port to be -bindable by a replacement gateway. +bindable by a replacement gateway. Gateway-restart stops the gateway gracefully +(Ctrl-C) with a persisted database, requires the same, then restarts it and checks +that the sandbox record is still listed, its workload is not recovered, and the +record can be deleted and its name reused. "@ Set-Content -Path (Join-Path $resultDir "summary.txt") -Value $summary -Encoding UTF8 Write-Host $summary -ForegroundColor ($(if ($verdict -eq "PASS") { "Green" } else { "Red" })) From 8305936d04472cad88658da545cf7de3bd606185 Mon Sep 17 00:00:00 2001 From: ashwind Date: Fri, 13 Nov 2026 23:44:51 +0530 Subject: [PATCH 06/10] docs(windows): document gateway exit and restart behavior for MXC Addresses review feedback on #5299: the Windows docs and build guidance should say that gateway exit terminates workloads and that MXC does not provide the stop-and-recover behavior of the Docker, Podman and VM drivers. Add a "Gateway exit and restart" section to the MXC driver README and replace the one-line limits entries there and in WINDOWS.md. They state that stop, delete, a graceful gateway stop and a gateway crash or hard kill end every sandbox process, workloads included; that MXC reports gateway_manages_lifecycle false, so the gateway skips its shutdown sweep and startup recovery for it; and that after a restart on the same database the sandbox record is still listed with a stale phase, no workload runs, and the record can be deleted and its name reused. The section says that behavior for a driver running as its own process is not defined yet. Add matching guidance to the build-openshell-mxc-windows skill, including how to report the behavior and that gateway-restart is not evidence for the per-generation job. The documented behavior was observed on Windows x64 with the gateway-kill and gateway-restart E2E scenarios; ARM64 has not been run. Refs #5298 Signed-off-by: ashwind --- .../build-openshell-mxc-windows/SKILL.md | 20 ++++++++++ WINDOWS.md | 11 ++++- crates/openshell-driver-mxc/README.md | 40 ++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 93d025a17e..45013efa3e 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -487,6 +487,26 @@ firewall modification is required. A failed host health check or workload request fails the test. This verifies direct localhost connectivity; retain the existing shared proxy SSRF controls. +The default `gateway-kill` and `gateway-restart` scenarios cover gateway loss. +`gateway-kill` hard-kills the gateway and requires every sandbox process under it +(the supervisor, `wxc-exec`, the boundary and the workload) to end and a +replacement gateway to bind the port. `gateway-restart` sends a real Ctrl-C with +a persisted SQLite database, requires a logged graceful shutdown and every +sandbox process to end, then restarts on the same database and requires the +record to still be listed, its workload not to be running, and the record to be +deletable with its name reusable. Use `-Scenario gateway-kill` or +`-Scenario gateway-restart` for focused checks. Report the behavior accurately: +gateway exit ends workloads and MXC has no restart recovery, unlike the Docker, +Podman and VM drivers, which stop and recover sandboxes on a graceful restart +(MXC reports `gateway_manages_lifecycle: false`). Never describe MXC as +stop-and-recover. A graceful stop ends workloads with or without the +per-generation job; the job is what covers a hard kill, so do not cite +`gateway-restart` as evidence for the job. If recovery is added, update +`gateway-restart`, the driver README and `WINDOWS.md` together. These scenarios +send Ctrl-C and track process trees, so run them on a host you can disturb; +they track only processes under their own gateway and leave other sandboxes +alone. + When reporting `windows:ci`, distinguish these categories: - Passed tests from the full x64 workspace test log. diff --git a/WINDOWS.md b/WINDOWS.md index cbd5be8b03..cac8b62803 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -28,7 +28,16 @@ under development and is not a production-qualified security boundary: Treat the gateway host as trusted. - Foreground main-process attachment and ConPTY resize are unavailable. Create command workloads with `--detach`. -- Gateway restart recovery for live MXC generations is not supported. +- Gateway exit ends sandbox workloads, and there is no restart recovery. When + the gateway process ends (a graceful stop with Ctrl-C, a crash, or a kill), the + sandbox processes it started end with it, workloads included. After the gateway + restarts on the same database, each sandbox record is still listed but its + workload is not restarted, and its phase is not updated to show that the + workload ended (for example `Provisioning` after a graceful stop, `Ready` after + a hard kill). Delete the record to reuse its name. The Docker, Podman and VM + drivers stop sandboxes on a graceful gateway stop and recover them on restart; + MXC does not provide that recovery. See + [gateway exit and restart](https://github.com/NVIDIA/OpenShell/blob/windows/crates/openshell-driver-mxc/README.md#gateway-exit-and-restart). See the Windows branch's [MXC driver documentation](https://github.com/NVIDIA/OpenShell/blob/windows/crates/openshell-driver-mxc/README.md) for the current capabilities and enforcement limits, and its diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 605052f5df..c8a7f69d49 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -222,7 +222,8 @@ not required; driver-owned guest TLS fields are rejected. | Exec, signals, retained output | Authenticated Sandbox Protocol; ConPTY resize is not yet supported | | Dynamic forwarding | Standard supervisor `ForwardTcp` path through sandbox loopback connect | | ETW/OCSF audit | Optional Windows Sandboxing ETW consumer | -| Gateway restart recovery | Not yet supported; live MXC generations remain in-memory | +| Gateway exit | Ends the sandbox processes, workloads included ([details](#gateway-exit-and-restart)) | +| Gateway restart recovery | Not supported; live MXC generations remain in-memory and are not recovered | Workloads receive the proxy CA in `NODE_EXTRA_CA_CERTS` and `DENO_CERT`, and a combined bundle in `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, and @@ -254,6 +255,43 @@ explicit UI policy (including `{}`) for drivers missing this capability; those drivers need no UI-specific handling. UI policy itself does not require the `allow_driver_config` opt-in. Recreate the sandbox to change these settings. +## Gateway exit and restart + +The driver runs inside the gateway. Each sandbox generation runs in its own +Windows job object with kill-on-close: the host supervisor and `wxc-exec` are +started inside it, and the boundary and workload processes below them inherit +it. When a generation stops, the job is terminated, and when the gateway process +ends for any reason Windows closes the job and ends everything in it. In +practice: + +- **Stop and delete** end the whole generation: supervisor, `wxc-exec`, boundary + and workload, not only the two direct children. +- **Graceful gateway stop** (Ctrl-C) ends every sandbox process, workloads + included, and frees the gateway port. +- **Gateway crash or hard kill** ends every sandbox process through the job, and + the listening socket is not inherited by any child, so a new gateway can bind + the port at once. + +MXC does not recover sandboxes across a gateway restart. The Docker, Podman and +VM drivers stop their sandboxes on a graceful gateway stop and recover them when +the gateway starts again; MXC reports `gateway_manages_lifecycle: false`, so the +gateway skips that shutdown sweep and startup recovery for it, and live +generations exist only in the driver's memory. After a restart on the same +database: + +- each sandbox record is still listed, with the phase it had when the gateway + went away (for example `Provisioning` after a graceful stop, `Ready` after a + hard kill); nothing updates it to show that the workload ended; +- no workload is running, and nothing restarts it; +- `openshell sandbox delete ` removes the record, and the name can then be + created again. + +Treat a sandbox that was running when the gateway exited as lost, delete its +record, and create it again. The `gateway-kill` and `gateway-restart` E2E +scenarios (see [Validation](#validation)) check this behavior. Behavior for a +driver that runs as its own process, separate from the gateway, is not defined +yet. + ## Validation Windows release build tasks stage the pinned, target-matching `libz3.dll` beside From 490571d3977c5b0d87a31fb08c54d33fc2bcf01d Mon Sep 17 00:00:00 2001 From: ashwind Date: Mon, 16 Nov 2026 12:26:35 +0530 Subject: [PATCH 07/10] test(windows): assert a clean exit in the gateway-restart scenario Addresses review feedback on #5299: the gateway-restart scenario only printed the gateway's exit code and treated the "Shutdown signal received" log line as proof of a graceful stop. That line is logged before cleanup is awaited, so a shutdown that began could still fail (nonzero exit) while the job removed the processes and a replacement gateway started, and the scenario would report a successful graceful restart. Decide the outcome in a small pure function, Get-GracefulStopFailure: the gateway must exit with code 0 and have logged the shutdown message; an unavailable exit code, a nonzero code (including 0xC000013A, the Ctrl-C default-handler exit) or a missing message is a failure, and a nonzero code is reported before the missing message. The scenario now checks it right after the gateway exits, before the process tree check and before the replacement gateway is started. Cover the failing cases in tasks/scripts/test-mxc-remote-tests.ps1, which loads the function from the runner by parsing it, like the existing pure helper regressions. The decision only protects the scenario if it is consulted at the right place, so the same script also checks the runner's syntax tree: exactly one Send-CtrlC and one check, the check after Ctrl-C, a throw on its result, and all of that before the replacement Start-Gw. Moving or dropping the call fails the script. Tests: test-mxc-remote-tests.ps1 passes (7 new checks: clean stop, nonzero exit after the message, 0xC000013A, exit 0 without the message, missing exit code, nonzero reported before the missing message, and the call order in the runner) on Windows x64 and native ARM64. Removing the throw from a copy of the runner makes the order check fail on both. The gateway-restart scenario still passes on both (gateway exit code 0, replacement gateway starts). With the check told the exit code was 1 (injected in a scratch copy of the runner), the scenario fails with "gateway exited with code 1 after Ctrl-C; a graceful shutdown must exit 0" and no replacement gateway is started, on both. The runs used a scratch copy of the runner whose readiness probe connects to the port (Get-NetTCPConnection under-reports listeners on the test hosts); the probe in the repository is unchanged. Refs #5298 Signed-off-by: ashwind --- .../examples/run-mxc-e2e.ps1 | 18 ++++++-- tasks/scripts/test-mxc-remote-tests.ps1 | 42 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index b507db710e..9b5b2264e0 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -606,6 +606,18 @@ public static class SendCtrlC { return $verdict[0] } +# Decide whether a gateway that was sent Ctrl-C shut down gracefully. Returns why it +# did not, or $null when it did. The "Shutdown signal received" log line only proves +# that shutdown began (it is logged before cleanup is awaited, and cleanup can still +# fail), so a graceful stop must also exit with code 0. Pure, so it is unit-tested in +# tasks/scripts/test-mxc-remote-tests.ps1. +function Get-GracefulStopFailure($ExitCode, [bool] $LoggedShutdown) { + if ($null -eq $ExitCode) { return 'gateway exit code is unavailable after Ctrl-C' } + if ($ExitCode -ne 0) { return "gateway exited with code $ExitCode after Ctrl-C; a graceful shutdown must exit 0" } + if (-not $LoggedShutdown) { return 'gateway exited without logging a graceful shutdown (it was not a graceful stop)' } + return $null +} + # Wait until a sandbox's whole process tree exists, then return it as # @{ pid = name }. Only processes under $gwProc, plus the process whose command # line carries $nonce, are included, so unrelated MXC sandboxes on the host are @@ -1183,9 +1195,9 @@ try { if ($sent -ne 'sent') { throw "could not send Ctrl-C to the gateway: $sent" } if (-not $gw.WaitForExit(60000)) { throw 'gateway did not exit within 60 s of Ctrl-C (no graceful shutdown)' } Info "gateway exited (exit code $($gw.ExitCode))" - if (-not (Select-String -Path $gwLog, $gwErrLog -Pattern 'Shutdown signal received' -Quiet -ErrorAction SilentlyContinue)) { - throw 'gateway exited without logging a graceful shutdown (it was not a graceful stop)' - } + $loggedShutdown = [bool](Select-String -Path $gwLog, $gwErrLog -Pattern 'Shutdown signal received' -Quiet -ErrorAction SilentlyContinue) + $stopFailure = Get-GracefulStopFailure $gw.ExitCode $loggedShutdown + if ($stopFailure) { throw $stopFailure } $survivors = @(Wait-TreeGone $victims 30) if ($survivors.Count -gt 0) { throw "sandbox processes survived a graceful gateway stop: $($survivors -join ', ')" diff --git a/tasks/scripts/test-mxc-remote-tests.ps1 b/tasks/scripts/test-mxc-remote-tests.ps1 index 0971360804..03c3fab3ec 100644 --- a/tasks/scripts/test-mxc-remote-tests.ps1 +++ b/tasks/scripts/test-mxc-remote-tests.ps1 @@ -76,4 +76,46 @@ try { Get-ChildItem -LiteralPath $cacheFixture -Directory -Recurse | Sort-Object { $_.FullName.Length } -Descending | ForEach-Object { Remove-Item -LiteralPath $_.FullName } Remove-Item -LiteralPath $cacheFixture } +# Graceful-stop decision used by the gateway-restart E2E scenario (pure function). +$e2eRunner = Join-Path $PSScriptRoot '../../crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1' +$tokens = $null +$errors = $null +$ast = [Management.Automation.Language.Parser]::ParseFile($e2eRunner, [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +foreach ($function in $ast.FindAll({ param($node) $node -is [Management.Automation.Language.FunctionDefinitionAst] }, $true)) { + if ($function.Name -eq 'Get-GracefulStopFailure') { Invoke-Expression $function.Extent.Text } +} +if (-not (Get-Command Get-GracefulStopFailure -ErrorAction SilentlyContinue)) { throw 'Get-GracefulStopFailure not found in the E2E runner.' } +if ($null -ne (Get-GracefulStopFailure 0 $true)) { throw 'Clean graceful stop rejected.' } +$count++ +if ((Get-GracefulStopFailure 1 $true) -notmatch 'code 1') { throw 'Nonzero exit after the shutdown message accepted.' } +$count++ +if ((Get-GracefulStopFailure -1073741510 $true) -notmatch 'code -1073741510') { throw 'Ctrl-C default-handler exit (0xC000013A) accepted as graceful.' } +$count++ +if (-not (Get-GracefulStopFailure 0 $false)) { throw 'Exit 0 without the shutdown message accepted.' } +$count++ +if ((Get-GracefulStopFailure $null $true) -notmatch 'unavailable') { throw 'Missing exit code accepted.' } +$count++ +if ((Get-GracefulStopFailure 1 $false) -notmatch 'code 1') { throw 'Nonzero exit must be reported before the missing log line.' } +$count++ +# The decision only protects the scenario if it is consulted after Ctrl-C, fails the +# scenario, and does so before the replacement gateway is started. Check that order +# in the runner's syntax tree so moving or dropping the call fails here. +function Get-CommandOffsets($Root, [string] $Name) { + return @($Root.FindAll({ param($node) $node -is [Management.Automation.Language.CommandAst] -and $node.GetCommandName() -eq $Name }, $true) | + ForEach-Object { $_.Extent.StartOffset }) +} +$ctrlC = @(Get-CommandOffsets $ast 'Send-CtrlC') +$checks = @(Get-CommandOffsets $ast 'Get-GracefulStopFailure') +$starts = @(Get-CommandOffsets $ast 'Start-Gw') +if ($ctrlC.Count -ne 1) { throw 'Expected exactly one Send-CtrlC call in the E2E runner.' } +if ($checks.Count -ne 1) { throw 'Expected exactly one Get-GracefulStopFailure call in the E2E runner.' } +if ($checks[0] -lt $ctrlC[0]) { throw 'The graceful-stop check must come after Ctrl-C is sent.' } +$restart = @($starts | Where-Object { $_ -gt $ctrlC[0] } | Select-Object -First 1) +if ($restart.Count -ne 1) { throw 'No replacement gateway start found after Ctrl-C.' } +if ($checks[0] -gt $restart[0]) { throw 'The graceful-stop check must come before the replacement gateway starts.' } +$throws = @($ast.FindAll({ param($node) $node -is [Management.Automation.Language.ThrowStatementAst] -and $node.Extent.Text -match 'stopFailure' }, $true) | + Where-Object { $_.Extent.StartOffset -gt $checks[0] -and $_.Extent.StartOffset -lt $restart[0] }) +if ($throws.Count -ne 1) { throw 'The graceful-stop check result must fail the scenario before the replacement gateway starts.' } +$count++ Write-Host "$count remote-script regression checks passed. No runtime qualification claimed." From d3f40a259d3e5852000e808ea5b2b647eb62a0ba Mon Sep 17 00:00:00 2001 From: ashwind Date: Tue, 17 Nov 2026 11:59:57 +0530 Subject: [PATCH 08/10] fix(windows): create sandbox processes already inside their job Addresses review feedback on #5299. The driver used to create the supervisor and wxc-exec suspended, assign them to the generation's kill-on-close job, then resume them. If the gateway was hard-killed between creation and assignment, that process was left running outside any job. Create each root process with CreateProcessW and PROC_THREAD_ATTRIBUTE_JOB_LIST so it is a member of the job from its first instruction; there is no window left. A handle list restricts what the child inherits to its own stdio. The new child module owns spawning (pipes, environment block, command-line quoting, exit watcher) and replaces tokio::process for these processes. Tests: unit tests cover born-in-job (with a negative control), a hard-killed owner leaving no process, an unusable job failing the spawn, and a stress test that hard-kills a host mid-spawn and counts processes. Reinstating the old create-then-assign order makes the stress test fail 6/6. 95 driver tests pass on x64 and ARM64, and the gateway-kill, gateway-restart and fs-rw scenarios pass against real MXC on both. Refs #5298 Signed-off-by: ashwind --- crates/openshell-driver-mxc/Cargo.toml | 2 +- crates/openshell-driver-mxc/README.md | 5 +- crates/openshell-driver-mxc/src/child.rs | 1087 +++++++++++++++++ crates/openshell-driver-mxc/src/driver.rs | 40 +- .../openshell-driver-mxc/src/etw_consumer.rs | 4 +- crates/openshell-driver-mxc/src/job.rs | 330 +++-- crates/openshell-driver-mxc/src/lib.rs | 2 + crates/openshell-driver-mxc/src/mxc.rs | 61 +- 8 files changed, 1356 insertions(+), 175 deletions(-) create mode 100644 crates/openshell-driver-mxc/src/child.rs diff --git a/crates/openshell-driver-mxc/Cargo.toml b/crates/openshell-driver-mxc/Cargo.toml index a34f3c5eb3..d3f071ff9b 100644 --- a/crates/openshell-driver-mxc/Cargo.toml +++ b/crates/openshell-driver-mxc/Cargo.toml @@ -37,7 +37,7 @@ uuid = { workspace = true } openshell-mxc-boundary = { path = "../openshell-mxc-boundary" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } -windows = { workspace = true, features = ["Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects"] } +windows = { workspace = true, features = ["Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Pipes", "Win32_UI_Shell"] } [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index c8a7f69d49..d190161ba9 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -259,8 +259,9 @@ drivers need no UI-specific handling. UI policy itself does not require the The driver runs inside the gateway. Each sandbox generation runs in its own Windows job object with kill-on-close: the host supervisor and `wxc-exec` are -started inside it, and the boundary and workload processes below them inherit -it. When a generation stops, the job is terminated, and when the gateway process +created inside it (the job is named in the `CreateProcessW` call, so no process +of a generation ever exists outside its job, even if the gateway is killed +mid-spawn), and the boundary and workload processes below them inherit it. When a generation stops, the job is terminated, and when the gateway process ends for any reason Windows closes the job and ends everything in it. In practice: diff --git a/crates/openshell-driver-mxc/src/child.rs b/crates/openshell-driver-mxc/src/child.rs new file mode 100644 index 0000000000..1f6dac2fc0 --- /dev/null +++ b/crates/openshell-driver-mxc/src/child.rs @@ -0,0 +1,1087 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Start a sandbox generation's processes inside their job object, atomically. +//! +//! The job is named in the `PROC_THREAD_ATTRIBUTE_JOB_LIST` attribute of the +//! `CreateProcessW` call, so the new process is a member of its job from the moment +//! it exists. There is no window between creating a process and assigning it to the +//! job in which a hard-killed driver could leave the process behind. The standard +//! library and tokio offer no stable way to set that attribute, so the processes the +//! driver starts for a generation (the host supervisor and `wxc-exec`) are created +//! here. +//! +//! [`ChildCommand`] describes the process in the same terms the driver already used +//! for `tokio::process::Command`, and [`GenerationChild`] offers the subset of +//! `tokio::process::Child` the driver relies on: `id`, `raw_handle`, `wait`, +//! `kill`, and async `stdout`/`stderr`. A child is killed when its +//! [`GenerationChild`] is dropped, like the former `kill_on_drop(true)`. +//! +//! Only the three standard handles are inherited (`PROC_THREAD_ATTRIBUTE_HANDLE_LIST`), +//! so no other handle of the host, such as a listening socket, leaks into a sandbox +//! process. + +// Process creation is Win32 FFI and needs `unsafe`; it is confined to this module. +#![allow(unsafe_code)] +// `pub(crate)` documents what the driver may use; this module is private. +#![allow(clippy::redundant_pub_crate)] + +use std::collections::BTreeMap; +use std::ffi::{OsStr, OsString, c_void}; +use std::io; +use std::mem::size_of; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::io::RawHandle; +use std::os::windows::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::ExitStatus; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tokio::net::windows::named_pipe::NamedPipeServer; +use tokio::sync::watch; +use windows::Win32::Foundation::{ + CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE, INVALID_HANDLE_VALUE, WAIT_FAILED, +}; +use windows::Win32::Security::SECURITY_ATTRIBUTES; +use windows::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, + FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_SHARE_MODE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING, PIPE_ACCESS_INBOUND, PIPE_ACCESS_OUTBOUND, +}; +use windows::Win32::System::Console::{ + GetStdHandle, STD_ERROR_HANDLE, STD_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, +}; +use windows::Win32::System::Pipes::{ + CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_WAIT, +}; +use windows::Win32::System::Threading::{ + CREATE_SUSPENDED, CREATE_UNICODE_ENVIRONMENT, CreateProcessW, DeleteProcThreadAttributeList, + EXTENDED_STARTUPINFO_PRESENT, GetCurrentProcess, GetExitCodeProcess, INFINITE, + InitializeProcThreadAttributeList, LPPROC_THREAD_ATTRIBUTE_LIST, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROC_THREAD_ATTRIBUTE_JOB_LIST, PROCESS_CREATION_FLAGS, + PROCESS_INFORMATION, STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, + UpdateProcThreadAttribute, WaitForSingleObject, +}; +use windows::core::{PCWSTR, PWSTR}; + +/// What a standard stream of a new process is connected to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChildStdio { + /// The `NUL` device. + Null, + /// A pipe whose parent end is exposed on the [`GenerationChild`]. + Piped, + /// The same stream as the host process. + Inherit, +} + +/// A process to start inside a generation's job. +#[derive(Clone, Debug)] +pub struct ChildCommand { + program: OsString, + args: Vec, + raw_args: Option, + envs: Vec<(OsString, OsString)>, + stdin: ChildStdio, + stdout: ChildStdio, + stderr: ChildStdio, +} + +impl ChildCommand { + pub(crate) fn new(program: impl AsRef) -> Self { + Self { + program: program.as_ref().to_os_string(), + args: Vec::new(), + raw_args: None, + envs: Vec::new(), + stdin: ChildStdio::Inherit, + stdout: ChildStdio::Inherit, + stderr: ChildStdio::Inherit, + } + } + + /// Append an argument; it is quoted so the new process parses it back unchanged. + pub(crate) fn arg(&mut self, arg: impl AsRef) -> &mut Self { + self.args.push(arg.as_ref().to_os_string()); + self + } + + #[cfg(test)] + pub(crate) fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + self.arg(arg); + } + self + } + + /// Append already-encoded arguments to the command line exactly as given. + pub(crate) fn raw_arg(&mut self, arg: impl AsRef) -> &mut Self { + self.raw_args = Some(arg.as_ref().to_os_string()); + self + } + + /// Set an environment variable for the new process, on top of the host's. + pub(crate) fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Self { + self.envs + .push((key.as_ref().to_os_string(), value.as_ref().to_os_string())); + self + } + + pub(crate) fn stdin(&mut self, stdio: ChildStdio) -> &mut Self { + self.stdin = stdio; + self + } + + pub(crate) fn stdout(&mut self, stdio: ChildStdio) -> &mut Self { + self.stdout = stdio; + self + } + + pub(crate) fn stderr(&mut self, stdio: ChildStdio) -> &mut Self { + self.stderr = stdio; + self + } + + /// The environment variables set explicitly on this command. + #[cfg(test)] + pub(crate) fn get_envs(&self) -> impl Iterator { + self.envs + .iter() + .map(|(k, v)| (k.as_os_str(), v.as_os_str())) + } +} + +/// An owned Win32 handle, closed on drop. +struct OwnedHandle(HANDLE); + +// SAFETY: a kernel handle has no thread affinity; every use below is thread-safe. +unsafe impl Send for OwnedHandle {} +// SAFETY: as above. +unsafe impl Sync for OwnedHandle {} + +impl Drop for OwnedHandle { + fn drop(&mut self) { + if !self.0.is_invalid() && !self.0.0.is_null() { + // SAFETY: the handle is owned and closed exactly once. + let _ = unsafe { CloseHandle(self.0) }; + } + } +} + +/// A process started inside its generation's job. +/// +/// It is killed when dropped if it is still running. +pub struct GenerationChild { + process: Arc, + pid: u32, + exit: watch::Receiver>, + /// Parent end of the child's standard input, when it was [`ChildStdio::Piped`]. + /// Held open for the child's lifetime so the child does not see end-of-file; the + /// driver never writes to it. + _stdin: Option, + pub(crate) stdout: Option, + pub(crate) stderr: Option, +} + +/// What the ETW consumer needs from a child to attribute its events. +pub trait ProcessHandle { + fn id(&self) -> Option; + fn raw_handle(&self) -> Option; +} + +impl ProcessHandle for GenerationChild { + fn id(&self) -> Option { + Self::id(self) + } + + fn raw_handle(&self) -> Option { + Self::raw_handle(self) + } +} + +impl ProcessHandle for tokio::process::Child { + fn id(&self) -> Option { + Self::id(self) + } + + fn raw_handle(&self) -> Option { + Self::raw_handle(self) + } +} + +impl GenerationChild { + /// The process ID. Always present; the `Option` mirrors `tokio::process::Child::id` + /// so the ETW consumer can take either through [`ProcessHandle`]. + #[allow(clippy::unnecessary_wraps)] + pub(crate) fn id(&self) -> Option { + Some(self.pid) + } + + /// The process handle, valid for as long as this value lives. Always present; see + /// [`GenerationChild::id`]. + #[allow(clippy::unnecessary_wraps)] + pub(crate) fn raw_handle(&self) -> Option { + Some(self.process.0.0) + } + + /// Ask the process to end now, without waiting for it. + pub(crate) fn start_kill(&mut self) -> io::Result<()> { + if self.exit.borrow().is_some() { + return Ok(()); + } + // SAFETY: the handle is owned by `self.process` and valid. + match unsafe { TerminateProcess(self.process.0, 1) } { + Ok(()) => Ok(()), + // Lost a race with the process ending by itself. + Err(_) if self.exit.borrow().is_some() => Ok(()), + Err(error) => Err(io::Error::from(error)), + } + } + + /// Wait for the process to end. Safe to cancel and to call again. + pub(crate) async fn wait(&mut self) -> io::Result { + loop { + let code = *self.exit.borrow(); + if let Some(code) = code { + return Ok(ExitStatus::from_raw(code)); + } + self.exit + .changed() + .await + .map_err(|_| io::Error::other("process exit watcher ended unexpectedly"))?; + } + } + + /// End the process and wait for it to be gone. + pub(crate) async fn kill(&mut self) -> io::Result<()> { + self.start_kill()?; + self.wait().await.map(|_| ()) + } +} + +impl Drop for GenerationChild { + fn drop(&mut self) { + let _ = self.start_kill(); + } +} + +static PIPE_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn inheritable() -> SECURITY_ATTRIBUTES { + SECURITY_ATTRIBUTES { + nLength: u32::try_from(size_of::()).unwrap_or(0), + lpSecurityDescriptor: std::ptr::null_mut(), + bInheritHandle: true.into(), + } +} + +fn to_wide(value: &OsStr) -> io::Result> { + let mut wide: Vec = value.encode_wide().collect(); + if wide.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a process argument, path or environment entry contains a NUL", + )); + } + wide.push(0); + Ok(wide) +} + +/// Append `arg` quoted so `CommandLineToArgvW` and the C runtime parse it back to `arg`. +fn append_arg(out: &mut Vec, arg: &OsStr) -> io::Result<()> { + let wide: Vec = arg.encode_wide().collect(); + if wide.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a process argument contains a NUL", + )); + } + let space = u16::from(b' '); + let tab = u16::from(b'\t'); + let quote = u16::from(b'"'); + let backslash = u16::from(b'\\'); + let needs_quotes = wide.is_empty() || wide.iter().any(|&c| c == space || c == tab); + if needs_quotes { + out.push(quote); + } + let mut backslashes = 0usize; + for &c in &wide { + if c == backslash { + backslashes += 1; + } else { + if c == quote { + // Double the backslashes before a quote, then escape the quote. + out.extend(std::iter::repeat_n(backslash, backslashes + 1)); + } + backslashes = 0; + } + out.push(c); + } + if needs_quotes { + // Backslashes at the end would otherwise escape the closing quote. + out.extend(std::iter::repeat_n(backslash, backslashes)); + out.push(quote); + } + Ok(()) +} + +/// The command line: the quoted program, the quoted arguments, then any raw text. +fn build_command_line(command: &ChildCommand, program: &Path) -> io::Result> { + let mut line: Vec = Vec::new(); + let program = program.as_os_str(); + if program.encode_wide().any(|c| c == u16::from(b'"')) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "program path contains a quote", + )); + } + // The program is always quoted: its first token is parsed differently from the rest. + line.push(u16::from(b'"')); + line.extend(program.encode_wide()); + line.push(u16::from(b'"')); + for arg in &command.args { + line.push(u16::from(b' ')); + append_arg(&mut line, arg)?; + } + if let Some(raw) = &command.raw_args { + let raw: Vec = raw.encode_wide().collect(); + if !raw.is_empty() { + line.push(u16::from(b' ')); + line.extend(raw); + } + } + if line.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "the command line contains a NUL", + )); + } + line.push(0); + Ok(line) +} + +/// Windows compares environment names case-insensitively. +fn env_sort_key(name: &OsStr) -> Vec { + char::decode_utf16(name.encode_wide()) + .flat_map(|unit| { + unit.map_or_else( + |_| Vec::new(), + |character| character.to_uppercase().collect::>(), + ) + }) + .flat_map(|character| { + let mut buffer = [0u16; 2]; + character.encode_utf16(&mut buffer).to_vec() + }) + .collect() +} + +/// The new process's environment: the host's, with `overrides` applied, as the +/// sorted UTF-16 block `CreateProcessW` requires. +fn build_env_block(overrides: &[(OsString, OsString)]) -> io::Result> { + let mut merged: BTreeMap, (OsString, OsString)> = BTreeMap::new(); + for (name, value) in std::env::vars_os() { + merged.insert(env_sort_key(&name), (name, value)); + } + for (name, value) in overrides { + merged.insert(env_sort_key(name), (name.clone(), value.clone())); + } + let mut block: Vec = Vec::new(); + for (name, value) in merged.values() { + let entry_start = block.len(); + block.extend(name.encode_wide()); + block.push(u16::from(b'=')); + block.extend(value.encode_wide()); + if block[entry_start..].contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "an environment entry contains a NUL", + )); + } + block.push(0); + } + if block.is_empty() { + block.push(0); + } + block.push(0); + Ok(block) +} + +/// Find the executable for `program`: as given when it has a directory, otherwise +/// in the `PATH` directories and then the system directory. The current directory +/// is deliberately not searched. +fn resolve_program(program: &OsStr) -> io::Result { + let candidates = |base: PathBuf| -> Vec { + if base.extension().is_some() { + vec![base] + } else { + let mut with_exe = base.clone().into_os_string(); + with_exe.push(".exe"); + vec![base, PathBuf::from(with_exe)] + } + }; + let given = Path::new(program); + if given.is_absolute() || given.components().count() > 1 { + return candidates(given.to_path_buf()) + .into_iter() + .find(|candidate| candidate.is_file()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("program not found: {}", given.display()), + ) + }); + } + let mut directories: Vec = std::env::var_os("PATH") + .map(|path| std::env::split_paths(&path).collect()) + .unwrap_or_default(); + if let Some(root) = std::env::var_os("SystemRoot") { + directories.push(PathBuf::from(root).join("System32")); + } + for directory in directories { + for candidate in candidates(directory.join(given)) { + if candidate.is_file() { + return Ok(candidate); + } + } + } + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("program not found: {}", given.display()), + )) +} + +/// The child-side handle of one standard stream, plus the parent end if piped. +struct StdioPair { + child: OwnedHandle, + parent: Option, +} + +fn open_nul(write: bool) -> io::Result { + let sa = inheritable(); + let access = if write { + FILE_GENERIC_WRITE.0 + } else { + FILE_GENERIC_READ.0 + }; + let name = to_wide(OsStr::new("NUL"))?; + // SAFETY: `name` is NUL-terminated and outlives the call; `sa` outlives it too. + let handle = unsafe { + CreateFileW( + PCWSTR(name.as_ptr()), + access, + FILE_SHARE_READ | FILE_SHARE_WRITE, + Some(&raw const sa), + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + None, + ) + } + .map_err(io::Error::from)?; + Ok(OwnedHandle(handle)) +} + +fn inherit_std(which: STD_HANDLE, write: bool) -> io::Result { + // SAFETY: plain Win32 calls; the duplicate is owned by the returned value. + unsafe { + let source = match GetStdHandle(which) { + Ok(handle) if !handle.is_invalid() && !handle.0.is_null() => handle, + _ => return open_nul(write), + }; + let mut duplicate = HANDLE::default(); + let process = GetCurrentProcess(); + match DuplicateHandle( + process, + source, + process, + &raw mut duplicate, + 0, + true, + DUPLICATE_SAME_ACCESS, + ) { + Ok(()) => Ok(OwnedHandle(duplicate)), + // The host's stream is not a duplicable handle (for example no console). + Err(_) => open_nul(write), + } + } +} + +/// A pipe with an overlapped parent end (usable by tokio) and an inheritable child end. +/// `child_writes` is true for stdout/stderr, false for stdin. +fn make_pipe(child_writes: bool) -> io::Result { + let name = format!( + r"\\.\pipe\openshell-mxc-{}-{}-{:016x}", + std::process::id(), + PIPE_COUNTER.fetch_add(1, Ordering::Relaxed), + rand::random::() + ); + let wide = to_wide(OsStr::new(&name))?; + let direction = if child_writes { + PIPE_ACCESS_INBOUND + } else { + PIPE_ACCESS_OUTBOUND + }; + // SAFETY: `wide` is NUL-terminated and outlives the calls; the handles are + // owned by the values they are stored in and closed once. + unsafe { + let server = CreateNamedPipeW( + PCWSTR(wide.as_ptr()), + direction | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 64 * 1024, + 64 * 1024, + 0, + None, + ); + if server == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let server = OwnedHandle(server); + let sa = inheritable(); + let access = if child_writes { + FILE_GENERIC_WRITE.0 + } else { + FILE_GENERIC_READ.0 + }; + let client = CreateFileW( + PCWSTR(wide.as_ptr()), + access, + FILE_SHARE_MODE(0), + Some(&raw const sa), + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + None, + ) + .map_err(io::Error::from)?; + let client = OwnedHandle(client); + let raw = server.0.0; + std::mem::forget(server); + // tokio takes ownership of the overlapped server end; on failure it is ours to close. + let parent = match NamedPipeServer::from_raw_handle(raw) { + Ok(parent) => parent, + Err(error) => { + let _ = CloseHandle(HANDLE(raw)); + return Err(error); + } + }; + Ok(StdioPair { + child: client, + parent: Some(parent), + }) + } +} + +fn open_stdio( + stdio: ChildStdio, + standard: STD_HANDLE, + child_writes: bool, +) -> io::Result { + match stdio { + ChildStdio::Null => Ok(StdioPair { + child: open_nul(child_writes)?, + parent: None, + }), + ChildStdio::Inherit => Ok(StdioPair { + child: inherit_std(standard, child_writes)?, + parent: None, + }), + ChildStdio::Piped => make_pipe(child_writes), + } +} + +/// A `PROC_THREAD_ATTRIBUTE_LIST`, deleted on drop. +struct AttributeList { + buffer: Vec, +} + +impl AttributeList { + fn new(count: u32) -> io::Result { + let mut size = 0usize; + // SAFETY: the first call only reports the size needed; failure is expected. + let _ = unsafe { InitializeProcThreadAttributeList(None, count, None, &raw mut size) }; + let mut buffer = vec![0u8; size]; + let list = LPPROC_THREAD_ATTRIBUTE_LIST(buffer.as_mut_ptr().cast()); + // SAFETY: `buffer` is `size` bytes, as the first call requested. + unsafe { InitializeProcThreadAttributeList(Some(list), count, None, &raw mut size) } + .map_err(io::Error::from)?; + Ok(Self { buffer }) + } + + fn list(&mut self) -> LPPROC_THREAD_ATTRIBUTE_LIST { + LPPROC_THREAD_ATTRIBUTE_LIST(self.buffer.as_mut_ptr().cast()) + } + + /// `value` must outlive this list (it is read by `CreateProcessW`). + fn set(&mut self, attribute: u32, value: &[T]) -> io::Result<()> { + // SAFETY: `value` is valid for the call and the caller keeps it alive. + unsafe { + UpdateProcThreadAttribute( + self.list(), + 0, + attribute as usize, + Some(value.as_ptr().cast::()), + size_of_val(value), + None, + None, + ) + } + .map_err(io::Error::from) + } +} + +impl Drop for AttributeList { + fn drop(&mut self) { + // SAFETY: the list was initialised in `new` and is deleted once. + unsafe { DeleteProcThreadAttributeList(self.list()) }; + } +} + +/// Create the process for `command` as a member of `job`. +/// +/// Must be called inside a tokio runtime (the pipe ends are registered with it). +pub fn spawn_in_job(job: HANDLE, command: &ChildCommand) -> io::Result { + create(job, command, false).map(|(child, _thread)| child) +} + +/// The initial thread of a process created suspended, to resume it. +#[cfg(test)] +pub struct SuspendedThread(OwnedHandle); + +#[cfg(test)] +impl SuspendedThread { + pub(crate) fn resume(self) -> io::Result<()> { + // SAFETY: the thread handle is valid and owned by `self`. + if unsafe { windows::Win32::System::Threading::ResumeThread(self.0.0) } == u32::MAX { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + +/// Like [`spawn_in_job`], but the process is created without starting it, so a test can +/// inspect it before it runs. +#[cfg(test)] +pub fn spawn_suspended_in_job( + job: HANDLE, + command: &ChildCommand, +) -> io::Result<(GenerationChild, SuspendedThread)> { + create(job, command, true).map(|(child, thread)| (child, SuspendedThread(thread))) +} + +fn create( + job: HANDLE, + command: &ChildCommand, + suspended: bool, +) -> io::Result<(GenerationChild, OwnedHandle)> { + let program = resolve_program(&command.program)?; + let mut command_line = build_command_line(command, &program)?; + let application = to_wide(program.as_os_str())?; + let environment = build_env_block(&command.envs)?; + + let StdioPair { + child: stdin_child, + parent: stdin_parent, + } = open_stdio(command.stdin, STD_INPUT_HANDLE, false)?; + let StdioPair { + child: stdout_child, + parent: stdout_parent, + } = open_stdio(command.stdout, STD_OUTPUT_HANDLE, true)?; + let StdioPair { + child: stderr_child, + parent: stderr_parent, + } = open_stdio(command.stderr, STD_ERROR_HANDLE, true)?; + + // The inherited-handle list may not contain duplicates. + let mut inherited: Vec = vec![stdin_child.0]; + for handle in [stdout_child.0, stderr_child.0] { + if !inherited.contains(&handle) { + inherited.push(handle); + } + } + let jobs = [job]; + let mut attributes = AttributeList::new(2)?; + attributes.set(PROC_THREAD_ATTRIBUTE_JOB_LIST, &jobs)?; + attributes.set(PROC_THREAD_ATTRIBUTE_HANDLE_LIST, &inherited)?; + + let mut startup = STARTUPINFOEXW::default(); + startup.StartupInfo.cb = u32::try_from(size_of::()).unwrap_or(0); + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.StartupInfo.hStdInput = stdin_child.0; + startup.StartupInfo.hStdOutput = stdout_child.0; + startup.StartupInfo.hStdError = stderr_child.0; + startup.lpAttributeList = attributes.list(); + + let mut flags: PROCESS_CREATION_FLAGS = + EXTENDED_STARTUPINFO_PRESENT | CREATE_UNICODE_ENVIRONMENT; + if suspended { + flags |= CREATE_SUSPENDED; + } + let mut info = PROCESS_INFORMATION::default(); + // SAFETY: every pointer refers to a value that outlives the call; the attribute + // list, handle arrays and strings are kept alive until after it returns. + unsafe { + CreateProcessW( + PCWSTR(application.as_ptr()), + Some(PWSTR(command_line.as_mut_ptr())), + None, + None, + true, + flags, + Some(environment.as_ptr().cast::()), + PCWSTR::null(), + &raw const startup.StartupInfo, + &raw mut info, + ) + } + .map_err(|error| { + io::Error::other(format!( + "create process {} in its sandbox job: {error}", + program.display() + )) + })?; + drop(attributes); + + let process = Arc::new(OwnedHandle(info.hProcess)); + let thread = OwnedHandle(info.hThread); + // The new process has its own copies of the child ends; close ours now so a reader + // sees end-of-file when the process ends. + drop((stdin_child, stdout_child, stderr_child)); + + let (sender, receiver) = watch::channel(None); + let watched = Arc::clone(&process); + let spawned = std::thread::Builder::new() + .name(format!("mxc-wait-{}", info.dwProcessId)) + .spawn(move || { + // SAFETY: `watched` keeps the process handle valid for the whole wait. + let mut code = 0u32; + unsafe { + if WaitForSingleObject(watched.0, INFINITE) != WAIT_FAILED { + let _ = GetExitCodeProcess(watched.0, &raw mut code); + } + } + let _ = sender.send(Some(code)); + }); + if let Err(error) = spawned { + // SAFETY: the process handle is valid; end the process nobody can watch. + let _ = unsafe { TerminateProcess(process.0, 1) }; + return Err(error); + } + + let child = GenerationChild { + process, + pid: info.dwProcessId, + exit: receiver, + _stdin: stdin_parent, + stdout: stdout_parent, + stderr: stderr_parent, + }; + Ok((child, thread)) +} + +#[cfg(test)] +#[allow(unsafe_code)] +mod tests { + use std::time::Duration; + + use tokio::io::AsyncReadExt as _; + use windows::Win32::Foundation::{HLOCAL, LocalFree, WAIT_TIMEOUT}; + use windows::Win32::Storage::FileSystem::CREATE_NEW; + use windows::Win32::System::Threading::{OpenProcess, PROCESS_SYNCHRONIZE}; + use windows::Win32::UI::Shell::CommandLineToArgvW; + + use super::*; + use crate::job::GenerationJob; + + fn quiet(program: &str, args: &[&str]) -> ChildCommand { + let mut command = ChildCommand::new(program); + command + .args(args) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + command + } + + fn is_alive(pid: u32) -> bool { + // SAFETY: plain Win32 calls; the handle is closed before returning. + unsafe { + let Ok(handle) = OpenProcess(PROCESS_SYNCHRONIZE, false, pid) else { + return false; + }; + let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT; + let _ = CloseHandle(handle); + alive + } + } + + /// Split a command line the way Windows does for every ordinary program. + fn windows_argv(line: &[u16]) -> Vec { + let mut count = 0i32; + // SAFETY: `line` is NUL-terminated; the returned array is freed once. + unsafe { + let argv = CommandLineToArgvW(PCWSTR(line.as_ptr()), &raw mut count); + assert!(!argv.is_null(), "CommandLineToArgvW failed"); + let parsed = (0..usize::try_from(count).unwrap()) + .map(|i| { + let pointer = (*argv.add(i)).0; + let mut len = 0; + while *pointer.add(len) != 0 { + len += 1; + } + OsString::from_wide(std::slice::from_raw_parts(pointer, len)) + }) + .collect(); + let _ = LocalFree(Some(HLOCAL(argv.cast()))); + parsed + } + } + + use std::os::windows::ffi::OsStringExt as _; + + #[test] + fn command_line_round_trips_through_the_windows_parser() { + let tricky = [ + "", + "plain", + "with space", + "tab\there", + r#"a"b"#, + r"a\b", + r"ends with backslash\", + r"ends with backslashes\\", + r"\\server\share\file", + r#"quote then backslash \" end"#, + r"\", + r#"say "hello" now"#, + r"C:\Program Files\Agent\agent.exe", + "unicode é 日本語", + ]; + let program = PathBuf::from(r"C:\Program Files\Host\tool.exe"); + let mut command = ChildCommand::new(&program); + for arg in tricky { + command.arg(arg); + } + let line = build_command_line(&command, &program).expect("command line"); + let parsed = windows_argv(&line); + let expected: Vec = std::iter::once(program.into_os_string()) + .chain(tricky.iter().map(OsString::from)) + .collect(); + assert_eq!(parsed, expected); + } + + #[test] + fn raw_arguments_are_appended_exactly() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.arg("one").raw_arg(r#"two "three four" /s"#); + let line = build_command_line(&command, &program).expect("command line"); + assert_eq!( + String::from_utf16_lossy(&line[..line.len() - 1]), + r#""C:\x\tool.exe" one two "three four" /s"# + ); + } + + #[test] + fn nul_in_an_argument_is_rejected() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.arg(OsString::from_wide(&[u16::from(b'a'), 0, u16::from(b'b')])); + assert!(build_command_line(&command, &program).is_err()); + } + + #[test] + fn environment_block_is_sorted_and_applies_overrides() { + let overrides = vec![ + (OsString::from("Openshell_B"), OsString::from("2")), + (OsString::from("OPENSHELL_a"), OsString::from("1")), + // Same name as the first, different case: the later value replaces it. + (OsString::from("openshell_b"), OsString::from("3")), + ]; + let block = build_env_block(&overrides).expect("block"); + assert!(block.ends_with(&[0, 0]), "the block ends with a double NUL"); + let entries: Vec = block + .split(|&unit| unit == 0) + .filter(|entry| !entry.is_empty()) + .map(String::from_utf16_lossy) + .collect(); + let ours: Vec<&String> = entries + .iter() + .filter(|entry| entry.to_ascii_lowercase().starts_with("openshell_")) + .collect(); + assert_eq!(ours, vec!["OPENSHELL_a=1", "openshell_b=3"]); + let keys: Vec = entries + .iter() + .map(|entry| entry.split('=').next().unwrap_or("").to_uppercase()) + .collect(); + let mut sorted = keys.clone(); + sorted.sort(); + assert_eq!( + keys, sorted, + "entries must be sorted by name, case-insensitively" + ); + } + + #[tokio::test] + async fn exit_status_is_reported() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/c", "exit 7"])) + .expect("spawn"); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("exited") + .unwrap(); + assert_eq!(status.code(), Some(7)); + // Waiting again returns the same answer. + assert_eq!(child.wait().await.unwrap().code(), Some(7)); + } + + #[tokio::test] + async fn piped_output_is_readable_until_the_child_exits() { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new("cmd.exe"); + command + .args(["/c", "echo out& echo err 1>&2"]) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped); + let mut child = job.spawn(&command).expect("spawn"); + let mut stdout = child.stdout.take().expect("stdout"); + let mut stderr = child.stderr.take().expect("stderr"); + let (mut out, mut err) = (String::new(), String::new()); + tokio::time::timeout(Duration::from_secs(10), async { + let _ = stdout.read_to_string(&mut out).await; + let _ = stderr.read_to_string(&mut err).await; + }) + .await + .expect("both streams reach end of file when the child exits"); + assert_eq!(out.trim(), "out"); + assert_eq!(err.trim(), "err"); + assert!(child.wait().await.unwrap().success()); + } + + #[tokio::test] + async fn environment_overrides_reach_the_child() { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new("cmd.exe"); + command + .args(["/c", "echo %OPENSHELL_CHILD_TEST%"]) + .env("OPENSHELL_CHILD_TEST", "value with spaces") + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Null); + let mut child = job.spawn(&command).expect("spawn"); + let mut out = String::new(); + let _ = child.stdout.take().unwrap().read_to_string(&mut out).await; + assert_eq!(out.trim(), "value with spaces"); + } + + #[tokio::test] + async fn a_missing_program_is_reported() { + let job = GenerationJob::new().expect("job"); + let error = job + .spawn(&quiet("definitely-not-a-real-binary", &[])) + .err() + .expect("must fail"); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn kill_ends_the_process_and_wait_reports_it() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .expect("spawn"); + tokio::time::timeout(Duration::from_secs(10), child.kill()) + .await + .expect("killed") + .unwrap(); + assert!(!is_alive(child.id().unwrap())); + // Killing an already-dead process is not an error. + child.start_kill().unwrap(); + } + + #[tokio::test] + async fn dropping_a_child_kills_it() { + let job = GenerationJob::new().expect("job"); + let child = job + .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .expect("spawn"); + let pid = child.id().unwrap(); + assert!(is_alive(pid)); + drop(child); + for _ in 0..100 { + if !is_alive(pid) { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the dropped child is still running"); + } + + /// An inheritable handle with exclusive access: while any other process holds a copy + /// the file cannot be deleted. + fn open_exclusive_inheritable(path: &Path) -> HANDLE { + let sa = inheritable(); + let name = to_wide(path.as_os_str()).unwrap(); + // SAFETY: `name` and `sa` outlive the call. + unsafe { + CreateFileW( + PCWSTR(name.as_ptr()), + FILE_GENERIC_WRITE.0, + FILE_SHARE_MODE(0), + Some(&raw const sa), + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + None, + ) + } + .expect("create inheritable file") + } + + /// Only the standard handles may reach a sandbox process. A handle the host holds + /// (here an inheritable file handle) must not be inherited, as the host's listening + /// socket once was. The control shows the test can detect an inherited handle. + #[tokio::test] + async fn only_the_standard_handles_are_inherited() { + let dir = std::env::temp_dir(); + let pid = std::process::id(); + + // Control: the ordinary way of starting a process inherits the handle. + let control = dir.join(format!("openshell-inherit-control-{pid}.tmp")); + let handle = open_exclusive_inheritable(&control); + let mut ordinary = tokio::process::Command::new("cmd.exe") + .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + .kill_on_drop(true) + .spawn() + .expect("ordinary child"); + // SAFETY: the handle is valid and closed once. + unsafe { CloseHandle(handle) }.unwrap(); + assert!( + std::fs::remove_file(&control).is_err(), + "control: the ordinary child should still hold the file open" + ); + let _ = ordinary.kill().await; + let _ = ordinary.wait().await; + std::fs::remove_file(&control).expect("control file deletable once the child is gone"); + + // Ours: the same handle is not inherited. + let ours = dir.join(format!("openshell-inherit-ours-{pid}.tmp")); + let job = GenerationJob::new().expect("job"); + let handle = open_exclusive_inheritable(&ours); + let child = job + .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .expect("spawn"); + // SAFETY: the handle is valid and closed once. + unsafe { CloseHandle(handle) }.unwrap(); + let deleted = std::fs::remove_file(&ours); + drop(child); + deleted.expect("a sandbox process must not hold the host's other handles"); + } +} diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index cf5e39e730..1a6dc6f139 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -11,6 +11,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use crate::child::{ChildCommand, ChildStdio, GenerationChild}; use base64::Engine as _; use futures::Stream; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; @@ -27,7 +28,6 @@ use openshell_sandbox_backend::boundary_protocol::{ }; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncBufReadExt as _, BufReader}; -use tokio::process::{Child, Command}; use tokio::sync::{Mutex, broadcast, mpsc, oneshot, watch}; use tokio_stream::wrappers::ReceiverStream; use tracing::{info, warn}; @@ -871,9 +871,9 @@ fn spawn_supervisor( auth_bundle_path: &Path, main: &openshell_core::sandbox_env::MainProcessConfig, job: &crate::job::GenerationJob, -) -> Result { - let mut command = supervisor_command(context, descriptor_path, auth_bundle_path, main)?; - job.spawn(&mut command) +) -> Result { + let command = supervisor_command(context, descriptor_path, auth_bundle_path, main)?; + job.spawn(&command) .map_err(|error| format!("start host openshell-supervisor: {error}")) } @@ -882,14 +882,14 @@ fn supervisor_command( descriptor_path: &Path, auth_bundle_path: &Path, main: &openshell_core::sandbox_env::MainProcessConfig, -) -> Result { +) -> Result { let main_process_spec = mxc_main_process_spec(main)?; - let mut command = Command::new(&context.config.supervisor_binary_path); + // A `GenerationChild` is killed when dropped, as `kill_on_drop(true)` did. + let mut command = ChildCommand::new(&context.config.supervisor_binary_path); command - .kill_on_drop(true) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped) .arg("--role") .arg("isolation-backend") .arg("--backend-descriptor-file") @@ -963,8 +963,8 @@ enum RuntimePairExit { } async fn monitor_runtime_pair( - mut boundary: Child, - mut supervisor: Child, + mut boundary: GenerationChild, + mut supervisor: GenerationChild, mut shutdown: oneshot::Receiver<()>, ) -> RuntimePairExit { tokio::select! { @@ -1543,7 +1543,7 @@ async fn wait_for_termination( } } -fn attach_child_logs(sandbox_name: &str, component: &'static str, child: &mut Child) { +fn attach_child_logs(sandbox_name: &str, component: &'static str, child: &mut GenerationChild) { if let Some(stdout) = child.stdout.take() { let sandbox_name = sandbox_name.to_string(); tokio::spawn(async move { @@ -1913,10 +1913,9 @@ mod tests { ) .unwrap(); let tls_dir = command - .as_std() .get_envs() .find(|(key, _)| *key == openshell_core::sandbox_env::PROXY_TLS_DIR) - .and_then(|(_, value)| value); + .map(|(_, value)| value); assert_eq!( tls_dir.map(Path::new), Some(host_state_dir.join(HOST_PROXY_TLS_DIR).as_path()) @@ -2841,17 +2840,20 @@ mod tests { #[tokio::test] async fn stop_terminates_and_reaps_both_runtime_processes() { + let job = crate::job::GenerationJob::new().expect("job"); let spawn = || { - Command::new("powershell.exe") + let mut command = ChildCommand::new("powershell.exe"); + command .args([ "-NoProfile", "-NonInteractive", "-Command", "Start-Sleep -Seconds 60", ]) - .kill_on_drop(true) - .spawn() - .expect("real test process") + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + job.spawn(&command).expect("real test process") }; let boundary = spawn(); let supervisor = spawn(); diff --git a/crates/openshell-driver-mxc/src/etw_consumer.rs b/crates/openshell-driver-mxc/src/etw_consumer.rs index d69d1e4630..de9952a326 100644 --- a/crates/openshell-driver-mxc/src/etw_consumer.rs +++ b/crates/openshell-driver-mxc/src/etw_consumer.rs @@ -1208,7 +1208,9 @@ const RETIRED_CORRELATION_TTL: Duration = PENDING_TTL; /// Query the kernel generation key for a spawned child while the driver still /// owns its process handle. This is the same value ETW appends under /// `EVENT_HEADER_EXT_TYPE_PROCESS_START_KEY`. -pub(crate) fn child_process_start_key(child: &tokio::process::Child) -> Result { +pub(crate) fn child_process_start_key( + child: &impl crate::child::ProcessHandle, +) -> Result { let raw_handle = child .raw_handle() .ok_or_else(|| "wxc-exec child handle is no longer available".to_string())?; diff --git a/crates/openshell-driver-mxc/src/job.rs b/crates/openshell-driver-mxc/src/job.rs index dee1aa3c2d..5e1df13ed3 100644 --- a/crates/openshell-driver-mxc/src/job.rs +++ b/crates/openshell-driver-mxc/src/job.rs @@ -12,26 +12,27 @@ //! driver's host process dies for any reason. Children inherit job membership //! and `BREAKAWAY_OK` is not set, so nothing can leave the job. //! +//! The root processes are created as members of the job +//! (`crate::child::spawn_in_job`), so there is no moment at which a process of the +//! generation exists outside its job, even if the driver is killed during the spawn. +//! //! The job lives with the driver rather than with whichever process hosts it, //! so the cleanup survives moving the driver out of the gateway. +// `pub(crate)` documents what the driver may use; this module is private. +#![allow(clippy::redundant_pub_crate)] + use std::io; use std::mem::size_of; -use tokio::process::{Child, Command}; use windows::Win32::Foundation::{CloseHandle, HANDLE}; -use windows::Win32::System::Diagnostics::ToolHelp::{ - CreateToolhelp32Snapshot, TH32CS_SNAPTHREAD, THREADENTRY32, Thread32First, Thread32Next, -}; use windows::Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, - JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, - SetInformationJobObject, TerminateJobObject, + CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JobObjectExtendedLimitInformation, SetInformationJobObject, TerminateJobObject, }; -use windows::Win32::System::Threading::{OpenThread, ResumeThread, THREAD_SUSPEND_RESUME}; use windows::core::PCWSTR; -const CREATE_SUSPENDED: u32 = 0x0000_0004; +use crate::child::{ChildCommand, GenerationChild}; /// Owns a kill-on-close job object. Dropping it ends every member process. pub struct GenerationJob { @@ -47,7 +48,7 @@ unsafe impl Sync for GenerationJob {} #[allow(unsafe_code)] // Win32 job object calls. impl GenerationJob { - pub fn new() -> io::Result { + pub(crate) fn new() -> io::Result { // SAFETY: plain Win32 calls with valid arguments; `info` outlives the // call and the size matches its type. No security attributes are // passed, so the handle is not inheritable and children cannot keep @@ -69,46 +70,29 @@ impl GenerationJob { } } - /// Spawn `command` as a member of this job, with no window in which it can - /// run (or start children) outside it. + /// Start `command` as a member of this job. /// - /// The process is created suspended, assigned to the job, then resumed. - /// If it cannot be assigned it is killed and an error is returned: a - /// sandbox whose processes could outlive the driver must not start. - pub fn spawn(&self, command: &mut Command) -> io::Result { - command.creation_flags(CREATE_SUSPENDED); - let mut child = command.spawn()?; - match self.adopt_and_resume(&child) { - Ok(()) => Ok(child), - Err(error) => { - // Still suspended, so it has run no code. `kill_on_drop` - // retries if this kill fails. - let _ = child.start_kill(); - Err(error) - } - } - } - - fn adopt_and_resume(&self, child: &Child) -> io::Result<()> { - let pid = child - .id() - .ok_or_else(|| io::Error::other("child exited before it could be assigned to a job"))?; - let raw = child - .raw_handle() - .ok_or_else(|| io::Error::other("child handle unavailable"))?; - // SAFETY: `raw` is the live process handle owned by `child`. - unsafe { AssignProcessToJobObject(self.handle, HANDLE(raw)) }.map_err(|error| { - io::Error::other(format!("assign process {pid} to its sandbox job: {error}")) - })?; - resume_process(pid) + /// The process is created inside the job (the job is named in the + /// `CreateProcessW` call), so it is never outside it. If it cannot be created + /// there, the spawn fails and no process exists: a sandbox whose processes could + /// outlive the driver must not start. + /// + /// Call inside a tokio runtime. + pub(crate) fn spawn(&self, command: &ChildCommand) -> io::Result { + crate::child::spawn_in_job(self.handle, command) } /// End every process in the job now. - pub fn terminate(&self) { + pub(crate) fn terminate(&self) { // SAFETY: valid job handle. An already-empty job is not an error worth // reporting. let _ = unsafe { TerminateJobObject(self.handle, 1) }; } + + #[cfg(test)] + pub(crate) fn raw(&self) -> HANDLE { + self.handle + } } impl Drop for GenerationJob { @@ -120,73 +104,39 @@ impl Drop for GenerationJob { } } -/// Resume every thread of a process created with `CREATE_SUSPENDED`. -#[allow(unsafe_code)] // Toolhelp and thread calls. -fn resume_process(pid: u32) -> io::Result<()> { - // SAFETY: the snapshot handle is closed on every path; `entry` outlives the - // calls and `dwSize` is set as the API requires. - unsafe { - let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0).map_err(io::Error::from)?; - let mut resumed = 0_u32; - let mut result = Ok(()); - let mut entry = THREADENTRY32 { - dwSize: u32::try_from(size_of::()).map_err(io::Error::other)?, - ..Default::default() - }; - let mut more = Thread32First(snapshot, &raw mut entry).is_ok(); - while more { - if entry.th32OwnerProcessID == pid { - match OpenThread(THREAD_SUSPEND_RESUME, false, entry.th32ThreadID) { - Ok(thread) => { - if ResumeThread(thread) == u32::MAX { - result = Err(io::Error::last_os_error()); - } else { - resumed += 1; - } - let _ = CloseHandle(thread); - } - Err(error) => result = Err(io::Error::from(error)), - } - } - more = Thread32Next(snapshot, &raw mut entry).is_ok(); - } - let _ = CloseHandle(snapshot); - result?; - if resumed == 0 { - return Err(io::Error::other(format!( - "no thread of process {pid} could be resumed" - ))); - } - Ok(()) - } -} - #[cfg(test)] +#[allow(unsafe_code)] mod tests { - use std::process::Stdio; use std::time::Duration; + use windows::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, + TH32CS_SNAPPROCESS, + }; + use windows::Win32::System::JobObjects::IsProcessInJob; + use super::*; + use crate::child::{ChildStdio, spawn_suspended_in_job}; - fn sleeper() -> Command { - let mut command = Command::new("cmd.exe"); + fn sleeper() -> ChildCommand { + let mut command = ChildCommand::new("cmd.exe"); command .args(["/c", "ping -n 30 127.0.0.1 >nul"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .kill_on_drop(true); + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); command } #[tokio::test] async fn spawned_process_runs_and_is_killed_by_terminate() { let job = GenerationJob::new().expect("job"); - let mut child = job.spawn(&mut sleeper()).expect("spawn in job"); - // Resumed: it must still be running (not stuck suspended, not exited). + let mut child = job.spawn(&sleeper()).expect("spawn in job"); tokio::time::sleep(Duration::from_millis(300)).await; assert!( - child.try_wait().unwrap().is_none(), + tokio::time::timeout(Duration::from_millis(1), child.wait()) + .await + .is_err(), "child should be running" ); job.terminate(); @@ -201,40 +151,77 @@ mod tests { async fn dropping_the_job_kills_the_whole_tree() { let job = GenerationJob::new().expect("job"); // cmd.exe starts ping.exe: a grandchild that inherits membership. - let mut child = job.spawn(&mut sleeper()).expect("spawn in job"); + let mut child = job.spawn(&sleeper()).expect("spawn in job"); tokio::time::sleep(Duration::from_millis(300)).await; drop(job); - let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + // Closing the job ends members with exit code 0, so only that the + // process ended (not how) can be asserted. + tokio::time::timeout(Duration::from_secs(10), child.wait()) .await .expect("child ended after the job closed") .unwrap(); - // Closing the job ends members with exit code 0, so only that the - // process ended (not how) can be asserted. - let _ = status; } - /// Fail closed: if the process cannot be assigned to the job, `spawn` - /// returns an error (and kills the still-suspended child) instead of - /// handing back a process that could outlive the driver. + /// Fail closed: if the process cannot be created inside its job, the spawn fails + /// instead of handing back a process that could outlive the driver. #[tokio::test] - async fn assignment_failure_fails_the_spawn() { - // An invalid job handle makes `AssignProcessToJobObject` fail. + async fn an_unusable_job_fails_the_spawn() { let job = GenerationJob { handle: HANDLE::default(), }; - let error = job.spawn(&mut sleeper()).expect_err("must fail closed"); + let error = job.spawn(&sleeper()).err().expect("must fail closed"); assert!( - error.to_string().contains("assign process"), + error.to_string().contains("sandbox job"), "unexpected error: {error}" ); std::mem::forget(job); // do not CloseHandle the invalid handle } - /// Set (by the test below) on the re-run of this test binary that plays the + /// The property the job exists for, and the case a review found: the process is a + /// member of its job from the moment it exists, before it has run one instruction. + /// A process created the ordinary way is not, so the check can fail. + #[tokio::test] + async fn child_is_a_member_of_its_job_before_it_runs() { + let job = GenerationJob::new().expect("job"); + let (child, thread) = + spawn_suspended_in_job(job.raw(), &sleeper()).expect("suspended spawn"); + let handle = HANDLE(child.raw_handle().expect("process handle")); + let mut in_job = windows::core::BOOL::default(); + // SAFETY: both handles are valid for the call. + unsafe { IsProcessInJob(handle, Some(job.raw()), &raw mut in_job) } + .expect("IsProcessInJob"); + assert!( + in_job.as_bool(), + "a freshly created, not yet running child must already be in the job" + ); + thread.resume().expect("resume"); + drop(child); + + // Negative control: a process created the ordinary way is not in the job. + let mut outsider = tokio::process::Command::new("cmd.exe") + .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + .kill_on_drop(true) + .spawn() + .expect("outsider"); + let outsider_handle = HANDLE(outsider.raw_handle().expect("outsider handle")); + let mut outsider_in_job = windows::core::BOOL::default(); + // SAFETY: both handles are valid for the call. + unsafe { IsProcessInJob(outsider_handle, Some(job.raw()), &raw mut outsider_in_job) } + .expect("IsProcessInJob"); + assert!( + !outsider_in_job.as_bool(), + "the check must be able to say no" + ); + let _ = outsider.kill().await; + } + + // ---- Hard-killing the process that owns the job ---- + + /// Set (by the tests below) on the re-run of this test binary that plays the /// role of the driver's host process. const HOST_ENV: &str = "OPENSHELL_JOB_TEST_HOST"; + const STRESS_ENV: &str = "OPENSHELL_JOB_TEST_STRESS_EXE"; - #[allow(unsafe_code)] fn is_alive(pid: u32) -> bool { use windows::Win32::Foundation::WAIT_TIMEOUT; use windows::Win32::System::Threading::{ @@ -251,31 +238,55 @@ mod tests { } } - #[allow(unsafe_code)] fn child_pids(parent: u32) -> Vec { - use windows::Win32::System::Diagnostics::ToolHelp::{ - PROCESSENTRY32, Process32First, Process32Next, TH32CS_SNAPPROCESS, - }; - // SAFETY: the snapshot is closed before returning; `entry.dwSize` is set. + // SAFETY: the snapshot is closed before returning; `dwSize` is set. unsafe { let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0).unwrap(); - let mut entry = PROCESSENTRY32 { - dwSize: u32::try_from(size_of::()).unwrap(), + let mut entry = PROCESSENTRY32W { + dwSize: u32::try_from(size_of::()).unwrap(), ..Default::default() }; let mut found = Vec::new(); - let mut more = Process32First(snapshot, &raw mut entry).is_ok(); + let mut more = Process32FirstW(snapshot, &raw mut entry).is_ok(); while more { if entry.th32ParentProcessID == parent { found.push(entry.th32ProcessID); } - more = Process32Next(snapshot, &raw mut entry).is_ok(); + more = Process32NextW(snapshot, &raw mut entry).is_ok(); } let _ = CloseHandle(snapshot); found } } + /// How many running processes have this executable file name. + fn count_named(exe_name: &str) -> usize { + // SAFETY: the snapshot is closed before returning; `dwSize` is set. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0).unwrap(); + let mut entry = PROCESSENTRY32W { + dwSize: u32::try_from(size_of::()).unwrap(), + ..Default::default() + }; + let mut count = 0; + let mut more = Process32FirstW(snapshot, &raw mut entry).is_ok(); + while more { + let len = entry + .szExeFile + .iter() + .position(|&c| c == 0) + .unwrap_or(entry.szExeFile.len()); + if String::from_utf16_lossy(&entry.szExeFile[..len]).eq_ignore_ascii_case(exe_name) + { + count += 1; + } + more = Process32NextW(snapshot, &raw mut entry).is_ok(); + } + let _ = CloseHandle(snapshot); + count + } + } + /// Plays the driver host: owns a job, starts a process tree in it, reports /// the PIDs, then waits to be killed. A no-op in a normal test run. #[tokio::test] @@ -284,7 +295,7 @@ mod tests { return; } let job = GenerationJob::new().expect("job"); - let child = job.spawn(&mut sleeper()).expect("spawn in job"); + let child = job.spawn(&sleeper()).expect("spawn in job"); let root = child.id().expect("pid"); tokio::time::sleep(Duration::from_millis(700)).await; // let cmd start ping let grandchildren = child_pids(root) @@ -303,11 +314,16 @@ mod tests { #[tokio::test] async fn hard_killing_the_job_owner_ends_the_whole_tree() { use std::io::{BufRead as _, BufReader}; + use std::process::Stdio; if std::env::var_os(HOST_ENV).is_some() { return; // inside the helper run } let mut host = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "job::tests::hard_kill_host_helper", "--nocapture"]) + .args([ + "--exact", + "job::tests::hard_kill_host_helper", + "--nocapture", + ]) .env(HOST_ENV, "1") .stdout(Stdio::piped()) .spawn() @@ -341,10 +357,80 @@ mod tests { panic!("processes survived the hard kill of their job owner: {tree:?}"); } + /// A host that starts generations as fast as it can, keeping a few alive, until killed. + /// A no-op in a normal test run. #[tokio::test] - async fn a_failed_spawn_is_reported_not_swallowed() { - let job = GenerationJob::new().expect("job"); - let mut command = Command::new("definitely-not-a-real-binary.exe"); - assert!(job.spawn(&mut command).is_err()); + async fn spawn_loop_host_helper() { + let Some(exe) = std::env::var_os(STRESS_ENV) else { + return; + }; + let mut live: std::collections::VecDeque<(GenerationJob, GenerationChild)> = + std::collections::VecDeque::new(); + println!("READY"); + loop { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new(&exe); + command + .args(["/c", "ping -n 60 127.0.0.1 >nul"]) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + if let Ok(child) = job.spawn(&command) { + live.push_back((job, child)); + } + if live.len() > 6 { + live.pop_front(); + } + } + } + + /// Hard-kill a host at many different moments while it is starting generations; no + /// process of any generation may survive, including one killed mid-spawn. + #[tokio::test] + async fn hard_killing_a_host_mid_spawn_leaves_no_process() { + use std::io::{BufRead as _, BufReader}; + use std::process::Stdio; + if std::env::var_os(HOST_ENV).is_some() || std::env::var_os(STRESS_ENV).is_some() { + return; // inside a helper run + } + // A copy of cmd.exe under a name nothing else has, so survivors are countable. + let name = format!("mxc-job-stress-{}.exe", std::process::id()); + let exe = std::env::temp_dir().join(&name); + let system = std::path::PathBuf::from(std::env::var_os("SystemRoot").unwrap()) + .join("System32/cmd.exe"); + std::fs::copy(system, &exe).expect("copy cmd.exe"); + + for trial in 0..12u64 { + let mut host = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "job::tests::spawn_loop_host_helper", + "--nocapture", + ]) + .env(STRESS_ENV, &exe) + .stdout(Stdio::piped()) + .spawn() + .expect("start stress host"); + let mut lines = BufReader::new(host.stdout.take().unwrap()).lines(); + while lines.next().expect("host exited early").unwrap() != "READY" {} + // Different, unrelated delays so the kill lands at different points. + tokio::time::sleep(Duration::from_millis(40 + (trial * 37) % 260)).await; + host.kill().expect("hard-kill the host"); + host.wait().unwrap(); + + let mut left = usize::MAX; + for _ in 0..100 { + left = count_named(&name); + if left == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert_eq!( + left, 0, + "trial {trial}: {left} process(es) survived the hard kill of the host" + ); + } + let _ = std::fs::remove_file(&exe); } } diff --git a/crates/openshell-driver-mxc/src/lib.rs b/crates/openshell-driver-mxc/src/lib.rs index fda0ea1af2..54f261f24d 100644 --- a/crates/openshell-driver-mxc/src/lib.rs +++ b/crates/openshell-driver-mxc/src/lib.rs @@ -17,6 +17,8 @@ #![allow(clippy::result_large_err)] +#[cfg(target_os = "windows")] +mod child; #[cfg(target_os = "windows")] mod driver; #[cfg(target_os = "windows")] diff --git a/crates/openshell-driver-mxc/src/mxc.rs b/crates/openshell-driver-mxc/src/mxc.rs index 1a6c025bab..cdc9ff90fe 100644 --- a/crates/openshell-driver-mxc/src/mxc.rs +++ b/crates/openshell-driver-mxc/src/mxc.rs @@ -3,12 +3,12 @@ //! `wxc-exec` `ProcessContainer` launcher and request types. +use crate::child::{ChildCommand, ChildStdio, GenerationChild}; use base64::Engine as _; use serde::Serialize; use std::net::SocketAddr; use std::path::PathBuf; use thiserror::Error; -use tokio::process::Command; use tracing::{debug, info}; /// Stable MXC schema version shared by the mapper and live launcher. @@ -265,15 +265,15 @@ impl WxcExecInvoker { self.mock } - fn real_command(&self) -> Command { - let mut command = Command::new(&self.exec_path); + /// Both processes in the runtime pair must die if their monitor is cancelled, not + /// only when it takes the explicit shutdown branch; a `GenerationChild` is killed + /// when dropped. + fn real_command(&self) -> ChildCommand { + let mut command = ChildCommand::new(&self.exec_path); command - // Both processes in the runtime pair must die if their monitor is - // cancelled, not only when it takes the explicit shutdown branch. - .kill_on_drop(true) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); + .stdin(ChildStdio::Piped) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped); command } @@ -282,7 +282,8 @@ impl WxcExecInvoker { fn mock_spawn_with_grants( process: &MxcProcess, grants: &[String], - ) -> Result { + job: &crate::job::GenerationJob, + ) -> Result { let cmd_norm = mock_normalize(&process.command_line); // The RFC 0012 boundary is launched with its granted bootstrap directory. // This check concerns bootstrap wiring, never workload authorization. @@ -298,11 +299,10 @@ impl WxcExecInvoker { |quoted| quoted.split_once('"'), ) .unwrap_or((&process.command_line, "")); - let mut cmd = Command::new(if in_policy { program } else { "cmd" }); - cmd.stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .kill_on_drop(true); + let mut cmd = ChildCommand::new(if in_policy { program } else { "cmd" }); + cmd.stdin(ChildStdio::Null) + .stdout(ChildStdio::Inherit) + .stderr(ChildStdio::Inherit); if in_policy { debug!(command = %process.command_line, "mock exec: in-policy, running agent"); // Retain the already-encoded Windows argv without another quoting pass. @@ -313,7 +313,7 @@ impl WxcExecInvoker { "echo Access is denied. (out-of-policy write blocked by AppContainer) 1>&2& exit 1", ); } - let child = cmd.spawn()?; + let child = job.spawn(&cmd)?; Ok(child) } @@ -335,7 +335,7 @@ impl WxcExecInvoker { network: Option, ui: Option, job: &crate::job::GenerationJob, - ) -> Result { + ) -> Result { let config = oneshot_config_json( container_id, &filesystem, @@ -350,7 +350,7 @@ impl WxcExecInvoker { .iter() .map(|p| mock_normalize(p)) .collect(); - return Self::mock_spawn_with_grants(&process, &grants); + return Self::mock_spawn_with_grants(&process, &grants, job); } let json = serde_json::to_string(&config)?; @@ -388,9 +388,9 @@ impl WxcExecInvoker { } info!(container_id = %container_id, "wxc-exec one-shot processContainer spawn"); - // Spawned inside the generation's job so the boundary and workload tree - // cannot outlive the driver; fails closed if the job cannot adopt it. - let child = job.spawn(&mut cmd)?; + // Created inside the generation's job, so the boundary and workload tree + // cannot outlive the driver; fails closed if it cannot be created there. + let child = job.spawn(&cmd)?; Ok(child) } } @@ -415,15 +415,16 @@ mod tests { let powershell = PathBuf::from(std::env::var("SystemRoot").unwrap()) .join("System32/WindowsPowerShell/v1.0/powershell.exe"); let invoker = WxcExecInvoker::new(powershell, false); - let child = invoker - .real_command() - .args([ - "-NoProfile", - "-NonInteractive", - "-Command", - "Start-Sleep -Seconds 60", - ]) - .spawn() + let job = crate::job::GenerationJob::new().expect("job"); + let mut command = invoker.real_command(); + command.args([ + "-NoProfile", + "-NonInteractive", + "-Command", + "Start-Sleep -Seconds 60", + ]); + let child = job + .spawn(&command) .expect("launch long-lived PowerShell process"); // SAFETY: OpenProcess supplies a synchronization-only handle for the // child, retained through termination and closed exactly once below. From ab1aa4d4df056fcc60f613a87a5f5c3c0d41ec36 Mon Sep 17 00:00:00 2001 From: ashwind Date: Tue, 17 Nov 2026 13:01:51 +0530 Subject: [PATCH 09/10] test(windows): quote gateway arguments in the MXC e2e runner Addresses review feedback on #5299. The gateway-restart scenario passes a SQLite URL built from the checkout path, and Start-Process -ArgumentList does not quote array elements, so a path containing a space was split into two arguments and the gateway refused to start. Build the argument string with the existing Quote-NativeArgument helper, as the other native launches in this script already do. Tests: from a checkout path containing a space, gateway-restart failed before this change (the gateway rejected the split URL) and passes after. It still passes from a path without spaces, and the script-level tests in test-mxc-remote-tests.ps1 are unchanged. Signed-off-by: ashwind --- crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index 9b5b2264e0..4ad323efce 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -336,8 +336,11 @@ function Start-Gw([string]$DbUrl = "sqlite::memory:") { # --config token: Start-Process -ArgumentList does not quote array elements, so a # config path containing a space gets split and the gateway's arg parser rejects it. $env:OPENSHELL_GATEWAY_CONFIG = $toml + # The database URL embeds the checkout path, so it can contain spaces: quote each + # argument ourselves (same helper as the other native launches in this script). + $gwArguments = @("--disable-tls", "--db-url", $DbUrl, "--port", $Port, "--log-level", "info") $p = Start-Process -FilePath $gateway ` - -ArgumentList @("--disable-tls", "--db-url", $DbUrl, "--port", $Port, "--log-level", "info") ` + -ArgumentList (($gwArguments | ForEach-Object { Quote-NativeArgument ([string]$_) }) -join ' ') ` -WorkingDirectory $here -PassThru -WindowStyle Hidden ` -RedirectStandardOutput $gwLog -RedirectStandardError $gwErrLog $deadline = (Get-Date).AddSeconds(30) From 62504183d2d4ff16f0beebc8102c54db7f69ff57 Mon Sep 17 00:00:00 2001 From: ashwind Date: Wed, 18 Nov 2026 23:06:10 +0530 Subject: [PATCH 10/10] fix(windows): close review findings in the process launcher Addresses review feedback on #5299. make_pipe closed the server end of a named pipe again when tokio failed to register it. tokio takes ownership of the handle before registering and closes it itself on failure, so this was a double close that could hit an unrelated handle another thread had just been given. The error is now propagated without a second close. A test seam (make_pipe_with) covers both halves: the launcher must not touch the handle after a failed registration, and tokio does close it. The environment-block test assumed the host had no other OPENSHELL_* variables. build_env_block now has a variant over an explicit host environment, so the test is hermetic, and a second test checks that the real host's variables are kept and sorted. A review of the launcher found further defects, fixed here: - start_kill reported "access denied" for a process that had just exited, before the exit watcher had published it. It now asks the process itself. - The exit watcher reported a clean exit when waiting for the process or reading its exit code failed. It now reports an error. - resolve_program tried a bare name before NAME.exe, searched the current directory through empty or relative PATH entries, ignored a PATH set for the new process, and gave a confusing failure for batch files. It now follows Windows: NAME.exe only, absolute PATH entries only, the new process's PATH, and a clear error for .bat and .cmd. - CreateProcessW errors lost their kind (not found, access denied); the kind is kept and attribute errors carry context. - The attribute list buffer is now pointer-aligned, raw_arg appends as documented, environment names that would corrupt the block are rejected, and the sort key upper-cases one UTF-16 unit at a time as Windows does. Test hardening: the handle-inheritance test no longer depends on a file lock (it raced with other tests' spawns and a cmd grandchild); it checks the one child with DuplicateHandle and CompareObjectHandles. cmd.exe is started with /d, the owner-kill test waits for the tree instead of a fixed 700 ms, and the mid-spawn stress test fails if no generation was ever started. Tests: every new test fails when its fix is reverted, including the mid-spawn stress test against the old create-then-assign order. 107 driver tests pass on Windows x64 and ARM64, in a clean and in a configured environment (OPENSHELL_* set, empty and relative PATH entries), with 1 and 32 test threads, and from a checkout path with spaces. gateway-kill, gateway-restart and fs-rw pass against real MXC on both. Refs #5298 Signed-off-by: ashwind --- crates/openshell-driver-mxc/src/child.rs | 636 ++++++++++++++++++----- crates/openshell-driver-mxc/src/job.rs | 45 +- 2 files changed, 551 insertions(+), 130 deletions(-) diff --git a/crates/openshell-driver-mxc/src/child.rs b/crates/openshell-driver-mxc/src/child.rs index 1f6dac2fc0..1a484476a6 100644 --- a/crates/openshell-driver-mxc/src/child.rs +++ b/crates/openshell-driver-mxc/src/child.rs @@ -41,7 +41,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use tokio::net::windows::named_pipe::NamedPipeServer; use tokio::sync::watch; use windows::Win32::Foundation::{ - CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE, INVALID_HANDLE_VALUE, WAIT_FAILED, + CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE, INVALID_HANDLE_VALUE, + WAIT_OBJECT_0, }; use windows::Win32::Security::SECURITY_ATTRIBUTES; use windows::Win32::Storage::FileSystem::{ @@ -119,9 +120,16 @@ impl ChildCommand { self } - /// Append already-encoded arguments to the command line exactly as given. + /// Append already-encoded arguments to the command line exactly as given, after the + /// quoted ones. Several calls are joined with a space. pub(crate) fn raw_arg(&mut self, arg: impl AsRef) -> &mut Self { - self.raw_args = Some(arg.as_ref().to_os_string()); + match &mut self.raw_args { + Some(existing) => { + existing.push(" "); + existing.push(arg.as_ref()); + } + None => self.raw_args = Some(arg.as_ref().to_os_string()), + } self } @@ -173,13 +181,22 @@ impl Drop for OwnedHandle { } } +/// How a process ended, as published by its exit-watcher thread. +#[derive(Clone, Copy)] +enum Exit { + Code(u32), + /// The watcher could not wait for the process or read its exit code. Reporting a + /// code here would claim a clean exit for a process that may still be running. + Unknown, +} + /// A process started inside its generation's job. /// /// It is killed when dropped if it is still running. pub struct GenerationChild { process: Arc, pid: u32, - exit: watch::Receiver>, + exit: watch::Receiver>, /// Parent end of the child's standard input, when it was [`ChildStdio::Piped`]. /// Held open for the child's lifetime so the child does not see end-of-file; the /// driver never writes to it. @@ -237,18 +254,30 @@ impl GenerationChild { // SAFETY: the handle is owned by `self.process` and valid. match unsafe { TerminateProcess(self.process.0, 1) } { Ok(()) => Ok(()), - // Lost a race with the process ending by itself. - Err(_) if self.exit.borrow().is_some() => Ok(()), + // Lost a race with the process ending by itself: `TerminateProcess` then fails + // with "access denied", possibly before the watcher thread has published it. + Err(_) if self.has_exited() => Ok(()), Err(error) => Err(io::Error::from(error)), } } + /// Whether the process has ended, asked of the process itself. + fn has_exited(&self) -> bool { + // SAFETY: the handle is owned by `self.process` and valid. + self.exit.borrow().is_some() + || unsafe { WaitForSingleObject(self.process.0, 0) } == WAIT_OBJECT_0 + } + /// Wait for the process to end. Safe to cancel and to call again. pub(crate) async fn wait(&mut self) -> io::Result { loop { - let code = *self.exit.borrow(); - if let Some(code) = code { - return Ok(ExitStatus::from_raw(code)); + let state = *self.exit.borrow(); + match state { + Some(Exit::Code(code)) => return Ok(ExitStatus::from_raw(code)), + Some(Exit::Unknown) => { + return Err(io::Error::other("could not read how the process ended")); + } + None => {} } self.exit .changed() @@ -368,15 +397,20 @@ fn build_command_line(command: &ChildCommand, program: &Path) -> io::Result Vec { char::decode_utf16(name.encode_wide()) - .flat_map(|unit| { - unit.map_or_else( - |_| Vec::new(), - |character| character.to_uppercase().collect::>(), - ) - }) - .flat_map(|character| { - let mut buffer = [0u16; 2]; - character.encode_utf16(&mut buffer).to_vec() + .flat_map(|unit| match unit { + Ok(character) => { + // Windows upper-cases one UTF-16 unit at a time and never lengthens a name + // ('ß' stays 'ß'), so only a one-to-one mapping counts. + let mut upper = character.to_uppercase(); + let mapped = match (upper.next(), upper.next()) { + (Some(single), None) => single, + _ => character, + }; + let mut buffer = [0u16; 2]; + mapped.encode_utf16(&mut buffer).to_vec() + } + // An unpaired surrogate is kept as it is. + Err(error) => vec![error.unpaired_surrogate()], }) .collect() } @@ -384,8 +418,29 @@ fn env_sort_key(name: &OsStr) -> Vec { /// The new process's environment: the host's, with `overrides` applied, as the /// sorted UTF-16 block `CreateProcessW` requires. fn build_env_block(overrides: &[(OsString, OsString)]) -> io::Result> { + build_env_block_from(std::env::vars_os(), overrides) +} + +/// [`build_env_block`] over an explicit host environment, so tests do not depend on the +/// environment they run in. +fn build_env_block_from( + host: impl IntoIterator, + overrides: &[(OsString, OsString)], +) -> io::Result> { + for (name, _) in overrides { + // An empty name or one holding `=` would corrupt the block. + if name.is_empty() || name.encode_wide().any(|unit| unit == u16::from(b'=')) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{} is not a valid environment variable name", + name.to_string_lossy() + ), + )); + } + } let mut merged: BTreeMap, (OsString, OsString)> = BTreeMap::new(); - for (name, value) in std::env::vars_os() { + for (name, value) in host { merged.insert(env_sort_key(&name), (name, value)); } for (name, value) in overrides { @@ -412,48 +467,58 @@ fn build_env_block(overrides: &[(OsString, OsString)]) -> io::Result> { Ok(block) } +/// The directories of `path` a program may be found in: absolute ones only. An empty or +/// relative entry would search the current directory, which is deliberately not searched. +fn usable_path_entries(path: &OsStr) -> Vec { + std::env::split_paths(path) + .filter(|directory| directory.is_absolute()) + .collect() +} + /// Find the executable for `program`: as given when it has a directory, otherwise -/// in the `PATH` directories and then the system directory. The current directory -/// is deliberately not searched. -fn resolve_program(program: &OsStr) -> io::Result { - let candidates = |base: PathBuf| -> Vec { +/// in the `PATH` directories (`path`, or the host's when `None`) and then the system +/// directory. The current directory is deliberately not searched. +/// +/// A name without an extension means `NAME.exe`, as it does to Windows: the bare name is +/// never tried, so an extension-less script of the same name cannot shadow the program. +fn resolve_program(program: &OsStr, path: Option<&OsStr>) -> io::Result { + let candidate = |base: PathBuf| -> PathBuf { if base.extension().is_some() { - vec![base] + base } else { - let mut with_exe = base.clone().into_os_string(); + let mut with_exe = base.into_os_string(); with_exe.push(".exe"); - vec![base, PathBuf::from(with_exe)] + PathBuf::from(with_exe) } }; + let not_found = || { + io::Error::new( + io::ErrorKind::NotFound, + format!("program not found: {}", Path::new(program).display()), + ) + }; let given = Path::new(program); if given.is_absolute() || given.components().count() > 1 { - return candidates(given.to_path_buf()) - .into_iter() - .find(|candidate| candidate.is_file()) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::NotFound, - format!("program not found: {}", given.display()), - ) - }); - } - let mut directories: Vec = std::env::var_os("PATH") - .map(|path| std::env::split_paths(&path).collect()) + let found = candidate(given.to_path_buf()); + return if found.is_file() { + Ok(found) + } else { + Err(not_found()) + }; + } + let mut directories = path + .map(OsStr::to_os_string) + .or_else(|| std::env::var_os("PATH")) + .map(|path| usable_path_entries(&path)) .unwrap_or_default(); if let Some(root) = std::env::var_os("SystemRoot") { directories.push(PathBuf::from(root).join("System32")); } - for directory in directories { - for candidate in candidates(directory.join(given)) { - if candidate.is_file() { - return Ok(candidate); - } - } - } - Err(io::Error::new( - io::ErrorKind::NotFound, - format!("program not found: {}", given.display()), - )) + directories + .into_iter() + .map(|directory| candidate(directory.join(given))) + .find(|found| found.is_file()) + .ok_or_else(not_found) } /// The child-side handle of one standard stream, plus the parent end if piped. @@ -514,6 +579,24 @@ fn inherit_std(which: STD_HANDLE, write: bool) -> io::Result { /// A pipe with an overlapped parent end (usable by tokio) and an inheritable child end. /// `child_writes` is true for stdout/stderr, false for stdin. fn make_pipe(child_writes: bool) -> io::Result { + // SAFETY: `make_pipe_with` hands over an open, overlapped pipe handle that nothing + // else owns or closes. + make_pipe_with(child_writes, |raw| unsafe { + NamedPipeServer::from_raw_handle(raw) + }) +} + +/// [`make_pipe`] with the step that registers the server end with tokio supplied by the +/// caller, so a test can make it fail. +/// +/// `register` takes ownership of the handle it is given **even when it fails**: tokio +/// wraps it in an owning pipe before registering it with the reactor and drops that pipe, +/// closing the handle, if registration fails. This function therefore must not close the +/// handle again on that path. +fn make_pipe_with( + child_writes: bool, + register: impl FnOnce(RawHandle) -> io::Result, +) -> io::Result { let name = format!( r"\\.\pipe\openshell-mxc-{}-{}-{:016x}", std::process::id(), @@ -562,14 +645,8 @@ fn make_pipe(child_writes: bool) -> io::Result { let client = OwnedHandle(client); let raw = server.0.0; std::mem::forget(server); - // tokio takes ownership of the overlapped server end; on failure it is ours to close. - let parent = match NamedPipeServer::from_raw_handle(raw) { - Ok(parent) => parent, - Err(error) => { - let _ = CloseHandle(HANDLE(raw)); - return Err(error); - } - }; + // `register` owns the server end from here on, whether or not it succeeds. + let parent = register(raw)?; Ok(StdioPair { child: client, parent: Some(parent), @@ -597,7 +674,8 @@ fn open_stdio( /// A `PROC_THREAD_ATTRIBUTE_LIST`, deleted on drop. struct AttributeList { - buffer: Vec, + /// Eight-byte units, so the list is aligned for pointers as Windows requires. + buffer: Vec, } impl AttributeList { @@ -605,7 +683,7 @@ impl AttributeList { let mut size = 0usize; // SAFETY: the first call only reports the size needed; failure is expected. let _ = unsafe { InitializeProcThreadAttributeList(None, count, None, &raw mut size) }; - let mut buffer = vec![0u8; size]; + let mut buffer = vec![0u64; size.div_ceil(size_of::())]; let list = LPPROC_THREAD_ATTRIBUTE_LIST(buffer.as_mut_ptr().cast()); // SAFETY: `buffer` is `size` bytes, as the first call requested. unsafe { InitializeProcThreadAttributeList(Some(list), count, None, &raw mut size) } @@ -679,7 +757,26 @@ fn create( command: &ChildCommand, suspended: bool, ) -> io::Result<(GenerationChild, OwnedHandle)> { - let program = resolve_program(&command.program)?; + // The new process's `PATH` decides where its program is looked for, as in std. + let path = command + .envs + .iter() + .rev() + .find(|(name, _)| name.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value.as_os_str()); + let program = resolve_program(&command.program, path)?; + if program + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("bat") || ext.eq_ignore_ascii_case("cmd")) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{} is a batch file; start it through cmd.exe /c", + program.display() + ), + )); + } let mut command_line = build_command_line(command, &program)?; let application = to_wide(program.as_os_str())?; let environment = build_env_block(&command.envs)?; @@ -706,8 +803,22 @@ fn create( } let jobs = [job]; let mut attributes = AttributeList::new(2)?; - attributes.set(PROC_THREAD_ATTRIBUTE_JOB_LIST, &jobs)?; - attributes.set(PROC_THREAD_ATTRIBUTE_HANDLE_LIST, &inherited)?; + attributes + .set(PROC_THREAD_ATTRIBUTE_JOB_LIST, &jobs) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("name the sandbox job for process creation: {error}"), + ) + })?; + attributes + .set(PROC_THREAD_ATTRIBUTE_HANDLE_LIST, &inherited) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("limit the handles the process inherits: {error}"), + ) + })?; let mut startup = STARTUPINFOEXW::default(); startup.StartupInfo.cb = u32::try_from(size_of::()).unwrap_or(0); @@ -740,10 +851,15 @@ fn create( ) } .map_err(|error| { - io::Error::other(format!( - "create process {} in its sandbox job: {error}", - program.display() - )) + let source = io::Error::from(error); + // Keep the kind (not found, access denied, bad image) for callers that match on it. + io::Error::new( + source.kind(), + format!( + "create process {} in its sandbox job: {source}", + program.display() + ), + ) })?; drop(attributes); @@ -759,13 +875,17 @@ fn create( .name(format!("mxc-wait-{}", info.dwProcessId)) .spawn(move || { // SAFETY: `watched` keeps the process handle valid for the whole wait. - let mut code = 0u32; - unsafe { - if WaitForSingleObject(watched.0, INFINITE) != WAIT_FAILED { - let _ = GetExitCodeProcess(watched.0, &raw mut code); + let exit = unsafe { + let mut code = 0u32; + if WaitForSingleObject(watched.0, INFINITE) == WAIT_OBJECT_0 + && GetExitCodeProcess(watched.0, &raw mut code).is_ok() + { + Exit::Code(code) + } else { + Exit::Unknown } - } - let _ = sender.send(Some(code)); + }; + let _ = sender.send(Some(exit)); }); if let Err(error) = spawned { // SAFETY: the process handle is valid; end the process nobody can watch. @@ -790,7 +910,9 @@ mod tests { use std::time::Duration; use tokio::io::AsyncReadExt as _; - use windows::Win32::Foundation::{HLOCAL, LocalFree, WAIT_TIMEOUT}; + use windows::Win32::Foundation::{ + CompareObjectHandles, GetHandleInformation, HLOCAL, LocalFree, WAIT_TIMEOUT, + }; use windows::Win32::Storage::FileSystem::CREATE_NEW; use windows::Win32::System::Threading::{OpenProcess, PROCESS_SYNCHRONIZE}; use windows::Win32::UI::Shell::CommandLineToArgvW; @@ -895,43 +1017,281 @@ mod tests { assert!(build_command_line(&command, &program).is_err()); } + /// The entries of an environment block as `NAME=value` strings. + fn block_entries(block: &[u16]) -> Vec { + block + .split(|&unit| unit == 0) + .filter(|entry| !entry.is_empty()) + .map(String::from_utf16_lossy) + .collect() + } + + /// The variable name of an environment entry. A name may itself start with `=` + /// (Windows keeps per-drive directories as `=C:=C:\dir`). + fn entry_name(entry: &str) -> &str { + let search_from = usize::from(entry.starts_with('=')); + entry[search_from..] + .find('=') + .map_or(entry, |at| &entry[..at + search_from]) + } + #[test] - fn environment_block_is_sorted_and_applies_overrides() { + fn environment_block_merges_host_and_overrides_in_windows_order() { + // A synthetic host, so the result does not depend on where the test runs: it + // deliberately holds an OPENSHELL_ setting, a drive-directory entry and a + // lower-case name. + let host = vec![ + (OsString::from("Path"), OsString::from(r"C:\x")), + ( + OsString::from("OPENSHELL_WXC_EXEC_PATH"), + OsString::from(r"C:\mxc\wxc-exec.exe"), + ), + (OsString::from("=C:"), OsString::from(r"C:\work")), + (OsString::from("openshell_b"), OsString::from("0")), + ]; let overrides = vec![ (OsString::from("Openshell_B"), OsString::from("2")), (OsString::from("OPENSHELL_a"), OsString::from("1")), // Same name as the first, different case: the later value replaces it. (OsString::from("openshell_b"), OsString::from("3")), ]; - let block = build_env_block(&overrides).expect("block"); + + let block = build_env_block_from(host, &overrides).expect("block"); + assert!(block.ends_with(&[0, 0]), "the block ends with a double NUL"); - let entries: Vec = block - .split(|&unit| unit == 0) - .filter(|entry| !entry.is_empty()) - .map(String::from_utf16_lossy) - .collect(); - let ours: Vec<&String> = entries - .iter() - .filter(|entry| entry.to_ascii_lowercase().starts_with("openshell_")) - .collect(); - assert_eq!(ours, vec!["OPENSHELL_a=1", "openshell_b=3"]); - let keys: Vec = entries + // Unrelated host entries are kept, overrides win over the host's own value, and + // the whole block is in case-insensitive name order. + assert_eq!( + block_entries(&block), + vec![ + r"=C:=C:\work", + "OPENSHELL_a=1", + "openshell_b=3", + r"OPENSHELL_WXC_EXEC_PATH=C:\mxc\wxc-exec.exe", + r"Path=C:\x", + ] + ); + } + + #[test] + fn environment_block_is_sorted_for_whatever_host_it_runs_on() { + // The real host: whatever it holds, the block must be in Windows order and every + // host variable must survive. + let block = build_env_block(&[]).expect("block"); + let entries = block_entries(&block); + let keys: Vec> = entries .iter() - .map(|entry| entry.split('=').next().unwrap_or("").to_uppercase()) + .map(|entry| env_sort_key(OsStr::new(entry_name(entry)))) .collect(); - let mut sorted = keys.clone(); - sorted.sort(); - assert_eq!( - keys, sorted, + assert!( + keys.windows(2).all(|pair| pair[0] <= pair[1]), "entries must be sorted by name, case-insensitively" ); + for (name, _) in std::env::vars_os() { + let wanted = env_sort_key(&name); + assert!( + keys.contains(&wanted), + "host variable {name:?} is missing from the block" + ); + } + } + + /// Whether `handle` is still an open handle of this process. + fn handle_is_open(handle: RawHandle) -> bool { + let mut flags = 0u32; + // SAFETY: a plain query; `flags` outlives the call. + unsafe { GetHandleInformation(HANDLE(handle), &raw mut flags) }.is_ok() + } + + #[test] + fn a_failed_pipe_registration_leaves_the_handle_to_its_new_owner() { + // `register` owns the handle even when it fails, so `make_pipe_with` must not + // close it again: a second close could hit an unrelated handle that another + // thread was given the same value. This registrar fails without closing, which + // makes a second close visible. + let seen = std::cell::Cell::new(std::ptr::null_mut()); + let result = make_pipe_with(true, |raw| { + seen.set(raw); + Err(io::Error::other("registration failed")) + }); + let raw = seen.get(); + assert!(!raw.is_null(), "the registrar was never reached"); + let still_open = handle_is_open(raw); + // SAFETY: the registrar did not close it, so it is ours to close here. + let _ = unsafe { CloseHandle(HANDLE(raw)) }; + + assert!(result.is_err(), "the registration error must be returned"); + assert!( + still_open, + "make_pipe_with closed a handle it no longer owned" + ); + } + + #[test] + fn tokio_closes_the_pipe_handle_itself_when_registration_fails() { + // The contract the test above relies on, against the real tokio: from a stopped + // runtime the registration fails, and the handle is closed by tokio. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + let handle = runtime.handle().clone(); + drop(runtime); + let _entered = handle.enter(); + + let seen = std::cell::Cell::new(std::ptr::null_mut()); + let result = make_pipe_with(true, |raw| { + seen.set(raw); + // SAFETY: `raw` is an open overlapped pipe handle that nothing else owns. + unsafe { NamedPipeServer::from_raw_handle(raw) } + }); + + assert!( + result.is_err(), + "registering with a stopped runtime must fail" + ); + let raw = seen.get(); + assert!(!raw.is_null(), "the registrar was never reached"); + assert!( + !handle_is_open(raw), + "tokio is expected to have closed the handle it could not register" + ); + } + + #[test] + fn raw_arguments_given_twice_are_both_kept() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.raw_arg("first").raw_arg(r#"second "third""#); + let line = build_command_line(&command, &program).expect("command line"); + assert_eq!( + String::from_utf16_lossy(&line[..line.len() - 1]), + r#""C:\x\tool.exe" first second "third""# + ); + } + + #[test] + fn environment_names_that_would_corrupt_the_block_are_rejected() { + for bad in ["", "A=B", "=X"] { + let overrides = vec![(OsString::from(bad), OsString::from("v"))]; + let error = build_env_block_from(Vec::new(), &overrides).expect_err(bad); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput, "{bad:?}"); + } + } + + #[test] + fn environment_names_sort_the_way_windows_compares_them() { + // One unit at a time: 'ß' does not become "SS", and a lone surrogate is kept. + assert_eq!(env_sort_key(OsStr::new("\u{df}")), vec![0x00DF]); + assert_eq!(env_sort_key(OsStr::new("a\u{df}")), vec![0x0041, 0x00DF]); + let lone = OsString::from_wide(&[0xD800]); + assert_eq!(env_sort_key(&lone), vec![0xD800]); + } + + #[test] + fn only_absolute_path_entries_are_searched() { + let path = OsString::from(r"C:\one;;.;relative\dir;D:\two"); + assert_eq!( + usable_path_entries(&path), + vec![PathBuf::from(r"C:\one"), PathBuf::from(r"D:\two")] + ); + } + + #[test] + fn a_program_name_without_an_extension_means_its_exe() { + let dir = std::env::temp_dir().join(format!("mxc-resolve-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + // A script-like file with the bare name must not shadow the executable. + std::fs::write(dir.join("tool"), b"not a program").unwrap(); + std::fs::write(dir.join("tool.exe"), b"MZ").unwrap(); + let path = OsString::from(dir.as_os_str()); + + let found = resolve_program(OsStr::new("tool"), Some(&path)); + std::fs::remove_file(dir.join("tool.exe")).unwrap(); + let only_bare = resolve_program(OsStr::new("tool"), Some(&path)); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(found.unwrap(), dir.join("tool.exe")); + assert_eq!(only_bare.unwrap_err().kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn the_new_process_path_decides_where_its_program_is_found() { + let dir = std::env::temp_dir().join(format!("mxc-pathenv-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let system = PathBuf::from(std::env::var_os("SystemRoot").unwrap()) + .join("System32") + .join("cmd.exe"); + std::fs::copy(system, dir.join("mxc-pathenv-tool.exe")).unwrap(); + + let job = GenerationJob::new().expect("job"); + let mut command = quiet("mxc-pathenv-tool", &["/d", "/c", "exit 5"]); + command.env("Path", &dir); + let mut child = job.spawn(&command).expect("found through the new PATH"); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("exited") + .unwrap(); + drop(child); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(status.code(), Some(5)); + } + + #[tokio::test] + async fn a_batch_file_is_refused_with_a_clear_error() { + let batch = std::env::temp_dir().join(format!("mxc-batch-{}.cmd", std::process::id())); + std::fs::write(&batch, b"@echo off").unwrap(); + let job = GenerationJob::new().expect("job"); + let error = job + .spawn(&quiet(batch.to_str().unwrap(), &[])) + .err() + .expect("must fail"); + let _ = std::fs::remove_file(&batch); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(error.to_string().contains("batch file"), "{error}"); + } + + #[tokio::test] + async fn killing_a_process_that_has_ended_is_not_an_error_before_the_watcher_reports() { + // `TerminateProcess` on a process that has ended fails with "access denied". The + // exit watcher publishes the exit a moment later, so `start_kill` must ask the + // process itself. A channel nobody publishes on stands in for a watcher that has + // not run yet, which makes the moment deterministic. + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 0"])) + .expect("spawn"); + let (_unused_sender, never_reports) = watch::channel(None); + child.exit = never_reports; + // SAFETY: the handle is owned by `child.process` and valid. + let ended = unsafe { WaitForSingleObject(child.process.0, 10_000) }; + assert_eq!(ended, WAIT_OBJECT_0, "the process should have ended"); + + child + .start_kill() + .expect("an ended process is not a kill failure"); + } + + #[tokio::test] + async fn an_exit_the_watcher_could_not_read_is_an_error_not_a_clean_exit() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 0"])) + .expect("spawn"); + let (_sender, unknown) = watch::channel(Some(Exit::Unknown)); + child.exit = unknown; + + let result = child.wait().await; + + assert!(result.is_err(), "reported {result:?} for an unknown exit"); } #[tokio::test] async fn exit_status_is_reported() { let job = GenerationJob::new().expect("job"); let mut child = job - .spawn(&quiet("cmd.exe", &["/c", "exit 7"])) + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 7"])) .expect("spawn"); let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) .await @@ -947,7 +1307,7 @@ mod tests { let job = GenerationJob::new().expect("job"); let mut command = ChildCommand::new("cmd.exe"); command - .args(["/c", "echo out& echo err 1>&2"]) + .args(["/d", "/c", "echo out& echo err 1>&2"]) .stdin(ChildStdio::Null) .stdout(ChildStdio::Piped) .stderr(ChildStdio::Piped); @@ -971,7 +1331,7 @@ mod tests { let job = GenerationJob::new().expect("job"); let mut command = ChildCommand::new("cmd.exe"); command - .args(["/c", "echo %OPENSHELL_CHILD_TEST%"]) + .args(["/d", "/c", "echo %OPENSHELL_CHILD_TEST%"]) .env("OPENSHELL_CHILD_TEST", "value with spaces") .stdin(ChildStdio::Null) .stdout(ChildStdio::Piped) @@ -996,7 +1356,10 @@ mod tests { async fn kill_ends_the_process_and_wait_reports_it() { let job = GenerationJob::new().expect("job"); let mut child = job - .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .spawn(&quiet( + "cmd.exe", + &["/d", "/c", "ping -n 30 127.0.0.1 >nul"], + )) .expect("spawn"); tokio::time::timeout(Duration::from_secs(10), child.kill()) .await @@ -1011,7 +1374,10 @@ mod tests { async fn dropping_a_child_kills_it() { let job = GenerationJob::new().expect("job"); let child = job - .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .spawn(&quiet( + "cmd.exe", + &["/d", "/c", "ping -n 30 127.0.0.1 >nul"], + )) .expect("spawn"); let pid = child.id().unwrap(); assert!(is_alive(pid)); @@ -1025,9 +1391,8 @@ mod tests { panic!("the dropped child is still running"); } - /// An inheritable handle with exclusive access: while any other process holds a copy - /// the file cannot be deleted. - fn open_exclusive_inheritable(path: &Path) -> HANDLE { + /// A handle to a scratch file that a child could inherit. + fn open_inheritable(path: &Path) -> HANDLE { let sa = inheritable(); let name = to_wide(path.as_os_str()).unwrap(); // SAFETY: `name` and `sa` outlive the call. @@ -1045,43 +1410,74 @@ mod tests { .expect("create inheritable file") } + /// Whether `child_process` holds a copy of `ours`. An inherited handle keeps its + /// value, so duplicate the handle of that value out of the child and check that it is + /// the same kernel object (another object that happens to have the same value does not + /// count). This looks at that one process only, so processes other tests start at the + /// same time cannot affect the answer. + fn child_holds_copy_of(child_process: HANDLE, ours: HANDLE) -> bool { + let mut copy = HANDLE::default(); + // SAFETY: both handles are valid; `copy` outlives the call and is closed below. + unsafe { + if DuplicateHandle( + child_process, + ours, + GetCurrentProcess(), + &raw mut copy, + 0, + false, + DUPLICATE_SAME_ACCESS, + ) + .is_err() + { + return false; + } + let same = CompareObjectHandles(ours, copy).as_bool(); + let _ = CloseHandle(copy); + same + } + } + /// Only the standard handles may reach a sandbox process. A handle the host holds /// (here an inheritable file handle) must not be inherited, as the host's listening - /// socket once was. The control shows the test can detect an inherited handle. + /// socket once was. The control shows the check can see an inherited handle. #[tokio::test] async fn only_the_standard_handles_are_inherited() { - let dir = std::env::temp_dir(); - let pid = std::process::id(); + let scratch = + std::env::temp_dir().join(format!("openshell-inherit-{}.tmp", std::process::id())); + let handle = open_inheritable(&scratch); // Control: the ordinary way of starting a process inherits the handle. - let control = dir.join(format!("openshell-inherit-control-{pid}.tmp")); - let handle = open_exclusive_inheritable(&control); - let mut ordinary = tokio::process::Command::new("cmd.exe") - .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + let mut ordinary = tokio::process::Command::new("ping.exe") + .args(["-n", "30", "127.0.0.1"]) + .stdout(std::process::Stdio::null()) .kill_on_drop(true) .spawn() .expect("ordinary child"); - // SAFETY: the handle is valid and closed once. - unsafe { CloseHandle(handle) }.unwrap(); - assert!( - std::fs::remove_file(&control).is_err(), - "control: the ordinary child should still hold the file open" - ); + let ordinary_process = HANDLE(ordinary.raw_handle().expect("process handle")); + let control = child_holds_copy_of(ordinary_process, handle); let _ = ordinary.kill().await; - let _ = ordinary.wait().await; - std::fs::remove_file(&control).expect("control file deletable once the child is gone"); // Ours: the same handle is not inherited. - let ours = dir.join(format!("openshell-inherit-ours-{pid}.tmp")); let job = GenerationJob::new().expect("job"); - let handle = open_exclusive_inheritable(&ours); let child = job - .spawn(&quiet("cmd.exe", &["/c", "ping -n 30 127.0.0.1 >nul"])) + .spawn(&quiet("ping.exe", &["-n", "30", "127.0.0.1"])) .expect("spawn"); + let ours_process = HANDLE(child.raw_handle().expect("process handle")); + let inherited = child_holds_copy_of(ours_process, handle); + drop(child); + // SAFETY: the handle is valid and closed once. unsafe { CloseHandle(handle) }.unwrap(); - let deleted = std::fs::remove_file(&ours); - drop(child); - deleted.expect("a sandbox process must not hold the host's other handles"); + let _ = std::fs::remove_file(&scratch); + + assert!( + control, + "control: an ordinary child should inherit the handle" + ); + assert!( + !inherited, + "a sandbox process must not hold the host's other handles" + ); } } diff --git a/crates/openshell-driver-mxc/src/job.rs b/crates/openshell-driver-mxc/src/job.rs index 5e1df13ed3..2916acd8ef 100644 --- a/crates/openshell-driver-mxc/src/job.rs +++ b/crates/openshell-driver-mxc/src/job.rs @@ -121,7 +121,7 @@ mod tests { fn sleeper() -> ChildCommand { let mut command = ChildCommand::new("cmd.exe"); command - .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + .args(["/d", "/c", "ping -n 30 127.0.0.1 >nul"]) .stdin(ChildStdio::Null) .stdout(ChildStdio::Null) .stderr(ChildStdio::Null); @@ -199,7 +199,7 @@ mod tests { // Negative control: a process created the ordinary way is not in the job. let mut outsider = tokio::process::Command::new("cmd.exe") - .args(["/c", "ping -n 30 127.0.0.1 >nul"]) + .args(["/d", "/c", "ping -n 30 127.0.0.1 >nul"]) .kill_on_drop(true) .spawn() .expect("outsider"); @@ -297,7 +297,11 @@ mod tests { let job = GenerationJob::new().expect("job"); let child = job.spawn(&sleeper()).expect("spawn in job"); let root = child.id().expect("pid"); - tokio::time::sleep(Duration::from_millis(700)).await; // let cmd start ping + // Let cmd start ping: wait for it, not for a fixed time a loaded host may exceed. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while child_pids(root).is_empty() && std::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } let grandchildren = child_pids(root) .iter() .map(u32::to_string) @@ -366,17 +370,30 @@ mod tests { }; let mut live: std::collections::VecDeque<(GenerationJob, GenerationChild)> = std::collections::VecDeque::new(); - println!("READY"); + let mut ready = false; loop { let job = GenerationJob::new().expect("job"); let mut command = ChildCommand::new(&exe); command - .args(["/c", "ping -n 60 127.0.0.1 >nul"]) + .args(["/d", "/c", "ping -n 60 127.0.0.1 >nul"]) .stdin(ChildStdio::Null) .stdout(ChildStdio::Null) .stderr(ChildStdio::Null); - if let Ok(child) = job.spawn(&command) { - live.push_back((job, child)); + match job.spawn(&command) { + Ok(child) => { + live.push_back((job, child)); + if !ready { + // Only now has a generation really been started, so a kill after + // this point can land in a real spawn. + println!("READY"); + ready = true; + } + } + Err(error) => { + // Failing to start would make the test pass without testing anything. + println!("SPAWN_FAILED {error}"); + std::process::exit(3); + } } if live.len() > 6 { live.pop_front(); @@ -396,8 +413,10 @@ mod tests { // A copy of cmd.exe under a name nothing else has, so survivors are countable. let name = format!("mxc-job-stress-{}.exe", std::process::id()); let exe = std::env::temp_dir().join(&name); - let system = std::path::PathBuf::from(std::env::var_os("SystemRoot").unwrap()) - .join("System32/cmd.exe"); + let system = std::path::PathBuf::from( + std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into()), + ) + .join("System32/cmd.exe"); std::fs::copy(system, &exe).expect("copy cmd.exe"); for trial in 0..12u64 { @@ -412,7 +431,13 @@ mod tests { .spawn() .expect("start stress host"); let mut lines = BufReader::new(host.stdout.take().unwrap()).lines(); - while lines.next().expect("host exited early").unwrap() != "READY" {} + loop { + let line = lines.next().expect("host exited early").unwrap(); + assert!(!line.starts_with("SPAWN_FAILED"), "{line}"); + if line == "READY" { + break; + } + } // Different, unrelated delays so the kill lands at different points. tokio::time::sleep(Duration::from_millis(40 + (trial * 37) % 260)).await; host.kill().expect("hard-kill the host");