diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index 2d62e43afe..a01b30a322 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -478,7 +478,7 @@ variable by default). Require both a nonempty response and the OpenShell proxy CA issuer; never replace these assertions with an insecure TLS request. This fixture permits unavailable CRL information using best-effort Schannel revocation for the ephemeral private CA; this is not revocation qualification. This -scenario is not part of the default seven offline-capable E2E scenarios. +scenario is not part of the default offline-capable E2E scenarios. The default `host-service` scenario runs the HTTP fixture on host `127.0.0.1` and a dynamic port, then checks two nonce-bound replies from direct curl requests @@ -489,6 +489,26 @@ firewall modification is required. A failed host health check or workload request fails the test. This verifies direct localhost connectivity; retain the existing shared proxy SSRF controls. +The default `gateway-kill` and `gateway-restart` scenarios cover gateway loss. +`gateway-kill` hard-kills the gateway and requires every sandbox process under it +(the supervisor, `wxc-exec`, the boundary and the workload) to end and a +replacement gateway to bind the port. `gateway-restart` sends a real Ctrl-C with +a persisted SQLite database, requires a logged graceful shutdown and every +sandbox process to end, then restarts on the same database and requires the +record to still be listed, its workload not to be running, and the record to be +deletable with its name reusable. Use `-Scenario gateway-kill` or +`-Scenario gateway-restart` for focused checks. Report the behavior accurately: +gateway exit ends workloads and MXC has no restart recovery, unlike the Docker, +Podman and VM drivers, which stop and recover sandboxes on a graceful restart +(MXC reports `gateway_manages_lifecycle: false`). Never describe MXC as +stop-and-recover. A graceful stop ends workloads with or without the +per-generation job; the job is what covers a hard kill, so do not cite +`gateway-restart` as evidence for the job. If recovery is added, update +`gateway-restart`, the driver README and `WINDOWS.md` together. These scenarios +send Ctrl-C and track process trees, so run them on a host you can disturb; +they track only processes under their own gateway and leave other sandboxes +alone. + When reporting `windows:ci`, distinguish these categories: - Passed tests from the full x64 workspace test log. diff --git a/Cargo.lock b/Cargo.lock index 3ef486c649..2f65326b16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3713,9 +3713,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "log", @@ -4870,6 +4870,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "windows", "wiremock", "x509-parser", ] diff --git a/WINDOWS.md b/WINDOWS.md index de5f4a1f60..88d110f867 100644 --- a/WINDOWS.md +++ b/WINDOWS.md @@ -28,7 +28,16 @@ under development and is not a production-qualified security boundary: Treat the gateway host as trusted. - Foreground main-process attachment and ConPTY resize are unavailable. Create command workloads with `--detach`. -- Gateway restart recovery for live MXC generations is not supported. +- Gateway exit ends sandbox workloads, and there is no restart recovery. When + the gateway process ends (a graceful stop with Ctrl-C, a crash, or a kill), the + sandbox processes it started end with it, workloads included. After the gateway + restarts on the same database, each sandbox record is still listed but its + workload is not restarted, and its phase is not updated to show that the + workload ended (for example `Provisioning` after a graceful stop, `Ready` after + a hard kill). Delete the record to reuse its name. The Docker, Podman and VM + drivers stop sandboxes on a graceful gateway stop and recover them on restart; + MXC does not provide that recovery. See + [gateway exit and restart](https://github.com/NVIDIA/OpenShell/blob/windows/crates/openshell-driver-mxc/README.md#gateway-exit-and-restart). See the Windows branch's [MXC driver documentation](https://github.com/NVIDIA/OpenShell/blob/windows/crates/openshell-driver-mxc/README.md) for the current capabilities and enforcement limits, and its diff --git a/crates/openshell-driver-mxc/Cargo.toml b/crates/openshell-driver-mxc/Cargo.toml index 94ace8c040..924624d47c 100644 --- a/crates/openshell-driver-mxc/Cargo.toml +++ b/crates/openshell-driver-mxc/Cargo.toml @@ -37,7 +37,7 @@ uuid = { workspace = true } openshell-mxc-boundary = { path = "../openshell-mxc-boundary" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } -windows = { workspace = true, features = ["Win32_System_Kernel", "Win32_System_Performance", "Win32_System_SystemInformation"] } +windows = { workspace = true, features = ["Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_Diagnostics_ToolHelp", "Win32_System_JobObjects", "Win32_System_Kernel", "Win32_System_Performance", "Win32_System_Pipes", "Win32_System_SystemInformation", "Win32_UI_Shell"] } [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 69bf6ab75f..2269eead34 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -222,7 +222,8 @@ not required; driver-owned guest TLS fields are rejected. | Exec, signals, retained output | Authenticated Sandbox Protocol; ConPTY resize is not yet supported | | Dynamic forwarding | Standard supervisor `ForwardTcp` path through sandbox loopback connect | | ETW/OCSF audit | Optional Windows Sandboxing ETW consumer | -| Gateway restart recovery | Not yet supported; live MXC generations remain in-memory | +| Gateway exit | Ends the sandbox processes, workloads included ([details](#gateway-exit-and-restart)) | +| Gateway restart recovery | Not supported; live MXC generations remain in-memory and are not recovered | Workloads receive the proxy CA in `NODE_EXTRA_CA_CERTS` and `DENO_CERT`, and a combined bundle in `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, and @@ -254,6 +255,44 @@ explicit UI policy (including `{}`) for drivers missing this capability; those drivers need no UI-specific handling. UI policy itself does not require the `allow_driver_config` opt-in. Recreate the sandbox to change these settings. +## Gateway exit and restart + +The driver runs inside the gateway. Each sandbox generation runs in its own +Windows job object with kill-on-close: the host supervisor and `wxc-exec` are +created inside it (the job is named in the `CreateProcessW` call, so no process +of a generation ever exists outside its job, even if the gateway is killed +mid-spawn), and the boundary and workload processes below them inherit it. When a generation stops, the job is terminated, and when the gateway process +ends for any reason Windows closes the job and ends everything in it. In +practice: + +- **Stop and delete** end the whole generation: supervisor, `wxc-exec`, boundary + and workload, not only the two direct children. +- **Graceful gateway stop** (Ctrl-C) ends every sandbox process, workloads + included, and frees the gateway port. +- **Gateway crash or hard kill** ends every sandbox process through the job, and + the listening socket is not inherited by any child, so a new gateway can bind + the port at once. + +MXC does not recover sandboxes across a gateway restart. The Docker, Podman and +VM drivers stop their sandboxes on a graceful gateway stop and recover them when +the gateway starts again; MXC reports `gateway_manages_lifecycle: false`, so the +gateway skips that shutdown sweep and startup recovery for it, and live +generations exist only in the driver's memory. After a restart on the same +database: + +- each sandbox record is still listed, with the phase it had when the gateway + went away (for example `Provisioning` after a graceful stop, `Ready` after a + hard kill); nothing updates it to show that the workload ended; +- no workload is running, and nothing restarts it; +- `openshell sandbox delete ` removes the record, and the name can then be + created again. + +Treat a sandbox that was running when the gateway exited as lost, delete its +record, and create it again. The `gateway-kill` and `gateway-restart` E2E +scenarios (see [Validation](#validation)) check this behavior. Behavior for a +driver that runs as its own process, separate from the gateway, is not defined +yet. + ## Validation Windows build tasks embed the CRT in all four executables and Z3 in the @@ -351,7 +390,7 @@ explicitly loads the boundary-injected CA bundle and records the peer certificat success requires a nonempty response and the OpenShell proxy CA issuer. It does not install a host trust root or disable CA/hostname verification. The fixture uses best-effort Schannel revocation because the ephemeral private CA has no CRL -distribution point; it does not qualify revocation enforcement. The default seven E2E +distribution point; it does not qualify revocation enforcement. The default E2E scenarios remain independent of Internet access. The `forwarding` scenario starts a TCP listener inside MXC, then verifies two @@ -375,12 +414,33 @@ host service is checked before launch and stopped after testing. Select it with `-Scenario host-service`; no Internet access, LAN address, or firewall changes are required. This verifies direct localhost connectivity. +The default `gateway-kill` scenario checks sandbox cleanup when the gateway dies +without warning. It creates a sandbox with a long-running workload, records the +sandbox process tree under that gateway, hard-kills the gateway, and requires +every recorded process to end and the gateway port to be bindable again (a +replacement gateway must start on it). Only processes under the scenario's own +gateway are tracked, so other sandboxes on the host are left alone. It relies on +each sandbox generation's kill-on-close job object; no workload outlives the +gateway. Select it with `-Scenario gateway-kill`. + +The default `gateway-restart` scenario covers a graceful gateway stop with +persisted state. It runs the gateway on a SQLite file, creates a sandbox with a +long-running workload, sends a real Ctrl-C, and requires the gateway to log a +graceful shutdown and exit, every recorded sandbox process to end, and a restart +on the same database to start. After the restart the sandbox record must still be +listed (the observed phase is `Provisioning` and does not change), its workload +must not be running (MXC has no restart recovery), and the record must be +deletable with its name reusable. The scenario pins today's no-recovery behavior, +so it needs updating when recovery is added. A graceful stop ends the workloads +with or without the per-generation job; the job is what covers a hard kill. +Select it with `-Scenario gateway-restart`. + The `ws-agent.yaml` and `openclaw-gateway.yaml` policies preserve the Windows branch's workload grants. `inference.yaml` is a template: replace its filesystem, curl executable, inference host, and port placeholders before use. Authorize curl, which owns the network socket, not its cmd launcher. These policies have parser/mapper regression coverage; they do not add live WebSocket, OpenClaw, -or provider-backed inference scenarios to the seven-scenario E2E runner. +or provider-backed inference scenarios to the E2E runner. With `-KeepRunning`, the runner restores the caller's environment but retains the disposable CLI/runtime directory for the live gateway and prints its path. diff --git a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 index a573388e9a..feae2bda41 100644 --- a/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 +++ b/crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1 @@ -37,6 +37,12 @@ # forwarding - two TCP request/reply exchanges through OpenShell ingress. # service-forwarding - named HTTP service routing and endpoint deletion. # host-service - workload reaches an HTTP service on host 127.0.0.1. +# gateway-kill - hard-killing the gateway ends every sandbox process and +# frees its port. processcontainer only. +# gateway-restart - a graceful gateway stop (Ctrl-C) ends every sandbox process; +# after a restart on the same database the sandbox record is +# still there but its workload is not recovered, and the +# record can be deleted and its name reused. processcontainer only. # https-ca - opt-in outbound HTTPS using the injected proxy CA. # socket-owner-normalized - opt in to equivalent Windows policy paths. # socket-owner-normalized-default - equivalent spelling is denied by default. @@ -317,8 +323,12 @@ function Test-PortListening { return @([Net.NetworkInformation.IPGlobalProperties]::GetIPGlobalProperties().GetActiveTcpListeners() | Where-Object { $_.Port -eq $Port }).Count -gt 0 } -function Start-Gw { +function Start-Gw([string]$DbUrl = "sqlite::memory:") { Remove-Item $gwLog, $gwErrLog -Force -ErrorAction SilentlyContinue + # A parent that ignores Ctrl-C (some ssh/service sessions do) passes that on to + # the gateway, which would then never see a graceful-stop request. Clear it for + # this process so the gateway starts with the default handling. + [void][OpenShellMxcCtrlC]::EnableCtrlC() # Ephemeral in-memory DB: this is a test harness, so it must NOT write sandbox # records to the persistent default store (%LOCALAPPDATA%\openshell\gateway\ # openshell.db). Without this, sandbox names persist across gateway restarts @@ -328,8 +338,11 @@ function Start-Gw { # --config token: Start-Process -ArgumentList does not quote array elements, so a # config path containing a space gets split and the gateway's arg parser rejects it. $env:OPENSHELL_GATEWAY_CONFIG = $toml + # The database URL embeds the checkout path, so it can contain spaces: quote each + # argument ourselves (same helper as the other native launches in this script). + $gwArguments = @("--disable-tls", "--db-url", $DbUrl, "--port", $Port, "--log-level", "info") $p = Start-Process -FilePath $gateway ` - -ArgumentList @("--disable-tls", "--db-url", "sqlite::memory:", "--port", $Port, "--log-level", "info") ` + -ArgumentList (($gwArguments | ForEach-Object { Quote-NativeArgument ([string]$_) }) -join ' ') ` -WorkingDirectory $here -PassThru -WindowStyle Hidden ` -RedirectStandardOutput $gwLog -RedirectStandardError $gwErrLog $deadline = (Get-Date).AddSeconds(30) @@ -339,6 +352,7 @@ function Start-Gw { throw "gateway exited early (code $($p.ExitCode)). See $gwLog." } if (Test-PortListening) { + $null = $p.Handle # keep the handle so ExitCode is available later return $p } Start-Sleep -Milliseconds 400 @@ -547,6 +561,119 @@ function Test-ServiceForwarding([string]$sandboxName, [string]$readyFile, [strin # Detect an agent *launch* failure (binary not found / not implemented) vs a # legitimate policy denial. Used to avoid false-passing a deny scenario when the # agent never actually ran. +if (-not ("OpenShellMxcCtrlC" -as [type])) { + Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; + +public static class OpenShellMxcCtrlC +{ + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + static extern bool SetConsoleCtrlHandler(IntPtr handler, bool add); + + // Restore default Ctrl-C handling for this process (and what it starts). + public static bool EnableCtrlC() { return SetConsoleCtrlHandler(IntPtr.Zero, false); } +} +'@ +} + +# Deliver a real Ctrl-C to another console process. Attaching to its console +# detaches the caller from its own, so this runs in a throwaway PowerShell. +function Send-CtrlC([int]$ProcessId) { + $script = @" +Add-Type -TypeDefinition @' +using System; using System.Runtime.InteropServices; +public static class SendCtrlC { + [DllImport("kernel32.dll", SetLastError=true)] static extern bool FreeConsole(); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool AttachConsole(uint pid); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool SetConsoleCtrlHandler(IntPtr h, bool add); + [DllImport("kernel32.dll", SetLastError=true)] static extern bool GenerateConsoleCtrlEvent(uint ev, uint grp); + public static string Run(uint pid) { + FreeConsole(); + if (!AttachConsole(pid)) return "AttachConsole failed: " + Marshal.GetLastWin32Error(); + SetConsoleCtrlHandler(IntPtr.Zero, true); + bool ok = GenerateConsoleCtrlEvent(0, 0); + int err = Marshal.GetLastWin32Error(); + FreeConsole(); + return ok ? "sent" : "GenerateConsoleCtrlEvent failed: " + err; + } +} +'@ +[SendCtrlC]::Run($ProcessId) +"@ + $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($script)) + $result = Invoke-NativeCaptured "powershell.exe" @("-NoProfile", "-NonInteractive", "-EncodedCommand", $encoded) 30 + # An encoded command also emits CLIXML progress noise on stderr; the verdict is + # the one line that starts with one of the helper's results. + $verdict = @($result.Output | ForEach-Object { "$_".Trim() } | Where-Object { $_ -match '^(sent|AttachConsole failed|GenerateConsoleCtrlEvent failed)' }) + if ($verdict.Count -eq 0) { return "no result from the Ctrl-C helper (exit $($result.ExitCode))" } + return $verdict[0] +} + +# Decide whether a gateway that was sent Ctrl-C shut down gracefully. Returns why it +# did not, or $null when it did. The "Shutdown signal received" log line only proves +# that shutdown began (it is logged before cleanup is awaited, and cleanup can still +# fail), so a graceful stop must also exit with code 0. Pure, so it is unit-tested in +# tasks/scripts/test-mxc-remote-tests.ps1. +function Get-GracefulStopFailure($ExitCode, [bool] $LoggedShutdown) { + if ($null -eq $ExitCode) { return 'gateway exit code is unavailable after Ctrl-C' } + if ($ExitCode -ne 0) { return "gateway exited with code $ExitCode after Ctrl-C; a graceful shutdown must exit 0" } + if (-not $LoggedShutdown) { return 'gateway exited without logging a graceful shutdown (it was not a graceful stop)' } + return $null +} + +# Wait until a sandbox's whole process tree exists, then return it as +# @{ pid = name }. Only processes under $gwProc, plus the process whose command +# line carries $nonce, are included, so unrelated MXC sandboxes on the host are +# never counted or touched. Throws if the tree does not come up. +function Wait-SandboxTree($gwProc, [string]$nonce) { + $victims = @{} + $deadline = (Get-Date).AddSeconds(60) + do { + Start-Sleep -Milliseconds 500 + $victims = @{} + foreach ($id in (Get-DescendantPids $gwProc.Id)) { + $proc = Get-Process -Id $id -ErrorAction SilentlyContinue + if ($proc) { $victims[$id] = $proc.ProcessName } + } + $marked = @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$nonce*" -and $_.ProcessId -ne $PID }) + foreach ($proc in $marked) { $victims[[int]$proc.ProcessId] = $proc.Name } + $names = @($victims.Values | ForEach-Object { $_ -replace '\.exe$', '' }) + $ready = ($names -contains 'wxc-exec') -and ($names -contains 'openshell-supervisor') -and ($marked.Count -gt 0) + } while (-not $ready -and (Get-Date) -lt $deadline) + if (-not $ready) { throw "sandbox process tree did not come up (saw: $(($victims.Values) -join ', '))" } + return $victims +} + +# Processes from a recorded tree that are still alive after waiting up to +# $seconds for them to end. +function Wait-TreeGone($victims, [int]$seconds = 30) { + $deadline = (Get-Date).AddSeconds($seconds) + do { + Start-Sleep -Milliseconds 500 + $survivors = @($victims.Keys | Where-Object { Get-Process -Id $_ -ErrorAction SilentlyContinue }) + } while ($survivors.Count -gt 0 -and (Get-Date) -lt $deadline) + return ($survivors | ForEach-Object { "$($victims[$_])($_)" }) +} + +# All descendants of a process, found by walking ParentProcessId links in one +# snapshot. Orphans keep their ParentProcessId, so the walk is still correct +# for a tree whose root has since died, but it must run while the root lives. +function Get-DescendantPids([int]$rootPid) { + $all = @(Get-CimInstance Win32_Process) + $found = New-Object System.Collections.Generic.HashSet[int] + $queue = New-Object System.Collections.Generic.Queue[int] + $queue.Enqueue($rootPid) + while ($queue.Count -gt 0) { + $parent = $queue.Dequeue() + foreach ($p in $all) { + if ($p.ParentProcessId -eq $parent -and $found.Add([int]$p.ProcessId)) { $queue.Enqueue([int]$p.ProcessId) } + } + } + return @($found) +} + function Launch-Failed([string]$gwText) { if ($null -eq $gwText) { return $false } return ($gwText -match 'CreateProcessW failed error:2' ` @@ -789,10 +916,20 @@ try { Name = 'host-service'; PolicyFile = Join-Path $policyDir 'host-service.yaml' SandboxId = 'host'; Backends = 'process_container'; Kind = 'host-service' Description = 'workload connects directly to an HTTP service on host 127.0.0.1' + }, + @{ + Name = 'gateway-kill'; PolicyFile = Join-Path $policyDir 'fs-rw.yaml' + SandboxId = 'gk'; Backends = 'process_container'; Kind = 'gateway-kill' + Description = 'hard-killing the gateway ends every sandbox process and frees its port' + }, + @{ + Name = 'gateway-restart'; PolicyFile = Join-Path $policyDir 'fs-rw.yaml' + SandboxId = 'gr'; Backends = 'process_container'; Kind = 'gateway-restart' + Description = 'graceful gateway stop ends sandbox processes; the record persists but is not recovered' } ) - # External connectivity is deliberately opt-in; the default seven scenarios + # External connectivity is deliberately opt-in; the default scenarios # remain runnable without Internet access. Use the same authenticated # gateway launch path as every other workload, not a bare driver fixture. if ($Scenario -eq 'https-ca') { @@ -923,6 +1060,11 @@ try { } else { $command = @($cmdExe, "/c", "echo denied 1> $denied") } + } elseif ($sc.Kind -in @('gateway-kill', 'gateway-restart')) { + # A long-running workload carrying a unique marker, so its process + # can be identified without matching unrelated cmd.exe instances. + $killNonce = [guid]::NewGuid().ToString('N') + $command = @($cmdExe, '/d', '/c', "for /l %i in (1,1,2000000000) do @rem $killNonce") } elseif ($sc.Kind -eq 'host-service') { Remove-Item -Path (Join-Path $DemoDir 'host-service.*') -Force -ErrorAction SilentlyContinue $batch = Join-Path $DemoDir 'host-service.cmd' @@ -981,7 +1123,11 @@ try { # policyUsed already lives in the result bundle and is the exact # rendered policy passed to OpenShell. - $gw = Start-Gw + # gateway-restart needs state that survives the gateway; every other + # scenario keeps the ephemeral in-memory database. + $persistedDb = Join-Path $resultDir "gateway.$($sc.Name).db" + $gwDbUrl = if ($sc.Kind -eq 'gateway-restart') { "sqlite:${persistedDb}?mode=rwc" } else { "sqlite::memory:" } + $gw = Start-Gw $gwDbUrl Info "gateway pid $($gw.Id)" Register-Cli @@ -997,7 +1143,7 @@ try { "--policy", [string]$policyUsed, "--driver-config-json", $driverConfig ) - $createArguments += if ($sc.Kind -eq 'host-service') { '--detach' } else { '--no-tty' } + $createArguments += if ($sc.Kind -in @('host-service', 'gateway-kill', 'gateway-restart')) { '--detach' } else { '--no-tty' } $createResult = Invoke-NativeCaptured $cli $createArguments $createOut = $createResult.Output $createExitCode = $createResult.ExitCode @@ -1021,6 +1167,80 @@ try { Bad "host-service: $($_.Exception.Message)" $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } } + } elseif ($sc.Kind -eq 'gateway-kill') { + try { + if ($createExitCode -ne 0) { throw "sandbox creation failed: $createOutStr" } + # Wait for the whole sandbox tree to exist, then record it. + $victims = Wait-SandboxTree $gw $killNonce + Info "tracking $($victims.Count) sandbox processes: $((($victims.GetEnumerator() | Sort-Object Name | ForEach-Object { "$($_.Value)($($_.Key))" })) -join ', ')" + + Info "hard-killing gateway pid $($gw.Id)" + Stop-Process -Id $gw.Id -Force + $gw.WaitForExit() + $survivors = @(Wait-TreeGone $victims 30) + if ($survivors.Count -gt 0) { + throw "sandbox processes survived the gateway's death: $($survivors -join ', ')" + } + # The port must be free again: a replacement gateway must bind it. + $gw = Start-Gw + Info "replacement gateway pid $($gw.Id) bound port $Port" + Ok "gateway-kill: all $($victims.Count) sandbox processes ended; port rebound" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'PASS'; Reason = "$($victims.Count) sandbox processes ended, port rebound" } + } catch { + Bad "gateway-kill: $($_.Exception.Message)" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } + } + } elseif ($sc.Kind -eq 'gateway-restart') { + # Documents today's behavior (MXC has no restart recovery): a graceful + # gateway stop ends the workloads, the persisted record is left + # behind, and nothing restarts it. Revisit the "not recovered" check + # when recovery is added. + try { + if ($createExitCode -ne 0) { throw "sandbox creation failed: $createOutStr" } + $victims = Wait-SandboxTree $gw $killNonce + Info "tracking $($victims.Count) sandbox processes: $((($victims.GetEnumerator() | Sort-Object Name | ForEach-Object { "$($_.Value)($($_.Key))" })) -join ', ')" + + Info "sending Ctrl-C to gateway pid $($gw.Id)" + $sent = Send-CtrlC $gw.Id + if ($sent -ne 'sent') { throw "could not send Ctrl-C to the gateway: $sent" } + if (-not $gw.WaitForExit(60000)) { throw 'gateway did not exit within 60 s of Ctrl-C (no graceful shutdown)' } + Info "gateway exited (exit code $($gw.ExitCode))" + $loggedShutdown = [bool](Select-String -Path $gwLog, $gwErrLog -Pattern 'Shutdown signal received' -Quiet -ErrorAction SilentlyContinue) + $stopFailure = Get-GracefulStopFailure $gw.ExitCode $loggedShutdown + if ($stopFailure) { throw $stopFailure } + $survivors = @(Wait-TreeGone $victims 30) + if ($survivors.Count -gt 0) { + throw "sandbox processes survived a graceful gateway stop: $($survivors -join ', ')" + } + Copy-Item $gwLog "$gwLog.first" -Force -ErrorAction SilentlyContinue + Copy-Item $gwErrLog "$gwErrLog.first" -Force -ErrorAction SilentlyContinue + + # Restart on the same database: the record persists; nothing recovers it. + $gw = Start-Gw $gwDbUrl + Info "restarted gateway pid $($gw.Id) on the same database" + $listed = Invoke-NativeCaptured $cli @('sandbox', 'list') + $row = @($listed.Output | Where-Object { $_ -match [regex]::Escape($sandboxName) }) | Select-Object -First 1 + if (-not $row) { throw "persisted sandbox record '$sandboxName' is missing after restart: $($listed.Output -join ' ')" } + Info "record after restart: $(($row -replace '\s+', ' ').Trim())" + Start-Sleep -Seconds 10 + $running = @(Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -like "*$killNonce*" -and $_.ProcessId -ne $PID }) + if ($running.Count -gt 0) { throw "workload was running after restart ($($running.Count) process(es)); recovery behavior changed, update this scenario and the Windows docs" } + + # The stale record must be manageable: delete it, then reuse the name. + $deleted = Invoke-NativeCaptured $cli @('sandbox', 'delete', $sandboxName) + if ($deleted.ExitCode -ne 0) { throw "deleting the stale record failed: $($deleted.Output -join ' ')" } + $listed = Invoke-NativeCaptured $cli @('sandbox', 'list') + if (@($listed.Output | Where-Object { $_ -match [regex]::Escape($sandboxName) }).Count -gt 0) { throw 'record still listed after delete' } + $again = Invoke-NativeCaptured $cli $createArguments + if ($again.ExitCode -ne 0) { throw "re-creating '$sandboxName' failed: $($again.Output -join ' ')" } + $null = Wait-SandboxTree $gw $killNonce + + Ok "gateway-restart: graceful stop ended $($victims.Count) processes; record persisted, not recovered; delete and re-create work" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'PASS'; Reason = "graceful stop ended $($victims.Count) processes; record persisted, not recovered, name reusable" } + } catch { + Bad "gateway-restart: $($_.Exception.Message)" + $results += [pscustomobject]@{ Scenario = $sc.Name; Result = 'FAIL'; Reason = $_.Exception.Message } + } } elseif ($sc.Kind -eq 'socket-owner') { try { if (-not (Wait-File $sc.ControlTarget 30)) { throw "parent cmd did not launch: $createOutStr" } @@ -1215,6 +1435,12 @@ admission and execution, not network enforcement. Forwarding proves managed ingress works, not that direct host ingress is blocked. Host-service verifies two exact direct HTTP replies from a service bound to host 127.0.0.1, with host-side request evidence. It checks localhost connectivity. +Gateway-kill creates a sandbox, hard-kills the gateway, and requires every process +that was under that gateway (and the marked workload) to end and the port to be +bindable by a replacement gateway. Gateway-restart stops the gateway gracefully +(Ctrl-C) with a persisted database, requires the same, then restarts it and checks +that the sandbox record is still listed, its workload is not recovered, and the +record can be deleted and its name reused. "@ Set-Content -Path (Join-Path $resultDir "summary.txt") -Value $summary -Encoding UTF8 Write-Host $summary -ForegroundColor ($(if ($verdict -eq "PASS") { "Green" } else { "Red" })) diff --git a/crates/openshell-driver-mxc/src/child.rs b/crates/openshell-driver-mxc/src/child.rs new file mode 100644 index 0000000000..1a484476a6 --- /dev/null +++ b/crates/openshell-driver-mxc/src/child.rs @@ -0,0 +1,1483 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Start a sandbox generation's processes inside their job object, atomically. +//! +//! The job is named in the `PROC_THREAD_ATTRIBUTE_JOB_LIST` attribute of the +//! `CreateProcessW` call, so the new process is a member of its job from the moment +//! it exists. There is no window between creating a process and assigning it to the +//! job in which a hard-killed driver could leave the process behind. The standard +//! library and tokio offer no stable way to set that attribute, so the processes the +//! driver starts for a generation (the host supervisor and `wxc-exec`) are created +//! here. +//! +//! [`ChildCommand`] describes the process in the same terms the driver already used +//! for `tokio::process::Command`, and [`GenerationChild`] offers the subset of +//! `tokio::process::Child` the driver relies on: `id`, `raw_handle`, `wait`, +//! `kill`, and async `stdout`/`stderr`. A child is killed when its +//! [`GenerationChild`] is dropped, like the former `kill_on_drop(true)`. +//! +//! Only the three standard handles are inherited (`PROC_THREAD_ATTRIBUTE_HANDLE_LIST`), +//! so no other handle of the host, such as a listening socket, leaks into a sandbox +//! process. + +// Process creation is Win32 FFI and needs `unsafe`; it is confined to this module. +#![allow(unsafe_code)] +// `pub(crate)` documents what the driver may use; this module is private. +#![allow(clippy::redundant_pub_crate)] + +use std::collections::BTreeMap; +use std::ffi::{OsStr, OsString, c_void}; +use std::io; +use std::mem::size_of; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::io::RawHandle; +use std::os::windows::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::process::ExitStatus; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tokio::net::windows::named_pipe::NamedPipeServer; +use tokio::sync::watch; +use windows::Win32::Foundation::{ + CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE, INVALID_HANDLE_VALUE, + WAIT_OBJECT_0, +}; +use windows::Win32::Security::SECURITY_ATTRIBUTES; +use windows::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, + FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_SHARE_MODE, FILE_SHARE_READ, FILE_SHARE_WRITE, + OPEN_EXISTING, PIPE_ACCESS_INBOUND, PIPE_ACCESS_OUTBOUND, +}; +use windows::Win32::System::Console::{ + GetStdHandle, STD_ERROR_HANDLE, STD_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, +}; +use windows::Win32::System::Pipes::{ + CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_WAIT, +}; +use windows::Win32::System::Threading::{ + CREATE_SUSPENDED, CREATE_UNICODE_ENVIRONMENT, CreateProcessW, DeleteProcThreadAttributeList, + EXTENDED_STARTUPINFO_PRESENT, GetCurrentProcess, GetExitCodeProcess, INFINITE, + InitializeProcThreadAttributeList, LPPROC_THREAD_ATTRIBUTE_LIST, + PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROC_THREAD_ATTRIBUTE_JOB_LIST, PROCESS_CREATION_FLAGS, + PROCESS_INFORMATION, STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, + UpdateProcThreadAttribute, WaitForSingleObject, +}; +use windows::core::{PCWSTR, PWSTR}; + +/// What a standard stream of a new process is connected to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChildStdio { + /// The `NUL` device. + Null, + /// A pipe whose parent end is exposed on the [`GenerationChild`]. + Piped, + /// The same stream as the host process. + Inherit, +} + +/// A process to start inside a generation's job. +#[derive(Clone, Debug)] +pub struct ChildCommand { + program: OsString, + args: Vec, + raw_args: Option, + envs: Vec<(OsString, OsString)>, + stdin: ChildStdio, + stdout: ChildStdio, + stderr: ChildStdio, +} + +impl ChildCommand { + pub(crate) fn new(program: impl AsRef) -> Self { + Self { + program: program.as_ref().to_os_string(), + args: Vec::new(), + raw_args: None, + envs: Vec::new(), + stdin: ChildStdio::Inherit, + stdout: ChildStdio::Inherit, + stderr: ChildStdio::Inherit, + } + } + + /// Append an argument; it is quoted so the new process parses it back unchanged. + pub(crate) fn arg(&mut self, arg: impl AsRef) -> &mut Self { + self.args.push(arg.as_ref().to_os_string()); + self + } + + #[cfg(test)] + pub(crate) fn args(&mut self, args: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + self.arg(arg); + } + self + } + + /// Append already-encoded arguments to the command line exactly as given, after the + /// quoted ones. Several calls are joined with a space. + pub(crate) fn raw_arg(&mut self, arg: impl AsRef) -> &mut Self { + match &mut self.raw_args { + Some(existing) => { + existing.push(" "); + existing.push(arg.as_ref()); + } + None => self.raw_args = Some(arg.as_ref().to_os_string()), + } + self + } + + /// Set an environment variable for the new process, on top of the host's. + pub(crate) fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Self { + self.envs + .push((key.as_ref().to_os_string(), value.as_ref().to_os_string())); + self + } + + pub(crate) fn stdin(&mut self, stdio: ChildStdio) -> &mut Self { + self.stdin = stdio; + self + } + + pub(crate) fn stdout(&mut self, stdio: ChildStdio) -> &mut Self { + self.stdout = stdio; + self + } + + pub(crate) fn stderr(&mut self, stdio: ChildStdio) -> &mut Self { + self.stderr = stdio; + self + } + + /// The environment variables set explicitly on this command. + #[cfg(test)] + pub(crate) fn get_envs(&self) -> impl Iterator { + self.envs + .iter() + .map(|(k, v)| (k.as_os_str(), v.as_os_str())) + } +} + +/// An owned Win32 handle, closed on drop. +struct OwnedHandle(HANDLE); + +// SAFETY: a kernel handle has no thread affinity; every use below is thread-safe. +unsafe impl Send for OwnedHandle {} +// SAFETY: as above. +unsafe impl Sync for OwnedHandle {} + +impl Drop for OwnedHandle { + fn drop(&mut self) { + if !self.0.is_invalid() && !self.0.0.is_null() { + // SAFETY: the handle is owned and closed exactly once. + let _ = unsafe { CloseHandle(self.0) }; + } + } +} + +/// How a process ended, as published by its exit-watcher thread. +#[derive(Clone, Copy)] +enum Exit { + Code(u32), + /// The watcher could not wait for the process or read its exit code. Reporting a + /// code here would claim a clean exit for a process that may still be running. + Unknown, +} + +/// A process started inside its generation's job. +/// +/// It is killed when dropped if it is still running. +pub struct GenerationChild { + process: Arc, + pid: u32, + exit: watch::Receiver>, + /// Parent end of the child's standard input, when it was [`ChildStdio::Piped`]. + /// Held open for the child's lifetime so the child does not see end-of-file; the + /// driver never writes to it. + _stdin: Option, + pub(crate) stdout: Option, + pub(crate) stderr: Option, +} + +/// What the ETW consumer needs from a child to attribute its events. +pub trait ProcessHandle { + fn id(&self) -> Option; + fn raw_handle(&self) -> Option; +} + +impl ProcessHandle for GenerationChild { + fn id(&self) -> Option { + Self::id(self) + } + + fn raw_handle(&self) -> Option { + Self::raw_handle(self) + } +} + +impl ProcessHandle for tokio::process::Child { + fn id(&self) -> Option { + Self::id(self) + } + + fn raw_handle(&self) -> Option { + Self::raw_handle(self) + } +} + +impl GenerationChild { + /// The process ID. Always present; the `Option` mirrors `tokio::process::Child::id` + /// so the ETW consumer can take either through [`ProcessHandle`]. + #[allow(clippy::unnecessary_wraps)] + pub(crate) fn id(&self) -> Option { + Some(self.pid) + } + + /// The process handle, valid for as long as this value lives. Always present; see + /// [`GenerationChild::id`]. + #[allow(clippy::unnecessary_wraps)] + pub(crate) fn raw_handle(&self) -> Option { + Some(self.process.0.0) + } + + /// Ask the process to end now, without waiting for it. + pub(crate) fn start_kill(&mut self) -> io::Result<()> { + if self.exit.borrow().is_some() { + return Ok(()); + } + // SAFETY: the handle is owned by `self.process` and valid. + match unsafe { TerminateProcess(self.process.0, 1) } { + Ok(()) => Ok(()), + // Lost a race with the process ending by itself: `TerminateProcess` then fails + // with "access denied", possibly before the watcher thread has published it. + Err(_) if self.has_exited() => Ok(()), + Err(error) => Err(io::Error::from(error)), + } + } + + /// Whether the process has ended, asked of the process itself. + fn has_exited(&self) -> bool { + // SAFETY: the handle is owned by `self.process` and valid. + self.exit.borrow().is_some() + || unsafe { WaitForSingleObject(self.process.0, 0) } == WAIT_OBJECT_0 + } + + /// Wait for the process to end. Safe to cancel and to call again. + pub(crate) async fn wait(&mut self) -> io::Result { + loop { + let state = *self.exit.borrow(); + match state { + Some(Exit::Code(code)) => return Ok(ExitStatus::from_raw(code)), + Some(Exit::Unknown) => { + return Err(io::Error::other("could not read how the process ended")); + } + None => {} + } + self.exit + .changed() + .await + .map_err(|_| io::Error::other("process exit watcher ended unexpectedly"))?; + } + } + + /// End the process and wait for it to be gone. + pub(crate) async fn kill(&mut self) -> io::Result<()> { + self.start_kill()?; + self.wait().await.map(|_| ()) + } +} + +impl Drop for GenerationChild { + fn drop(&mut self) { + let _ = self.start_kill(); + } +} + +static PIPE_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn inheritable() -> SECURITY_ATTRIBUTES { + SECURITY_ATTRIBUTES { + nLength: u32::try_from(size_of::()).unwrap_or(0), + lpSecurityDescriptor: std::ptr::null_mut(), + bInheritHandle: true.into(), + } +} + +fn to_wide(value: &OsStr) -> io::Result> { + let mut wide: Vec = value.encode_wide().collect(); + if wide.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a process argument, path or environment entry contains a NUL", + )); + } + wide.push(0); + Ok(wide) +} + +/// Append `arg` quoted so `CommandLineToArgvW` and the C runtime parse it back to `arg`. +fn append_arg(out: &mut Vec, arg: &OsStr) -> io::Result<()> { + let wide: Vec = arg.encode_wide().collect(); + if wide.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "a process argument contains a NUL", + )); + } + let space = u16::from(b' '); + let tab = u16::from(b'\t'); + let quote = u16::from(b'"'); + let backslash = u16::from(b'\\'); + let needs_quotes = wide.is_empty() || wide.iter().any(|&c| c == space || c == tab); + if needs_quotes { + out.push(quote); + } + let mut backslashes = 0usize; + for &c in &wide { + if c == backslash { + backslashes += 1; + } else { + if c == quote { + // Double the backslashes before a quote, then escape the quote. + out.extend(std::iter::repeat_n(backslash, backslashes + 1)); + } + backslashes = 0; + } + out.push(c); + } + if needs_quotes { + // Backslashes at the end would otherwise escape the closing quote. + out.extend(std::iter::repeat_n(backslash, backslashes)); + out.push(quote); + } + Ok(()) +} + +/// The command line: the quoted program, the quoted arguments, then any raw text. +fn build_command_line(command: &ChildCommand, program: &Path) -> io::Result> { + let mut line: Vec = Vec::new(); + let program = program.as_os_str(); + if program.encode_wide().any(|c| c == u16::from(b'"')) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "program path contains a quote", + )); + } + // The program is always quoted: its first token is parsed differently from the rest. + line.push(u16::from(b'"')); + line.extend(program.encode_wide()); + line.push(u16::from(b'"')); + for arg in &command.args { + line.push(u16::from(b' ')); + append_arg(&mut line, arg)?; + } + if let Some(raw) = &command.raw_args { + let raw: Vec = raw.encode_wide().collect(); + if !raw.is_empty() { + line.push(u16::from(b' ')); + line.extend(raw); + } + } + if line.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "the command line contains a NUL", + )); + } + line.push(0); + Ok(line) +} + +/// Windows compares environment names case-insensitively. +fn env_sort_key(name: &OsStr) -> Vec { + char::decode_utf16(name.encode_wide()) + .flat_map(|unit| match unit { + Ok(character) => { + // Windows upper-cases one UTF-16 unit at a time and never lengthens a name + // ('ß' stays 'ß'), so only a one-to-one mapping counts. + let mut upper = character.to_uppercase(); + let mapped = match (upper.next(), upper.next()) { + (Some(single), None) => single, + _ => character, + }; + let mut buffer = [0u16; 2]; + mapped.encode_utf16(&mut buffer).to_vec() + } + // An unpaired surrogate is kept as it is. + Err(error) => vec![error.unpaired_surrogate()], + }) + .collect() +} + +/// The new process's environment: the host's, with `overrides` applied, as the +/// sorted UTF-16 block `CreateProcessW` requires. +fn build_env_block(overrides: &[(OsString, OsString)]) -> io::Result> { + build_env_block_from(std::env::vars_os(), overrides) +} + +/// [`build_env_block`] over an explicit host environment, so tests do not depend on the +/// environment they run in. +fn build_env_block_from( + host: impl IntoIterator, + overrides: &[(OsString, OsString)], +) -> io::Result> { + for (name, _) in overrides { + // An empty name or one holding `=` would corrupt the block. + if name.is_empty() || name.encode_wide().any(|unit| unit == u16::from(b'=')) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{} is not a valid environment variable name", + name.to_string_lossy() + ), + )); + } + } + let mut merged: BTreeMap, (OsString, OsString)> = BTreeMap::new(); + for (name, value) in host { + merged.insert(env_sort_key(&name), (name, value)); + } + for (name, value) in overrides { + merged.insert(env_sort_key(name), (name.clone(), value.clone())); + } + let mut block: Vec = Vec::new(); + for (name, value) in merged.values() { + let entry_start = block.len(); + block.extend(name.encode_wide()); + block.push(u16::from(b'=')); + block.extend(value.encode_wide()); + if block[entry_start..].contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "an environment entry contains a NUL", + )); + } + block.push(0); + } + if block.is_empty() { + block.push(0); + } + block.push(0); + Ok(block) +} + +/// The directories of `path` a program may be found in: absolute ones only. An empty or +/// relative entry would search the current directory, which is deliberately not searched. +fn usable_path_entries(path: &OsStr) -> Vec { + std::env::split_paths(path) + .filter(|directory| directory.is_absolute()) + .collect() +} + +/// Find the executable for `program`: as given when it has a directory, otherwise +/// in the `PATH` directories (`path`, or the host's when `None`) and then the system +/// directory. The current directory is deliberately not searched. +/// +/// A name without an extension means `NAME.exe`, as it does to Windows: the bare name is +/// never tried, so an extension-less script of the same name cannot shadow the program. +fn resolve_program(program: &OsStr, path: Option<&OsStr>) -> io::Result { + let candidate = |base: PathBuf| -> PathBuf { + if base.extension().is_some() { + base + } else { + let mut with_exe = base.into_os_string(); + with_exe.push(".exe"); + PathBuf::from(with_exe) + } + }; + let not_found = || { + io::Error::new( + io::ErrorKind::NotFound, + format!("program not found: {}", Path::new(program).display()), + ) + }; + let given = Path::new(program); + if given.is_absolute() || given.components().count() > 1 { + let found = candidate(given.to_path_buf()); + return if found.is_file() { + Ok(found) + } else { + Err(not_found()) + }; + } + let mut directories = path + .map(OsStr::to_os_string) + .or_else(|| std::env::var_os("PATH")) + .map(|path| usable_path_entries(&path)) + .unwrap_or_default(); + if let Some(root) = std::env::var_os("SystemRoot") { + directories.push(PathBuf::from(root).join("System32")); + } + directories + .into_iter() + .map(|directory| candidate(directory.join(given))) + .find(|found| found.is_file()) + .ok_or_else(not_found) +} + +/// The child-side handle of one standard stream, plus the parent end if piped. +struct StdioPair { + child: OwnedHandle, + parent: Option, +} + +fn open_nul(write: bool) -> io::Result { + let sa = inheritable(); + let access = if write { + FILE_GENERIC_WRITE.0 + } else { + FILE_GENERIC_READ.0 + }; + let name = to_wide(OsStr::new("NUL"))?; + // SAFETY: `name` is NUL-terminated and outlives the call; `sa` outlives it too. + let handle = unsafe { + CreateFileW( + PCWSTR(name.as_ptr()), + access, + FILE_SHARE_READ | FILE_SHARE_WRITE, + Some(&raw const sa), + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + None, + ) + } + .map_err(io::Error::from)?; + Ok(OwnedHandle(handle)) +} + +fn inherit_std(which: STD_HANDLE, write: bool) -> io::Result { + // SAFETY: plain Win32 calls; the duplicate is owned by the returned value. + unsafe { + let source = match GetStdHandle(which) { + Ok(handle) if !handle.is_invalid() && !handle.0.is_null() => handle, + _ => return open_nul(write), + }; + let mut duplicate = HANDLE::default(); + let process = GetCurrentProcess(); + match DuplicateHandle( + process, + source, + process, + &raw mut duplicate, + 0, + true, + DUPLICATE_SAME_ACCESS, + ) { + Ok(()) => Ok(OwnedHandle(duplicate)), + // The host's stream is not a duplicable handle (for example no console). + Err(_) => open_nul(write), + } + } +} + +/// A pipe with an overlapped parent end (usable by tokio) and an inheritable child end. +/// `child_writes` is true for stdout/stderr, false for stdin. +fn make_pipe(child_writes: bool) -> io::Result { + // SAFETY: `make_pipe_with` hands over an open, overlapped pipe handle that nothing + // else owns or closes. + make_pipe_with(child_writes, |raw| unsafe { + NamedPipeServer::from_raw_handle(raw) + }) +} + +/// [`make_pipe`] with the step that registers the server end with tokio supplied by the +/// caller, so a test can make it fail. +/// +/// `register` takes ownership of the handle it is given **even when it fails**: tokio +/// wraps it in an owning pipe before registering it with the reactor and drops that pipe, +/// closing the handle, if registration fails. This function therefore must not close the +/// handle again on that path. +fn make_pipe_with( + child_writes: bool, + register: impl FnOnce(RawHandle) -> io::Result, +) -> io::Result { + let name = format!( + r"\\.\pipe\openshell-mxc-{}-{}-{:016x}", + std::process::id(), + PIPE_COUNTER.fetch_add(1, Ordering::Relaxed), + rand::random::() + ); + let wide = to_wide(OsStr::new(&name))?; + let direction = if child_writes { + PIPE_ACCESS_INBOUND + } else { + PIPE_ACCESS_OUTBOUND + }; + // SAFETY: `wide` is NUL-terminated and outlives the calls; the handles are + // owned by the values they are stored in and closed once. + unsafe { + let server = CreateNamedPipeW( + PCWSTR(wide.as_ptr()), + direction | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 64 * 1024, + 64 * 1024, + 0, + None, + ); + if server == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + let server = OwnedHandle(server); + let sa = inheritable(); + let access = if child_writes { + FILE_GENERIC_WRITE.0 + } else { + FILE_GENERIC_READ.0 + }; + let client = CreateFileW( + PCWSTR(wide.as_ptr()), + access, + FILE_SHARE_MODE(0), + Some(&raw const sa), + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + None, + ) + .map_err(io::Error::from)?; + let client = OwnedHandle(client); + let raw = server.0.0; + std::mem::forget(server); + // `register` owns the server end from here on, whether or not it succeeds. + let parent = register(raw)?; + Ok(StdioPair { + child: client, + parent: Some(parent), + }) + } +} + +fn open_stdio( + stdio: ChildStdio, + standard: STD_HANDLE, + child_writes: bool, +) -> io::Result { + match stdio { + ChildStdio::Null => Ok(StdioPair { + child: open_nul(child_writes)?, + parent: None, + }), + ChildStdio::Inherit => Ok(StdioPair { + child: inherit_std(standard, child_writes)?, + parent: None, + }), + ChildStdio::Piped => make_pipe(child_writes), + } +} + +/// A `PROC_THREAD_ATTRIBUTE_LIST`, deleted on drop. +struct AttributeList { + /// Eight-byte units, so the list is aligned for pointers as Windows requires. + buffer: Vec, +} + +impl AttributeList { + fn new(count: u32) -> io::Result { + let mut size = 0usize; + // SAFETY: the first call only reports the size needed; failure is expected. + let _ = unsafe { InitializeProcThreadAttributeList(None, count, None, &raw mut size) }; + let mut buffer = vec![0u64; size.div_ceil(size_of::())]; + let list = LPPROC_THREAD_ATTRIBUTE_LIST(buffer.as_mut_ptr().cast()); + // SAFETY: `buffer` is `size` bytes, as the first call requested. + unsafe { InitializeProcThreadAttributeList(Some(list), count, None, &raw mut size) } + .map_err(io::Error::from)?; + Ok(Self { buffer }) + } + + fn list(&mut self) -> LPPROC_THREAD_ATTRIBUTE_LIST { + LPPROC_THREAD_ATTRIBUTE_LIST(self.buffer.as_mut_ptr().cast()) + } + + /// `value` must outlive this list (it is read by `CreateProcessW`). + fn set(&mut self, attribute: u32, value: &[T]) -> io::Result<()> { + // SAFETY: `value` is valid for the call and the caller keeps it alive. + unsafe { + UpdateProcThreadAttribute( + self.list(), + 0, + attribute as usize, + Some(value.as_ptr().cast::()), + size_of_val(value), + None, + None, + ) + } + .map_err(io::Error::from) + } +} + +impl Drop for AttributeList { + fn drop(&mut self) { + // SAFETY: the list was initialised in `new` and is deleted once. + unsafe { DeleteProcThreadAttributeList(self.list()) }; + } +} + +/// Create the process for `command` as a member of `job`. +/// +/// Must be called inside a tokio runtime (the pipe ends are registered with it). +pub fn spawn_in_job(job: HANDLE, command: &ChildCommand) -> io::Result { + create(job, command, false).map(|(child, _thread)| child) +} + +/// The initial thread of a process created suspended, to resume it. +#[cfg(test)] +pub struct SuspendedThread(OwnedHandle); + +#[cfg(test)] +impl SuspendedThread { + pub(crate) fn resume(self) -> io::Result<()> { + // SAFETY: the thread handle is valid and owned by `self`. + if unsafe { windows::Win32::System::Threading::ResumeThread(self.0.0) } == u32::MAX { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + +/// Like [`spawn_in_job`], but the process is created without starting it, so a test can +/// inspect it before it runs. +#[cfg(test)] +pub fn spawn_suspended_in_job( + job: HANDLE, + command: &ChildCommand, +) -> io::Result<(GenerationChild, SuspendedThread)> { + create(job, command, true).map(|(child, thread)| (child, SuspendedThread(thread))) +} + +fn create( + job: HANDLE, + command: &ChildCommand, + suspended: bool, +) -> io::Result<(GenerationChild, OwnedHandle)> { + // The new process's `PATH` decides where its program is looked for, as in std. + let path = command + .envs + .iter() + .rev() + .find(|(name, _)| name.eq_ignore_ascii_case("PATH")) + .map(|(_, value)| value.as_os_str()); + let program = resolve_program(&command.program, path)?; + if program + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("bat") || ext.eq_ignore_ascii_case("cmd")) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{} is a batch file; start it through cmd.exe /c", + program.display() + ), + )); + } + let mut command_line = build_command_line(command, &program)?; + let application = to_wide(program.as_os_str())?; + let environment = build_env_block(&command.envs)?; + + let StdioPair { + child: stdin_child, + parent: stdin_parent, + } = open_stdio(command.stdin, STD_INPUT_HANDLE, false)?; + let StdioPair { + child: stdout_child, + parent: stdout_parent, + } = open_stdio(command.stdout, STD_OUTPUT_HANDLE, true)?; + let StdioPair { + child: stderr_child, + parent: stderr_parent, + } = open_stdio(command.stderr, STD_ERROR_HANDLE, true)?; + + // The inherited-handle list may not contain duplicates. + let mut inherited: Vec = vec![stdin_child.0]; + for handle in [stdout_child.0, stderr_child.0] { + if !inherited.contains(&handle) { + inherited.push(handle); + } + } + let jobs = [job]; + let mut attributes = AttributeList::new(2)?; + attributes + .set(PROC_THREAD_ATTRIBUTE_JOB_LIST, &jobs) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("name the sandbox job for process creation: {error}"), + ) + })?; + attributes + .set(PROC_THREAD_ATTRIBUTE_HANDLE_LIST, &inherited) + .map_err(|error| { + io::Error::new( + error.kind(), + format!("limit the handles the process inherits: {error}"), + ) + })?; + + let mut startup = STARTUPINFOEXW::default(); + startup.StartupInfo.cb = u32::try_from(size_of::()).unwrap_or(0); + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + startup.StartupInfo.hStdInput = stdin_child.0; + startup.StartupInfo.hStdOutput = stdout_child.0; + startup.StartupInfo.hStdError = stderr_child.0; + startup.lpAttributeList = attributes.list(); + + let mut flags: PROCESS_CREATION_FLAGS = + EXTENDED_STARTUPINFO_PRESENT | CREATE_UNICODE_ENVIRONMENT; + if suspended { + flags |= CREATE_SUSPENDED; + } + let mut info = PROCESS_INFORMATION::default(); + // SAFETY: every pointer refers to a value that outlives the call; the attribute + // list, handle arrays and strings are kept alive until after it returns. + unsafe { + CreateProcessW( + PCWSTR(application.as_ptr()), + Some(PWSTR(command_line.as_mut_ptr())), + None, + None, + true, + flags, + Some(environment.as_ptr().cast::()), + PCWSTR::null(), + &raw const startup.StartupInfo, + &raw mut info, + ) + } + .map_err(|error| { + let source = io::Error::from(error); + // Keep the kind (not found, access denied, bad image) for callers that match on it. + io::Error::new( + source.kind(), + format!( + "create process {} in its sandbox job: {source}", + program.display() + ), + ) + })?; + drop(attributes); + + let process = Arc::new(OwnedHandle(info.hProcess)); + let thread = OwnedHandle(info.hThread); + // The new process has its own copies of the child ends; close ours now so a reader + // sees end-of-file when the process ends. + drop((stdin_child, stdout_child, stderr_child)); + + let (sender, receiver) = watch::channel(None); + let watched = Arc::clone(&process); + let spawned = std::thread::Builder::new() + .name(format!("mxc-wait-{}", info.dwProcessId)) + .spawn(move || { + // SAFETY: `watched` keeps the process handle valid for the whole wait. + let exit = unsafe { + let mut code = 0u32; + if WaitForSingleObject(watched.0, INFINITE) == WAIT_OBJECT_0 + && GetExitCodeProcess(watched.0, &raw mut code).is_ok() + { + Exit::Code(code) + } else { + Exit::Unknown + } + }; + let _ = sender.send(Some(exit)); + }); + if let Err(error) = spawned { + // SAFETY: the process handle is valid; end the process nobody can watch. + let _ = unsafe { TerminateProcess(process.0, 1) }; + return Err(error); + } + + let child = GenerationChild { + process, + pid: info.dwProcessId, + exit: receiver, + _stdin: stdin_parent, + stdout: stdout_parent, + stderr: stderr_parent, + }; + Ok((child, thread)) +} + +#[cfg(test)] +#[allow(unsafe_code)] +mod tests { + use std::time::Duration; + + use tokio::io::AsyncReadExt as _; + use windows::Win32::Foundation::{ + CompareObjectHandles, GetHandleInformation, HLOCAL, LocalFree, WAIT_TIMEOUT, + }; + use windows::Win32::Storage::FileSystem::CREATE_NEW; + use windows::Win32::System::Threading::{OpenProcess, PROCESS_SYNCHRONIZE}; + use windows::Win32::UI::Shell::CommandLineToArgvW; + + use super::*; + use crate::job::GenerationJob; + + fn quiet(program: &str, args: &[&str]) -> ChildCommand { + let mut command = ChildCommand::new(program); + command + .args(args) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + command + } + + fn is_alive(pid: u32) -> bool { + // SAFETY: plain Win32 calls; the handle is closed before returning. + unsafe { + let Ok(handle) = OpenProcess(PROCESS_SYNCHRONIZE, false, pid) else { + return false; + }; + let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT; + let _ = CloseHandle(handle); + alive + } + } + + /// Split a command line the way Windows does for every ordinary program. + fn windows_argv(line: &[u16]) -> Vec { + let mut count = 0i32; + // SAFETY: `line` is NUL-terminated; the returned array is freed once. + unsafe { + let argv = CommandLineToArgvW(PCWSTR(line.as_ptr()), &raw mut count); + assert!(!argv.is_null(), "CommandLineToArgvW failed"); + let parsed = (0..usize::try_from(count).unwrap()) + .map(|i| { + let pointer = (*argv.add(i)).0; + let mut len = 0; + while *pointer.add(len) != 0 { + len += 1; + } + OsString::from_wide(std::slice::from_raw_parts(pointer, len)) + }) + .collect(); + let _ = LocalFree(Some(HLOCAL(argv.cast()))); + parsed + } + } + + use std::os::windows::ffi::OsStringExt as _; + + #[test] + fn command_line_round_trips_through_the_windows_parser() { + let tricky = [ + "", + "plain", + "with space", + "tab\there", + r#"a"b"#, + r"a\b", + r"ends with backslash\", + r"ends with backslashes\\", + r"\\server\share\file", + r#"quote then backslash \" end"#, + r"\", + r#"say "hello" now"#, + r"C:\Program Files\Agent\agent.exe", + "unicode é 日本語", + ]; + let program = PathBuf::from(r"C:\Program Files\Host\tool.exe"); + let mut command = ChildCommand::new(&program); + for arg in tricky { + command.arg(arg); + } + let line = build_command_line(&command, &program).expect("command line"); + let parsed = windows_argv(&line); + let expected: Vec = std::iter::once(program.into_os_string()) + .chain(tricky.iter().map(OsString::from)) + .collect(); + assert_eq!(parsed, expected); + } + + #[test] + fn raw_arguments_are_appended_exactly() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.arg("one").raw_arg(r#"two "three four" /s"#); + let line = build_command_line(&command, &program).expect("command line"); + assert_eq!( + String::from_utf16_lossy(&line[..line.len() - 1]), + r#""C:\x\tool.exe" one two "three four" /s"# + ); + } + + #[test] + fn nul_in_an_argument_is_rejected() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.arg(OsString::from_wide(&[u16::from(b'a'), 0, u16::from(b'b')])); + assert!(build_command_line(&command, &program).is_err()); + } + + /// The entries of an environment block as `NAME=value` strings. + fn block_entries(block: &[u16]) -> Vec { + block + .split(|&unit| unit == 0) + .filter(|entry| !entry.is_empty()) + .map(String::from_utf16_lossy) + .collect() + } + + /// The variable name of an environment entry. A name may itself start with `=` + /// (Windows keeps per-drive directories as `=C:=C:\dir`). + fn entry_name(entry: &str) -> &str { + let search_from = usize::from(entry.starts_with('=')); + entry[search_from..] + .find('=') + .map_or(entry, |at| &entry[..at + search_from]) + } + + #[test] + fn environment_block_merges_host_and_overrides_in_windows_order() { + // A synthetic host, so the result does not depend on where the test runs: it + // deliberately holds an OPENSHELL_ setting, a drive-directory entry and a + // lower-case name. + let host = vec![ + (OsString::from("Path"), OsString::from(r"C:\x")), + ( + OsString::from("OPENSHELL_WXC_EXEC_PATH"), + OsString::from(r"C:\mxc\wxc-exec.exe"), + ), + (OsString::from("=C:"), OsString::from(r"C:\work")), + (OsString::from("openshell_b"), OsString::from("0")), + ]; + let overrides = vec![ + (OsString::from("Openshell_B"), OsString::from("2")), + (OsString::from("OPENSHELL_a"), OsString::from("1")), + // Same name as the first, different case: the later value replaces it. + (OsString::from("openshell_b"), OsString::from("3")), + ]; + + let block = build_env_block_from(host, &overrides).expect("block"); + + assert!(block.ends_with(&[0, 0]), "the block ends with a double NUL"); + // Unrelated host entries are kept, overrides win over the host's own value, and + // the whole block is in case-insensitive name order. + assert_eq!( + block_entries(&block), + vec![ + r"=C:=C:\work", + "OPENSHELL_a=1", + "openshell_b=3", + r"OPENSHELL_WXC_EXEC_PATH=C:\mxc\wxc-exec.exe", + r"Path=C:\x", + ] + ); + } + + #[test] + fn environment_block_is_sorted_for_whatever_host_it_runs_on() { + // The real host: whatever it holds, the block must be in Windows order and every + // host variable must survive. + let block = build_env_block(&[]).expect("block"); + let entries = block_entries(&block); + let keys: Vec> = entries + .iter() + .map(|entry| env_sort_key(OsStr::new(entry_name(entry)))) + .collect(); + assert!( + keys.windows(2).all(|pair| pair[0] <= pair[1]), + "entries must be sorted by name, case-insensitively" + ); + for (name, _) in std::env::vars_os() { + let wanted = env_sort_key(&name); + assert!( + keys.contains(&wanted), + "host variable {name:?} is missing from the block" + ); + } + } + + /// Whether `handle` is still an open handle of this process. + fn handle_is_open(handle: RawHandle) -> bool { + let mut flags = 0u32; + // SAFETY: a plain query; `flags` outlives the call. + unsafe { GetHandleInformation(HANDLE(handle), &raw mut flags) }.is_ok() + } + + #[test] + fn a_failed_pipe_registration_leaves_the_handle_to_its_new_owner() { + // `register` owns the handle even when it fails, so `make_pipe_with` must not + // close it again: a second close could hit an unrelated handle that another + // thread was given the same value. This registrar fails without closing, which + // makes a second close visible. + let seen = std::cell::Cell::new(std::ptr::null_mut()); + let result = make_pipe_with(true, |raw| { + seen.set(raw); + Err(io::Error::other("registration failed")) + }); + let raw = seen.get(); + assert!(!raw.is_null(), "the registrar was never reached"); + let still_open = handle_is_open(raw); + // SAFETY: the registrar did not close it, so it is ours to close here. + let _ = unsafe { CloseHandle(HANDLE(raw)) }; + + assert!(result.is_err(), "the registration error must be returned"); + assert!( + still_open, + "make_pipe_with closed a handle it no longer owned" + ); + } + + #[test] + fn tokio_closes_the_pipe_handle_itself_when_registration_fails() { + // The contract the test above relies on, against the real tokio: from a stopped + // runtime the registration fails, and the handle is closed by tokio. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + let handle = runtime.handle().clone(); + drop(runtime); + let _entered = handle.enter(); + + let seen = std::cell::Cell::new(std::ptr::null_mut()); + let result = make_pipe_with(true, |raw| { + seen.set(raw); + // SAFETY: `raw` is an open overlapped pipe handle that nothing else owns. + unsafe { NamedPipeServer::from_raw_handle(raw) } + }); + + assert!( + result.is_err(), + "registering with a stopped runtime must fail" + ); + let raw = seen.get(); + assert!(!raw.is_null(), "the registrar was never reached"); + assert!( + !handle_is_open(raw), + "tokio is expected to have closed the handle it could not register" + ); + } + + #[test] + fn raw_arguments_given_twice_are_both_kept() { + let program = PathBuf::from(r"C:\x\tool.exe"); + let mut command = ChildCommand::new(&program); + command.raw_arg("first").raw_arg(r#"second "third""#); + let line = build_command_line(&command, &program).expect("command line"); + assert_eq!( + String::from_utf16_lossy(&line[..line.len() - 1]), + r#""C:\x\tool.exe" first second "third""# + ); + } + + #[test] + fn environment_names_that_would_corrupt_the_block_are_rejected() { + for bad in ["", "A=B", "=X"] { + let overrides = vec![(OsString::from(bad), OsString::from("v"))]; + let error = build_env_block_from(Vec::new(), &overrides).expect_err(bad); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput, "{bad:?}"); + } + } + + #[test] + fn environment_names_sort_the_way_windows_compares_them() { + // One unit at a time: 'ß' does not become "SS", and a lone surrogate is kept. + assert_eq!(env_sort_key(OsStr::new("\u{df}")), vec![0x00DF]); + assert_eq!(env_sort_key(OsStr::new("a\u{df}")), vec![0x0041, 0x00DF]); + let lone = OsString::from_wide(&[0xD800]); + assert_eq!(env_sort_key(&lone), vec![0xD800]); + } + + #[test] + fn only_absolute_path_entries_are_searched() { + let path = OsString::from(r"C:\one;;.;relative\dir;D:\two"); + assert_eq!( + usable_path_entries(&path), + vec![PathBuf::from(r"C:\one"), PathBuf::from(r"D:\two")] + ); + } + + #[test] + fn a_program_name_without_an_extension_means_its_exe() { + let dir = std::env::temp_dir().join(format!("mxc-resolve-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + // A script-like file with the bare name must not shadow the executable. + std::fs::write(dir.join("tool"), b"not a program").unwrap(); + std::fs::write(dir.join("tool.exe"), b"MZ").unwrap(); + let path = OsString::from(dir.as_os_str()); + + let found = resolve_program(OsStr::new("tool"), Some(&path)); + std::fs::remove_file(dir.join("tool.exe")).unwrap(); + let only_bare = resolve_program(OsStr::new("tool"), Some(&path)); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(found.unwrap(), dir.join("tool.exe")); + assert_eq!(only_bare.unwrap_err().kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn the_new_process_path_decides_where_its_program_is_found() { + let dir = std::env::temp_dir().join(format!("mxc-pathenv-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let system = PathBuf::from(std::env::var_os("SystemRoot").unwrap()) + .join("System32") + .join("cmd.exe"); + std::fs::copy(system, dir.join("mxc-pathenv-tool.exe")).unwrap(); + + let job = GenerationJob::new().expect("job"); + let mut command = quiet("mxc-pathenv-tool", &["/d", "/c", "exit 5"]); + command.env("Path", &dir); + let mut child = job.spawn(&command).expect("found through the new PATH"); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("exited") + .unwrap(); + drop(child); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(status.code(), Some(5)); + } + + #[tokio::test] + async fn a_batch_file_is_refused_with_a_clear_error() { + let batch = std::env::temp_dir().join(format!("mxc-batch-{}.cmd", std::process::id())); + std::fs::write(&batch, b"@echo off").unwrap(); + let job = GenerationJob::new().expect("job"); + let error = job + .spawn(&quiet(batch.to_str().unwrap(), &[])) + .err() + .expect("must fail"); + let _ = std::fs::remove_file(&batch); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert!(error.to_string().contains("batch file"), "{error}"); + } + + #[tokio::test] + async fn killing_a_process_that_has_ended_is_not_an_error_before_the_watcher_reports() { + // `TerminateProcess` on a process that has ended fails with "access denied". The + // exit watcher publishes the exit a moment later, so `start_kill` must ask the + // process itself. A channel nobody publishes on stands in for a watcher that has + // not run yet, which makes the moment deterministic. + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 0"])) + .expect("spawn"); + let (_unused_sender, never_reports) = watch::channel(None); + child.exit = never_reports; + // SAFETY: the handle is owned by `child.process` and valid. + let ended = unsafe { WaitForSingleObject(child.process.0, 10_000) }; + assert_eq!(ended, WAIT_OBJECT_0, "the process should have ended"); + + child + .start_kill() + .expect("an ended process is not a kill failure"); + } + + #[tokio::test] + async fn an_exit_the_watcher_could_not_read_is_an_error_not_a_clean_exit() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 0"])) + .expect("spawn"); + let (_sender, unknown) = watch::channel(Some(Exit::Unknown)); + child.exit = unknown; + + let result = child.wait().await; + + assert!(result.is_err(), "reported {result:?} for an unknown exit"); + } + + #[tokio::test] + async fn exit_status_is_reported() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet("cmd.exe", &["/d", "/c", "exit 7"])) + .expect("spawn"); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("exited") + .unwrap(); + assert_eq!(status.code(), Some(7)); + // Waiting again returns the same answer. + assert_eq!(child.wait().await.unwrap().code(), Some(7)); + } + + #[tokio::test] + async fn piped_output_is_readable_until_the_child_exits() { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new("cmd.exe"); + command + .args(["/d", "/c", "echo out& echo err 1>&2"]) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped); + let mut child = job.spawn(&command).expect("spawn"); + let mut stdout = child.stdout.take().expect("stdout"); + let mut stderr = child.stderr.take().expect("stderr"); + let (mut out, mut err) = (String::new(), String::new()); + tokio::time::timeout(Duration::from_secs(10), async { + let _ = stdout.read_to_string(&mut out).await; + let _ = stderr.read_to_string(&mut err).await; + }) + .await + .expect("both streams reach end of file when the child exits"); + assert_eq!(out.trim(), "out"); + assert_eq!(err.trim(), "err"); + assert!(child.wait().await.unwrap().success()); + } + + #[tokio::test] + async fn environment_overrides_reach_the_child() { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new("cmd.exe"); + command + .args(["/d", "/c", "echo %OPENSHELL_CHILD_TEST%"]) + .env("OPENSHELL_CHILD_TEST", "value with spaces") + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Null); + let mut child = job.spawn(&command).expect("spawn"); + let mut out = String::new(); + let _ = child.stdout.take().unwrap().read_to_string(&mut out).await; + assert_eq!(out.trim(), "value with spaces"); + } + + #[tokio::test] + async fn a_missing_program_is_reported() { + let job = GenerationJob::new().expect("job"); + let error = job + .spawn(&quiet("definitely-not-a-real-binary", &[])) + .err() + .expect("must fail"); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn kill_ends_the_process_and_wait_reports_it() { + let job = GenerationJob::new().expect("job"); + let mut child = job + .spawn(&quiet( + "cmd.exe", + &["/d", "/c", "ping -n 30 127.0.0.1 >nul"], + )) + .expect("spawn"); + tokio::time::timeout(Duration::from_secs(10), child.kill()) + .await + .expect("killed") + .unwrap(); + assert!(!is_alive(child.id().unwrap())); + // Killing an already-dead process is not an error. + child.start_kill().unwrap(); + } + + #[tokio::test] + async fn dropping_a_child_kills_it() { + let job = GenerationJob::new().expect("job"); + let child = job + .spawn(&quiet( + "cmd.exe", + &["/d", "/c", "ping -n 30 127.0.0.1 >nul"], + )) + .expect("spawn"); + let pid = child.id().unwrap(); + assert!(is_alive(pid)); + drop(child); + for _ in 0..100 { + if !is_alive(pid) { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("the dropped child is still running"); + } + + /// A handle to a scratch file that a child could inherit. + fn open_inheritable(path: &Path) -> HANDLE { + let sa = inheritable(); + let name = to_wide(path.as_os_str()).unwrap(); + // SAFETY: `name` and `sa` outlive the call. + unsafe { + CreateFileW( + PCWSTR(name.as_ptr()), + FILE_GENERIC_WRITE.0, + FILE_SHARE_MODE(0), + Some(&raw const sa), + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + None, + ) + } + .expect("create inheritable file") + } + + /// Whether `child_process` holds a copy of `ours`. An inherited handle keeps its + /// value, so duplicate the handle of that value out of the child and check that it is + /// the same kernel object (another object that happens to have the same value does not + /// count). This looks at that one process only, so processes other tests start at the + /// same time cannot affect the answer. + fn child_holds_copy_of(child_process: HANDLE, ours: HANDLE) -> bool { + let mut copy = HANDLE::default(); + // SAFETY: both handles are valid; `copy` outlives the call and is closed below. + unsafe { + if DuplicateHandle( + child_process, + ours, + GetCurrentProcess(), + &raw mut copy, + 0, + false, + DUPLICATE_SAME_ACCESS, + ) + .is_err() + { + return false; + } + let same = CompareObjectHandles(ours, copy).as_bool(); + let _ = CloseHandle(copy); + same + } + } + + /// Only the standard handles may reach a sandbox process. A handle the host holds + /// (here an inheritable file handle) must not be inherited, as the host's listening + /// socket once was. The control shows the check can see an inherited handle. + #[tokio::test] + async fn only_the_standard_handles_are_inherited() { + let scratch = + std::env::temp_dir().join(format!("openshell-inherit-{}.tmp", std::process::id())); + let handle = open_inheritable(&scratch); + + // Control: the ordinary way of starting a process inherits the handle. + let mut ordinary = tokio::process::Command::new("ping.exe") + .args(["-n", "30", "127.0.0.1"]) + .stdout(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("ordinary child"); + let ordinary_process = HANDLE(ordinary.raw_handle().expect("process handle")); + let control = child_holds_copy_of(ordinary_process, handle); + let _ = ordinary.kill().await; + + // Ours: the same handle is not inherited. + let job = GenerationJob::new().expect("job"); + let child = job + .spawn(&quiet("ping.exe", &["-n", "30", "127.0.0.1"])) + .expect("spawn"); + let ours_process = HANDLE(child.raw_handle().expect("process handle")); + let inherited = child_holds_copy_of(ours_process, handle); + drop(child); + + // SAFETY: the handle is valid and closed once. + unsafe { CloseHandle(handle) }.unwrap(); + let _ = std::fs::remove_file(&scratch); + + assert!( + control, + "control: an ordinary child should inherit the handle" + ); + assert!( + !inherited, + "a sandbox process must not hold the host's other handles" + ); + } +} diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index b2ae367bfd..1a6dc6f139 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -11,6 +11,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use crate::child::{ChildCommand, ChildStdio, GenerationChild}; use base64::Engine as _; use futures::Stream; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; @@ -27,7 +28,6 @@ use openshell_sandbox_backend::boundary_protocol::{ }; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncBufReadExt as _, BufReader}; -use tokio::process::{Child, Command}; use tokio::sync::{Mutex, broadcast, mpsc, oneshot, watch}; use tokio_stream::wrappers::ReceiverStream; use tracing::{info, warn}; @@ -757,7 +757,13 @@ async fn run_lifecycle_inner( // ProcessContainer launch so unrelated local processes cannot squat it // during the more expensive host-side setup. drop(proxy_reservation); - let mut supervisor = spawn_supervisor(context, &descriptor_path, &auth_bundle_path, &main)?; + // One kill-on-close job per generation owns the supervisor and the whole + // boundary tree. Every early return below drops it, which ends whatever was + // already started. + let job = crate::job::GenerationJob::new() + .map_err(|error| format!("create sandbox job object: {error}"))?; + let mut supervisor = + spawn_supervisor(context, &descriptor_path, &auth_bundle_path, &main, &job)?; let sandbox_command = encode_windows_command_line(&[ context.config.sandbox_binary_path.clone(), "--bootstrap".to_string(), @@ -795,6 +801,7 @@ async fn run_lifecycle_inner( process, Some(network), mapped.ui, + &job, ) .await .map_err(|error| format!("start MXC ProcessContainer: {error}"))?; @@ -838,6 +845,10 @@ async fn run_lifecycle_inner( } drop(startup_guard); let result = monitor_runtime_pair(boundary, supervisor, shutdown_rx).await; + // Whatever ended the pair, end the rest of the tree (workload, conhost, + // grandchildren) that `Child::kill` on the two roots does not reach. + job.terminate(); + drop(job); let _ = terminated_tx.send(true); if let Ok(mut attribution) = context.attribution.lock() { attribution.forget(&context.sandbox.id); @@ -859,9 +870,10 @@ fn spawn_supervisor( descriptor_path: &Path, auth_bundle_path: &Path, main: &openshell_core::sandbox_env::MainProcessConfig, -) -> Result { - supervisor_command(context, descriptor_path, auth_bundle_path, main)? - .spawn() + job: &crate::job::GenerationJob, +) -> Result { + let command = supervisor_command(context, descriptor_path, auth_bundle_path, main)?; + job.spawn(&command) .map_err(|error| format!("start host openshell-supervisor: {error}")) } @@ -870,14 +882,14 @@ fn supervisor_command( descriptor_path: &Path, auth_bundle_path: &Path, main: &openshell_core::sandbox_env::MainProcessConfig, -) -> Result { +) -> Result { let main_process_spec = mxc_main_process_spec(main)?; - let mut command = Command::new(&context.config.supervisor_binary_path); + // A `GenerationChild` is killed when dropped, as `kill_on_drop(true)` did. + let mut command = ChildCommand::new(&context.config.supervisor_binary_path); command - .kill_on_drop(true) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped) .arg("--role") .arg("isolation-backend") .arg("--backend-descriptor-file") @@ -951,8 +963,8 @@ enum RuntimePairExit { } async fn monitor_runtime_pair( - mut boundary: Child, - mut supervisor: Child, + mut boundary: GenerationChild, + mut supervisor: GenerationChild, mut shutdown: oneshot::Receiver<()>, ) -> RuntimePairExit { tokio::select! { @@ -1531,7 +1543,7 @@ async fn wait_for_termination( } } -fn attach_child_logs(sandbox_name: &str, component: &'static str, child: &mut Child) { +fn attach_child_logs(sandbox_name: &str, component: &'static str, child: &mut GenerationChild) { if let Some(stdout) = child.stdout.take() { let sandbox_name = sandbox_name.to_string(); tokio::spawn(async move { @@ -1901,10 +1913,9 @@ mod tests { ) .unwrap(); let tls_dir = command - .as_std() .get_envs() .find(|(key, _)| *key == openshell_core::sandbox_env::PROXY_TLS_DIR) - .and_then(|(_, value)| value); + .map(|(_, value)| value); assert_eq!( tls_dir.map(Path::new), Some(host_state_dir.join(HOST_PROXY_TLS_DIR).as_path()) @@ -2829,17 +2840,20 @@ mod tests { #[tokio::test] async fn stop_terminates_and_reaps_both_runtime_processes() { + let job = crate::job::GenerationJob::new().expect("job"); let spawn = || { - Command::new("powershell.exe") + let mut command = ChildCommand::new("powershell.exe"); + command .args([ "-NoProfile", "-NonInteractive", "-Command", "Start-Sleep -Seconds 60", ]) - .kill_on_drop(true) - .spawn() - .expect("real test process") + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + job.spawn(&command).expect("real test process") }; let boundary = spawn(); let supervisor = spawn(); diff --git a/crates/openshell-driver-mxc/src/etw_consumer.rs b/crates/openshell-driver-mxc/src/etw_consumer.rs index 018a208a9e..e0339e50b8 100644 --- a/crates/openshell-driver-mxc/src/etw_consumer.rs +++ b/crates/openshell-driver-mxc/src/etw_consumer.rs @@ -1305,7 +1305,9 @@ const RETIRED_CORRELATION_TTL: Duration = PENDING_TTL; /// Query the kernel generation key for a spawned child while the driver still /// owns its process handle. This is the same value ETW appends under /// `EVENT_HEADER_EXT_TYPE_PROCESS_START_KEY`. -pub(crate) fn child_process_start_key(child: &tokio::process::Child) -> Result { +pub(crate) fn child_process_start_key( + child: &impl crate::child::ProcessHandle, +) -> Result { let raw_handle = child .raw_handle() .ok_or_else(|| "wxc-exec child handle is no longer available".to_string())?; diff --git a/crates/openshell-driver-mxc/src/job.rs b/crates/openshell-driver-mxc/src/job.rs new file mode 100644 index 0000000000..2916acd8ef --- /dev/null +++ b/crates/openshell-driver-mxc/src/job.rs @@ -0,0 +1,461 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One kill-on-close job object per sandbox generation. +//! +//! A generation's process tree is `openshell-supervisor`, `wxc-exec`, the +//! in-container `openshell-windows-sandbox` and the workload with its +//! children. Putting the root processes in a job with +//! `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` makes Windows end the whole tree when +//! the last handle to the job closes: on stop/delete (explicit +//! [`GenerationJob::terminate`]), when the lifecycle task ends, or when the +//! driver's host process dies for any reason. Children inherit job membership +//! and `BREAKAWAY_OK` is not set, so nothing can leave the job. +//! +//! The root processes are created as members of the job +//! (`crate::child::spawn_in_job`), so there is no moment at which a process of the +//! generation exists outside its job, even if the driver is killed during the spawn. +//! +//! The job lives with the driver rather than with whichever process hosts it, +//! so the cleanup survives moving the driver out of the gateway. + +// `pub(crate)` documents what the driver may use; this module is private. +#![allow(clippy::redundant_pub_crate)] + +use std::io; +use std::mem::size_of; + +use windows::Win32::Foundation::{CloseHandle, HANDLE}; +use windows::Win32::System::JobObjects::{ + CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JobObjectExtendedLimitInformation, SetInformationJobObject, TerminateJobObject, +}; +use windows::core::PCWSTR; + +use crate::child::{ChildCommand, GenerationChild}; + +/// Owns a kill-on-close job object. Dropping it ends every member process. +pub struct GenerationJob { + handle: HANDLE, +} + +// SAFETY: a job object handle is a process-wide kernel handle with no thread +// affinity; every operation used here is thread-safe. +#[allow(unsafe_code)] +unsafe impl Send for GenerationJob {} +#[allow(unsafe_code)] +unsafe impl Sync for GenerationJob {} + +#[allow(unsafe_code)] // Win32 job object calls. +impl GenerationJob { + pub(crate) fn new() -> io::Result { + // SAFETY: plain Win32 calls with valid arguments; `info` outlives the + // call and the size matches its type. No security attributes are + // passed, so the handle is not inheritable and children cannot keep + // the job open. + unsafe { + let handle = CreateJobObjectW(None, PCWSTR::null()).map_err(io::Error::from)?; + let job = Self { handle }; + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + SetInformationJobObject( + job.handle, + JobObjectExtendedLimitInformation, + (&raw const info).cast(), + u32::try_from(size_of::()) + .map_err(io::Error::other)?, + ) + .map_err(io::Error::from)?; + Ok(job) + } + } + + /// Start `command` as a member of this job. + /// + /// The process is created inside the job (the job is named in the + /// `CreateProcessW` call), so it is never outside it. If it cannot be created + /// there, the spawn fails and no process exists: a sandbox whose processes could + /// outlive the driver must not start. + /// + /// Call inside a tokio runtime. + pub(crate) fn spawn(&self, command: &ChildCommand) -> io::Result { + crate::child::spawn_in_job(self.handle, command) + } + + /// End every process in the job now. + pub(crate) fn terminate(&self) { + // SAFETY: valid job handle. An already-empty job is not an error worth + // reporting. + let _ = unsafe { TerminateJobObject(self.handle, 1) }; + } + + #[cfg(test)] + pub(crate) fn raw(&self) -> HANDLE { + self.handle + } +} + +impl Drop for GenerationJob { + #[allow(unsafe_code)] + fn drop(&mut self) { + // Closing the last handle triggers KILL_ON_JOB_CLOSE. + // SAFETY: `handle` came from `CreateJobObjectW` and is closed once. + let _ = unsafe { CloseHandle(self.handle) }; + } +} + +#[cfg(test)] +#[allow(unsafe_code)] +mod tests { + use std::time::Duration; + + use windows::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, PROCESSENTRY32W, Process32FirstW, Process32NextW, + TH32CS_SNAPPROCESS, + }; + use windows::Win32::System::JobObjects::IsProcessInJob; + + use super::*; + use crate::child::{ChildStdio, spawn_suspended_in_job}; + + fn sleeper() -> ChildCommand { + let mut command = ChildCommand::new("cmd.exe"); + command + .args(["/d", "/c", "ping -n 30 127.0.0.1 >nul"]) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + command + } + + #[tokio::test] + async fn spawned_process_runs_and_is_killed_by_terminate() { + let job = GenerationJob::new().expect("job"); + let mut child = job.spawn(&sleeper()).expect("spawn in job"); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + tokio::time::timeout(Duration::from_millis(1), child.wait()) + .await + .is_err(), + "child should be running" + ); + job.terminate(); + let status = tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("child ended after terminate") + .unwrap(); + assert!(!status.success()); + } + + #[tokio::test] + async fn dropping_the_job_kills_the_whole_tree() { + let job = GenerationJob::new().expect("job"); + // cmd.exe starts ping.exe: a grandchild that inherits membership. + let mut child = job.spawn(&sleeper()).expect("spawn in job"); + tokio::time::sleep(Duration::from_millis(300)).await; + drop(job); + // Closing the job ends members with exit code 0, so only that the + // process ended (not how) can be asserted. + tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("child ended after the job closed") + .unwrap(); + } + + /// Fail closed: if the process cannot be created inside its job, the spawn fails + /// instead of handing back a process that could outlive the driver. + #[tokio::test] + async fn an_unusable_job_fails_the_spawn() { + let job = GenerationJob { + handle: HANDLE::default(), + }; + let error = job.spawn(&sleeper()).err().expect("must fail closed"); + assert!( + error.to_string().contains("sandbox job"), + "unexpected error: {error}" + ); + std::mem::forget(job); // do not CloseHandle the invalid handle + } + + /// The property the job exists for, and the case a review found: the process is a + /// member of its job from the moment it exists, before it has run one instruction. + /// A process created the ordinary way is not, so the check can fail. + #[tokio::test] + async fn child_is_a_member_of_its_job_before_it_runs() { + let job = GenerationJob::new().expect("job"); + let (child, thread) = + spawn_suspended_in_job(job.raw(), &sleeper()).expect("suspended spawn"); + let handle = HANDLE(child.raw_handle().expect("process handle")); + let mut in_job = windows::core::BOOL::default(); + // SAFETY: both handles are valid for the call. + unsafe { IsProcessInJob(handle, Some(job.raw()), &raw mut in_job) } + .expect("IsProcessInJob"); + assert!( + in_job.as_bool(), + "a freshly created, not yet running child must already be in the job" + ); + thread.resume().expect("resume"); + drop(child); + + // Negative control: a process created the ordinary way is not in the job. + let mut outsider = tokio::process::Command::new("cmd.exe") + .args(["/d", "/c", "ping -n 30 127.0.0.1 >nul"]) + .kill_on_drop(true) + .spawn() + .expect("outsider"); + let outsider_handle = HANDLE(outsider.raw_handle().expect("outsider handle")); + let mut outsider_in_job = windows::core::BOOL::default(); + // SAFETY: both handles are valid for the call. + unsafe { IsProcessInJob(outsider_handle, Some(job.raw()), &raw mut outsider_in_job) } + .expect("IsProcessInJob"); + assert!( + !outsider_in_job.as_bool(), + "the check must be able to say no" + ); + let _ = outsider.kill().await; + } + + // ---- Hard-killing the process that owns the job ---- + + /// Set (by the tests below) on the re-run of this test binary that plays the + /// role of the driver's host process. + const HOST_ENV: &str = "OPENSHELL_JOB_TEST_HOST"; + const STRESS_ENV: &str = "OPENSHELL_JOB_TEST_STRESS_EXE"; + + fn is_alive(pid: u32) -> bool { + use windows::Win32::Foundation::WAIT_TIMEOUT; + use windows::Win32::System::Threading::{ + OpenProcess, PROCESS_SYNCHRONIZE, WaitForSingleObject, + }; + // SAFETY: plain Win32 calls; the handle is closed before returning. + unsafe { + let Ok(handle) = OpenProcess(PROCESS_SYNCHRONIZE, false, pid) else { + return false; // no such process any more + }; + let alive = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT; + let _ = CloseHandle(handle); + alive + } + } + + fn child_pids(parent: u32) -> Vec { + // SAFETY: the snapshot is closed before returning; `dwSize` is set. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0).unwrap(); + let mut entry = PROCESSENTRY32W { + dwSize: u32::try_from(size_of::()).unwrap(), + ..Default::default() + }; + let mut found = Vec::new(); + let mut more = Process32FirstW(snapshot, &raw mut entry).is_ok(); + while more { + if entry.th32ParentProcessID == parent { + found.push(entry.th32ProcessID); + } + more = Process32NextW(snapshot, &raw mut entry).is_ok(); + } + let _ = CloseHandle(snapshot); + found + } + } + + /// How many running processes have this executable file name. + fn count_named(exe_name: &str) -> usize { + // SAFETY: the snapshot is closed before returning; `dwSize` is set. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0).unwrap(); + let mut entry = PROCESSENTRY32W { + dwSize: u32::try_from(size_of::()).unwrap(), + ..Default::default() + }; + let mut count = 0; + let mut more = Process32FirstW(snapshot, &raw mut entry).is_ok(); + while more { + let len = entry + .szExeFile + .iter() + .position(|&c| c == 0) + .unwrap_or(entry.szExeFile.len()); + if String::from_utf16_lossy(&entry.szExeFile[..len]).eq_ignore_ascii_case(exe_name) + { + count += 1; + } + more = Process32NextW(snapshot, &raw mut entry).is_ok(); + } + let _ = CloseHandle(snapshot); + count + } + } + + /// Plays the driver host: owns a job, starts a process tree in it, reports + /// the PIDs, then waits to be killed. A no-op in a normal test run. + #[tokio::test] + async fn hard_kill_host_helper() { + if std::env::var_os(HOST_ENV).is_none() { + return; + } + let job = GenerationJob::new().expect("job"); + let child = job.spawn(&sleeper()).expect("spawn in job"); + let root = child.id().expect("pid"); + // Let cmd start ping: wait for it, not for a fixed time a loaded host may exceed. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while child_pids(root).is_empty() && std::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + let grandchildren = child_pids(root) + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + println!("TREE {root} {grandchildren}"); + tokio::time::sleep(Duration::from_mins(2)).await; + drop(job); + } + + /// The property the job exists for: when the process that owns the job is + /// hard-killed (no destructors, no cleanup code), Windows still ends every + /// process in the tree, grandchildren included. + #[tokio::test] + async fn hard_killing_the_job_owner_ends_the_whole_tree() { + use std::io::{BufRead as _, BufReader}; + use std::process::Stdio; + if std::env::var_os(HOST_ENV).is_some() { + return; // inside the helper run + } + let mut host = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "job::tests::hard_kill_host_helper", + "--nocapture", + ]) + .env(HOST_ENV, "1") + .stdout(Stdio::piped()) + .spawn() + .expect("start helper host"); + let mut lines = BufReader::new(host.stdout.take().unwrap()).lines(); + let tree: Vec = loop { + let line = lines.next().expect("helper exited early").unwrap(); + if let Some(rest) = line.strip_prefix("TREE ") { + break rest + .split([' ', ',']) + .filter(|p| !p.is_empty()) + .map(|p| p.parse().unwrap()) + .collect(); + } + }; + assert!( + tree.len() >= 2, + "expected the root and at least one grandchild, got {tree:?}" + ); + assert!(tree.iter().all(|&pid| is_alive(pid)), "tree should be live"); + + host.kill().expect("hard-kill the host"); + host.wait().unwrap(); + + for _ in 0..100 { + if tree.iter().all(|&pid| !is_alive(pid)) { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("processes survived the hard kill of their job owner: {tree:?}"); + } + + /// A host that starts generations as fast as it can, keeping a few alive, until killed. + /// A no-op in a normal test run. + #[tokio::test] + async fn spawn_loop_host_helper() { + let Some(exe) = std::env::var_os(STRESS_ENV) else { + return; + }; + let mut live: std::collections::VecDeque<(GenerationJob, GenerationChild)> = + std::collections::VecDeque::new(); + let mut ready = false; + loop { + let job = GenerationJob::new().expect("job"); + let mut command = ChildCommand::new(&exe); + command + .args(["/d", "/c", "ping -n 60 127.0.0.1 >nul"]) + .stdin(ChildStdio::Null) + .stdout(ChildStdio::Null) + .stderr(ChildStdio::Null); + match job.spawn(&command) { + Ok(child) => { + live.push_back((job, child)); + if !ready { + // Only now has a generation really been started, so a kill after + // this point can land in a real spawn. + println!("READY"); + ready = true; + } + } + Err(error) => { + // Failing to start would make the test pass without testing anything. + println!("SPAWN_FAILED {error}"); + std::process::exit(3); + } + } + if live.len() > 6 { + live.pop_front(); + } + } + } + + /// Hard-kill a host at many different moments while it is starting generations; no + /// process of any generation may survive, including one killed mid-spawn. + #[tokio::test] + async fn hard_killing_a_host_mid_spawn_leaves_no_process() { + use std::io::{BufRead as _, BufReader}; + use std::process::Stdio; + if std::env::var_os(HOST_ENV).is_some() || std::env::var_os(STRESS_ENV).is_some() { + return; // inside a helper run + } + // A copy of cmd.exe under a name nothing else has, so survivors are countable. + let name = format!("mxc-job-stress-{}.exe", std::process::id()); + let exe = std::env::temp_dir().join(&name); + let system = std::path::PathBuf::from( + std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into()), + ) + .join("System32/cmd.exe"); + std::fs::copy(system, &exe).expect("copy cmd.exe"); + + for trial in 0..12u64 { + let mut host = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "job::tests::spawn_loop_host_helper", + "--nocapture", + ]) + .env(STRESS_ENV, &exe) + .stdout(Stdio::piped()) + .spawn() + .expect("start stress host"); + let mut lines = BufReader::new(host.stdout.take().unwrap()).lines(); + loop { + let line = lines.next().expect("host exited early").unwrap(); + assert!(!line.starts_with("SPAWN_FAILED"), "{line}"); + if line == "READY" { + break; + } + } + // Different, unrelated delays so the kill lands at different points. + tokio::time::sleep(Duration::from_millis(40 + (trial * 37) % 260)).await; + host.kill().expect("hard-kill the host"); + host.wait().unwrap(); + + let mut left = usize::MAX; + for _ in 0..100 { + left = count_named(&name); + if left == 0 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert_eq!( + left, 0, + "trial {trial}: {left} process(es) survived the hard kill of the host" + ); + } + let _ = std::fs::remove_file(&exe); + } +} diff --git a/crates/openshell-driver-mxc/src/lib.rs b/crates/openshell-driver-mxc/src/lib.rs index 9b0850b456..54f261f24d 100644 --- a/crates/openshell-driver-mxc/src/lib.rs +++ b/crates/openshell-driver-mxc/src/lib.rs @@ -17,6 +17,8 @@ #![allow(clippy::result_large_err)] +#[cfg(target_os = "windows")] +mod child; #[cfg(target_os = "windows")] mod driver; #[cfg(target_os = "windows")] @@ -24,6 +26,8 @@ mod grpc; #[cfg(target_os = "windows")] mod isolation; #[cfg(target_os = "windows")] +mod job; +#[cfg(target_os = "windows")] mod mxc; #[cfg(target_os = "windows")] mod policy; diff --git a/crates/openshell-driver-mxc/src/mxc.rs b/crates/openshell-driver-mxc/src/mxc.rs index f252493e2d..cdc9ff90fe 100644 --- a/crates/openshell-driver-mxc/src/mxc.rs +++ b/crates/openshell-driver-mxc/src/mxc.rs @@ -3,12 +3,12 @@ //! `wxc-exec` `ProcessContainer` launcher and request types. +use crate::child::{ChildCommand, ChildStdio, GenerationChild}; use base64::Engine as _; use serde::Serialize; use std::net::SocketAddr; use std::path::PathBuf; use thiserror::Error; -use tokio::process::Command; use tracing::{debug, info}; /// Stable MXC schema version shared by the mapper and live launcher. @@ -265,15 +265,15 @@ impl WxcExecInvoker { self.mock } - fn real_command(&self) -> Command { - let mut command = Command::new(&self.exec_path); + /// Both processes in the runtime pair must die if their monitor is cancelled, not + /// only when it takes the explicit shutdown branch; a `GenerationChild` is killed + /// when dropped. + fn real_command(&self) -> ChildCommand { + let mut command = ChildCommand::new(&self.exec_path); command - // Both processes in the runtime pair must die if their monitor is - // cancelled, not only when it takes the explicit shutdown branch. - .kill_on_drop(true) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); + .stdin(ChildStdio::Piped) + .stdout(ChildStdio::Piped) + .stderr(ChildStdio::Piped); command } @@ -282,7 +282,8 @@ impl WxcExecInvoker { fn mock_spawn_with_grants( process: &MxcProcess, grants: &[String], - ) -> Result { + job: &crate::job::GenerationJob, + ) -> Result { let cmd_norm = mock_normalize(&process.command_line); // The RFC 0012 boundary is launched with its granted bootstrap directory. // This check concerns bootstrap wiring, never workload authorization. @@ -298,11 +299,10 @@ impl WxcExecInvoker { |quoted| quoted.split_once('"'), ) .unwrap_or((&process.command_line, "")); - let mut cmd = Command::new(if in_policy { program } else { "cmd" }); - cmd.stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()) - .kill_on_drop(true); + let mut cmd = ChildCommand::new(if in_policy { program } else { "cmd" }); + cmd.stdin(ChildStdio::Null) + .stdout(ChildStdio::Inherit) + .stderr(ChildStdio::Inherit); if in_policy { debug!(command = %process.command_line, "mock exec: in-policy, running agent"); // Retain the already-encoded Windows argv without another quoting pass. @@ -313,7 +313,7 @@ impl WxcExecInvoker { "echo Access is denied. (out-of-policy write blocked by AppContainer) 1>&2& exit 1", ); } - let child = cmd.spawn()?; + let child = job.spawn(&cmd)?; Ok(child) } @@ -325,6 +325,7 @@ impl WxcExecInvoker { /// default-deny, so a write to any ungranted path is denied by the OS. /// /// **Stdout is raw agent output; the exit code is the agent's own exit code.** + #[allow(clippy::too_many_arguments)] pub async fn run_oneshot( &self, container_id: &str, @@ -333,7 +334,8 @@ impl WxcExecInvoker { process: MxcProcess, network: Option, ui: Option, - ) -> Result { + job: &crate::job::GenerationJob, + ) -> Result { let config = oneshot_config_json( container_id, &filesystem, @@ -348,7 +350,7 @@ impl WxcExecInvoker { .iter() .map(|p| mock_normalize(p)) .collect(); - return Self::mock_spawn_with_grants(&process, &grants); + return Self::mock_spawn_with_grants(&process, &grants, job); } let json = serde_json::to_string(&config)?; @@ -386,7 +388,9 @@ impl WxcExecInvoker { } info!(container_id = %container_id, "wxc-exec one-shot processContainer spawn"); - let child = cmd.spawn()?; + // Created inside the generation's job, so the boundary and workload tree + // cannot outlive the driver; fails closed if it cannot be created there. + let child = job.spawn(&cmd)?; Ok(child) } } @@ -411,15 +415,16 @@ mod tests { let powershell = PathBuf::from(std::env::var("SystemRoot").unwrap()) .join("System32/WindowsPowerShell/v1.0/powershell.exe"); let invoker = WxcExecInvoker::new(powershell, false); - let child = invoker - .real_command() - .args([ - "-NoProfile", - "-NonInteractive", - "-Command", - "Start-Sleep -Seconds 60", - ]) - .spawn() + let job = crate::job::GenerationJob::new().expect("job"); + let mut command = invoker.real_command(); + command.args([ + "-NoProfile", + "-NonInteractive", + "-Command", + "Start-Sleep -Seconds 60", + ]); + let child = job + .spawn(&command) .expect("launch long-lived PowerShell process"); // SAFETY: OpenProcess supplies a synchronization-only handle for the // child, retained through termination and closed exactly once below. diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index f8fcf0abf4..b34aa16ae3 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -145,5 +145,9 @@ wiremock = "0.6" # `testing` provides InMemorySpanExporter, so span assertions do not need a # collector, a network hop, or a flush barrier. opentelemetry_sdk = { workspace = true, features = ["testing"] } +# Windows-only tests read the inherit flag of the gateway's listening socket. +[target.'cfg(target_os = "windows")'.dev-dependencies] +windows = { workspace = true } + [lints] workspace = true diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index afc2f4a763..3ebfc3ec02 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -23,3 +23,49 @@ pub async fn bind_gateway_listener(address: SocketAddr) -> Result bool { + let handle = HANDLE(listener.as_raw_socket() as *mut c_void); + let mut flags = 0u32; + // SAFETY: `listener` owns a live socket handle and `flags` outlives the call. + unsafe { GetHandleInformation(handle, &raw mut flags) }.expect("GetHandleInformation"); + flags & HANDLE_FLAG_INHERIT.0 != 0 + } + + #[tokio::test] + async fn bound_gateway_listener_is_not_inheritable() { + let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + assert!( + !is_inheritable(&bound.listener), + "child processes would inherit the gateway's listening socket" + ); + } + + #[tokio::test] + async fn listener_accepts_connections() { + let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + let client = tokio::net::TcpStream::connect(bound.address); + let (accepted, connected) = tokio::join!(bound.listener.accept(), client); + accepted.expect("accept"); + connected.expect("connect"); + } +} diff --git a/tasks/scripts/test-mxc-remote-tests.ps1 b/tasks/scripts/test-mxc-remote-tests.ps1 index 0971360804..03c3fab3ec 100644 --- a/tasks/scripts/test-mxc-remote-tests.ps1 +++ b/tasks/scripts/test-mxc-remote-tests.ps1 @@ -76,4 +76,46 @@ try { Get-ChildItem -LiteralPath $cacheFixture -Directory -Recurse | Sort-Object { $_.FullName.Length } -Descending | ForEach-Object { Remove-Item -LiteralPath $_.FullName } Remove-Item -LiteralPath $cacheFixture } +# Graceful-stop decision used by the gateway-restart E2E scenario (pure function). +$e2eRunner = Join-Path $PSScriptRoot '../../crates/openshell-driver-mxc/examples/run-mxc-e2e.ps1' +$tokens = $null +$errors = $null +$ast = [Management.Automation.Language.Parser]::ParseFile($e2eRunner, [ref]$tokens, [ref]$errors) +if ($errors.Count) { throw ($errors | Out-String) } +foreach ($function in $ast.FindAll({ param($node) $node -is [Management.Automation.Language.FunctionDefinitionAst] }, $true)) { + if ($function.Name -eq 'Get-GracefulStopFailure') { Invoke-Expression $function.Extent.Text } +} +if (-not (Get-Command Get-GracefulStopFailure -ErrorAction SilentlyContinue)) { throw 'Get-GracefulStopFailure not found in the E2E runner.' } +if ($null -ne (Get-GracefulStopFailure 0 $true)) { throw 'Clean graceful stop rejected.' } +$count++ +if ((Get-GracefulStopFailure 1 $true) -notmatch 'code 1') { throw 'Nonzero exit after the shutdown message accepted.' } +$count++ +if ((Get-GracefulStopFailure -1073741510 $true) -notmatch 'code -1073741510') { throw 'Ctrl-C default-handler exit (0xC000013A) accepted as graceful.' } +$count++ +if (-not (Get-GracefulStopFailure 0 $false)) { throw 'Exit 0 without the shutdown message accepted.' } +$count++ +if ((Get-GracefulStopFailure $null $true) -notmatch 'unavailable') { throw 'Missing exit code accepted.' } +$count++ +if ((Get-GracefulStopFailure 1 $false) -notmatch 'code 1') { throw 'Nonzero exit must be reported before the missing log line.' } +$count++ +# The decision only protects the scenario if it is consulted after Ctrl-C, fails the +# scenario, and does so before the replacement gateway is started. Check that order +# in the runner's syntax tree so moving or dropping the call fails here. +function Get-CommandOffsets($Root, [string] $Name) { + return @($Root.FindAll({ param($node) $node -is [Management.Automation.Language.CommandAst] -and $node.GetCommandName() -eq $Name }, $true) | + ForEach-Object { $_.Extent.StartOffset }) +} +$ctrlC = @(Get-CommandOffsets $ast 'Send-CtrlC') +$checks = @(Get-CommandOffsets $ast 'Get-GracefulStopFailure') +$starts = @(Get-CommandOffsets $ast 'Start-Gw') +if ($ctrlC.Count -ne 1) { throw 'Expected exactly one Send-CtrlC call in the E2E runner.' } +if ($checks.Count -ne 1) { throw 'Expected exactly one Get-GracefulStopFailure call in the E2E runner.' } +if ($checks[0] -lt $ctrlC[0]) { throw 'The graceful-stop check must come after Ctrl-C is sent.' } +$restart = @($starts | Where-Object { $_ -gt $ctrlC[0] } | Select-Object -First 1) +if ($restart.Count -ne 1) { throw 'No replacement gateway start found after Ctrl-C.' } +if ($checks[0] -gt $restart[0]) { throw 'The graceful-stop check must come before the replacement gateway starts.' } +$throws = @($ast.FindAll({ param($node) $node -is [Management.Automation.Language.ThrowStatementAst] -and $node.Extent.Text -match 'stopFailure' }, $true) | + Where-Object { $_.Extent.StartOffset -gt $checks[0] -and $_.Extent.StartOffset -lt $restart[0] }) +if ($throws.Count -ne 1) { throw 'The graceful-stop check result must fail the scenario before the replacement gateway starts.' } +$count++ Write-Host "$count remote-script regression checks passed. No runtime qualification claimed."