From ac158ebe143d11e410d3cfb71bf8b3e5d01612eb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:31:40 -0800 Subject: [PATCH 01/17] test(testkit): add providers, run_python, last_json, policy readers and binaries_yaml Shared helpers for suites that run client scripts in a sandbox: attach providers from the sandbox builder, run a Python script and read its JSON result, read a sandbox's policy revisions and active policy, and write a binaries list into hand-written policy YAML. Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/context.rs | 119 +++++++++++++++++++++++++++ tests/testkit/testkit/src/policy.rs | 34 ++++++++ 2 files changed, 153 insertions(+) diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index d2c4520500..4c78fe9167 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -397,6 +397,16 @@ impl SandboxBuilder { self } + /// Attach these providers to the sandbox. + pub fn providers(mut self, providers: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.spec.providers = providers.into_iter().map(Into::into).collect(); + self + } + /// Create the sandbox with this policy instead of the gateway's default. pub fn policy(mut self, policy: proto::SandboxPolicy) -> Self { self.policy = Some(policy); @@ -542,6 +552,19 @@ impl ExecOutput { Ok(self) } + /// The last line of stdout that parses as JSON, such as the result a + /// client script prints after its other output. + pub fn last_json(&self) -> Result { + last_json_line(&self.stdout).ok_or_else(|| { + Error::msg(format!( + "{} printed no JSON line; stdout: {}; stderr: {}", + self.description, + self.stdout.trim(), + self.stderr.trim() + )) + }) + } + fn outcome(&self) -> Outcome<'_> { Outcome { what: &self.description, @@ -552,6 +575,26 @@ impl ExecOutput { } } +/// Only objects and arrays count, so a line that happens to parse, such as +/// a number, isn't taken for the result. +fn last_json_line(stdout: &str) -> Option { + stdout.lines().rev().find_map(|line| { + serde_json::from_str(line.trim()) + .ok() + .filter(|value: &serde_json::Value| value.is_object() || value.is_array()) + }) +} + +/// A script's first line that isn't blank or a license header, which names +/// it in reports instead of its full text. +fn script_summary(script: &str) -> &str { + script + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && !line.starts_with("# SPDX-")) + .unwrap_or_default() +} + impl Sandbox { pub fn name(&self) -> &str { &self.name @@ -725,6 +768,41 @@ impl Sandbox { }) } + /// Run `script` with `python3 -c` and `args`, as in a sandbox from + /// [`crate::python_image`]. Fails unless it exits 0. Read a JSON result + /// with [`ExecOutput::last_json`]. + pub async fn run_python(&self, script: &str, args: &[&str]) -> Result { + let mut command = vec!["python3", "-c", script]; + command.extend_from_slice(args); + let mut output = self.exec(command).await?; + output.description = format!( + "python3 script {:?} with {args:?} in sandbox {}", + script_summary(script), + self.name + ); + output.success() + } + + /// The sandbox's policy revisions, from `policy list --output json`. + pub async fn policy_revisions(&self) -> Result { + self.handles + .cli + .run(["policy", "list", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// The sandbox's active policy, from `policy get --full --output json`. + pub async fn active_policy(&self) -> Result { + self.handles + .cli + .run(["policy", "get", &self.name, "--full", "--output", "json"]) + .await? + .success()? + .json() + } + /// Run `command` every 250 ms until `ready` accepts its output, and return /// that output. An exec error fails at once; a timeout reports the last /// output. @@ -873,6 +951,47 @@ fn generate_run_id() -> String { mod tests { use super::*; + fn exec_output(stdout: &str) -> ExecOutput { + ExecOutput { + description: "exec in sandbox".to_string(), + exit_code: 0, + stdout: stdout.to_string(), + stderr: String::new(), + } + } + + #[test] + fn last_json_takes_the_last_object_or_array_line() { + let output = exec_output("{\"first\": 1}\nprogress\n {\"last\": true} \n42\nDONE\n"); + assert_eq!( + output.last_json().expect("a JSON line"), + serde_json::json!({"last": true}) + ); + assert_eq!( + exec_output("[1, 2]\n").last_json().expect("a JSON line"), + serde_json::json!([1, 2]) + ); + } + + #[test] + fn last_json_reports_output_without_a_json_line() { + let error = exec_output("ECHO_OK\n7\n") + .last_json() + .expect_err("no object or array"); + assert!(error.to_string().contains("ECHO_OK"), "{error}"); + } + + #[test] + fn script_summary_skips_blank_and_license_lines() { + let script = "\n# SPDX-FileCopyrightText: x\n# SPDX-License-Identifier: y\n\n# Sends a request.\nimport json\n"; + assert_eq!(script_summary(script), "# Sends a request."); + assert_eq!( + script_summary("import sys; print(1)"), + "import sys; print(1)" + ); + assert_eq!(script_summary(""), ""); + } + #[test] fn policy_create_request_keeps_the_spec_and_adds_the_policy() { let spec = SandboxSpec { diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..54f441637d 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -75,6 +75,18 @@ pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { .context("serialize the policy") } +/// `binaries` as the flow-style list a network policy's `binaries:` key +/// takes in YAML, such as `[{path: "/usr/bin/python*"}]`. Use it in policy +/// YAML a test writes by hand, for example with +/// `binaries_yaml(PYTHON_BINARIES)`. +pub fn binaries_yaml(binaries: &[&str]) -> String { + let paths: Vec = binaries + .iter() + .map(|path| format!("{{path: {}}}", serde_json::Value::from(*path))) + .collect(); + format!("[{}]", paths.join(", ")) +} + /// Settings for an endpoint from [`allow`], named as in policy YAML. pub trait Endpoint { /// `protocol: rest` with `access`. @@ -185,4 +197,26 @@ mod tests { .expect("the serialized policy parses"); assert_eq!(round_trip, expected); } + + /// The list parses as the `binaries` of a policy, the same as + /// [`allow`] adds them. + #[test] + fn binaries_yaml_lists_each_path() { + let binaries = ["/usr/bin/python*", "/sandbox/.uv/python/*/bin/python*"]; + let yaml = format!( + "{BASE}network_policies: + upstream: + name: upstream + endpoints: + - {{host: example.test, port: 8080}} + binaries: {} +", + binaries_yaml(&binaries) + ); + let parsed = openshell_policy::parse_sandbox_policy(&yaml).expect("the YAML parses"); + let mut expected = base(); + allow(&mut expected, "upstream", "example.test", 8080, &binaries); + assert_eq!(parsed, expected); + assert_eq!(binaries_yaml(&[]), "[]"); + } } From a9a6c59c93eeb028ccefb1f58b8f2a72c5f2c7a4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:31:49 -0800 Subject: [PATCH 02/17] test(host-server): add TEST_HOST Signed-off-by: Matthew Grossman --- tests/support/host-server/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 8ac8528570..f10a0e0632 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -21,6 +21,9 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request}; pub use websocket::WebSocketRecorder; +/// The name a sandbox reaches these servers by. +pub const TEST_HOST: &str = "host.openshell.internal"; + const WAIT_TIMEOUT: Duration = Duration::from_secs(10); const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); From 42c3656cc261781caa67503f7ac26d8659745556 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:36:39 -0800 Subject: [PATCH 03/17] test(support): add sandbox-clients crate with connect and http probes A shared home for the clients tests run inside a sandbox. One probe script connects, optionally sends bytes, and reads until the connection ends; connect(), http() and http_raw() run it and return typed results. HTTP parsing happens in Rust and is unit-tested. Requests send Connection: close by default, and raw sends stop after two idle seconds, so a keep-alive server can't stall a read. Signed-off-by: Matthew Grossman --- tests/support/Cargo.lock | 9 + tests/support/Cargo.toml | 2 +- tests/support/sandbox-clients/Cargo.toml | 17 + .../sandbox-clients/scripts/tcp_probe.py | 88 ++++ tests/support/sandbox-clients/src/lib.rs | 53 +++ tests/support/sandbox-clients/src/message.rs | 435 ++++++++++++++++++ tests/support/sandbox-clients/src/probe.rs | 345 ++++++++++++++ 7 files changed, 948 insertions(+), 1 deletion(-) create mode 100644 tests/support/sandbox-clients/Cargo.toml create mode 100644 tests/support/sandbox-clients/scripts/tcp_probe.py create mode 100644 tests/support/sandbox-clients/src/lib.rs create mode 100644 tests/support/sandbox-clients/src/message.rs create mode 100644 tests/support/sandbox-clients/src/probe.rs diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index bc18df94cd..a51182ae34 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -3182,6 +3182,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/support/Cargo.toml b/tests/support/Cargo.toml index 4c7c495f97..e3dae4234d 100644 --- a/tests/support/Cargo.toml +++ b/tests/support/Cargo.toml @@ -5,4 +5,4 @@ # environment runs. Suites depend on them by path. [workspace] resolver = "2" -members = ["container-engine", "host-server", "keycloak", "podman"] +members = ["container-engine", "host-server", "keycloak", "podman", "sandbox-clients"] diff --git a/tests/support/sandbox-clients/Cargo.toml b/tests/support/sandbox-clients/Cargo.toml new file mode 100644 index 0000000000..9a7455e7c0 --- /dev/null +++ b/tests/support/sandbox-clients/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "sandbox-clients-support" +description = "Clients that tests run in a sandbox, and typed probes for their results" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../testkit/testkit" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py new file mode 100644 index 0000000000..e1209729cd --- /dev/null +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects to argv[1]:argv[2], sends the bytes hex-encoded in argv[3] ("-" +# to only connect), and reads until the connection ends. Prints how each step +# went as one JSON line; failures to connect or read are results, not errors. +# +# argv[4]: connect timeout in seconds. +# argv[5]: read timeout in seconds, for all of the reading. +# argv[6]: idle timeout in seconds: stop reading after this long without +# bytes. 0 waits for the read timeout instead. +import json +import socket +import sys +import time + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = None if sys.argv[3] == "-" else bytes.fromhex(sys.argv[3]) +CONNECT_TIMEOUT = float(sys.argv[4]) +READ_TIMEOUT = float(sys.argv[5]) +IDLE_TIMEOUT = float(sys.argv[6]) + +result = { + "connect": None, + "errno": None, + "error": None, + "connect_ms": None, + "read_end": None, + "read_errno": None, + "read_error": None, + "received_hex": "", +} + + +def millis(since): + return int((time.monotonic() - since) * 1000) + + +def exchange(sock): + """Send PAYLOAD and read until the connection ends. Returns the bytes + read and how reading ended.""" + received = bytearray() + try: + sock.sendall(PAYLOAD) + deadline = time.monotonic() + READ_TIMEOUT + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return received, "timeout" + idle = 0 < IDLE_TIMEOUT < remaining + sock.settimeout(IDLE_TIMEOUT if idle else remaining) + try: + chunk = sock.recv(65536) + except socket.timeout: + return received, "idle" if idle else "timeout" + if not chunk: + return received, "closed" + received += chunk + except ConnectionResetError: + return received, "reset" + except OSError as error: + result["read_errno"] = error.errno + result["read_error"] = repr(error) + return received, "failed" + + +start = time.monotonic() +try: + sock = socket.create_connection((HOST, PORT), timeout=CONNECT_TIMEOUT) +except socket.timeout: + result["connect"] = "timeout" +except OSError as error: + result["connect"] = "failed" + result["errno"] = error.errno + result["error"] = repr(error) +else: + result["connect"] = "ok" +result["connect_ms"] = millis(start) + +if result["connect"] == "ok": + with sock: + if PAYLOAD is not None: + received, read_end = exchange(sock) + result["received_hex"] = received.hex() + result["read_end"] = read_end + +print(json.dumps(result, sort_keys=True), flush=True) diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs new file mode 100644 index 0000000000..392507aa43 --- /dev/null +++ b/tests/support/sandbox-clients/src/lib.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Clients that tests run inside a sandbox, and typed results for them. +//! +//! The clients run in a sandbox created from [`testkit::python_image`], with a +//! policy that lets [`testkit::PYTHON_BINARIES`] reach the target: +//! +//! ```ignore +//! use host_server_support::{HttpRecorder, TEST_HOST}; +//! use sandbox_clients_support::{Request, http}; +//! use testkit::{PYTHON_BINARIES, policy}; +//! +//! let server = HttpRecorder::start("hello").await?; +//! let mut allowed = policy::base(); +//! policy::allow(&mut allowed, "server", TEST_HOST, server.port, PYTHON_BINARIES); +//! let sandbox = cx +//! .sandbox("probe") +//! .image(testkit::python_image()) +//! .command(["sleep", "infinity"]) +//! .policy(allowed) +//! .create_ready() +//! .await?; +//! let reply = http(&sandbox, TEST_HOST, server.port, &Request::get("/")).await?; +//! testkit::ensure!(reply.status() == Some(200), "{reply:?}"); +//! ``` +//! +//! - [`connect`] opens a TCP connection and reports whether it connected, +//! the errno it failed with, or a timeout, and how long it took. +//! - [`http`] sends a [`Request`], and [`http_raw`] sends bytes as they are. +//! Both report how the connection went, the bytes that came back, and the +//! parsed [`Response`]. +//! - [`scripts`] holds the client scripts, for tests that run one with +//! [`testkit::Sandbox::run_python`] and read its result with +//! [`testkit::ExecOutput::last_json`]. Protocol clients, such as `GraphQL` or +//! JSON-RPC ones, belong there too, so every suite can use them. +//! +//! Both probes use a raw socket. Sandboxes set no proxy variables, so this is +//! the same path a library such as `urllib` takes. + +mod message; +mod probe; + +pub use message::{Http, Request, Response}; +pub use probe::{Connect, ConnectOutcome, ReadEnd, connect, http, http_raw}; + +/// Client scripts, as text to pass to [`testkit::Sandbox::run_python`]. +pub mod scripts { + /// Connects to a host and port, optionally sends bytes and reads the + /// reply, and prints the outcome as one JSON line. [`crate::connect`] and + /// [`crate::http`] run it and parse its output. + pub const TCP_PROBE: &str = include_str!("../scripts/tcp_probe.py"); +} diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs new file mode 100644 index 0000000000..91ee9f3337 --- /dev/null +++ b/tests/support/sandbox-clients/src/message.rs @@ -0,0 +1,435 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP requests for [`crate::http`] and the replies it reads. + +use std::fmt; +use std::time::Duration; + +use crate::probe::{ConnectOutcome, Probe, ReadEnd}; + +/// An HTTP/1.1 request for [`crate::http`]. +/// +/// It sends `Host` for the target host and port, `Content-Length` for a body +/// or a method that takes one, and `Connection: close`, unless the request +/// sets those headers itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Request { + method: String, + target: String, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Request { + /// `GET target`. + pub fn get(target: impl Into) -> Self { + Self::new("GET", target) + } + + /// `method target`. The target is usually a path, such as `/allowed`, + /// but may be absolute, such as `http://host:port/allowed`. + pub fn new(method: impl Into, target: impl Into) -> Self { + Self { + method: method.into(), + target: target.into(), + headers: Vec::new(), + body: Vec::new(), + } + } + + /// Add a header. A header added more than once is sent more than once. + #[must_use] + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + /// Send `body` with the request. + #[must_use] + pub fn body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } + + pub(crate) fn is_head(&self) -> bool { + self.method.eq_ignore_ascii_case("HEAD") + } + + /// The request as bytes on the wire to `host:port`. + pub(crate) fn encode(&self, host: &str, port: u16) -> Vec { + let has = |name: &str| { + self.headers + .iter() + .any(|(header, _)| header.eq_ignore_ascii_case(name)) + }; + let mut head = format!("{} {} HTTP/1.1\r\n", self.method, self.target); + if !has("Host") { + if host.contains(':') { + head.push_str(&format!("Host: [{host}]:{port}\r\n")); + } else { + head.push_str(&format!("Host: {host}:{port}\r\n")); + } + } + for (name, value) in &self.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let takes_body = ["POST", "PUT", "PATCH"] + .iter() + .any(|method| self.method.eq_ignore_ascii_case(method)); + if (takes_body || !self.body.is_empty()) + && !has("Content-Length") + && !has("Transfer-Encoding") + { + head.push_str(&format!("Content-Length: {}\r\n", self.body.len())); + } + if !has("Connection") { + head.push_str("Connection: close\r\n"); + } + head.push_str("\r\n"); + let mut bytes = head.into_bytes(); + bytes.extend_from_slice(&self.body); + bytes + } +} + +/// The result of [`crate::http`] or [`crate::http_raw`]. +#[derive(Clone, PartialEq, Eq)] +pub struct Http { + pub connect: ConnectOutcome, + /// How long the connect took. + pub elapsed: Duration, + /// How reading the reply ended, or none if the connect failed. + pub read_end: Option, + /// Every byte that came back. + pub received: Vec, + /// The reply parsed as an HTTP response, or none if nothing came back or + /// it isn't one. Only the first response is parsed. + pub response: Option, +} + +impl Http { + pub(crate) fn from_probe(probe: Probe, head: bool) -> Self { + let response = parse_response(&probe.received, head); + Self { + connect: probe.connect, + elapsed: probe.connect_elapsed, + read_end: probe.read_end, + received: probe.received, + response, + } + } + + /// The response's status, if one came back. + pub fn status(&self) -> Option { + self.response.as_ref().map(|response| response.status) + } + + /// [`Http::received`] as text, with invalid UTF-8 replaced. + pub fn received_text(&self) -> String { + String::from_utf8_lossy(&self.received).into_owned() + } +} + +impl fmt::Debug for Http { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Http") + .field("connect", &self.connect) + .field("elapsed", &self.elapsed) + .field("read_end", &self.read_end) + .field("received", &self.received_text()) + .field("response", &self.response) + .finish() + } +} + +/// An HTTP response. +#[derive(Clone, PartialEq, Eq)] +pub struct Response { + pub status: u16, + /// The headers, in order, as they arrived. + pub headers: Vec<(String, String)>, + /// The body, with chunked transfer encoding removed. + pub body: Vec, +} + +impl Response { + /// The first header named `name`, ignoring case. + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header, _)| header.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// The body as text, with invalid UTF-8 replaced. + pub fn body_text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } +} + +impl fmt::Debug for Response { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Response") + .field("status", &self.status) + .field("headers", &self.headers) + .field("body", &self.body_text()) + .finish() + } +} + +/// The first HTTP response in `bytes`, or none if they don't start with a +/// complete status line and headers. `head` says the request was `HEAD`, so +/// the response has no body. +/// +/// The body ends where chunked encoding or `Content-Length` says, or at the +/// end of `bytes` without either. A truncated body is returned as far as it +/// arrived. +fn parse_response(bytes: &[u8], head: bool) -> Option { + let end = find(bytes, b"\r\n\r\n")?; + let text = std::str::from_utf8(&bytes[..end]).ok()?; + let mut lines = text.split("\r\n"); + let status = parse_status_line(lines.next()?)?; + let headers: Vec<(String, String)> = lines + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + Some((name.trim().to_string(), value.trim().to_string())) + }) + .collect(); + let rest = &bytes[end + 4..]; + let mut response = Response { + status, + headers, + body: Vec::new(), + }; + let bodiless = head || (100..200).contains(&status) || status == 204 || status == 304; + if !bodiless { + response.body = if response + .header("Transfer-Encoding") + .is_some_and(|encoding| encoding.to_ascii_lowercase().contains("chunked")) + { + dechunk(rest) + } else if let Some(length) = response + .header("Content-Length") + .and_then(|length| length.parse::().ok()) + { + rest[..length.min(rest.len())].to_vec() + } else { + rest.to_vec() + }; + } + Some(response) +} + +/// The status code in `HTTP/1.x NNN reason`. +fn parse_status_line(line: &str) -> Option { + let mut parts = line.splitn(3, ' '); + let version = parts.next()?; + let code = parts.next()?; + if !version.starts_with("HTTP/1.") || code.len() != 3 { + return None; + } + code.parse().ok() +} + +/// The data in a chunked body, up to the last complete chunk if it is cut +/// short. +fn dechunk(mut rest: &[u8]) -> Vec { + let mut body = Vec::new(); + while let Some(line_end) = find(rest, b"\r\n") { + let size_line = String::from_utf8_lossy(&rest[..line_end]); + let size = size_line.split(';').next().unwrap_or_default().trim(); + let Ok(size) = usize::from_str_radix(size, 16) else { + break; + }; + rest = &rest[line_end + 2..]; + if size == 0 { + break; + } + let available = size.min(rest.len()); + body.extend_from_slice(&rest[..available]); + if available < size { + break; + } + rest = rest.get(size + 2..).unwrap_or_default(); + } + body +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn text(bytes: &[u8]) -> &str { + std::str::from_utf8(bytes).expect("UTF-8") + } + + #[test] + fn a_get_asks_the_server_to_close() { + let request = Request::get("/allowed").encode("host.openshell.internal", 8080); + assert_eq!( + text(&request), + "GET /allowed HTTP/1.1\r\nHost: host.openshell.internal:8080\r\nConnection: close\r\n\r\n" + ); + } + + #[test] + fn a_post_sends_its_length_and_body() { + let request = Request::new("POST", "/token") + .header("Content-Type", "text/plain") + .body("secret") + .encode("10.0.0.1", 80); + assert_eq!( + text(&request), + "POST /token HTTP/1.1\r\nHost: 10.0.0.1:80\r\nContent-Type: text/plain\r\n\ + Content-Length: 6\r\nConnection: close\r\n\r\nsecret" + ); + let empty = Request::new("post", "/").encode("h", 1); + assert!(text(&empty).contains("Content-Length: 0\r\n")); + } + + #[test] + fn headers_the_request_sets_replace_the_defaults() { + let request = Request::get("http://h:1/x") + .header("host", "other") + .header("connection", "keep-alive") + .encode("h", 1); + assert_eq!( + text(&request), + "GET http://h:1/x HTTP/1.1\r\nhost: other\r\nconnection: keep-alive\r\n\r\n" + ); + let chunked = Request::new("POST", "/") + .header("Transfer-Encoding", "chunked") + .body("0\r\n\r\n") + .encode("h", 1); + assert!(!text(&chunked).contains("Content-Length")); + } + + #[test] + fn an_ipv6_host_is_bracketed() { + let request = Request::get("/").encode("::1", 80); + assert!(text(&request).contains("Host: [::1]:80\r\n")); + } + + #[test] + fn a_response_with_content_length() { + let response = parse_response( + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\ncontent-length: 5\r\n\r\nhello extra", + false, + ) + .expect("a response"); + assert_eq!(response.status, 200); + assert_eq!(response.header("Content-Type"), Some("text/plain")); + assert_eq!(response.header("CONTENT-LENGTH"), Some("5")); + assert_eq!(response.body, b"hello"); + } + + #[test] + fn a_response_without_content_length_reads_to_the_end() { + let response = + parse_response(b"HTTP/1.0 200 OK\r\n\r\nall of it", false).expect("a response"); + assert_eq!(response.body, b"all of it"); + } + + #[test] + fn a_chunked_response_is_decoded() { + let response = parse_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5;ext=1\r\nhello\r\n6\r\n world\r\n0\r\n\r\n", + false, + ) + .expect("a response"); + assert_eq!(response.body, b"hello world"); + } + + #[test] + fn a_truncated_body_is_kept_as_far_as_it_arrived() { + let chunked = parse_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\na\r\nworl", + false, + ) + .expect("a response"); + assert_eq!(chunked.body, b"helloworl"); + let sized = parse_response(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nhel", false) + .expect("a response"); + assert_eq!(sized.body, b"hel"); + } + + #[test] + fn proxy_denials_without_a_body() { + let empty = parse_response( + b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n", + false, + ) + .expect("a response"); + assert_eq!(empty.status, 403); + assert!(empty.body.is_empty()); + let closed = + parse_response(b"HTTP/1.1 502 Bad Gateway\r\n\r\n", false).expect("a response"); + assert_eq!(closed.status, 502); + assert!(closed.body.is_empty()); + } + + #[test] + fn bodiless_statuses_and_head_have_no_body() { + for status in [ + "101 Switching Protocols", + "204 No Content", + "304 Not Modified", + ] { + let bytes = format!("HTTP/1.1 {status}\r\nContent-Length: 4\r\n\r\nnext"); + let response = parse_response(bytes.as_bytes(), false).expect("a response"); + assert!(response.body.is_empty(), "{status}"); + } + let head = parse_response(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n", true) + .expect("a response"); + assert!(head.body.is_empty()); + } + + #[test] + fn non_http_or_incomplete_bytes_are_not_a_response() { + assert_eq!(parse_response(b"", false), None); + assert_eq!(parse_response(b"HTTP/1.1 200 OK\r\nContent-", false), None); + assert_eq!( + parse_response(b"\x00\xff\x13\x37not-http\r\n\r\n", false), + None + ); + assert_eq!(parse_response(b"SSH-2.0-OpenSSH\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/1.1 20 OK\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/2 200\r\n\r\n", false), None); + } + + #[test] + fn http_parses_what_the_probe_received() { + let probe = Probe { + connect: ConnectOutcome::Connected, + connect_elapsed: Duration::from_millis(3), + read_end: Some(ReadEnd::Closed), + received: b"HTTP/1.1 403 Forbidden\r\nContent-Length: 2\r\n\r\nno".to_vec(), + }; + let http = Http::from_probe(probe, false); + assert_eq!(http.status(), Some(403)); + assert_eq!( + http.response.as_ref().expect("a response").body_text(), + "no" + ); + assert!(http.received_text().starts_with("HTTP/1.1 403")); + + let reset = Http::from_probe( + Probe { + connect: ConnectOutcome::Connected, + connect_elapsed: Duration::ZERO, + read_end: Some(ReadEnd::Reset), + received: Vec::new(), + }, + false, + ); + assert_eq!(reset.status(), None); + assert_eq!(reset.read_end, Some(ReadEnd::Reset)); + } +} diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs new file mode 100644 index 0000000000..ad94d3c4fe --- /dev/null +++ b/tests/support/sandbox-clients/src/probe.rs @@ -0,0 +1,345 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runs [`scripts::TCP_PROBE`] in a sandbox and turns its JSON line into +//! typed results. + +use std::time::Duration; + +use serde::Deserialize; +use testkit::{Result, ResultExt as _, Sandbox}; + +use crate::message::{Http, Request}; +use crate::scripts; + +/// How long a connect may take, as the clients the probes replace allowed. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// How long reading a reply may take in all. +const READ_TIMEOUT: Duration = Duration::from_secs(15); +/// How long [`http_raw`] waits for more bytes before it stops reading. Raw +/// bytes may leave the server waiting for more, so the connection may never +/// close. +const RAW_IDLE_TIMEOUT: Duration = Duration::from_secs(2); + +const EPERM: i32 = 1; +const EACCES: i32 = 13; + +/// How a connect ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConnectOutcome { + Connected, + /// The connect failed with `errno`, or none for an error without one. + /// `error` is Python's description of it. + Failed { + errno: Option, + error: String, + }, + TimedOut, +} + +impl ConnectOutcome { + pub fn is_connected(&self) -> bool { + matches!(self, Self::Connected) + } + + /// The errno a failed connect reported. + pub fn errno(&self) -> Option { + match self { + Self::Failed { errno, .. } => *errno, + Self::Connected | Self::TimedOut => None, + } + } + + /// Whether the connect failed with `EACCES` or `EPERM`, as the sandbox + /// denies a connection. + pub fn is_permission_denied(&self) -> bool { + matches!(self.errno(), Some(EACCES | EPERM)) + } +} + +/// The result of [`connect`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Connect { + pub outcome: ConnectOutcome, + /// How long the connect took, until it connected, failed, or timed out. + pub elapsed: Duration, +} + +/// How reading a reply ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadEnd { + /// The peer closed the connection. + Closed, + /// The peer reset the connection (`ECONNRESET`). + Reset, + /// No bytes arrived for the idle timeout of [`http_raw`]. + Idle, + /// The read timeout ran out before the connection ended. + TimedOut, + /// Sending or reading failed some other way. + Failed { errno: Option, error: String }, +} + +/// Open a TCP connection from `sandbox` to `host:port`, then close it. +/// A connect that fails or times out is a result, not an error. +pub async fn connect(sandbox: &Sandbox, host: &str, port: u16) -> Result { + let probe = run(sandbox, host, port, None, Duration::ZERO).await?; + Ok(Connect { + outcome: probe.connect, + elapsed: probe.connect_elapsed, + }) +} + +/// Send `request` from `sandbox` to `host:port` and read the reply until the +/// server closes the connection. The request asks it to, with +/// `Connection: close`, unless it sets its own `Connection` header. +pub async fn http(sandbox: &Sandbox, host: &str, port: u16, request: &Request) -> Result { + let bytes = request.encode(host, port); + let probe = run(sandbox, host, port, Some(&bytes), Duration::ZERO).await?; + Ok(Http::from_probe(probe, request.is_head())) +} + +/// Send `bytes` as they are from `sandbox` to `host:port`, and read the reply +/// until the connection ends or goes quiet for two seconds. Use it for +/// requests [`Request`] can't make, or bytes that aren't HTTP at all. +pub async fn http_raw(sandbox: &Sandbox, host: &str, port: u16, bytes: &[u8]) -> Result { + let probe = run(sandbox, host, port, Some(bytes), RAW_IDLE_TIMEOUT).await?; + Ok(Http::from_probe(probe, false)) +} + +/// What the probe script reported. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Probe { + pub(crate) connect: ConnectOutcome, + pub(crate) connect_elapsed: Duration, + pub(crate) read_end: Option, + pub(crate) received: Vec, +} + +async fn run( + sandbox: &Sandbox, + host: &str, + port: u16, + payload: Option<&[u8]>, + idle_timeout: Duration, +) -> Result { + let payload = payload.map_or_else(|| "-".to_string(), to_hex); + let args = [ + host.to_string(), + port.to_string(), + payload, + CONNECT_TIMEOUT.as_secs_f64().to_string(), + READ_TIMEOUT.as_secs_f64().to_string(), + idle_timeout.as_secs_f64().to_string(), + ]; + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = sandbox + .run_python(scripts::TCP_PROBE, &args) + .await + .context(format!( + "probe {host}:{port} from sandbox {}", + sandbox.name() + ))?; + parse_probe(output.last_json()?) +} + +#[derive(Deserialize)] +struct ProbeJson { + connect: ConnectJson, + errno: Option, + error: Option, + connect_ms: u64, + read_end: Option, + read_errno: Option, + read_error: Option, + received_hex: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ConnectJson { + Ok, + Failed, + Timeout, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ReadEndJson { + Closed, + Reset, + Idle, + Timeout, + Failed, +} + +fn parse_probe(value: serde_json::Value) -> Result { + let json: ProbeJson = serde_json::from_value(value.clone()) + .context(format!("read the probe's result {value}"))?; + let connect = match json.connect { + ConnectJson::Ok => ConnectOutcome::Connected, + ConnectJson::Failed => ConnectOutcome::Failed { + errno: json.errno, + error: json.error.unwrap_or_default(), + }, + ConnectJson::Timeout => ConnectOutcome::TimedOut, + }; + let read_end = json.read_end.map(|end| match end { + ReadEndJson::Closed => ReadEnd::Closed, + ReadEndJson::Reset => ReadEnd::Reset, + ReadEndJson::Idle => ReadEnd::Idle, + ReadEndJson::Timeout => ReadEnd::TimedOut, + ReadEndJson::Failed => ReadEnd::Failed { + errno: json.read_errno, + error: json.read_error.clone().unwrap_or_default(), + }, + }); + Ok(Probe { + connect, + connect_elapsed: Duration::from_millis(json.connect_ms), + read_end, + received: from_hex(&json.received_hex)?, + }) +} + +fn to_hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +fn from_hex(hex: &str) -> Result> { + testkit::ensure!( + hex.len().is_multiple_of(2), + "the probe's received_hex has odd length" + ); + (0..hex.len()) + .step_by(2) + .map(|at| { + hex.get(at..at + 2) + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!("the probe's received_hex {hex:?} isn't hex")) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn probe_json(overrides: serde_json::Value) -> serde_json::Value { + let mut value = json!({ + "connect": "ok", + "errno": null, + "error": null, + "connect_ms": 12, + "read_end": null, + "read_errno": null, + "read_error": null, + "received_hex": "", + }); + for (key, field) in overrides.as_object().expect("an object") { + value[key] = field.clone(); + } + value + } + + #[test] + fn a_connect_only_probe_has_no_read() { + let probe = parse_probe(probe_json(json!({}))).expect("parses"); + assert_eq!( + probe, + Probe { + connect: ConnectOutcome::Connected, + connect_elapsed: Duration::from_millis(12), + read_end: None, + received: Vec::new(), + } + ); + } + + #[test] + fn a_failed_connect_keeps_its_errno() { + let probe = parse_probe(probe_json(json!({ + "connect": "failed", + "errno": 13, + "error": "PermissionError(13, 'Permission denied')", + }))) + .expect("parses"); + assert_eq!(probe.connect.errno(), Some(13)); + assert!(probe.connect.is_permission_denied()); + assert!(!probe.connect.is_connected()); + + let refused = parse_probe(probe_json(json!({"connect": "failed", "errno": 111}))) + .expect("parses") + .connect; + assert!(!refused.is_permission_denied()); + let eperm = parse_probe(probe_json(json!({"connect": "failed", "errno": 1}))) + .expect("parses") + .connect; + assert!(eperm.is_permission_denied()); + } + + #[test] + fn a_connect_timeout_is_not_a_failure() { + let probe = parse_probe(probe_json( + json!({"connect": "timeout", "connect_ms": 10_000}), + )) + .expect("parses"); + assert_eq!(probe.connect, ConnectOutcome::TimedOut); + assert_eq!(probe.connect.errno(), None); + assert_eq!(probe.connect_elapsed, Duration::from_secs(10)); + } + + #[test] + fn each_read_end_is_distinct() { + for (name, end) in [ + ("closed", ReadEnd::Closed), + ("reset", ReadEnd::Reset), + ("idle", ReadEnd::Idle), + ("timeout", ReadEnd::TimedOut), + ] { + let probe = parse_probe(probe_json( + json!({"read_end": name, "received_hex": "6869"}), + )) + .expect("parses"); + assert_eq!(probe.read_end, Some(end), "{name}"); + assert_eq!(probe.received, b"hi"); + } + let failed = parse_probe(probe_json(json!({ + "read_end": "failed", + "read_errno": 32, + "read_error": "BrokenPipeError(32, 'Broken pipe')", + }))) + .expect("parses"); + assert_eq!( + failed.read_end, + Some(ReadEnd::Failed { + errno: Some(32), + error: "BrokenPipeError(32, 'Broken pipe')".to_string(), + }) + ); + } + + #[test] + fn an_unknown_outcome_is_an_error() { + assert!(parse_probe(probe_json(json!({"connect": "maybe"}))).is_err()); + assert!(parse_probe(probe_json(json!({"read_end": "later"}))).is_err()); + assert!(parse_probe(json!({"connect": "ok"})).is_err()); + } + + #[test] + fn hex_round_trips_and_rejects_bad_input() { + let bytes = [0x00, 0xff, 0x13, 0x37, b'a']; + assert_eq!(to_hex(&bytes), "00ff133761"); + assert_eq!(from_hex(&to_hex(&bytes)).expect("hex"), bytes); + assert!(from_hex("abc").is_err()); + assert!(from_hex("zz").is_err()); + } +} From eeb87f9990c692f88a3f8a39b180503755b1da08 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:38:09 -0800 Subject: [PATCH 04/17] test(features): use shared helpers in host-gateway The test uses TEST_HOST and the sandbox-clients http probe in place of its own Python client. Its main changes from sh -c "exec sleep infinity" to sleep infinity, which leaves the same final process. Signed-off-by: Matthew Grossman --- .../host-gateway/tests/host_gateway.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 6c864b0e01..f18c4950f2 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -9,7 +9,7 @@ //! Python test image ([`testkit::python_image`]) that requests a page from that //! server through the name. -use host_server_support::HttpRecorder; +use host_server_support::{HttpRecorder, TEST_HOST}; use testkit::policy; use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; @@ -20,14 +20,14 @@ use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; /// behind a symlink. const PYTHON: &str = "python3"; -/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and +/// Requests `/` from the host named in `argv[1]` on the port in `argv[2]` and /// prints the response. The server listens before the sandbox is created, /// and the network is up before the main process starts, so the client /// doesn't retry. const CLIENT: &str = r#" import socket, sys -host, port = "host.openshell.internal", int(sys.argv[1]) +host, port = sys.argv[1], int(sys.argv[2]) with socket.create_connection((host, port), timeout=10) as conn: conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) response = b"" @@ -54,7 +54,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result .cli_sandbox("hg") .image(image.clone()) .policy_yaml(policy::to_yaml(&policy(server.port))?) - .command([PYTHON, "-c", CLIENT, &server.port.to_string()]) + .command([PYTHON, "-c", CLIENT, TEST_HOST, &server.port.to_string()]) .run() .await? .success() @@ -71,17 +71,11 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result } /// A sandbox policy that allows the image's Python ([`PYTHON`]) to -/// connect to `host.openshell.internal` on `port`. The endpoint names only the +/// connect to [`TEST_HOST`] on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. fn policy(port: u16) -> proto::SandboxPolicy { let mut policy = policy::base(); - policy::allow( - &mut policy, - "host_server", - "host.openshell.internal", - port, - PYTHON_BINARIES, - ); + policy::allow(&mut policy, "host_server", TEST_HOST, port, PYTHON_BINARIES); policy } From 77e1a6d0fb7dde728c5278a6c2b41d3f11624f38 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:39:35 -0800 Subject: [PATCH 05/17] test(suites): drop ready_sandbox wrappers and unify long-running commands Tests create a long-running sandbox with the builder chain directly instead of a per-file ready_sandbox helper. The cli and policy-advisor mains change from sh -c "exec sleep infinity" to sleep infinity, which leaves the same final process. connect-ssh-supervision keeps its sh -lc mains. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/cli.rs | 2 +- tests/suites/conformance/src/exec_cli.rs | 19 +++++++------- .../suites/conformance/src/policy_advisor.rs | 4 +-- tests/suites/conformance/src/ssh.rs | 19 +++++++------- .../tests/connect_ssh_supervision.rs | 25 ++++++++----------- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index 169894def0..342a515b41 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -64,7 +64,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .cli_sandbox("cl") - .command(["sh", "-c", "exec sleep infinity"]) + .command(["sleep", "infinity"]) .create_ready() .await?; let name = sandbox.name().to_string(); diff --git a/tests/suites/conformance/src/exec_cli.rs b/tests/suites/conformance/src/exec_cli.rs index 35924cb351..e854277abc 100644 --- a/tests/suites/conformance/src/exec_cli.rs +++ b/tests/suites/conformance/src/exec_cli.rs @@ -21,7 +21,11 @@ const STDIN_LIMIT: usize = 4 * 1024 * 1024; #[testkit::test] async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result { const BYTES: usize = 8 * 1024 * 1024; - let sandbox = ready_sandbox(cx, "lo").await?; + let sandbox = cx + .sandbox("lo") + .command(["sleep", "infinity"]) + .create_ready() + .await?; for stream in ["stdout", "stderr"] { let script = match stream { @@ -88,7 +92,11 @@ async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result /// read fails the exec. #[testkit::test] async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> testkit::Result { - let sandbox = ready_sandbox(cx, "si").await?; + let sandbox = cx + .sandbox("si") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let wc = exec_args(&sandbox, &["wc", "-c"]); for size in [5, 1024 * 1024, STDIN_LIMIT] { @@ -165,13 +173,6 @@ async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> te Ok(()) } -async fn ready_sandbox(cx: &Context, suffix: &str) -> testkit::Result { - cx.sandbox(suffix) - .command(["sleep", "infinity"]) - .create_ready() - .await -} - /// `sandbox exec` of `command` without a TTY or login shell. fn exec_args(sandbox: &Sandbox, command: &[&str]) -> Vec { [ diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 82ff54cc09..31acc4fa04 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -224,9 +224,7 @@ async fn create( suffix: &str, policy: Option, ) -> testkit::Result { - let mut builder = cx - .sandbox(suffix) - .command(["sh", "-c", "exec sleep infinity"]); + let mut builder = cx.sandbox(suffix).command(["sleep", "infinity"]); if let Some(policy) = policy { builder = builder.policy(policy); } diff --git a/tests/suites/conformance/src/ssh.rs b/tests/suites/conformance/src/ssh.rs index 61dada7621..729ef3b51d 100644 --- a/tests/suites/conformance/src/ssh.rs +++ b/tests/suites/conformance/src/ssh.rs @@ -35,7 +35,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( cli.gateway().name.is_some(), "this test needs a named gateway for `sandbox ssh-config`; set OPENSHELL_GATEWAY" ); - let sandbox = ready_sandbox(cx).await?; + let sandbox = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let name = sandbox.name().to_string(); let original = fingerprint(cx, &sandbox).await?; @@ -110,7 +114,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( .wait_deleted(&name, testkit::DELETE_TIMEOUT, Some(sandbox.id())) .await .context(format!("wait for sandbox {name} to be deleted"))?; - let replacement = ready_sandbox(cx).await?; + let replacement = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let replaced = fingerprint(cx, &replacement).await?; testkit::ensure!( replaced != original, @@ -119,13 +127,6 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( Ok(()) } -async fn ready_sandbox(cx: &Context) -> testkit::Result { - cx.sandbox("hk") - .command(["sleep", "infinity"]) - .create_ready() - .await -} - /// The sandbox's host key fingerprint, the same in get and list. async fn fingerprint(cx: &Context, sandbox: &Sandbox) -> testkit::Result { let name = sandbox.name(); diff --git a/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs index ec6faca9d6..f911f7819d 100644 --- a/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs +++ b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs @@ -62,7 +62,7 @@ mod linux { use nix::sys::signal::{Signal, kill}; use nix::unistd::Pid; - use testkit::{Context, ResultExt as _, Sandbox}; + use testkit::{Context, ResultExt as _}; use super::ATTACH_TIMEOUT; @@ -75,7 +75,11 @@ mod linux { const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; pub(super) async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { - let sandbox = ready_sandbox(cx, "tr", &["sh", "-lc", ECHO_MAIN]).await?; + let sandbox = cx + .sandbox("tr") + .command(["sh", "-lc", ECHO_MAIN]) + .create_ready() + .await?; let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?; let first = connect.stdout_line(ATTACH_TIMEOUT).await?; testkit::ensure!( @@ -120,7 +124,11 @@ mod linux { pub(super) async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( cx: &mut Context, ) -> testkit::Result { - let sandbox = ready_sandbox(cx, "term", &["sh", "-lc", "exec sleep infinity"]).await?; + let sandbox = cx + .sandbox("term") + .command(["sh", "-lc", "exec sleep infinity"]) + .create_ready() + .await?; let pty = nix::pty::openpty(None, None) .map_err(std::io::Error::from) .context("open a pseudo-terminal")?; @@ -158,17 +166,6 @@ mod linux { .await } - async fn ready_sandbox( - cx: &Context, - suffix: &str, - command: &[&str], - ) -> testkit::Result { - cx.sandbox(suffix) - .command(command.iter().copied()) - .create_ready() - .await - } - fn send(pid: u32, signal: Signal) -> testkit::Result { let pid = i32::try_from(pid) .map_err(|error| testkit::Error::msg(format!("pid {pid}: {error}")))?; From 8a6c29aa9a664309d8b4b6c9089b1752cd9b8da9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:39:55 -0800 Subject: [PATCH 06/17] docs(testkit): list the shared sandbox helpers Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 9cc6120620..4d12a7eefb 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -65,12 +65,19 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then `create()` it or `create_ready()` to also wait for Ready. Set its image - with `image(..)` and its policy with `policy(..)`. The returned + with `image(..)`, its policy with `policy(..)` and its providers with + `providers(..)`. A long-running sandbox is + `.command(["sleep", "infinity"])`; add `.image(testkit::python_image())` + when a test runs Python clients in it. The returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set - a setting, read `sandbox get` JSON with `get_json()`, and delete. Its - `expect_listed`, `expect_absent` and `wait_absent` check list and get. - Exec output has `success()`, `failure()` and `expect_stdout()`. + a setting, read `sandbox get` JSON with `get_json()`, read its policy + revisions and active policy with `policy_revisions()` and + `active_policy()`, and delete. Its `expect_listed`, `expect_absent` and + `wait_absent` check list and get. `run_python(script, args)` runs a script + with `python3 -c` and fails unless it exits 0. Exec output has `success()`, + `failure()`, `expect_stdout()`, and `last_json()` for the last JSON object + or array line of stdout, such as a client script's result. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, @@ -110,7 +117,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. (binaries apply to every endpoint under `name`) that `policy::Endpoint` configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` - for the CLI. + for the CLI. `policy::binaries_yaml(binaries)` writes a `binaries:` list + into policy YAML a test writes by hand. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a @@ -118,6 +126,11 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. +- In `tests/support`: `host-server` starts servers on the machine that runs + the tests, which sandboxes reach as `host_server_support::TEST_HOST`, and + `sandbox-clients` runs clients in a sandbox: `connect`, `http` and + `http_raw` return typed results, and `scripts` holds client scripts for + `run_python`. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. - `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a From 54c15c989cd975f85cf046c356242561614b8c01 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:00:37 -0800 Subject: [PATCH 07/17] test(suites): use shared sandbox and provider helpers in cli, cdi and keycloak tests The cli tests read sandbox get JSON through Sandbox::get_json, the cdi scenario creates its sandbox with cli_sandbox, and the keycloak refresh test installs its provider with install_provider instead of importing the profile and creating the provider by hand. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/cli.rs | 18 ++---- tests/suites/features/Cargo.lock | 1 - tests/suites/features/cdi/src/lib.rs | 21 ++----- .../keycloak-provider-refresh/Cargo.toml | 1 - .../tests/provider_refresh.rs | 60 +++---------------- 5 files changed, 18 insertions(+), 83 deletions(-) diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index 342a515b41..9f87643763 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -8,7 +8,6 @@ //! check that the CLI commands for create, get, list, exec, stop, start, and //! delete work against the same gateway. -use serde_json::Value; use testkit::Context; /// Delay before an attached main prints and exits, so the CLI attaches while @@ -112,8 +111,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> "sandbox {completed} is not listed after its main exited; sandboxes are kept by default" ); // The phase can trail the CLI's exit by a status update. - cx.adopt(&completed).await?.wait_finished().await?; - let got = get_json(cx, &completed).await?; + let sandbox = cx.adopt(&completed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; testkit::ensure!( got["phase"] == "Completed", "sandbox {completed} is in phase {}; expected Completed", @@ -136,8 +136,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> output.stderr.trim() ); expect_streamed(&output, "failed-main")?; - cx.adopt(&failed).await?.wait_finished().await?; - let got = get_json(cx, &failed).await?; + let sandbox = cx.adopt(&failed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; testkit::ensure!( got["phase"] == "Error" && got["exit_code"] == 7, "sandbox {failed} is in phase {} with exit code {}; expected Error with 7", @@ -198,13 +199,6 @@ fn expect_streamed(output: &testkit::CliOutput, marker: &str) -> testkit::Result Ok(()) } -async fn get_json(cx: &Context, name: &str) -> testkit::Result { - cx.cli(["sandbox", "get", name, "--output", "json"]) - .await? - .success()? - .json() -} - /// Wait until `sandbox list` no longer shows the sandbox. async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result { testkit::poll_until( diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4c65ad4021..28471bcfdc 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1833,7 +1833,6 @@ version = "0.0.0" dependencies = [ "keycloak-support", "openshell-testkit", - "tempfile", ] [[package]] diff --git a/tests/suites/features/cdi/src/lib.rs b/tests/suites/features/cdi/src/lib.rs index 46a94f3c86..ddadd385ef 100644 --- a/tests/suites/features/cdi/src/lib.rs +++ b/tests/suites/features/cdi/src/lib.rs @@ -374,26 +374,13 @@ pub async fn scenario(cx: &Context, name: &str) -> Result { let Scenario { device, checks } = load(name)?; let driver = compute_driver(cx).await?; - let sandbox = cx.name("cdi"); - cx.track(&sandbox); let config = json!({driver: {"cdi_devices": [device]}}).to_string(); let probe = probe_script(&checks); let output = cx - .cli([ - "sandbox", - "create", - "--name", - &sandbox, - "--no-tty", - "--gpu", - "1", - "--driver-config-json", - &config, - "--", - "sh", - "-c", - &probe, - ]) + .cli_sandbox("cdi") + .args(["--gpu", "1", "--driver-config-json", &config]) + .command(["sh", "-c", &probe]) + .run() .await?; let output = output diff --git a/tests/suites/features/keycloak-provider-refresh/Cargo.toml b/tests/suites/features/keycloak-provider-refresh/Cargo.toml index e326fef5cb..1932a3f374 100644 --- a/tests/suites/features/keycloak-provider-refresh/Cargo.toml +++ b/tests/suites/features/keycloak-provider-refresh/Cargo.toml @@ -14,4 +14,3 @@ publish = false [dependencies] keycloak-support = { path = "../../../support/keycloak" } openshell-testkit = { path = "../../../testkit/testkit" } -tempfile = "3" diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index a617c24843..9a8afa886d 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -6,10 +6,8 @@ //! Keycloak is only the provider's token issuer here; the CLI still reaches //! the gateway the usual way. -use std::io::Write as _; - use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, ResultExt as _}; +use testkit::{CliOutput, Context}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -22,32 +20,12 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - let provider = kc.name("kc-refresh"); let profile_id = kc.name("kc-refresh-profile"); let grant = kc.keycloak().password_grant().await?; - let profile = write_profile(&profile_id, kc.keycloak())?; - let profile_path = profile.path().to_string_lossy().into_owned(); - - // Register each delete before its create, so cleanup also covers a create - // the gateway applied but the CLI reported as failed. Both deletes succeed - // when there is nothing to delete. - delete_on_cleanup(&kc, "profile", &profile_id); - kc.cli(["profile", "import", "--file", &profile_path]) - .await? - .success()?; - delete_on_cleanup(&kc, "provider", &provider); - kc.cli_with_env( + kc.install_provider( + &provider, + &profile(&profile_id, kc.keycloak()), &[(CREDENTIAL_KEY, &grant.access_token)], - [ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--credential", - CREDENTIAL_KEY, - ], ) - .await? - .success()?; + .await?; let client_id = format!("client_id={}", kc.keycloak().client_id()); kc.cli_with_env( &[("KEYCLOAK_REFRESH_TOKEN", &grant.refresh_token)], @@ -117,27 +95,13 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - Ok(()) } -/// Delete the `kind` named `name` when the test ends, since testkit only -/// cleans up sandboxes by itself. -fn delete_on_cleanup(cx: &Context, kind: &'static str, name: &str) { - let cli = cx.cli_handle(); - let name = name.to_string(); - cx.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &name]).await?.success()?; - Ok(()) - }); -} - fn combined(output: &CliOutput) -> String { format!("{}{}", output.stdout, output.stderr) } -fn write_profile( - profile_id: &str, - keycloak: &keycloak_support::Keycloak, -) -> testkit::Result { +fn profile(profile_id: &str, keycloak: &keycloak_support::Keycloak) -> String { let issuer_token_url = keycloak.token_endpoint(); - let profile = format!( + format!( r#"id: {profile_id} display_name: Keycloak provider refresh test category: other @@ -168,13 +132,5 @@ endpoints: binaries: - /usr/bin/curl "# - ); - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .context("create provider profile")?; - file.write_all(profile.as_bytes()) - .and_then(|()| file.flush()) - .context("write provider profile")?; - Ok(file) + ) } From c11a90d0abd179ba45f6e7bd0761c565a97c03c3 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:10:44 -0800 Subject: [PATCH 08/17] test: trim unit tests for the shared sandbox helpers The e2e suites exercise the helpers. Keep only the HTTP parser tests for chunked bodies and non-HTTP bytes, which no e2e test reaches yet. Signed-off-by: Matthew Grossman --- tests/support/sandbox-clients/src/message.rs | 144 ------------------- tests/support/sandbox-clients/src/probe.rs | 117 --------------- tests/testkit/testkit/src/context.rs | 41 ------ tests/testkit/testkit/src/policy.rs | 22 --- 4 files changed, 324 deletions(-) diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs index 91ee9f3337..e74b6fca1a 100644 --- a/tests/support/sandbox-clients/src/message.rs +++ b/tests/support/sandbox-clients/src/message.rs @@ -266,77 +266,6 @@ fn find(haystack: &[u8], needle: &[u8]) -> Option { mod tests { use super::*; - fn text(bytes: &[u8]) -> &str { - std::str::from_utf8(bytes).expect("UTF-8") - } - - #[test] - fn a_get_asks_the_server_to_close() { - let request = Request::get("/allowed").encode("host.openshell.internal", 8080); - assert_eq!( - text(&request), - "GET /allowed HTTP/1.1\r\nHost: host.openshell.internal:8080\r\nConnection: close\r\n\r\n" - ); - } - - #[test] - fn a_post_sends_its_length_and_body() { - let request = Request::new("POST", "/token") - .header("Content-Type", "text/plain") - .body("secret") - .encode("10.0.0.1", 80); - assert_eq!( - text(&request), - "POST /token HTTP/1.1\r\nHost: 10.0.0.1:80\r\nContent-Type: text/plain\r\n\ - Content-Length: 6\r\nConnection: close\r\n\r\nsecret" - ); - let empty = Request::new("post", "/").encode("h", 1); - assert!(text(&empty).contains("Content-Length: 0\r\n")); - } - - #[test] - fn headers_the_request_sets_replace_the_defaults() { - let request = Request::get("http://h:1/x") - .header("host", "other") - .header("connection", "keep-alive") - .encode("h", 1); - assert_eq!( - text(&request), - "GET http://h:1/x HTTP/1.1\r\nhost: other\r\nconnection: keep-alive\r\n\r\n" - ); - let chunked = Request::new("POST", "/") - .header("Transfer-Encoding", "chunked") - .body("0\r\n\r\n") - .encode("h", 1); - assert!(!text(&chunked).contains("Content-Length")); - } - - #[test] - fn an_ipv6_host_is_bracketed() { - let request = Request::get("/").encode("::1", 80); - assert!(text(&request).contains("Host: [::1]:80\r\n")); - } - - #[test] - fn a_response_with_content_length() { - let response = parse_response( - b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\ncontent-length: 5\r\n\r\nhello extra", - false, - ) - .expect("a response"); - assert_eq!(response.status, 200); - assert_eq!(response.header("Content-Type"), Some("text/plain")); - assert_eq!(response.header("CONTENT-LENGTH"), Some("5")); - assert_eq!(response.body, b"hello"); - } - - #[test] - fn a_response_without_content_length_reads_to_the_end() { - let response = - parse_response(b"HTTP/1.0 200 OK\r\n\r\nall of it", false).expect("a response"); - assert_eq!(response.body, b"all of it"); - } - #[test] fn a_chunked_response_is_decoded() { let response = parse_response( @@ -347,50 +276,6 @@ mod tests { assert_eq!(response.body, b"hello world"); } - #[test] - fn a_truncated_body_is_kept_as_far_as_it_arrived() { - let chunked = parse_response( - b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\na\r\nworl", - false, - ) - .expect("a response"); - assert_eq!(chunked.body, b"helloworl"); - let sized = parse_response(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nhel", false) - .expect("a response"); - assert_eq!(sized.body, b"hel"); - } - - #[test] - fn proxy_denials_without_a_body() { - let empty = parse_response( - b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n", - false, - ) - .expect("a response"); - assert_eq!(empty.status, 403); - assert!(empty.body.is_empty()); - let closed = - parse_response(b"HTTP/1.1 502 Bad Gateway\r\n\r\n", false).expect("a response"); - assert_eq!(closed.status, 502); - assert!(closed.body.is_empty()); - } - - #[test] - fn bodiless_statuses_and_head_have_no_body() { - for status in [ - "101 Switching Protocols", - "204 No Content", - "304 Not Modified", - ] { - let bytes = format!("HTTP/1.1 {status}\r\nContent-Length: 4\r\n\r\nnext"); - let response = parse_response(bytes.as_bytes(), false).expect("a response"); - assert!(response.body.is_empty(), "{status}"); - } - let head = parse_response(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n", true) - .expect("a response"); - assert!(head.body.is_empty()); - } - #[test] fn non_http_or_incomplete_bytes_are_not_a_response() { assert_eq!(parse_response(b"", false), None); @@ -403,33 +288,4 @@ mod tests { assert_eq!(parse_response(b"HTTP/1.1 20 OK\r\n\r\n", false), None); assert_eq!(parse_response(b"HTTP/2 200\r\n\r\n", false), None); } - - #[test] - fn http_parses_what_the_probe_received() { - let probe = Probe { - connect: ConnectOutcome::Connected, - connect_elapsed: Duration::from_millis(3), - read_end: Some(ReadEnd::Closed), - received: b"HTTP/1.1 403 Forbidden\r\nContent-Length: 2\r\n\r\nno".to_vec(), - }; - let http = Http::from_probe(probe, false); - assert_eq!(http.status(), Some(403)); - assert_eq!( - http.response.as_ref().expect("a response").body_text(), - "no" - ); - assert!(http.received_text().starts_with("HTTP/1.1 403")); - - let reset = Http::from_probe( - Probe { - connect: ConnectOutcome::Connected, - connect_elapsed: Duration::ZERO, - read_end: Some(ReadEnd::Reset), - received: Vec::new(), - }, - false, - ); - assert_eq!(reset.status(), None); - assert_eq!(reset.read_end, Some(ReadEnd::Reset)); - } } diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs index ad94d3c4fe..c967d24b00 100644 --- a/tests/support/sandbox-clients/src/probe.rs +++ b/tests/support/sandbox-clients/src/probe.rs @@ -226,120 +226,3 @@ fn from_hex(hex: &str) -> Result> { }) .collect() } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::*; - - fn probe_json(overrides: serde_json::Value) -> serde_json::Value { - let mut value = json!({ - "connect": "ok", - "errno": null, - "error": null, - "connect_ms": 12, - "read_end": null, - "read_errno": null, - "read_error": null, - "received_hex": "", - }); - for (key, field) in overrides.as_object().expect("an object") { - value[key] = field.clone(); - } - value - } - - #[test] - fn a_connect_only_probe_has_no_read() { - let probe = parse_probe(probe_json(json!({}))).expect("parses"); - assert_eq!( - probe, - Probe { - connect: ConnectOutcome::Connected, - connect_elapsed: Duration::from_millis(12), - read_end: None, - received: Vec::new(), - } - ); - } - - #[test] - fn a_failed_connect_keeps_its_errno() { - let probe = parse_probe(probe_json(json!({ - "connect": "failed", - "errno": 13, - "error": "PermissionError(13, 'Permission denied')", - }))) - .expect("parses"); - assert_eq!(probe.connect.errno(), Some(13)); - assert!(probe.connect.is_permission_denied()); - assert!(!probe.connect.is_connected()); - - let refused = parse_probe(probe_json(json!({"connect": "failed", "errno": 111}))) - .expect("parses") - .connect; - assert!(!refused.is_permission_denied()); - let eperm = parse_probe(probe_json(json!({"connect": "failed", "errno": 1}))) - .expect("parses") - .connect; - assert!(eperm.is_permission_denied()); - } - - #[test] - fn a_connect_timeout_is_not_a_failure() { - let probe = parse_probe(probe_json( - json!({"connect": "timeout", "connect_ms": 10_000}), - )) - .expect("parses"); - assert_eq!(probe.connect, ConnectOutcome::TimedOut); - assert_eq!(probe.connect.errno(), None); - assert_eq!(probe.connect_elapsed, Duration::from_secs(10)); - } - - #[test] - fn each_read_end_is_distinct() { - for (name, end) in [ - ("closed", ReadEnd::Closed), - ("reset", ReadEnd::Reset), - ("idle", ReadEnd::Idle), - ("timeout", ReadEnd::TimedOut), - ] { - let probe = parse_probe(probe_json( - json!({"read_end": name, "received_hex": "6869"}), - )) - .expect("parses"); - assert_eq!(probe.read_end, Some(end), "{name}"); - assert_eq!(probe.received, b"hi"); - } - let failed = parse_probe(probe_json(json!({ - "read_end": "failed", - "read_errno": 32, - "read_error": "BrokenPipeError(32, 'Broken pipe')", - }))) - .expect("parses"); - assert_eq!( - failed.read_end, - Some(ReadEnd::Failed { - errno: Some(32), - error: "BrokenPipeError(32, 'Broken pipe')".to_string(), - }) - ); - } - - #[test] - fn an_unknown_outcome_is_an_error() { - assert!(parse_probe(probe_json(json!({"connect": "maybe"}))).is_err()); - assert!(parse_probe(probe_json(json!({"read_end": "later"}))).is_err()); - assert!(parse_probe(json!({"connect": "ok"})).is_err()); - } - - #[test] - fn hex_round_trips_and_rejects_bad_input() { - let bytes = [0x00, 0xff, 0x13, 0x37, b'a']; - assert_eq!(to_hex(&bytes), "00ff133761"); - assert_eq!(from_hex(&to_hex(&bytes)).expect("hex"), bytes); - assert!(from_hex("abc").is_err()); - assert!(from_hex("zz").is_err()); - } -} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 4c78fe9167..ae9cf4dbcb 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -951,47 +951,6 @@ fn generate_run_id() -> String { mod tests { use super::*; - fn exec_output(stdout: &str) -> ExecOutput { - ExecOutput { - description: "exec in sandbox".to_string(), - exit_code: 0, - stdout: stdout.to_string(), - stderr: String::new(), - } - } - - #[test] - fn last_json_takes_the_last_object_or_array_line() { - let output = exec_output("{\"first\": 1}\nprogress\n {\"last\": true} \n42\nDONE\n"); - assert_eq!( - output.last_json().expect("a JSON line"), - serde_json::json!({"last": true}) - ); - assert_eq!( - exec_output("[1, 2]\n").last_json().expect("a JSON line"), - serde_json::json!([1, 2]) - ); - } - - #[test] - fn last_json_reports_output_without_a_json_line() { - let error = exec_output("ECHO_OK\n7\n") - .last_json() - .expect_err("no object or array"); - assert!(error.to_string().contains("ECHO_OK"), "{error}"); - } - - #[test] - fn script_summary_skips_blank_and_license_lines() { - let script = "\n# SPDX-FileCopyrightText: x\n# SPDX-License-Identifier: y\n\n# Sends a request.\nimport json\n"; - assert_eq!(script_summary(script), "# Sends a request."); - assert_eq!( - script_summary("import sys; print(1)"), - "import sys; print(1)" - ); - assert_eq!(script_summary(""), ""); - } - #[test] fn policy_create_request_keeps_the_spec_and_adds_the_policy() { let spec = SandboxSpec { diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index 54f441637d..c62c930528 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -197,26 +197,4 @@ mod tests { .expect("the serialized policy parses"); assert_eq!(round_trip, expected); } - - /// The list parses as the `binaries` of a policy, the same as - /// [`allow`] adds them. - #[test] - fn binaries_yaml_lists_each_path() { - let binaries = ["/usr/bin/python*", "/sandbox/.uv/python/*/bin/python*"]; - let yaml = format!( - "{BASE}network_policies: - upstream: - name: upstream - endpoints: - - {{host: example.test, port: 8080}} - binaries: {} -", - binaries_yaml(&binaries) - ); - let parsed = openshell_policy::parse_sandbox_policy(&yaml).expect("the YAML parses"); - let mut expected = base(); - allow(&mut expected, "upstream", "example.test", 8080, &binaries); - assert_eq!(parsed, expected); - assert_eq!(binaries_yaml(&[]), "[]"); - } } From c54c9d962dcb28b1c1a5350e1ba7b7c58745a0cc Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:14:09 -0800 Subject: [PATCH 09/17] test(features): leave the keycloak refresh test to #5657 Signed-off-by: Matthew Grossman #5657 makes the same install_provider change in that file along with its typed profile, so this branch leaves it unchanged to avoid a conflict. --- tests/suites/features/Cargo.lock | 1 + .../keycloak-provider-refresh/Cargo.toml | 1 + .../tests/provider_refresh.rs | 60 ++++++++++++++++--- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 28471bcfdc..4c65ad4021 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1833,6 +1833,7 @@ version = "0.0.0" dependencies = [ "keycloak-support", "openshell-testkit", + "tempfile", ] [[package]] diff --git a/tests/suites/features/keycloak-provider-refresh/Cargo.toml b/tests/suites/features/keycloak-provider-refresh/Cargo.toml index 1932a3f374..e326fef5cb 100644 --- a/tests/suites/features/keycloak-provider-refresh/Cargo.toml +++ b/tests/suites/features/keycloak-provider-refresh/Cargo.toml @@ -14,3 +14,4 @@ publish = false [dependencies] keycloak-support = { path = "../../../support/keycloak" } openshell-testkit = { path = "../../../testkit/testkit" } +tempfile = "3" diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index 9a8afa886d..a617c24843 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -6,8 +6,10 @@ //! Keycloak is only the provider's token issuer here; the CLI still reaches //! the gateway the usual way. +use std::io::Write as _; + use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context}; +use testkit::{CliOutput, Context, ResultExt as _}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -20,12 +22,32 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - let provider = kc.name("kc-refresh"); let profile_id = kc.name("kc-refresh-profile"); let grant = kc.keycloak().password_grant().await?; - kc.install_provider( - &provider, - &profile(&profile_id, kc.keycloak()), + let profile = write_profile(&profile_id, kc.keycloak())?; + let profile_path = profile.path().to_string_lossy().into_owned(); + + // Register each delete before its create, so cleanup also covers a create + // the gateway applied but the CLI reported as failed. Both deletes succeed + // when there is nothing to delete. + delete_on_cleanup(&kc, "profile", &profile_id); + kc.cli(["profile", "import", "--file", &profile_path]) + .await? + .success()?; + delete_on_cleanup(&kc, "provider", &provider); + kc.cli_with_env( &[(CREDENTIAL_KEY, &grant.access_token)], + [ + "provider", + "create", + "--name", + &provider, + "--type", + &profile_id, + "--credential", + CREDENTIAL_KEY, + ], ) - .await?; + .await? + .success()?; let client_id = format!("client_id={}", kc.keycloak().client_id()); kc.cli_with_env( &[("KEYCLOAK_REFRESH_TOKEN", &grant.refresh_token)], @@ -95,13 +117,27 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - Ok(()) } +/// Delete the `kind` named `name` when the test ends, since testkit only +/// cleans up sandboxes by itself. +fn delete_on_cleanup(cx: &Context, kind: &'static str, name: &str) { + let cli = cx.cli_handle(); + let name = name.to_string(); + cx.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &name]).await?.success()?; + Ok(()) + }); +} + fn combined(output: &CliOutput) -> String { format!("{}{}", output.stdout, output.stderr) } -fn profile(profile_id: &str, keycloak: &keycloak_support::Keycloak) -> String { +fn write_profile( + profile_id: &str, + keycloak: &keycloak_support::Keycloak, +) -> testkit::Result { let issuer_token_url = keycloak.token_endpoint(); - format!( + let profile = format!( r#"id: {profile_id} display_name: Keycloak provider refresh test category: other @@ -132,5 +168,13 @@ endpoints: binaries: - /usr/bin/curl "# - ) + ); + let mut file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .context("create provider profile")?; + file.write_all(profile.as_bytes()) + .and_then(|()| file.flush()) + .context("write provider profile")?; + Ok(file) } From 1a3fdc8cb23a1bdb34fbe2752e10a86145d6cfe1 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:18:07 -0800 Subject: [PATCH 10/17] test(testkit): fail early on sandbox names the gateway rejects The gateway limits sandbox names to 19 characters, which leaves 5 for a suffix after test--. cx.sandbox, cx.cli_sandbox and cx.track now panic at the test's call with that explanation, instead of the test failing later with the gateway's validation error. Context::name is unchanged, since provider and profile names have no such limit. Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 6 +++- tests/testkit/testkit/src/context.rs | 43 ++++++++++++++++++++++++++-- tests/testkit/testkit/src/lib.rs | 4 +-- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 4d12a7eefb..1566290f18 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -63,7 +63,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. `Context` currently provides: -- `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then +- `cx.sandbox(suffix)` to build a sandbox with a run-unique name + (`test--`), then `create()` it or `create_ready()` to also wait for Ready. Set its image with `image(..)`, its policy with `policy(..)` and its providers with `providers(..)`. A long-running sandbox is @@ -82,6 +83,9 @@ fails. A killed test process skips cleanup, so names are unique to the run. can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, and `run()` returns the attached CLI output. +- Sandbox names are limited to `testkit::MAX_SANDBOX_NAME_LEN` (19) + characters, so a sandbox suffix can have at most 5. `cx.sandbox`, + `cx.cli_sandbox` and `cx.track` panic on a longer name. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index ae9cf4dbcb..fb7207ea8b 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -177,12 +177,18 @@ impl Context { } /// Start describing a sandbox named `cx.name(suffix)`. + /// + /// # Panics + /// + /// If the name is longer than the gateway allows; see + /// [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn sandbox(&self, suffix: &str) -> SandboxBuilder { SandboxBuilder { handles: self.handles.clone(), policy: None, spec: SandboxSpec { - name: Some(self.name(suffix)), + name: Some(self.sandbox_name(suffix)), ..SandboxSpec::default() }, } @@ -192,8 +198,20 @@ impl Context { /// `openshell` CLI creates. Use it for what only `sandbox create` /// offers, such as a `--policy` file or `--no-keep`; prefer /// [`Context::sandbox`] otherwise. + /// + /// # Panics + /// + /// As [`Context::sandbox`]. + #[track_caller] pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder { - CliSandboxBuilder::new(self.handles.clone(), self.name(suffix)) + CliSandboxBuilder::new(self.handles.clone(), self.sandbox_name(suffix)) + } + + #[track_caller] + fn sandbox_name(&self, suffix: &str) -> String { + let name = self.name(suffix); + check_sandbox_name(&name); + name } /// Track a sandbox the test is about to create through the CLI, so cleanup @@ -203,7 +221,13 @@ impl Context { /// Cleanup deletes the sandbox and waits until it is gone, at the point /// in the cleanup order where it was first tracked. Tracking a sandbox /// again, for example by adopting it, doesn't move it. + /// + /// # Panics + /// + /// If `name` is longer than [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn track(&self, name: &str) { + check_sandbox_name(name); self.handles.track(name); } @@ -938,6 +962,21 @@ impl Context { } } +/// The longest sandbox name the gateway accepts. With `test--`, a +/// sandbox suffix can have at most 5 characters. +pub const MAX_SANDBOX_NAME_LEN: usize = 19; + +/// Panic, at the test's call, on a sandbox name the gateway would reject, so +/// the test fails with this message instead of the gateway's. +#[track_caller] +fn check_sandbox_name(name: &str) { + assert!( + name.len() <= MAX_SANDBOX_NAME_LEN, + "sandbox name {name:?} has {} characters; the gateway allows {MAX_SANDBOX_NAME_LEN}, so a suffix after `test--` can have at most 5", + name.len() + ); +} + /// Short, lowercase, and DNS-label safe, so names work on every driver. fn generate_run_id() -> String { const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7c233ce834..e86b7c3b14 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -49,8 +49,8 @@ pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; pub use cli_child::CliChild; pub use cli_sandbox::CliSandboxBuilder; pub use context::{ - Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, - SandboxBuilder, + Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, MAX_SANDBOX_NAME_LEN, READY_TIMEOUT, + STOP_TIMEOUT, Sandbox, SandboxBuilder, }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; From 6eae90b9566d3afa73fb4eeaf55a636c53815dc1 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 10:12:24 -0800 Subject: [PATCH 11/17] docs(testkit): document how to write a sandbox client The testkit README now explains why clients run in the sandbox, the contract a client script follows, and when a script moves to sandbox-clients. tcp_probe.py points to it, sandbox-clients says it holds network clients only, TESTING.md points Rust tests to run_python, and run_python suggests python_image() when the sandbox has no python3. Signed-off-by: Matthew Grossman --- TESTING.md | 5 +++++ tests/support/sandbox-clients/Cargo.toml | 2 +- .../sandbox-clients/scripts/tcp_probe.py | 5 +++++ tests/support/sandbox-clients/src/lib.rs | 7 ++++++- tests/testkit/README.md | 21 +++++++++++++++++++ tests/testkit/testkit/src/context.rs | 10 ++++++++- 6 files changed, 47 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index 5a8a3942aa..9d0f2bc29b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -180,6 +180,11 @@ by `OPENSHELL_BIN`. See [tests/testkit/README.md](tests/testkit/README.md) for how to write and run tests. +To run Python in a testkit sandbox, use `Sandbox::run_python` with a sandbox +from `testkit::python_image()`, not the Python SDK's `exec_python` described +under Python E2E. The README's "Writing a sandbox client" section covers +scripts that probe the network from inside a sandbox. + The conformance crate also builds `openshell-conformance`, a program that runs the conformance suite against a gateway without cargo or nextest (`openshell-conformance list`, `ready`, `run`). It covers only the diff --git a/tests/support/sandbox-clients/Cargo.toml b/tests/support/sandbox-clients/Cargo.toml index 9a7455e7c0..96dfb07ac3 100644 --- a/tests/support/sandbox-clients/Cargo.toml +++ b/tests/support/sandbox-clients/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "sandbox-clients-support" -description = "Clients that tests run in a sandbox, and typed probes for their results" +description = "Network clients that tests run in a sandbox, and typed probes for their results" version = "0.0.0" edition = "2024" rust-version = "1.94" diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py index e1209729cd..d497b42aed 100644 --- a/tests/support/sandbox-clients/scripts/tcp_probe.py +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -1,6 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Runs inside the sandbox, so the sandbox policy applies to its connection. +# It follows the contract in tests/testkit/README.md ("Writing a sandbox +# client"): standard library only, input from argv, one JSON line out, and +# parsing in Rust (src/probe.rs). +# # Connects to argv[1]:argv[2], sends the bytes hex-encoded in argv[3] ("-" # to only connect), and reads until the connection ends. Prints how each step # went as one JSON line; failures to connect or read are results, not errors. diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs index 392507aa43..0be0298727 100644 --- a/tests/support/sandbox-clients/src/lib.rs +++ b/tests/support/sandbox-clients/src/lib.rs @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Clients that tests run inside a sandbox, and typed results for them. +//! Network clients that tests run inside a sandbox, and typed results for +//! them. Each runs in the sandbox, so the sandbox policy applies to it. File +//! system or system-call probes belong in a sibling crate. +//! +//! See "Writing a sandbox client" in `tests/testkit/README.md` for the +//! contract a client script follows. //! //! The clients run in a sandbox created from [`testkit::python_image`], with a //! policy that lets [`testkit::PYTHON_BINARIES`] reach the target: diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 1566290f18..d655ae68a8 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -142,6 +142,27 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `poll_until` for waiting on other state, and `with_deadline(what, timeout, future)` for a step that must finish sooner than its own timeout. +## Writing a sandbox client + +A test that checks what a sandbox can reach runs its client inside the +sandbox, because the sandbox policy applies only to processes there. A client +script: + +- uses only Python's standard library, and runs with `sandbox.run_python(script, + args)` in a sandbox from `testkit::python_image()`; +- takes its input as arguments; +- prints its result as one JSON line, which the test reads with + `output.last_json()`. A refused connection or a failed read is a result, + not a crash: the script exits non-zero only on a bug; +- leaves parsing and assertions to Rust. + +A script only one test uses can stay beside that test, as a `const` or an +`include_str!`. Move it to `tests/support/sandbox-clients/scripts/`, with a +constant in its `scripts` module, once a second test or suite needs it, so no +suite reads another workspace's files. If its result has a fixed shape, add a +typed probe there too, like `connect` and `http`. `sandbox-clients` holds +network clients; put file-system or system-call probes in a sibling crate. + ## Running against a gateway Tests find the gateway the same way the `openshell` CLI does: diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index fb7207ea8b..23b99c8e32 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -804,7 +804,15 @@ impl Sandbox { script_summary(script), self.name ); - output.success() + let missing_python = + output.exit_code == 127 || output.stderr.contains("python3: not found"); + output.success().map_err(|error| { + if missing_python { + error.context("the sandbox has no python3; create it from testkit::python_image()") + } else { + error + } + }) } /// The sandbox's policy revisions, from `policy list --output json`. From b67830ef09f29c05e0728c8838fdeb9a88729458 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 11:04:58 -0800 Subject: [PATCH 12/17] test(conformance): use the shared TEST_HOST in credential gating Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 2d67926ef0..24bffdbff9 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -10,32 +10,31 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, TEST_HOST, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{ Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, }; -const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -/// POSTs a body containing the provider's token to `TEST_HOST` on the port -/// in `argv[1]`, directly rather than through a proxy. +/// POSTs a body containing the provider's token to the host in `argv[1]` on +/// the port in `argv[2]`, directly rather than through a proxy. const BODY_CLIENT: &str = r#" import os, sys, urllib.request body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() -request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) +request = urllib.request.Request(f"http://{sys.argv[1]}:{sys.argv[2]}/token", data=body, headers={"Content-Type": "text/plain"}) urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() "#; -/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one -/// binary frame, and waits briefly for the connection to be closed. +/// Upgrades to WebSocket at the host in `argv[1]` on the port in `argv[2]`, +/// sends one binary frame, and waits briefly for the connection to be closed. const WEBSOCKET_CLIENT: &str = r#" import base64, os, socket, sys -host, port = "host.openshell.internal", int(sys.argv[1]) +host, port = sys.argv[1], int(sys.argv[2]) with socket.create_connection((host, port), timeout=10) as sock: key = base64.b64encode(os.urandom(16)).decode() sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) @@ -336,11 +335,12 @@ async fn expect_rejected( } } -/// Run `script` with the image's Python and `port` as its argument. +/// Run `script` with the image's Python, and [`TEST_HOST`] and `port` as its +/// arguments. async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { let port = port.to_string(); sandbox - .exec(["python3", "-c", script, &port]) + .exec(["python3", "-c", script, TEST_HOST, &port]) .await? .success() .context("run the client in the sandbox")?; From 06b770c9d034ab85ce9a0b5dac293e5411d0b088 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 11:04:58 -0800 Subject: [PATCH 13/17] fix(support): report DNS failures from the connect probe as dns, not errno Signed-off-by: Matthew Grossman --- tests/support/sandbox-clients/scripts/tcp_probe.py | 7 ++++++- tests/support/sandbox-clients/src/probe.rs | 11 ++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py index d497b42aed..adfe47d742 100644 --- a/tests/support/sandbox-clients/scripts/tcp_probe.py +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -29,6 +29,7 @@ result = { "connect": None, "errno": None, + "dns": False, "error": None, "connect_ms": None, "read_end": None, @@ -77,7 +78,11 @@ def exchange(sock): result["connect"] = "timeout" except OSError as error: result["connect"] = "failed" - result["errno"] = error.errno + # A resolver failure's code is an EAI_* value, not an errno. + if isinstance(error, socket.gaierror): + result["dns"] = True + else: + result["errno"] = error.errno result["error"] = repr(error) else: result["connect"] = "ok" diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs index c967d24b00..10dc5a0b01 100644 --- a/tests/support/sandbox-clients/src/probe.rs +++ b/tests/support/sandbox-clients/src/probe.rs @@ -28,10 +28,12 @@ const EACCES: i32 = 13; #[derive(Debug, Clone, PartialEq, Eq)] pub enum ConnectOutcome { Connected, - /// The connect failed with `errno`, or none for an error without one. + /// The connect failed with `errno`, or none for an error without one, + /// such as a name that didn't resolve ([`ConnectOutcome::is_dns_failure`]). /// `error` is Python's description of it. Failed { errno: Option, + dns: bool, error: String, }, TimedOut, @@ -50,6 +52,11 @@ impl ConnectOutcome { } } + /// Whether the connect failed because the name didn't resolve. + pub fn is_dns_failure(&self) -> bool { + matches!(self, Self::Failed { dns: true, .. }) + } + /// Whether the connect failed with `EACCES` or `EPERM`, as the sandbox /// denies a connection. pub fn is_permission_denied(&self) -> bool { @@ -147,6 +154,7 @@ async fn run( struct ProbeJson { connect: ConnectJson, errno: Option, + dns: bool, error: Option, connect_ms: u64, read_end: Option, @@ -180,6 +188,7 @@ fn parse_probe(value: serde_json::Value) -> Result { ConnectJson::Ok => ConnectOutcome::Connected, ConnectJson::Failed => ConnectOutcome::Failed { errno: json.errno, + dns: json.dns, error: json.error.unwrap_or_default(), }, ConnectJson::Timeout => ConnectOutcome::TimedOut, From 9ecf6b746743c5e5272d57c8ae1ed679afc0b715 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:16:56 -0800 Subject: [PATCH 14/17] refactor(support): drop the unused is_dns_failure helper Signed-off-by: Matthew Grossman --- tests/support/sandbox-clients/src/probe.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs index 10dc5a0b01..8e0abf9631 100644 --- a/tests/support/sandbox-clients/src/probe.rs +++ b/tests/support/sandbox-clients/src/probe.rs @@ -29,8 +29,8 @@ const EACCES: i32 = 13; pub enum ConnectOutcome { Connected, /// The connect failed with `errno`, or none for an error without one, - /// such as a name that didn't resolve ([`ConnectOutcome::is_dns_failure`]). - /// `error` is Python's description of it. + /// such as a name that didn't resolve (`dns`). `error` is Python's + /// description of it. Failed { errno: Option, dns: bool, @@ -52,11 +52,6 @@ impl ConnectOutcome { } } - /// Whether the connect failed because the name didn't resolve. - pub fn is_dns_failure(&self) -> bool { - matches!(self, Self::Failed { dns: true, .. }) - } - /// Whether the connect failed with `EACCES` or `EPERM`, as the sandbox /// denies a connection. pub fn is_permission_denied(&self) -> bool { From e38f314ce1c0e753d8b9987b3d401dfa11ea6cf9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:16:56 -0800 Subject: [PATCH 15/17] refactor(support): parse HTTP replies with httparse Signed-off-by: Matthew Grossman --- tests/support/Cargo.lock | 1 + tests/support/sandbox-clients/Cargo.toml | 1 + tests/support/sandbox-clients/src/message.rs | 49 +++++++------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index a51182ae34..83e6c075e1 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -3186,6 +3186,7 @@ dependencies = [ name = "sandbox-clients-support" version = "0.0.0" dependencies = [ + "httparse", "openshell-testkit", "serde", "serde_json", diff --git a/tests/support/sandbox-clients/Cargo.toml b/tests/support/sandbox-clients/Cargo.toml index 96dfb07ac3..7a52251539 100644 --- a/tests/support/sandbox-clients/Cargo.toml +++ b/tests/support/sandbox-clients/Cargo.toml @@ -12,6 +12,7 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +httparse = "1" openshell-testkit = { path = "../../testkit/testkit" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs index e74b6fca1a..17079c2f8d 100644 --- a/tests/support/sandbox-clients/src/message.rs +++ b/tests/support/sandbox-clients/src/message.rs @@ -186,17 +186,21 @@ impl fmt::Debug for Response { /// end of `bytes` without either. A truncated body is returned as far as it /// arrived. fn parse_response(bytes: &[u8], head: bool) -> Option { - let end = find(bytes, b"\r\n\r\n")?; - let text = std::str::from_utf8(&bytes[..end]).ok()?; - let mut lines = text.split("\r\n"); - let status = parse_status_line(lines.next()?)?; - let headers: Vec<(String, String)> = lines - .filter_map(|line| { - let (name, value) = line.split_once(':')?; - Some((name.trim().to_string(), value.trim().to_string())) + let mut headers = [httparse::EMPTY_HEADER; 64]; + let mut parsed = httparse::Response::new(&mut headers); + let httparse::Status::Complete(end) = parsed.parse(bytes).ok()? else { + return None; + }; + let status = parsed.code?; + let headers = parsed + .headers + .iter() + .map(|header| { + let value = String::from_utf8_lossy(header.value).into_owned(); + (header.name.to_string(), value) }) .collect(); - let rest = &bytes[end + 4..]; + let rest = &bytes[end..]; let mut response = Response { status, headers, @@ -221,28 +225,13 @@ fn parse_response(bytes: &[u8], head: bool) -> Option { Some(response) } -/// The status code in `HTTP/1.x NNN reason`. -fn parse_status_line(line: &str) -> Option { - let mut parts = line.splitn(3, ' '); - let version = parts.next()?; - let code = parts.next()?; - if !version.starts_with("HTTP/1.") || code.len() != 3 { - return None; - } - code.parse().ok() -} - /// The data in a chunked body, up to the last complete chunk if it is cut /// short. fn dechunk(mut rest: &[u8]) -> Vec { let mut body = Vec::new(); - while let Some(line_end) = find(rest, b"\r\n") { - let size_line = String::from_utf8_lossy(&rest[..line_end]); - let size = size_line.split(';').next().unwrap_or_default().trim(); - let Ok(size) = usize::from_str_radix(size, 16) else { - break; - }; - rest = &rest[line_end + 2..]; + while let Ok(httparse::Status::Complete((start, size))) = httparse::parse_chunk_size(rest) { + let size = usize::try_from(size).unwrap_or(usize::MAX); + rest = &rest[start..]; if size == 0 { break; } @@ -256,12 +245,6 @@ fn dechunk(mut rest: &[u8]) -> Vec { body } -fn find(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - #[cfg(test)] mod tests { use super::*; From 38d122cd538f265b74d0af741e07646cabea75e0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:22:16 -0800 Subject: [PATCH 16/17] fix(support): report a stalled send from the connect probe as a timeout Signed-off-by: Matthew Grossman --- tests/support/sandbox-clients/scripts/tcp_probe.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py index adfe47d742..4013192e01 100644 --- a/tests/support/sandbox-clients/scripts/tcp_probe.py +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -11,7 +11,7 @@ # went as one JSON line; failures to connect or read are results, not errors. # # argv[4]: connect timeout in seconds. -# argv[5]: read timeout in seconds, for all of the reading. +# argv[5]: timeout in seconds for sending and all of the reading. # argv[6]: idle timeout in seconds: stop reading after this long without # bytes. 0 waits for the read timeout instead. import json @@ -47,9 +47,10 @@ def exchange(sock): """Send PAYLOAD and read until the connection ends. Returns the bytes read and how reading ended.""" received = bytearray() + deadline = time.monotonic() + READ_TIMEOUT try: + sock.settimeout(READ_TIMEOUT) sock.sendall(PAYLOAD) - deadline = time.monotonic() + READ_TIMEOUT while True: remaining = deadline - time.monotonic() if remaining <= 0: @@ -63,6 +64,8 @@ def exchange(sock): if not chunk: return received, "closed" received += chunk + except socket.timeout: + return received, "timeout" except ConnectionResetError: return received, "reset" except OSError as error: From a81fb3264dfa55bcea9c2b2472af695b3a1ee36a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:22:16 -0800 Subject: [PATCH 17/17] docs(testkit): link the sandbox client guide and the server name limit Signed-off-by: Matthew Grossman --- TESTING.md | 4 ++-- tests/support/sandbox-clients/src/message.rs | 4 +++- tests/testkit/testkit/src/context.rs | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/TESTING.md b/TESTING.md index 9d0f2bc29b..1fd252b182 100644 --- a/TESTING.md +++ b/TESTING.md @@ -182,8 +182,8 @@ tests. To run Python in a testkit sandbox, use `Sandbox::run_python` with a sandbox from `testkit::python_image()`, not the Python SDK's `exec_python` described -under Python E2E. The README's "Writing a sandbox client" section covers -scripts that probe the network from inside a sandbox. +under Python E2E. [Writing a sandbox client](tests/testkit/README.md#writing-a-sandbox-client) +covers scripts that probe the network from inside a sandbox. The conformance crate also builds `openshell-conformance`, a program that runs the conformance suite against a gateway without cargo or nextest diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs index 17079c2f8d..81c6fb8665 100644 --- a/tests/support/sandbox-clients/src/message.rs +++ b/tests/support/sandbox-clients/src/message.rs @@ -149,7 +149,9 @@ pub struct Response { pub status: u16, /// The headers, in order, as they arrived. pub headers: Vec<(String, String)>, - /// The body, with chunked transfer encoding removed. + /// The body, with chunked transfer encoding removed. A body cut short is + /// returned as far as it arrived; [`Http::read_end`] says why reading + /// stopped. pub body: Vec, } diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 23b99c8e32..0a671483ae 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -970,8 +970,9 @@ impl Context { } } -/// The longest sandbox name the gateway accepts. With `test--`, a -/// sandbox suffix can have at most 5 characters. +/// The longest sandbox name the gateway accepts, the server's +/// `MAX_ROUTABLE_NAME_LEN` in `crates/openshell-server/src/grpc/mod.rs`. With +/// `test--`, a sandbox suffix can have at most 5 characters. pub const MAX_SANDBOX_NAME_LEN: usize = 19; /// Panic, at the test's call, on a sandbox name the gateway would reject, so