diff --git a/TESTING.md b/TESTING.md index 5a8a3942aa..1fd252b182 100644 --- a/TESTING.md +++ b/TESTING.md @@ -180,6 +180,11 @@ by `OPENSHELL_BIN`. See [tests/testkit/README.md](tests/testkit/README.md) for how to write and run tests. +To run Python in a testkit sandbox, use `Sandbox::run_python` with a sandbox +from `testkit::python_image()`, not the Python SDK's `exec_python` described +under Python E2E. [Writing a sandbox client](tests/testkit/README.md#writing-a-sandbox-client) +covers scripts that probe the network from inside a sandbox. + The conformance crate also builds `openshell-conformance`, a program that runs the conformance suite against a gateway without cargo or nextest (`openshell-conformance list`, `ready`, `run`). It covers only the diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index 169894def0..9f87643763 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -8,7 +8,6 @@ //! check that the CLI commands for create, get, list, exec, stop, start, and //! delete work against the same gateway. -use serde_json::Value; use testkit::Context; /// Delay before an attached main prints and exits, so the CLI attaches while @@ -64,7 +63,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .cli_sandbox("cl") - .command(["sh", "-c", "exec sleep infinity"]) + .command(["sleep", "infinity"]) .create_ready() .await?; let name = sandbox.name().to_string(); @@ -112,8 +111,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> "sandbox {completed} is not listed after its main exited; sandboxes are kept by default" ); // The phase can trail the CLI's exit by a status update. - cx.adopt(&completed).await?.wait_finished().await?; - let got = get_json(cx, &completed).await?; + let sandbox = cx.adopt(&completed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; testkit::ensure!( got["phase"] == "Completed", "sandbox {completed} is in phase {}; expected Completed", @@ -136,8 +136,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> output.stderr.trim() ); expect_streamed(&output, "failed-main")?; - cx.adopt(&failed).await?.wait_finished().await?; - let got = get_json(cx, &failed).await?; + let sandbox = cx.adopt(&failed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; testkit::ensure!( got["phase"] == "Error" && got["exit_code"] == 7, "sandbox {failed} is in phase {} with exit code {}; expected Error with 7", @@ -198,13 +199,6 @@ fn expect_streamed(output: &testkit::CliOutput, marker: &str) -> testkit::Result Ok(()) } -async fn get_json(cx: &Context, name: &str) -> testkit::Result { - cx.cli(["sandbox", "get", name, "--output", "json"]) - .await? - .success()? - .json() -} - /// Wait until `sandbox list` no longer shows the sandbox. async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result { testkit::poll_until( diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 2d67926ef0..24bffdbff9 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -10,32 +10,31 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, TEST_HOST, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{ Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, }; -const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -/// POSTs a body containing the provider's token to `TEST_HOST` on the port -/// in `argv[1]`, directly rather than through a proxy. +/// POSTs a body containing the provider's token to the host in `argv[1]` on +/// the port in `argv[2]`, directly rather than through a proxy. const BODY_CLIENT: &str = r#" import os, sys, urllib.request body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() -request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) +request = urllib.request.Request(f"http://{sys.argv[1]}:{sys.argv[2]}/token", data=body, headers={"Content-Type": "text/plain"}) urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() "#; -/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one -/// binary frame, and waits briefly for the connection to be closed. +/// Upgrades to WebSocket at the host in `argv[1]` on the port in `argv[2]`, +/// sends one binary frame, and waits briefly for the connection to be closed. const WEBSOCKET_CLIENT: &str = r#" import base64, os, socket, sys -host, port = "host.openshell.internal", int(sys.argv[1]) +host, port = sys.argv[1], int(sys.argv[2]) with socket.create_connection((host, port), timeout=10) as sock: key = base64.b64encode(os.urandom(16)).decode() sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) @@ -336,11 +335,12 @@ async fn expect_rejected( } } -/// Run `script` with the image's Python and `port` as its argument. +/// Run `script` with the image's Python, and [`TEST_HOST`] and `port` as its +/// arguments. async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { let port = port.to_string(); sandbox - .exec(["python3", "-c", script, &port]) + .exec(["python3", "-c", script, TEST_HOST, &port]) .await? .success() .context("run the client in the sandbox")?; diff --git a/tests/suites/conformance/src/exec_cli.rs b/tests/suites/conformance/src/exec_cli.rs index 35924cb351..e854277abc 100644 --- a/tests/suites/conformance/src/exec_cli.rs +++ b/tests/suites/conformance/src/exec_cli.rs @@ -21,7 +21,11 @@ const STDIN_LIMIT: usize = 4 * 1024 * 1024; #[testkit::test] async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result { const BYTES: usize = 8 * 1024 * 1024; - let sandbox = ready_sandbox(cx, "lo").await?; + let sandbox = cx + .sandbox("lo") + .command(["sleep", "infinity"]) + .create_ready() + .await?; for stream in ["stdout", "stderr"] { let script = match stream { @@ -88,7 +92,11 @@ async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result /// read fails the exec. #[testkit::test] async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> testkit::Result { - let sandbox = ready_sandbox(cx, "si").await?; + let sandbox = cx + .sandbox("si") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let wc = exec_args(&sandbox, &["wc", "-c"]); for size in [5, 1024 * 1024, STDIN_LIMIT] { @@ -165,13 +173,6 @@ async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> te Ok(()) } -async fn ready_sandbox(cx: &Context, suffix: &str) -> testkit::Result { - cx.sandbox(suffix) - .command(["sleep", "infinity"]) - .create_ready() - .await -} - /// `sandbox exec` of `command` without a TTY or login shell. fn exec_args(sandbox: &Sandbox, command: &[&str]) -> Vec { [ diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 82ff54cc09..31acc4fa04 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -224,9 +224,7 @@ async fn create( suffix: &str, policy: Option, ) -> testkit::Result { - let mut builder = cx - .sandbox(suffix) - .command(["sh", "-c", "exec sleep infinity"]); + let mut builder = cx.sandbox(suffix).command(["sleep", "infinity"]); if let Some(policy) = policy { builder = builder.policy(policy); } diff --git a/tests/suites/conformance/src/ssh.rs b/tests/suites/conformance/src/ssh.rs index 61dada7621..729ef3b51d 100644 --- a/tests/suites/conformance/src/ssh.rs +++ b/tests/suites/conformance/src/ssh.rs @@ -35,7 +35,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( cli.gateway().name.is_some(), "this test needs a named gateway for `sandbox ssh-config`; set OPENSHELL_GATEWAY" ); - let sandbox = ready_sandbox(cx).await?; + let sandbox = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let name = sandbox.name().to_string(); let original = fingerprint(cx, &sandbox).await?; @@ -110,7 +114,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( .wait_deleted(&name, testkit::DELETE_TIMEOUT, Some(sandbox.id())) .await .context(format!("wait for sandbox {name} to be deleted"))?; - let replacement = ready_sandbox(cx).await?; + let replacement = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; let replaced = fingerprint(cx, &replacement).await?; testkit::ensure!( replaced != original, @@ -119,13 +127,6 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( Ok(()) } -async fn ready_sandbox(cx: &Context) -> testkit::Result { - cx.sandbox("hk") - .command(["sleep", "infinity"]) - .create_ready() - .await -} - /// The sandbox's host key fingerprint, the same in get and list. async fn fingerprint(cx: &Context, sandbox: &Sandbox) -> testkit::Result { let name = sandbox.name(); diff --git a/tests/suites/features/cdi/src/lib.rs b/tests/suites/features/cdi/src/lib.rs index 46a94f3c86..ddadd385ef 100644 --- a/tests/suites/features/cdi/src/lib.rs +++ b/tests/suites/features/cdi/src/lib.rs @@ -374,26 +374,13 @@ pub async fn scenario(cx: &Context, name: &str) -> Result { let Scenario { device, checks } = load(name)?; let driver = compute_driver(cx).await?; - let sandbox = cx.name("cdi"); - cx.track(&sandbox); let config = json!({driver: {"cdi_devices": [device]}}).to_string(); let probe = probe_script(&checks); let output = cx - .cli([ - "sandbox", - "create", - "--name", - &sandbox, - "--no-tty", - "--gpu", - "1", - "--driver-config-json", - &config, - "--", - "sh", - "-c", - &probe, - ]) + .cli_sandbox("cdi") + .args(["--gpu", "1", "--driver-config-json", &config]) + .command(["sh", "-c", &probe]) + .run() .await?; let output = output diff --git a/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs index ec6faca9d6..f911f7819d 100644 --- a/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs +++ b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs @@ -62,7 +62,7 @@ mod linux { use nix::sys::signal::{Signal, kill}; use nix::unistd::Pid; - use testkit::{Context, ResultExt as _, Sandbox}; + use testkit::{Context, ResultExt as _}; use super::ATTACH_TIMEOUT; @@ -75,7 +75,11 @@ mod linux { const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; pub(super) async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { - let sandbox = ready_sandbox(cx, "tr", &["sh", "-lc", ECHO_MAIN]).await?; + let sandbox = cx + .sandbox("tr") + .command(["sh", "-lc", ECHO_MAIN]) + .create_ready() + .await?; let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?; let first = connect.stdout_line(ATTACH_TIMEOUT).await?; testkit::ensure!( @@ -120,7 +124,11 @@ mod linux { pub(super) async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( cx: &mut Context, ) -> testkit::Result { - let sandbox = ready_sandbox(cx, "term", &["sh", "-lc", "exec sleep infinity"]).await?; + let sandbox = cx + .sandbox("term") + .command(["sh", "-lc", "exec sleep infinity"]) + .create_ready() + .await?; let pty = nix::pty::openpty(None, None) .map_err(std::io::Error::from) .context("open a pseudo-terminal")?; @@ -158,17 +166,6 @@ mod linux { .await } - async fn ready_sandbox( - cx: &Context, - suffix: &str, - command: &[&str], - ) -> testkit::Result { - cx.sandbox(suffix) - .command(command.iter().copied()) - .create_ready() - .await - } - fn send(pid: u32, signal: Signal) -> testkit::Result { let pid = i32::try_from(pid) .map_err(|error| testkit::Error::msg(format!("pid {pid}: {error}")))?; diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 6c864b0e01..f18c4950f2 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -9,7 +9,7 @@ //! Python test image ([`testkit::python_image`]) that requests a page from that //! server through the name. -use host_server_support::HttpRecorder; +use host_server_support::{HttpRecorder, TEST_HOST}; use testkit::policy; use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; @@ -20,14 +20,14 @@ use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; /// behind a symlink. const PYTHON: &str = "python3"; -/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and +/// Requests `/` from the host named in `argv[1]` on the port in `argv[2]` and /// prints the response. The server listens before the sandbox is created, /// and the network is up before the main process starts, so the client /// doesn't retry. const CLIENT: &str = r#" import socket, sys -host, port = "host.openshell.internal", int(sys.argv[1]) +host, port = sys.argv[1], int(sys.argv[2]) with socket.create_connection((host, port), timeout=10) as conn: conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) response = b"" @@ -54,7 +54,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result .cli_sandbox("hg") .image(image.clone()) .policy_yaml(policy::to_yaml(&policy(server.port))?) - .command([PYTHON, "-c", CLIENT, &server.port.to_string()]) + .command([PYTHON, "-c", CLIENT, TEST_HOST, &server.port.to_string()]) .run() .await? .success() @@ -71,17 +71,11 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result } /// A sandbox policy that allows the image's Python ([`PYTHON`]) to -/// connect to `host.openshell.internal` on `port`. The endpoint names only the +/// connect to [`TEST_HOST`] on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. fn policy(port: u16) -> proto::SandboxPolicy { let mut policy = policy::base(); - policy::allow( - &mut policy, - "host_server", - "host.openshell.internal", - port, - PYTHON_BINARIES, - ); + policy::allow(&mut policy, "host_server", TEST_HOST, port, PYTHON_BINARIES); policy } diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index bc18df94cd..83e6c075e1 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -3182,6 +3182,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "httparse", + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/support/Cargo.toml b/tests/support/Cargo.toml index 4c7c495f97..e3dae4234d 100644 --- a/tests/support/Cargo.toml +++ b/tests/support/Cargo.toml @@ -5,4 +5,4 @@ # environment runs. Suites depend on them by path. [workspace] resolver = "2" -members = ["container-engine", "host-server", "keycloak", "podman"] +members = ["container-engine", "host-server", "keycloak", "podman", "sandbox-clients"] diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 8ac8528570..f10a0e0632 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -21,6 +21,9 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request}; pub use websocket::WebSocketRecorder; +/// The name a sandbox reaches these servers by. +pub const TEST_HOST: &str = "host.openshell.internal"; + const WAIT_TIMEOUT: Duration = Duration::from_secs(10); const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/tests/support/sandbox-clients/Cargo.toml b/tests/support/sandbox-clients/Cargo.toml new file mode 100644 index 0000000000..7a52251539 --- /dev/null +++ b/tests/support/sandbox-clients/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "sandbox-clients-support" +description = "Network clients that tests run in a sandbox, and typed probes for their results" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +httparse = "1" +openshell-testkit = { path = "../../testkit/testkit" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py new file mode 100644 index 0000000000..4013192e01 --- /dev/null +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs inside the sandbox, so the sandbox policy applies to its connection. +# It follows the contract in tests/testkit/README.md ("Writing a sandbox +# client"): standard library only, input from argv, one JSON line out, and +# parsing in Rust (src/probe.rs). +# +# Connects to argv[1]:argv[2], sends the bytes hex-encoded in argv[3] ("-" +# to only connect), and reads until the connection ends. Prints how each step +# went as one JSON line; failures to connect or read are results, not errors. +# +# argv[4]: connect timeout in seconds. +# argv[5]: timeout in seconds for sending and all of the reading. +# argv[6]: idle timeout in seconds: stop reading after this long without +# bytes. 0 waits for the read timeout instead. +import json +import socket +import sys +import time + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = None if sys.argv[3] == "-" else bytes.fromhex(sys.argv[3]) +CONNECT_TIMEOUT = float(sys.argv[4]) +READ_TIMEOUT = float(sys.argv[5]) +IDLE_TIMEOUT = float(sys.argv[6]) + +result = { + "connect": None, + "errno": None, + "dns": False, + "error": None, + "connect_ms": None, + "read_end": None, + "read_errno": None, + "read_error": None, + "received_hex": "", +} + + +def millis(since): + return int((time.monotonic() - since) * 1000) + + +def exchange(sock): + """Send PAYLOAD and read until the connection ends. Returns the bytes + read and how reading ended.""" + received = bytearray() + deadline = time.monotonic() + READ_TIMEOUT + try: + sock.settimeout(READ_TIMEOUT) + sock.sendall(PAYLOAD) + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return received, "timeout" + idle = 0 < IDLE_TIMEOUT < remaining + sock.settimeout(IDLE_TIMEOUT if idle else remaining) + try: + chunk = sock.recv(65536) + except socket.timeout: + return received, "idle" if idle else "timeout" + if not chunk: + return received, "closed" + received += chunk + except socket.timeout: + return received, "timeout" + except ConnectionResetError: + return received, "reset" + except OSError as error: + result["read_errno"] = error.errno + result["read_error"] = repr(error) + return received, "failed" + + +start = time.monotonic() +try: + sock = socket.create_connection((HOST, PORT), timeout=CONNECT_TIMEOUT) +except socket.timeout: + result["connect"] = "timeout" +except OSError as error: + result["connect"] = "failed" + # A resolver failure's code is an EAI_* value, not an errno. + if isinstance(error, socket.gaierror): + result["dns"] = True + else: + result["errno"] = error.errno + result["error"] = repr(error) +else: + result["connect"] = "ok" +result["connect_ms"] = millis(start) + +if result["connect"] == "ok": + with sock: + if PAYLOAD is not None: + received, read_end = exchange(sock) + result["received_hex"] = received.hex() + result["read_end"] = read_end + +print(json.dumps(result, sort_keys=True), flush=True) diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs new file mode 100644 index 0000000000..0be0298727 --- /dev/null +++ b/tests/support/sandbox-clients/src/lib.rs @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Network clients that tests run inside a sandbox, and typed results for +//! them. Each runs in the sandbox, so the sandbox policy applies to it. File +//! system or system-call probes belong in a sibling crate. +//! +//! See "Writing a sandbox client" in `tests/testkit/README.md` for the +//! contract a client script follows. +//! +//! The clients run in a sandbox created from [`testkit::python_image`], with a +//! policy that lets [`testkit::PYTHON_BINARIES`] reach the target: +//! +//! ```ignore +//! use host_server_support::{HttpRecorder, TEST_HOST}; +//! use sandbox_clients_support::{Request, http}; +//! use testkit::{PYTHON_BINARIES, policy}; +//! +//! let server = HttpRecorder::start("hello").await?; +//! let mut allowed = policy::base(); +//! policy::allow(&mut allowed, "server", TEST_HOST, server.port, PYTHON_BINARIES); +//! let sandbox = cx +//! .sandbox("probe") +//! .image(testkit::python_image()) +//! .command(["sleep", "infinity"]) +//! .policy(allowed) +//! .create_ready() +//! .await?; +//! let reply = http(&sandbox, TEST_HOST, server.port, &Request::get("/")).await?; +//! testkit::ensure!(reply.status() == Some(200), "{reply:?}"); +//! ``` +//! +//! - [`connect`] opens a TCP connection and reports whether it connected, +//! the errno it failed with, or a timeout, and how long it took. +//! - [`http`] sends a [`Request`], and [`http_raw`] sends bytes as they are. +//! Both report how the connection went, the bytes that came back, and the +//! parsed [`Response`]. +//! - [`scripts`] holds the client scripts, for tests that run one with +//! [`testkit::Sandbox::run_python`] and read its result with +//! [`testkit::ExecOutput::last_json`]. Protocol clients, such as `GraphQL` or +//! JSON-RPC ones, belong there too, so every suite can use them. +//! +//! Both probes use a raw socket. Sandboxes set no proxy variables, so this is +//! the same path a library such as `urllib` takes. + +mod message; +mod probe; + +pub use message::{Http, Request, Response}; +pub use probe::{Connect, ConnectOutcome, ReadEnd, connect, http, http_raw}; + +/// Client scripts, as text to pass to [`testkit::Sandbox::run_python`]. +pub mod scripts { + /// Connects to a host and port, optionally sends bytes and reads the + /// reply, and prints the outcome as one JSON line. [`crate::connect`] and + /// [`crate::http`] run it and parse its output. + pub const TCP_PROBE: &str = include_str!("../scripts/tcp_probe.py"); +} diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs new file mode 100644 index 0000000000..81c6fb8665 --- /dev/null +++ b/tests/support/sandbox-clients/src/message.rs @@ -0,0 +1,276 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP requests for [`crate::http`] and the replies it reads. + +use std::fmt; +use std::time::Duration; + +use crate::probe::{ConnectOutcome, Probe, ReadEnd}; + +/// An HTTP/1.1 request for [`crate::http`]. +/// +/// It sends `Host` for the target host and port, `Content-Length` for a body +/// or a method that takes one, and `Connection: close`, unless the request +/// sets those headers itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Request { + method: String, + target: String, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Request { + /// `GET target`. + pub fn get(target: impl Into) -> Self { + Self::new("GET", target) + } + + /// `method target`. The target is usually a path, such as `/allowed`, + /// but may be absolute, such as `http://host:port/allowed`. + pub fn new(method: impl Into, target: impl Into) -> Self { + Self { + method: method.into(), + target: target.into(), + headers: Vec::new(), + body: Vec::new(), + } + } + + /// Add a header. A header added more than once is sent more than once. + #[must_use] + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + /// Send `body` with the request. + #[must_use] + pub fn body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } + + pub(crate) fn is_head(&self) -> bool { + self.method.eq_ignore_ascii_case("HEAD") + } + + /// The request as bytes on the wire to `host:port`. + pub(crate) fn encode(&self, host: &str, port: u16) -> Vec { + let has = |name: &str| { + self.headers + .iter() + .any(|(header, _)| header.eq_ignore_ascii_case(name)) + }; + let mut head = format!("{} {} HTTP/1.1\r\n", self.method, self.target); + if !has("Host") { + if host.contains(':') { + head.push_str(&format!("Host: [{host}]:{port}\r\n")); + } else { + head.push_str(&format!("Host: {host}:{port}\r\n")); + } + } + for (name, value) in &self.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let takes_body = ["POST", "PUT", "PATCH"] + .iter() + .any(|method| self.method.eq_ignore_ascii_case(method)); + if (takes_body || !self.body.is_empty()) + && !has("Content-Length") + && !has("Transfer-Encoding") + { + head.push_str(&format!("Content-Length: {}\r\n", self.body.len())); + } + if !has("Connection") { + head.push_str("Connection: close\r\n"); + } + head.push_str("\r\n"); + let mut bytes = head.into_bytes(); + bytes.extend_from_slice(&self.body); + bytes + } +} + +/// The result of [`crate::http`] or [`crate::http_raw`]. +#[derive(Clone, PartialEq, Eq)] +pub struct Http { + pub connect: ConnectOutcome, + /// How long the connect took. + pub elapsed: Duration, + /// How reading the reply ended, or none if the connect failed. + pub read_end: Option, + /// Every byte that came back. + pub received: Vec, + /// The reply parsed as an HTTP response, or none if nothing came back or + /// it isn't one. Only the first response is parsed. + pub response: Option, +} + +impl Http { + pub(crate) fn from_probe(probe: Probe, head: bool) -> Self { + let response = parse_response(&probe.received, head); + Self { + connect: probe.connect, + elapsed: probe.connect_elapsed, + read_end: probe.read_end, + received: probe.received, + response, + } + } + + /// The response's status, if one came back. + pub fn status(&self) -> Option { + self.response.as_ref().map(|response| response.status) + } + + /// [`Http::received`] as text, with invalid UTF-8 replaced. + pub fn received_text(&self) -> String { + String::from_utf8_lossy(&self.received).into_owned() + } +} + +impl fmt::Debug for Http { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Http") + .field("connect", &self.connect) + .field("elapsed", &self.elapsed) + .field("read_end", &self.read_end) + .field("received", &self.received_text()) + .field("response", &self.response) + .finish() + } +} + +/// An HTTP response. +#[derive(Clone, PartialEq, Eq)] +pub struct Response { + pub status: u16, + /// The headers, in order, as they arrived. + pub headers: Vec<(String, String)>, + /// The body, with chunked transfer encoding removed. A body cut short is + /// returned as far as it arrived; [`Http::read_end`] says why reading + /// stopped. + pub body: Vec, +} + +impl Response { + /// The first header named `name`, ignoring case. + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header, _)| header.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// The body as text, with invalid UTF-8 replaced. + pub fn body_text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } +} + +impl fmt::Debug for Response { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Response") + .field("status", &self.status) + .field("headers", &self.headers) + .field("body", &self.body_text()) + .finish() + } +} + +/// The first HTTP response in `bytes`, or none if they don't start with a +/// complete status line and headers. `head` says the request was `HEAD`, so +/// the response has no body. +/// +/// The body ends where chunked encoding or `Content-Length` says, or at the +/// end of `bytes` without either. A truncated body is returned as far as it +/// arrived. +fn parse_response(bytes: &[u8], head: bool) -> Option { + let mut headers = [httparse::EMPTY_HEADER; 64]; + let mut parsed = httparse::Response::new(&mut headers); + let httparse::Status::Complete(end) = parsed.parse(bytes).ok()? else { + return None; + }; + let status = parsed.code?; + let headers = parsed + .headers + .iter() + .map(|header| { + let value = String::from_utf8_lossy(header.value).into_owned(); + (header.name.to_string(), value) + }) + .collect(); + let rest = &bytes[end..]; + let mut response = Response { + status, + headers, + body: Vec::new(), + }; + let bodiless = head || (100..200).contains(&status) || status == 204 || status == 304; + if !bodiless { + response.body = if response + .header("Transfer-Encoding") + .is_some_and(|encoding| encoding.to_ascii_lowercase().contains("chunked")) + { + dechunk(rest) + } else if let Some(length) = response + .header("Content-Length") + .and_then(|length| length.parse::().ok()) + { + rest[..length.min(rest.len())].to_vec() + } else { + rest.to_vec() + }; + } + Some(response) +} + +/// The data in a chunked body, up to the last complete chunk if it is cut +/// short. +fn dechunk(mut rest: &[u8]) -> Vec { + let mut body = Vec::new(); + while let Ok(httparse::Status::Complete((start, size))) = httparse::parse_chunk_size(rest) { + let size = usize::try_from(size).unwrap_or(usize::MAX); + rest = &rest[start..]; + if size == 0 { + break; + } + let available = size.min(rest.len()); + body.extend_from_slice(&rest[..available]); + if available < size { + break; + } + rest = rest.get(size + 2..).unwrap_or_default(); + } + body +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_chunked_response_is_decoded() { + let response = parse_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5;ext=1\r\nhello\r\n6\r\n world\r\n0\r\n\r\n", + false, + ) + .expect("a response"); + assert_eq!(response.body, b"hello world"); + } + + #[test] + fn non_http_or_incomplete_bytes_are_not_a_response() { + assert_eq!(parse_response(b"", false), None); + assert_eq!(parse_response(b"HTTP/1.1 200 OK\r\nContent-", false), None); + assert_eq!( + parse_response(b"\x00\xff\x13\x37not-http\r\n\r\n", false), + None + ); + assert_eq!(parse_response(b"SSH-2.0-OpenSSH\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/1.1 20 OK\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/2 200\r\n\r\n", false), None); + } +} diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs new file mode 100644 index 0000000000..8e0abf9631 --- /dev/null +++ b/tests/support/sandbox-clients/src/probe.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runs [`scripts::TCP_PROBE`] in a sandbox and turns its JSON line into +//! typed results. + +use std::time::Duration; + +use serde::Deserialize; +use testkit::{Result, ResultExt as _, Sandbox}; + +use crate::message::{Http, Request}; +use crate::scripts; + +/// How long a connect may take, as the clients the probes replace allowed. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// How long reading a reply may take in all. +const READ_TIMEOUT: Duration = Duration::from_secs(15); +/// How long [`http_raw`] waits for more bytes before it stops reading. Raw +/// bytes may leave the server waiting for more, so the connection may never +/// close. +const RAW_IDLE_TIMEOUT: Duration = Duration::from_secs(2); + +const EPERM: i32 = 1; +const EACCES: i32 = 13; + +/// How a connect ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConnectOutcome { + Connected, + /// The connect failed with `errno`, or none for an error without one, + /// such as a name that didn't resolve (`dns`). `error` is Python's + /// description of it. + Failed { + errno: Option, + dns: bool, + error: String, + }, + TimedOut, +} + +impl ConnectOutcome { + pub fn is_connected(&self) -> bool { + matches!(self, Self::Connected) + } + + /// The errno a failed connect reported. + pub fn errno(&self) -> Option { + match self { + Self::Failed { errno, .. } => *errno, + Self::Connected | Self::TimedOut => None, + } + } + + /// Whether the connect failed with `EACCES` or `EPERM`, as the sandbox + /// denies a connection. + pub fn is_permission_denied(&self) -> bool { + matches!(self.errno(), Some(EACCES | EPERM)) + } +} + +/// The result of [`connect`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Connect { + pub outcome: ConnectOutcome, + /// How long the connect took, until it connected, failed, or timed out. + pub elapsed: Duration, +} + +/// How reading a reply ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadEnd { + /// The peer closed the connection. + Closed, + /// The peer reset the connection (`ECONNRESET`). + Reset, + /// No bytes arrived for the idle timeout of [`http_raw`]. + Idle, + /// The read timeout ran out before the connection ended. + TimedOut, + /// Sending or reading failed some other way. + Failed { errno: Option, error: String }, +} + +/// Open a TCP connection from `sandbox` to `host:port`, then close it. +/// A connect that fails or times out is a result, not an error. +pub async fn connect(sandbox: &Sandbox, host: &str, port: u16) -> Result { + let probe = run(sandbox, host, port, None, Duration::ZERO).await?; + Ok(Connect { + outcome: probe.connect, + elapsed: probe.connect_elapsed, + }) +} + +/// Send `request` from `sandbox` to `host:port` and read the reply until the +/// server closes the connection. The request asks it to, with +/// `Connection: close`, unless it sets its own `Connection` header. +pub async fn http(sandbox: &Sandbox, host: &str, port: u16, request: &Request) -> Result { + let bytes = request.encode(host, port); + let probe = run(sandbox, host, port, Some(&bytes), Duration::ZERO).await?; + Ok(Http::from_probe(probe, request.is_head())) +} + +/// Send `bytes` as they are from `sandbox` to `host:port`, and read the reply +/// until the connection ends or goes quiet for two seconds. Use it for +/// requests [`Request`] can't make, or bytes that aren't HTTP at all. +pub async fn http_raw(sandbox: &Sandbox, host: &str, port: u16, bytes: &[u8]) -> Result { + let probe = run(sandbox, host, port, Some(bytes), RAW_IDLE_TIMEOUT).await?; + Ok(Http::from_probe(probe, false)) +} + +/// What the probe script reported. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Probe { + pub(crate) connect: ConnectOutcome, + pub(crate) connect_elapsed: Duration, + pub(crate) read_end: Option, + pub(crate) received: Vec, +} + +async fn run( + sandbox: &Sandbox, + host: &str, + port: u16, + payload: Option<&[u8]>, + idle_timeout: Duration, +) -> Result { + let payload = payload.map_or_else(|| "-".to_string(), to_hex); + let args = [ + host.to_string(), + port.to_string(), + payload, + CONNECT_TIMEOUT.as_secs_f64().to_string(), + READ_TIMEOUT.as_secs_f64().to_string(), + idle_timeout.as_secs_f64().to_string(), + ]; + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = sandbox + .run_python(scripts::TCP_PROBE, &args) + .await + .context(format!( + "probe {host}:{port} from sandbox {}", + sandbox.name() + ))?; + parse_probe(output.last_json()?) +} + +#[derive(Deserialize)] +struct ProbeJson { + connect: ConnectJson, + errno: Option, + dns: bool, + error: Option, + connect_ms: u64, + read_end: Option, + read_errno: Option, + read_error: Option, + received_hex: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ConnectJson { + Ok, + Failed, + Timeout, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ReadEndJson { + Closed, + Reset, + Idle, + Timeout, + Failed, +} + +fn parse_probe(value: serde_json::Value) -> Result { + let json: ProbeJson = serde_json::from_value(value.clone()) + .context(format!("read the probe's result {value}"))?; + let connect = match json.connect { + ConnectJson::Ok => ConnectOutcome::Connected, + ConnectJson::Failed => ConnectOutcome::Failed { + errno: json.errno, + dns: json.dns, + error: json.error.unwrap_or_default(), + }, + ConnectJson::Timeout => ConnectOutcome::TimedOut, + }; + let read_end = json.read_end.map(|end| match end { + ReadEndJson::Closed => ReadEnd::Closed, + ReadEndJson::Reset => ReadEnd::Reset, + ReadEndJson::Idle => ReadEnd::Idle, + ReadEndJson::Timeout => ReadEnd::TimedOut, + ReadEndJson::Failed => ReadEnd::Failed { + errno: json.read_errno, + error: json.read_error.clone().unwrap_or_default(), + }, + }); + Ok(Probe { + connect, + connect_elapsed: Duration::from_millis(json.connect_ms), + read_end, + received: from_hex(&json.received_hex)?, + }) +} + +fn to_hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +fn from_hex(hex: &str) -> Result> { + testkit::ensure!( + hex.len().is_multiple_of(2), + "the probe's received_hex has odd length" + ); + (0..hex.len()) + .step_by(2) + .map(|at| { + hex.get(at..at + 2) + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!("the probe's received_hex {hex:?} isn't hex")) + }) + }) + .collect() +} diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 9cc6120620..d655ae68a8 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -63,18 +63,29 @@ fails. A killed test process skips cleanup, so names are unique to the run. `Context` currently provides: -- `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then +- `cx.sandbox(suffix)` to build a sandbox with a run-unique name + (`test--`), then `create()` it or `create_ready()` to also wait for Ready. Set its image - with `image(..)` and its policy with `policy(..)`. The returned + with `image(..)`, its policy with `policy(..)` and its providers with + `providers(..)`. A long-running sandbox is + `.command(["sleep", "infinity"])`; add `.image(testkit::python_image())` + when a test runs Python clients in it. The returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set - a setting, read `sandbox get` JSON with `get_json()`, and delete. Its - `expect_listed`, `expect_absent` and `wait_absent` check list and get. - Exec output has `success()`, `failure()` and `expect_stdout()`. + a setting, read `sandbox get` JSON with `get_json()`, read its policy + revisions and active policy with `policy_revisions()` and + `active_policy()`, and delete. Its `expect_listed`, `expect_absent` and + `wait_absent` check list and get. `run_python(script, args)` runs a script + with `python3 -c` and fails unless it exits 0. Exec output has `success()`, + `failure()`, `expect_stdout()`, and `last_json()` for the last JSON object + or array line of stdout, such as a client script's result. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, and `run()` returns the attached CLI output. +- Sandbox names are limited to `testkit::MAX_SANDBOX_NAME_LEN` (19) + characters, so a sandbox suffix can have at most 5. `cx.sandbox`, + `cx.cli_sandbox` and `cx.track` panic on a longer name. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned @@ -110,7 +121,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. (binaries apply to every endpoint under `name`) that `policy::Endpoint` configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` - for the CLI. + for the CLI. `policy::binaries_yaml(binaries)` writes a `binaries:` list + into policy YAML a test writes by hand. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a @@ -118,6 +130,11 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. +- In `tests/support`: `host-server` starts servers on the machine that runs + the tests, which sandboxes reach as `host_server_support::TEST_HOST`, and + `sandbox-clients` runs clients in a sandbox: `connect`, `http` and + `http_raw` return typed results, and `scripts` holds client scripts for + `run_python`. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. - `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a @@ -125,6 +142,27 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `poll_until` for waiting on other state, and `with_deadline(what, timeout, future)` for a step that must finish sooner than its own timeout. +## Writing a sandbox client + +A test that checks what a sandbox can reach runs its client inside the +sandbox, because the sandbox policy applies only to processes there. A client +script: + +- uses only Python's standard library, and runs with `sandbox.run_python(script, + args)` in a sandbox from `testkit::python_image()`; +- takes its input as arguments; +- prints its result as one JSON line, which the test reads with + `output.last_json()`. A refused connection or a failed read is a result, + not a crash: the script exits non-zero only on a bug; +- leaves parsing and assertions to Rust. + +A script only one test uses can stay beside that test, as a `const` or an +`include_str!`. Move it to `tests/support/sandbox-clients/scripts/`, with a +constant in its `scripts` module, once a second test or suite needs it, so no +suite reads another workspace's files. If its result has a fixed shape, add a +typed probe there too, like `connect` and `http`. `sandbox-clients` holds +network clients; put file-system or system-call probes in a sibling crate. + ## Running against a gateway Tests find the gateway the same way the `openshell` CLI does: diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index d2c4520500..0a671483ae 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -177,12 +177,18 @@ impl Context { } /// Start describing a sandbox named `cx.name(suffix)`. + /// + /// # Panics + /// + /// If the name is longer than the gateway allows; see + /// [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn sandbox(&self, suffix: &str) -> SandboxBuilder { SandboxBuilder { handles: self.handles.clone(), policy: None, spec: SandboxSpec { - name: Some(self.name(suffix)), + name: Some(self.sandbox_name(suffix)), ..SandboxSpec::default() }, } @@ -192,8 +198,20 @@ impl Context { /// `openshell` CLI creates. Use it for what only `sandbox create` /// offers, such as a `--policy` file or `--no-keep`; prefer /// [`Context::sandbox`] otherwise. + /// + /// # Panics + /// + /// As [`Context::sandbox`]. + #[track_caller] pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder { - CliSandboxBuilder::new(self.handles.clone(), self.name(suffix)) + CliSandboxBuilder::new(self.handles.clone(), self.sandbox_name(suffix)) + } + + #[track_caller] + fn sandbox_name(&self, suffix: &str) -> String { + let name = self.name(suffix); + check_sandbox_name(&name); + name } /// Track a sandbox the test is about to create through the CLI, so cleanup @@ -203,7 +221,13 @@ impl Context { /// Cleanup deletes the sandbox and waits until it is gone, at the point /// in the cleanup order where it was first tracked. Tracking a sandbox /// again, for example by adopting it, doesn't move it. + /// + /// # Panics + /// + /// If `name` is longer than [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn track(&self, name: &str) { + check_sandbox_name(name); self.handles.track(name); } @@ -397,6 +421,16 @@ impl SandboxBuilder { self } + /// Attach these providers to the sandbox. + pub fn providers(mut self, providers: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.spec.providers = providers.into_iter().map(Into::into).collect(); + self + } + /// Create the sandbox with this policy instead of the gateway's default. pub fn policy(mut self, policy: proto::SandboxPolicy) -> Self { self.policy = Some(policy); @@ -542,6 +576,19 @@ impl ExecOutput { Ok(self) } + /// The last line of stdout that parses as JSON, such as the result a + /// client script prints after its other output. + pub fn last_json(&self) -> Result { + last_json_line(&self.stdout).ok_or_else(|| { + Error::msg(format!( + "{} printed no JSON line; stdout: {}; stderr: {}", + self.description, + self.stdout.trim(), + self.stderr.trim() + )) + }) + } + fn outcome(&self) -> Outcome<'_> { Outcome { what: &self.description, @@ -552,6 +599,26 @@ impl ExecOutput { } } +/// Only objects and arrays count, so a line that happens to parse, such as +/// a number, isn't taken for the result. +fn last_json_line(stdout: &str) -> Option { + stdout.lines().rev().find_map(|line| { + serde_json::from_str(line.trim()) + .ok() + .filter(|value: &serde_json::Value| value.is_object() || value.is_array()) + }) +} + +/// A script's first line that isn't blank or a license header, which names +/// it in reports instead of its full text. +fn script_summary(script: &str) -> &str { + script + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && !line.starts_with("# SPDX-")) + .unwrap_or_default() +} + impl Sandbox { pub fn name(&self) -> &str { &self.name @@ -725,6 +792,49 @@ impl Sandbox { }) } + /// Run `script` with `python3 -c` and `args`, as in a sandbox from + /// [`crate::python_image`]. Fails unless it exits 0. Read a JSON result + /// with [`ExecOutput::last_json`]. + pub async fn run_python(&self, script: &str, args: &[&str]) -> Result { + let mut command = vec!["python3", "-c", script]; + command.extend_from_slice(args); + let mut output = self.exec(command).await?; + output.description = format!( + "python3 script {:?} with {args:?} in sandbox {}", + script_summary(script), + self.name + ); + let missing_python = + output.exit_code == 127 || output.stderr.contains("python3: not found"); + output.success().map_err(|error| { + if missing_python { + error.context("the sandbox has no python3; create it from testkit::python_image()") + } else { + error + } + }) + } + + /// The sandbox's policy revisions, from `policy list --output json`. + pub async fn policy_revisions(&self) -> Result { + self.handles + .cli + .run(["policy", "list", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// The sandbox's active policy, from `policy get --full --output json`. + pub async fn active_policy(&self) -> Result { + self.handles + .cli + .run(["policy", "get", &self.name, "--full", "--output", "json"]) + .await? + .success()? + .json() + } + /// Run `command` every 250 ms until `ready` accepts its output, and return /// that output. An exec error fails at once; a timeout reports the last /// output. @@ -860,6 +970,22 @@ impl Context { } } +/// The longest sandbox name the gateway accepts, the server's +/// `MAX_ROUTABLE_NAME_LEN` in `crates/openshell-server/src/grpc/mod.rs`. With +/// `test--`, a sandbox suffix can have at most 5 characters. +pub const MAX_SANDBOX_NAME_LEN: usize = 19; + +/// Panic, at the test's call, on a sandbox name the gateway would reject, so +/// the test fails with this message instead of the gateway's. +#[track_caller] +fn check_sandbox_name(name: &str) { + assert!( + name.len() <= MAX_SANDBOX_NAME_LEN, + "sandbox name {name:?} has {} characters; the gateway allows {MAX_SANDBOX_NAME_LEN}, so a suffix after `test--` can have at most 5", + name.len() + ); +} + /// Short, lowercase, and DNS-label safe, so names work on every driver. fn generate_run_id() -> String { const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7c233ce834..e86b7c3b14 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -49,8 +49,8 @@ pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; pub use cli_child::CliChild; pub use cli_sandbox::CliSandboxBuilder; pub use context::{ - Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, - SandboxBuilder, + Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, MAX_SANDBOX_NAME_LEN, READY_TIMEOUT, + STOP_TIMEOUT, Sandbox, SandboxBuilder, }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..c62c930528 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -75,6 +75,18 @@ pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { .context("serialize the policy") } +/// `binaries` as the flow-style list a network policy's `binaries:` key +/// takes in YAML, such as `[{path: "/usr/bin/python*"}]`. Use it in policy +/// YAML a test writes by hand, for example with +/// `binaries_yaml(PYTHON_BINARIES)`. +pub fn binaries_yaml(binaries: &[&str]) -> String { + let paths: Vec = binaries + .iter() + .map(|path| format!("{{path: {}}}", serde_json::Value::from(*path))) + .collect(); + format!("[{}]", paths.join(", ")) +} + /// Settings for an endpoint from [`allow`], named as in policy YAML. pub trait Endpoint { /// `protocol: rest` with `access`.