Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
ac158eb
test(testkit): add providers, run_python, last_json, policy readers a…
matthewgrossman Nov 18, 2026
a9a6c59
test(host-server): add TEST_HOST
matthewgrossman Nov 18, 2026
42c3656
test(support): add sandbox-clients crate with connect and http probes
matthewgrossman Nov 18, 2026
eeb87f9
test(features): use shared helpers in host-gateway
matthewgrossman Nov 18, 2026
77e1a6d
test(suites): drop ready_sandbox wrappers and unify long-running comm…
matthewgrossman Nov 18, 2026
8a6c29a
docs(testkit): list the shared sandbox helpers
matthewgrossman Nov 18, 2026
54c15c9
test(suites): use shared sandbox and provider helpers in cli, cdi and…
matthewgrossman Nov 18, 2026
c11a90d
test: trim unit tests for the shared sandbox helpers
matthewgrossman Nov 18, 2026
c54c9d9
test(features): leave the keycloak refresh test to #5657
matthewgrossman Nov 18, 2026
1a3fdc8
test(testkit): fail early on sandbox names the gateway rejects
matthewgrossman Nov 18, 2026
6eae90b
docs(testkit): document how to write a sandbox client
matthewgrossman Nov 18, 2026
b67830e
test(conformance): use the shared TEST_HOST in credential gating
matthewgrossman Nov 18, 2026
06b770c
fix(support): report DNS failures from the connect probe as dns, not …
matthewgrossman Nov 18, 2026
9ecf6b7
refactor(support): drop the unused is_dns_failure helper
matthewgrossman Nov 18, 2026
e38f314
refactor(support): parse HTTP replies with httparse
matthewgrossman Nov 18, 2026
38d122c
fix(support): report a stalled send from the connect probe as a timeout
matthewgrossman Nov 18, 2026
a81fb32
docs(testkit): link the sandbox client guide and the server name limit
matthewgrossman Nov 18, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,11 @@ by `OPENSHELL_BIN`. See
[tests/testkit/README.md](tests/testkit/README.md) for how to write and run
tests.

To run Python in a testkit sandbox, use `Sandbox::run_python` with a sandbox
from `testkit::python_image()`, not the Python SDK's `exec_python` described
under Python E2E. [Writing a sandbox client](tests/testkit/README.md#writing-a-sandbox-client)
covers scripts that probe the network from inside a sandbox.

The conformance crate also builds `openshell-conformance`, a program that
runs the conformance suite against a gateway without cargo or nextest
(`openshell-conformance list`, `ready`, `run`). It covers only the
Expand Down
20 changes: 7 additions & 13 deletions tests/suites/conformance/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
//! check that the CLI commands for create, get, list, exec, stop, start, and
//! delete work against the same gateway.

use serde_json::Value;
use testkit::Context;

/// Delay before an attached main prints and exits, so the CLI attaches while
Expand Down Expand Up @@ -64,7 +63,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result {
async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result {
let sandbox = cx
.cli_sandbox("cl")
.command(["sh", "-c", "exec sleep infinity"])
.command(["sleep", "infinity"])
.create_ready()
.await?;
let name = sandbox.name().to_string();
Expand Down Expand Up @@ -112,8 +111,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) ->
"sandbox {completed} is not listed after its main exited; sandboxes are kept by default"
);
// The phase can trail the CLI's exit by a status update.
cx.adopt(&completed).await?.wait_finished().await?;
let got = get_json(cx, &completed).await?;
let sandbox = cx.adopt(&completed).await?;
sandbox.wait_finished().await?;
let got = sandbox.get_json().await?;
testkit::ensure!(
got["phase"] == "Completed",
"sandbox {completed} is in phase {}; expected Completed",
Expand All @@ -136,8 +136,9 @@ async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) ->
output.stderr.trim()
);
expect_streamed(&output, "failed-main")?;
cx.adopt(&failed).await?.wait_finished().await?;
let got = get_json(cx, &failed).await?;
let sandbox = cx.adopt(&failed).await?;
sandbox.wait_finished().await?;
let got = sandbox.get_json().await?;
testkit::ensure!(
got["phase"] == "Error" && got["exit_code"] == 7,
"sandbox {failed} is in phase {} with exit code {}; expected Error with 7",
Expand Down Expand Up @@ -198,13 +199,6 @@ fn expect_streamed(output: &testkit::CliOutput, marker: &str) -> testkit::Result
Ok(())
}

async fn get_json(cx: &Context, name: &str) -> testkit::Result<Value> {
cx.cli(["sandbox", "get", name, "--output", "json"])
.await?
.success()?
.json()
}

/// Wait until `sandbox list` no longer shows the sandbox.
async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result {
testkit::poll_until(
Expand Down
20 changes: 10 additions & 10 deletions tests/suites/conformance/src/credential_gating.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,32 +10,31 @@

use std::time::Duration;

use host_server_support::{HttpRecorder, Request, WebSocketRecorder};
use host_server_support::{HttpRecorder, Request, TEST_HOST, WebSocketRecorder};
use testkit::policy::{self, Access, Endpoint as _};
use testkit::sdk::raw::proto;
use testkit::{
Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile,
};

const TEST_HOST: &str = "host.openshell.internal";
const TOKEN_ENV: &str = "E2E_GATING_TOKEN";
const TEST_SECRET: &str = "e2e-gating-secret-value";
const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:";

/// POSTs a body containing the provider's token to `TEST_HOST` on the port
/// in `argv[1]`, directly rather than through a proxy.
/// POSTs a body containing the provider's token to the host in `argv[1]` on
/// the port in `argv[2]`, directly rather than through a proxy.
const BODY_CLIENT: &str = r#"
import os, sys, urllib.request
body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode()
request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"})
request = urllib.request.Request(f"http://{sys.argv[1]}:{sys.argv[2]}/token", data=body, headers={"Content-Type": "text/plain"})
urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read()
"#;

/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one
/// binary frame, and waits briefly for the connection to be closed.
/// Upgrades to WebSocket at the host in `argv[1]` on the port in `argv[2]`,
/// sends one binary frame, and waits briefly for the connection to be closed.
const WEBSOCKET_CLIENT: &str = r#"
import base64, os, socket, sys
host, port = "host.openshell.internal", int(sys.argv[1])
host, port = sys.argv[1], int(sys.argv[2])
with socket.create_connection((host, port), timeout=10) as sock:
key = base64.b64encode(os.urandom(16)).decode()
sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode())
Expand Down Expand Up @@ -336,11 +335,12 @@ async fn expect_rejected(
}
}

/// Run `script` with the image's Python and `port` as its argument.
/// Run `script` with the image's Python, and [`TEST_HOST`] and `port` as its
/// arguments.
async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result {
let port = port.to_string();
sandbox
.exec(["python3", "-c", script, &port])
.exec(["python3", "-c", script, TEST_HOST, &port])
.await?
.success()
.context("run the client in the sandbox")?;
Expand Down
19 changes: 10 additions & 9 deletions tests/suites/conformance/src/exec_cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ const STDIN_LIMIT: usize = 4 * 1024 * 1024;
#[testkit::test]
async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result {
const BYTES: usize = 8 * 1024 * 1024;
let sandbox = ready_sandbox(cx, "lo").await?;
let sandbox = cx
.sandbox("lo")
.command(["sleep", "infinity"])
.create_ready()
.await?;

for stream in ["stdout", "stderr"] {
let script = match stream {
Expand Down Expand Up @@ -88,7 +92,11 @@ async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result
/// read fails the exec.
#[testkit::test]
async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> testkit::Result {
let sandbox = ready_sandbox(cx, "si").await?;
let sandbox = cx
.sandbox("si")
.command(["sleep", "infinity"])
.create_ready()
.await?;
let wc = exec_args(&sandbox, &["wc", "-c"]);

for size in [5, 1024 * 1024, STDIN_LIMIT] {
Expand Down Expand Up @@ -165,13 +173,6 @@ async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> te
Ok(())
}

async fn ready_sandbox(cx: &Context, suffix: &str) -> testkit::Result<Sandbox> {
cx.sandbox(suffix)
.command(["sleep", "infinity"])
.create_ready()
.await
}

/// `sandbox exec` of `command` without a TTY or login shell.
fn exec_args(sandbox: &Sandbox, command: &[&str]) -> Vec<String> {
[
Expand Down
4 changes: 1 addition & 3 deletions tests/suites/conformance/src/policy_advisor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,7 @@ async fn create(
suffix: &str,
policy: Option<proto::SandboxPolicy>,
) -> testkit::Result<Sandbox> {
let mut builder = cx
.sandbox(suffix)
.command(["sh", "-c", "exec sleep infinity"]);
let mut builder = cx.sandbox(suffix).command(["sleep", "infinity"]);
if let Some(policy) = policy {
builder = builder.policy(policy);
}
Expand Down
19 changes: 10 additions & 9 deletions tests/suites/conformance/src/ssh.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate(
cli.gateway().name.is_some(),
"this test needs a named gateway for `sandbox ssh-config`; set OPENSHELL_GATEWAY"
);
let sandbox = ready_sandbox(cx).await?;
let sandbox = cx
.sandbox("hk")
.command(["sleep", "infinity"])
.create_ready()
.await?;
let name = sandbox.name().to_string();
let original = fingerprint(cx, &sandbox).await?;

Expand Down Expand Up @@ -110,7 +114,11 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate(
.wait_deleted(&name, testkit::DELETE_TIMEOUT, Some(sandbox.id()))
.await
.context(format!("wait for sandbox {name} to be deleted"))?;
let replacement = ready_sandbox(cx).await?;
let replacement = cx
.sandbox("hk")
.command(["sleep", "infinity"])
.create_ready()
.await?;
let replaced = fingerprint(cx, &replacement).await?;
testkit::ensure!(
replaced != original,
Expand All @@ -119,13 +127,6 @@ async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate(
Ok(())
}

async fn ready_sandbox(cx: &Context) -> testkit::Result<Sandbox> {
cx.sandbox("hk")
.command(["sleep", "infinity"])
.create_ready()
.await
}

/// The sandbox's host key fingerprint, the same in get and list.
async fn fingerprint(cx: &Context, sandbox: &Sandbox) -> testkit::Result<String> {
let name = sandbox.name();
Expand Down
21 changes: 4 additions & 17 deletions tests/suites/features/cdi/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -374,26 +374,13 @@ pub async fn scenario(cx: &Context, name: &str) -> Result {
let Scenario { device, checks } = load(name)?;
let driver = compute_driver(cx).await?;

let sandbox = cx.name("cdi");
cx.track(&sandbox);
let config = json!({driver: {"cdi_devices": [device]}}).to_string();
let probe = probe_script(&checks);
let output = cx
.cli([
"sandbox",
"create",
"--name",
&sandbox,
"--no-tty",
"--gpu",
"1",
"--driver-config-json",
&config,
"--",
"sh",
"-c",
&probe,
])
.cli_sandbox("cdi")
.args(["--gpu", "1", "--driver-config-json", &config])
.command(["sh", "-c", &probe])
.run()
.await?;

let output = output
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ mod linux {

use nix::sys::signal::{Signal, kill};
use nix::unistd::Pid;
use testkit::{Context, ResultExt as _, Sandbox};
use testkit::{Context, ResultExt as _};

use super::ATTACH_TIMEOUT;

Expand All @@ -75,7 +75,11 @@ mod linux {
const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#;

pub(super) async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result {
let sandbox = ready_sandbox(cx, "tr", &["sh", "-lc", ECHO_MAIN]).await?;
let sandbox = cx
.sandbox("tr")
.command(["sh", "-lc", ECHO_MAIN])
.create_ready()
.await?;
let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?;
let first = connect.stdout_line(ATTACH_TIMEOUT).await?;
testkit::ensure!(
Expand Down Expand Up @@ -120,7 +124,11 @@ mod linux {
pub(super) async fn connect_forwards_pid_targeted_termination_and_reaps_ssh(
cx: &mut Context,
) -> testkit::Result {
let sandbox = ready_sandbox(cx, "term", &["sh", "-lc", "exec sleep infinity"]).await?;
let sandbox = cx
.sandbox("term")
.command(["sh", "-lc", "exec sleep infinity"])
.create_ready()
.await?;
let pty = nix::pty::openpty(None, None)
.map_err(std::io::Error::from)
.context("open a pseudo-terminal")?;
Expand Down Expand Up @@ -158,17 +166,6 @@ mod linux {
.await
}

async fn ready_sandbox(
cx: &Context,
suffix: &str,
command: &[&str],
) -> testkit::Result<Sandbox> {
cx.sandbox(suffix)
.command(command.iter().copied())
.create_ready()
.await
}

fn send(pid: u32, signal: Signal) -> testkit::Result {
let pid = i32::try_from(pid)
.map_err(|error| testkit::Error::msg(format!("pid {pid}: {error}")))?;
Expand Down
18 changes: 6 additions & 12 deletions tests/suites/features/host-gateway/tests/host_gateway.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
//! Python test image ([`testkit::python_image`]) that requests a page from that
//! server through the name.

use host_server_support::HttpRecorder;
use host_server_support::{HttpRecorder, TEST_HOST};
use testkit::policy;
use testkit::sdk::raw::proto;
use testkit::{Context, PYTHON_BINARIES, ResultExt as _};
Expand All @@ -20,14 +20,14 @@ use testkit::{Context, PYTHON_BINARIES, ResultExt as _};
/// behind a symlink.
const PYTHON: &str = "python3";

/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and
/// Requests `/` from the host named in `argv[1]` on the port in `argv[2]` and
/// prints the response. The server listens before the sandbox is created,
/// and the network is up before the main process starts, so the client
/// doesn't retry.
const CLIENT: &str = r#"
import socket, sys

host, port = "host.openshell.internal", int(sys.argv[1])
host, port = sys.argv[1], int(sys.argv[2])
with socket.create_connection((host, port), timeout=10) as conn:
conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode())
response = b""
Expand All @@ -54,7 +54,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result
.cli_sandbox("hg")
.image(image.clone())
.policy_yaml(policy::to_yaml(&policy(server.port))?)
.command([PYTHON, "-c", CLIENT, &server.port.to_string()])
.command([PYTHON, "-c", CLIENT, TEST_HOST, &server.port.to_string()])
.run()
.await?
.success()
Expand All @@ -71,17 +71,11 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result
}

/// A sandbox policy that allows the image's Python ([`PYTHON`]) to
/// connect to `host.openshell.internal` on `port`. The endpoint names only the
/// connect to [`TEST_HOST`] on `port`. The endpoint names only the
/// host and port, as the policy docs describe, so the driver's address for the
/// name is what makes it reachable, not an `allowed_ips` range.
fn policy(port: u16) -> proto::SandboxPolicy {
let mut policy = policy::base();
policy::allow(
&mut policy,
"host_server",
"host.openshell.internal",
port,
PYTHON_BINARIES,
);
policy::allow(&mut policy, "host_server", TEST_HOST, port, PYTHON_BINARIES);
policy
}
10 changes: 10 additions & 0 deletions tests/support/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/support/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
# environment runs. Suites depend on them by path.
[workspace]
resolver = "2"
members = ["container-engine", "host-server", "keycloak", "podman"]
members = ["container-engine", "host-server", "keycloak", "podman", "sandbox-clients"]
3 changes: 3 additions & 0 deletions tests/support/host-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ use tokio::net::TcpListener;
pub use http::{HttpRecorder, Request};
pub use websocket::WebSocketRecorder;

/// The name a sandbox reaches these servers by.
pub const TEST_HOST: &str = "host.openshell.internal";

const WAIT_TIMEOUT: Duration = Duration::from_secs(10);
const WAIT_INTERVAL: Duration = Duration::from_millis(100);
const READ_TIMEOUT: Duration = Duration::from_secs(10);
Expand Down
Loading
Loading